Merge branch 'master' into master

backend/mempool-config.sample.json

@@ -68,7 +68,8 @@
     "DATABASE": "mempool",
     "USERNAME": "mempool",
     "PASSWORD": "mempool",
-    "TIMEOUT": 180000
+    "TIMEOUT": 180000,
+    "PID_DIR": ""
   },
   "SYSLOG": {
     "ENABLED": true,

backend/src/__fixtures__/mempool-config.template.json

@@ -69,6 +69,7 @@
     "DATABASE": "__DATABASE_DATABASE__",
     "USERNAME": "__DATABASE_USERNAME__",
     "PASSWORD": "__DATABASE_PASSWORD__",
+    "PID_DIR": "__DATABASE_PID_FILE__",
     "TIMEOUT": 3000
   },
   "SYSLOG": {

backend/src/__tests__/config.test.ts

@@ -84,6 +84,7 @@ describe('Mempool Backend Config', () => {
         USERNAME: 'mempool',
         PASSWORD: 'mempool',
         TIMEOUT: 180000,
+        PID_DIR: ''
       });
 
       expect(config.SYSLOG).toStrictEqual({

backend/src/api/database-migration.ts

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 65;
+  private static currentVersion = 66;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -553,6 +553,11 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `blocks_audits` ADD accelerated_txs JSON DEFAULT "[]"');
       await this.updateToSchemaVersion(65);
     }
+
+    if (databaseSchemaVersion < 66) {
+      await this.$executeQuery('ALTER TABLE `statistics` ADD min_fee FLOAT UNSIGNED DEFAULT NULL');
+      await this.updateToSchemaVersion(66);
+    }
   }
 
   /**

backend/src/api/statistics/statistics-api.ts

@@ -15,6 +15,7 @@ class StatisticsApi {
               mempool_byte_weight,
               fee_data,
               total_fee,
+              min_fee,
               vsize_1,
               vsize_2,
               vsize_3,
@@ -54,7 +55,7 @@ class StatisticsApi {
               vsize_1800,
               vsize_2000
             )
-            VALUES (NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+            VALUES (NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)`;
       const [result]: any = await DB.query(query);
       return result.insertId;
@@ -73,6 +74,7 @@ class StatisticsApi {
               mempool_byte_weight,
               fee_data,
               total_fee,
+              min_fee,
               vsize_1,
               vsize_2,
               vsize_3,
@@ -112,7 +114,7 @@ class StatisticsApi {
               vsize_1800,
               vsize_2000
             )
-            VALUES (${statistics.added}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
+            VALUES (${statistics.added}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
                ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
 
       const params: (string | number)[] = [
@@ -122,6 +124,7 @@ class StatisticsApi {
         statistics.mempool_byte_weight,
         statistics.fee_data,
         statistics.total_fee,
+        statistics.min_fee,
         statistics.vsize_1,
         statistics.vsize_2,
         statistics.vsize_3,
@@ -173,6 +176,7 @@ class StatisticsApi {
       UNIX_TIMESTAMP(added) as added,
       CAST(avg(unconfirmed_transactions) as DOUBLE) as unconfirmed_transactions,
       CAST(avg(vbytes_per_second) as DOUBLE) as vbytes_per_second,
+      CAST(avg(min_fee) as DOUBLE) as min_fee,
       CAST(avg(vsize_1) as DOUBLE) as vsize_1,
       CAST(avg(vsize_2) as DOUBLE) as vsize_2,
       CAST(avg(vsize_3) as DOUBLE) as vsize_3,
@@ -222,6 +226,7 @@ class StatisticsApi {
       UNIX_TIMESTAMP(added) as added,
       CAST(avg(unconfirmed_transactions) as DOUBLE) as unconfirmed_transactions,
       CAST(avg(vbytes_per_second) as DOUBLE) as vbytes_per_second,
+      CAST(avg(min_fee) as DOUBLE) as min_fee,
       vsize_1,
       vsize_2,
       vsize_3,
@@ -407,6 +412,7 @@ class StatisticsApi {
         vbytes_per_second: s.vbytes_per_second,
         mempool_byte_weight: s.mempool_byte_weight,
         total_fee: s.total_fee,
+        min_fee: s.min_fee,
         vsizes: [
           s.vsize_1,
           s.vsize_2,

backend/src/api/statistics/statistics.ts

@@ -89,6 +89,9 @@ class Statistics {
       }
     });
 
+    // get minFee and convert to sats/vb
+    const minFee = memPool.getMempoolInfo().mempoolminfee * 100000;
+
     try {
       const insertId = await statisticsApi.$create({
         added: 'NOW()',
@@ -98,6 +101,7 @@ class Statistics {
         mempool_byte_weight: totalWeight,
         total_fee: totalFee,
         fee_data: '',
+        min_fee: minFee,
         vsize_1: weightVsizeFees['1'] || 0,
         vsize_2: weightVsizeFees['2'] || 0,
         vsize_3: weightVsizeFees['3'] || 0,

backend/src/config.ts

@@ -93,6 +93,7 @@ interface IConfig {
     USERNAME: string;
     PASSWORD: string;
     TIMEOUT: number;
+    PID_DIR: string;
   };
   SYSLOG: {
     ENABLED: boolean;
@@ -219,6 +220,7 @@ const defaults: IConfig = {
     'USERNAME': 'mempool',
     'PASSWORD': 'mempool',
     'TIMEOUT': 180000,
+    'PID_DIR': '',
   },
   'SYSLOG': {
     'ENABLED': true,

backend/src/database.ts

@@ -1,3 +1,5 @@
+import * as fs from 'fs';
+import path from 'path';
 import config from './config';
 import { createPool, Pool, PoolConnection } from 'mysql2/promise';
 import logger from './logger';
@@ -101,6 +103,33 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr
     }
   }
 
+  public getPidLock(): boolean {
+    const filePath = path.join(config.DATABASE.PID_DIR || __dirname, `/mempool-${config.DATABASE.DATABASE}.pid`);
+    if (fs.existsSync(filePath)) {
+      const pid = fs.readFileSync(filePath).toString();
+      if (pid !== `${process.pid}`) {
+        const msg = `Already running on PID ${pid} (or pid file '${filePath}' is stale)`;
+        logger.err(msg);
+        throw new Error(msg);
+      } else {
+        return true;
+      }
+    } else {
+      fs.writeFileSync(filePath, `${process.pid}`);
+      return true;
+    }
+  }
+
+  public releasePidLock(): void {
+    const filePath = path.join(config.DATABASE.PID_DIR || __dirname, `/mempool-${config.DATABASE.DATABASE}.pid`);
+    if (fs.existsSync(filePath)) {
+      const pid = fs.readFileSync(filePath).toString();
+      if (pid === `${process.pid}`) {
+        fs.unlinkSync(filePath);
+      }
+    }
+  }
+
   private async getPool(): Promise<Pool> {
     if (this.pool === null) {
       this.pool = createPool(this.poolConfig);

backend/src/index.ts

@@ -91,11 +91,18 @@ class Server {
   async startServer(worker = false): Promise<void> {
     logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);
 
+    // Register cleanup listeners for exit events
+    ['exit', 'SIGINT', 'SIGTERM', 'SIGUSR1', 'SIGUSR2', 'uncaughtException', 'unhandledRejection'].forEach(event => {
+      process.on(event, () => { this.onExit(event); });
+    });
+
     if (config.MEMPOOL.BACKEND === 'esplora') {
       bitcoinApi.startHealthChecks();
     }
 
     if (config.DATABASE.ENABLED) {
+      DB.getPidLock();
+
       await DB.checkDbConnection();
       try {
         if (process.env.npm_config_reindex_blocks === 'true') { // Re-index requests
@@ -306,6 +313,15 @@ class Server {
       this.lastHeapLogTime = now;
     }
   }
+
+  onExit(exitEvent): void {
+    if (config.DATABASE.ENABLED) {
+      DB.releasePidLock();
+    }
+    process.exit(0);
+  }
 }
 
+
+
 ((): Server => new Server())();

backend/src/mempool.interfaces.ts

@@ -300,6 +300,7 @@ export interface Statistic {
   total_fee: number;
   mempool_byte_weight: number;
   fee_data: string;
+  min_fee: number;
 
   vsize_1: number;
   vsize_2: number;
@@ -346,6 +347,7 @@ export interface OptimizedStatistic {
   vbytes_per_second: number;
   total_fee: number;
   mempool_byte_weight: number;
+  min_fee: number;
   vsizes: number[];
 }
 

docker/backend/mempool-config.json

@@ -69,7 +69,8 @@
     "DATABASE": "__DATABASE_DATABASE__",
     "USERNAME": "__DATABASE_USERNAME__",
     "PASSWORD": "__DATABASE_PASSWORD__",
-    "TIMEOUT": __DATABASE_TIMEOUT__
+    "TIMEOUT": __DATABASE_TIMEOUT__,
+    "PID_DIR": "__DATABASE_PID_DIR__",
   },
   "SYSLOG": {
     "ENABLED": __SYSLOG_ENABLED__,

docker/backend/start.sh

@@ -71,6 +71,7 @@ __DATABASE_DATABASE__=${DATABASE_DATABASE:=mempool}
 __DATABASE_USERNAME__=${DATABASE_USERNAME:=mempool}
 __DATABASE_PASSWORD__=${DATABASE_PASSWORD:=mempool}
 __DATABASE_TIMEOUT__=${DATABASE_TIMEOUT:=180000}
+__DATABASE_PID_DIR__=${DATABASE_PID_DIR:=""}
 
 # SYSLOG
 __SYSLOG_ENABLED__=${SYSLOG_ENABLED:=false}
@@ -209,6 +210,7 @@ sed -i "s!__DATABASE_DATABASE__!${__DATABASE_DATABASE__}!g" mempool-config.json
 sed -i "s!__DATABASE_USERNAME__!${__DATABASE_USERNAME__}!g" mempool-config.json
 sed -i "s!__DATABASE_PASSWORD__!${__DATABASE_PASSWORD__}!g" mempool-config.json
 sed -i "s!__DATABASE_TIMEOUT__!${__DATABASE_TIMEOUT__}!g" mempool-config.json
+sed -i "s!__DATABASE_PID_DIR__!${__DATABASE_PID_DIR__}!g" mempool-config.json
 
 sed -i "s!__SYSLOG_ENABLED__!${__SYSLOG_ENABLED__}!g" mempool-config.json
 sed -i "s!__SYSLOG_HOST__!${__SYSLOG_HOST__}!g" mempool-config.json

frontend/src/app/components/incoming-transactions-graph/incoming-transactions-graph.component.ts

@@ -37,6 +37,7 @@ export class IncomingTransactionsGraphComponent implements OnInit, OnChanges, On
   };
   windowPreference: string;
   chartInstance: any = undefined;
+  MA: number[][] = [];
   weightMode: boolean = false;
   rateUnitSub: Subscription;
 
@@ -62,6 +63,7 @@ export class IncomingTransactionsGraphComponent implements OnInit, OnChanges, On
       return;
     }
     this.windowPreference = this.windowPreferenceOverride ? this.windowPreferenceOverride : this.storageService.getValue('graphWindowPreference');
+    this.MA = this.calculateMA(this.data.series[0]);
     this.mountChart();
   }
 
@@ -72,7 +74,101 @@ export class IncomingTransactionsGraphComponent implements OnInit, OnChanges, On
     this.isLoading = false;
   }
 
+  /// calculate the moving average of maData
+  calculateMA(maData): number[][] {
+    //update const variables that are not changed
+    const ma: number[][] = [];
+    let sum = 0;
+    let i = 0;
+    const len = maData.length;
+
+    //Adjust window length based on the length of the data
+    //5% appeared as a good amount from tests
+    //TODO: make this a text box in the UI
+    const maWindowLen = Math.ceil(len * 0.05);
+
+    //calculate the center of the moving average window
+    const center = Math.floor(maWindowLen / 2);
+
+    //calculate the centered moving average
+    for (i = center; i < len - center; i++) {
+      sum = 0;
+      //build out ma as we loop through the data
+      ma[i] = [];
+      ma[i].push(maData[i][0]);
+      for (let j = i - center; j <= i + center; j++) {
+        sum += maData[j][1];
+      }
+
+      ma[i].push(sum / maWindowLen);
+    }
+
+    //return the moving average array
+    return ma;
+  }
+
   mountChart(): void {
+    //create an array for the echart series
+    //similar to how it is done in mempool-graph.component.ts
+    const seriesGraph = [];
+    seriesGraph.push({
+      zlevel: 0,
+      name: 'data',
+      data: this.data.series[0],
+      type: 'line',
+      smooth: false,
+      showSymbol: false,
+      symbol: 'none',
+      lineStyle: {
+        width: 3,
+      },
+      markLine: {
+        silent: true,
+        symbol: 'none',
+        lineStyle: {
+          color: '#fff',
+          opacity: 1,
+          width: 2,
+        },
+        data: [{
+          yAxis: 1667,
+          label: {
+            show: false,
+            color: '#ffffff',
+          }
+        }],
+      }
+    },
+    {
+      zlevel: 0,
+      name: 'MA',
+      data: this.MA,
+      type: 'line',
+      smooth: false,
+      showSymbol: false,
+      symbol: 'none',
+      lineStyle: {
+        width: 1,
+        color: "white",
+      },
+      markLine: {
+        silent: true,
+        symbol: 'none',
+        lineStyle: {
+          color: '#fff',
+          opacity: 1,
+          width: 2,
+        },
+        data: [{
+          yAxis: 1667,
+          label: {
+            show: false,
+            color: '#ffffff',
+          }
+        }],
+      }
+    });
+
     this.mempoolStatsChartOption = {
       grid: {
         height: this.height,
@@ -126,13 +222,17 @@ export class IncomingTransactionsGraphComponent implements OnInit, OnChanges, On
           const colorSpan = (color: string) => `<span class="indicator" style="background-color: ` + color + `"></span>`;
           let itemFormatted = '<div class="title">' + axisValueLabel + '</div>';
           params.map((item: any, index: number) => {
+
+            //Do no include MA in tooltip legend!
+            if (item.seriesName !== 'MA') {
               if (index < 26) {
                 itemFormatted += `<div class="item">
                   <div class="indicator-container">${colorSpan(item.color)}</div>
                   <div class="grow"></div>
-                <div class="value">${formatNumber(this.weightMode ? item.value[1] * 4 : item.value[1], this.locale, '1.0-0')} <span class="symbol">${this.weightMode ? 'WU' : 'vB'}/s</span></div>
+                  <div class="value">${formatNumber(item.value[1], this.locale, '1.0-0')}<span class="symbol">vB/s</span></div>
                 </div>`;
               }
+            }
           });
           return `<div class="tx-wrapper-tooltip-chart ${(this.template === 'advanced') ? 'tx-wrapper-tooltip-chart-advanced' : ''}">${itemFormatted}</div>`;
         }
@@ -171,35 +271,7 @@ export class IncomingTransactionsGraphComponent implements OnInit, OnChanges, On
           }
         }
       },
-      series: [
+      series: seriesGraph,
-        {
-          zlevel: 0,
-          data: this.data.series[0],
-          type: 'line',
-          smooth: false,
-          showSymbol: false,
-          symbol: 'none',
-          lineStyle: {
-            width: 3,
-          },
-          markLine: {
-            silent: true,
-            symbol: 'none',
-            lineStyle: {
-              color: '#fff',
-              opacity: 1,
-              width: 2,
-            },
-            data: [{
-              yAxis: 1667,
-              label: {
-                show: false,
-                color: '#ffffff',
-              }
-            }],
-          }
-        },
-      ],
       visualMap: {
         show: false,
         top: 50,