From f295392dcab8c2b8c7beef54503c8cb5790e9578 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 15 Aug 2023 20:54:03 +0900 Subject: [PATCH 1/5] Record purging rate in statistics --- backend/src/api/database-migration.ts | 7 ++++++- backend/src/api/statistics/statistics-api.ts | 10 ++++++++-- backend/src/api/statistics/statistics.ts | 4 ++++ backend/src/mempool.interfaces.ts | 2 ++ 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index b7dc39493..89ef7a7be 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 65; + private static currentVersion = 66; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -553,6 +553,11 @@ class DatabaseMigration { await this.$executeQuery('ALTER TABLE `blocks_audits` ADD accelerated_txs JSON DEFAULT "[]"'); await this.updateToSchemaVersion(65); } + + if (databaseSchemaVersion < 66) { + await this.$executeQuery('ALTER TABLE `statistics` ADD min_fee FLOAT UNSIGNED DEFAULT NULL'); + await this.updateToSchemaVersion(66); + } } /** diff --git a/backend/src/api/statistics/statistics-api.ts b/backend/src/api/statistics/statistics-api.ts index 9df12d704..d76b77a37 100644 --- a/backend/src/api/statistics/statistics-api.ts +++ b/backend/src/api/statistics/statistics-api.ts @@ -15,6 +15,7 @@ class StatisticsApi { mempool_byte_weight, fee_data, total_fee, + min_fee, vsize_1, vsize_2, vsize_3, @@ -54,7 +55,7 @@ class StatisticsApi { vsize_1800, vsize_2000 ) - VALUES (NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + VALUES (NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)`; const [result]: any = await DB.query(query); return result.insertId; @@ -73,6 +74,7 @@ class StatisticsApi { mempool_byte_weight, fee_data, total_fee, + min_fee, vsize_1, vsize_2, vsize_3, @@ -112,7 +114,7 @@ class StatisticsApi { vsize_1800, vsize_2000 ) - VALUES (${statistics.added}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + VALUES (${statistics.added}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`; const params: (string | number)[] = [ @@ -122,6 +124,7 @@ class StatisticsApi { statistics.mempool_byte_weight, statistics.fee_data, statistics.total_fee, + statistics.min_fee, statistics.vsize_1, statistics.vsize_2, statistics.vsize_3, @@ -172,6 +175,7 @@ class StatisticsApi { return `SELECT UNIX_TIMESTAMP(added) as added, CAST(avg(vbytes_per_second) as DOUBLE) as vbytes_per_second, + CAST(avg(min_fee) as DOUBLE) as min_fee, CAST(avg(vsize_1) as DOUBLE) as vsize_1, CAST(avg(vsize_2) as DOUBLE) as vsize_2, CAST(avg(vsize_3) as DOUBLE) as vsize_3, @@ -220,6 +224,7 @@ class StatisticsApi { return `SELECT UNIX_TIMESTAMP(added) as added, CAST(avg(vbytes_per_second) as DOUBLE) as vbytes_per_second, + CAST(avg(min_fee) as DOUBLE) as min_fee, vsize_1, vsize_2, vsize_3, @@ -404,6 +409,7 @@ class StatisticsApi { vbytes_per_second: s.vbytes_per_second, mempool_byte_weight: s.mempool_byte_weight, total_fee: s.total_fee, + min_fee: s.min_fee, vsizes: [ s.vsize_1, s.vsize_2, diff --git a/backend/src/api/statistics/statistics.ts b/backend/src/api/statistics/statistics.ts index 27554f36d..494777aad 100644 --- a/backend/src/api/statistics/statistics.ts +++ b/backend/src/api/statistics/statistics.ts @@ -89,6 +89,9 @@ class Statistics { } }); + // get minFee and convert to sats/vb + const minFee = memPool.getMempoolInfo().mempoolminfee * 100000; + try { const insertId = await statisticsApi.$create({ added: 'NOW()', @@ -98,6 +101,7 @@ class Statistics { mempool_byte_weight: totalWeight, total_fee: totalFee, fee_data: '', + min_fee: minFee, vsize_1: weightVsizeFees['1'] || 0, vsize_2: weightVsizeFees['2'] || 0, vsize_3: weightVsizeFees['3'] || 0, diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index c08846191..db04ded43 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -299,6 +299,7 @@ export interface Statistic { total_fee: number; mempool_byte_weight: number; fee_data: string; + min_fee: number; vsize_1: number; vsize_2: number; @@ -345,6 +346,7 @@ export interface OptimizedStatistic { vbytes_per_second: number; total_fee: number; mempool_byte_weight: number; + min_fee: number; vsizes: number[]; } From 7dad00523ff53a9ad49730c3421b880bdaecc979 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 28 Aug 2023 19:20:58 +0900 Subject: [PATCH 2/5] Implement pid file & checks --- backend/mempool-config.sample.json | 3 ++- backend/src/__tests__/config.test.ts | 1 + backend/src/config.ts | 2 ++ backend/src/database.ts | 29 ++++++++++++++++++++++++++++ backend/src/index.ts | 14 ++++++++++++++ docker/backend/mempool-config.json | 3 ++- docker/backend/start.sh | 2 ++ 7 files changed, 52 insertions(+), 2 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 00fe95cc5..4bbf6cfad 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -68,7 +68,8 @@ "DATABASE": "mempool", "USERNAME": "mempool", "PASSWORD": "mempool", - "TIMEOUT": 180000 + "TIMEOUT": 180000, + "PID_DIR": "" }, "SYSLOG": { "ENABLED": true, diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 8097a2465..1a21cd99b 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -84,6 +84,7 @@ describe('Mempool Backend Config', () => { USERNAME: 'mempool', PASSWORD: 'mempool', TIMEOUT: 180000, + PID_DIR: '' }); expect(config.SYSLOG).toStrictEqual({ diff --git a/backend/src/config.ts b/backend/src/config.ts index ed320d957..9ded762fa 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -93,6 +93,7 @@ interface IConfig { USERNAME: string; PASSWORD: string; TIMEOUT: number; + PID_DIR: string; }; SYSLOG: { ENABLED: boolean; @@ -219,6 +220,7 @@ const defaults: IConfig = { 'USERNAME': 'mempool', 'PASSWORD': 'mempool', 'TIMEOUT': 180000, + 'PID_DIR': '', }, 'SYSLOG': { 'ENABLED': true, diff --git a/backend/src/database.ts b/backend/src/database.ts index 6ad545fda..c27f28d23 100644 --- a/backend/src/database.ts +++ b/backend/src/database.ts @@ -1,3 +1,5 @@ +import * as fs from 'fs'; +import path from 'path'; import config from './config'; import { createPool, Pool, PoolConnection } from 'mysql2/promise'; import logger from './logger'; @@ -101,6 +103,33 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr } } + public getPidLock(): boolean { + const filePath = path.join(config.DATABASE.PID_DIR || __dirname, `/mempool-${config.DATABASE.DATABASE}.pid`); + if (fs.existsSync(filePath)) { + const pid = fs.readFileSync(filePath).toString(); + if (pid !== `${process.pid}`) { + const msg = `Already running on PID ${pid} (or pid file '${filePath}' is stale)`; + logger.err(msg); + throw new Error(msg); + } else { + return true; + } + } else { + fs.writeFileSync(filePath, `${process.pid}`); + return true; + } + } + + public releasePidLock(): void { + const filePath = path.join(config.DATABASE.PID_DIR || __dirname, `/mempool-${config.DATABASE.DATABASE}.pid`); + if (fs.existsSync(filePath)) { + const pid = fs.readFileSync(filePath).toString(); + if (pid === `${process.pid}`) { + fs.unlinkSync(filePath); + } + } + } + private async getPool(): Promise { if (this.pool === null) { this.pool = createPool(this.poolConfig); diff --git a/backend/src/index.ts b/backend/src/index.ts index 9d0fa07f5..a8af35d1c 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -91,11 +91,18 @@ class Server { async startServer(worker = false): Promise { logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`); + // Register cleanup listeners for exit events + ['exit', 'SIGINT', 'SIGTERM', 'SIGUSR1', 'SIGUSR2', 'uncaughtException', 'unhandledRejection'].forEach(event => { + process.on(event, () => { this.onExit(event); }); + }); + if (config.MEMPOOL.BACKEND === 'esplora') { bitcoinApi.startHealthChecks(); } if (config.DATABASE.ENABLED) { + DB.getPidLock(); + await DB.checkDbConnection(); try { if (process.env.npm_config_reindex_blocks === 'true') { // Re-index requests @@ -306,6 +313,13 @@ class Server { this.lastHeapLogTime = now; } } + + onExit(exitEvent): void { + DB.releasePidLock(); + process.exit(0); + } } + + ((): Server => new Server())(); diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index aa084133f..5642b36d3 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -69,7 +69,8 @@ "DATABASE": "__DATABASE_DATABASE__", "USERNAME": "__DATABASE_USERNAME__", "PASSWORD": "__DATABASE_PASSWORD__", - "TIMEOUT": __DATABASE_TIMEOUT__ + "TIMEOUT": __DATABASE_TIMEOUT__, + "PID_DIR": "__PID_DIR__", }, "SYSLOG": { "ENABLED": __SYSLOG_ENABLED__, diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 1bfb06c24..938a5c26f 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -71,6 +71,7 @@ __DATABASE_DATABASE__=${DATABASE_DATABASE:=mempool} __DATABASE_USERNAME__=${DATABASE_USERNAME:=mempool} __DATABASE_PASSWORD__=${DATABASE_PASSWORD:=mempool} __DATABASE_TIMEOUT__=${DATABASE_TIMEOUT:=180000} +__DATABASE_PID_DIR__=${DATABASE_PID_DIR:=""} # SYSLOG __SYSLOG_ENABLED__=${SYSLOG_ENABLED:=false} @@ -209,6 +210,7 @@ sed -i "s!__DATABASE_DATABASE__!${__DATABASE_DATABASE__}!g" mempool-config.json sed -i "s!__DATABASE_USERNAME__!${__DATABASE_USERNAME__}!g" mempool-config.json sed -i "s!__DATABASE_PASSWORD__!${__DATABASE_PASSWORD__}!g" mempool-config.json sed -i "s!__DATABASE_TIMEOUT__!${__DATABASE_TIMEOUT__}!g" mempool-config.json +sed -i "s!__DATABASE_PID_DIR__!${__DATABASE_PID_DIR__}!g" mempool-config.json sed -i "s!__SYSLOG_ENABLED__!${__SYSLOG_ENABLED__}!g" mempool-config.json sed -i "s!__SYSLOG_HOST__!${__SYSLOG_HOST__}!g" mempool-config.json From 0808640ec0d346a427cf87a07a8dc8317bfff0c1 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 1 Oct 2023 16:38:45 +0100 Subject: [PATCH 3/5] Check database enabled before releasing PID lock file --- backend/src/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index a8af35d1c..e7e1afa3d 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -315,7 +315,9 @@ class Server { } onExit(exitEvent): void { - DB.releasePidLock(); + if (config.DATABASE.ENABLED) { + DB.releasePidLock(); + } process.exit(0); } } From 75c50416f5b0b6fd603cecbfebf6d84bfd99d0bf Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 1 Oct 2023 16:43:21 +0100 Subject: [PATCH 4/5] Fix PID docker config --- backend/src/__fixtures__/mempool-config.template.json | 1 + docker/backend/mempool-config.json | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 1b6c8d411..652536d7a 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -69,6 +69,7 @@ "DATABASE": "__DATABASE_DATABASE__", "USERNAME": "__DATABASE_USERNAME__", "PASSWORD": "__DATABASE_PASSWORD__", + "PID_DIR": "__DATABASE_PID_FILE__", "TIMEOUT": 3000 }, "SYSLOG": { diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 5642b36d3..457eccd4a 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -70,7 +70,7 @@ "USERNAME": "__DATABASE_USERNAME__", "PASSWORD": "__DATABASE_PASSWORD__", "TIMEOUT": __DATABASE_TIMEOUT__, - "PID_DIR": "__PID_DIR__", + "PID_DIR": "__DATABASE_PID_DIR__", }, "SYSLOG": { "ENABLED": __SYSLOG_ENABLED__, From 76621464f12e272edbc364ee41f2f5a97eb28caf Mon Sep 17 00:00:00 2001 From: Peter Date: Wed, 11 Oct 2023 08:09:06 -0400 Subject: [PATCH 5/5] ma vbytes --- .../incoming-transactions-graph.component.ts | 144 +++++++++++++----- 1 file changed, 108 insertions(+), 36 deletions(-) diff --git a/frontend/src/app/components/incoming-transactions-graph/incoming-transactions-graph.component.ts b/frontend/src/app/components/incoming-transactions-graph/incoming-transactions-graph.component.ts index 219811e9c..3b93cb686 100644 --- a/frontend/src/app/components/incoming-transactions-graph/incoming-transactions-graph.component.ts +++ b/frontend/src/app/components/incoming-transactions-graph/incoming-transactions-graph.component.ts @@ -37,6 +37,7 @@ export class IncomingTransactionsGraphComponent implements OnInit, OnChanges, On }; windowPreference: string; chartInstance: any = undefined; + MA: number[][] = []; weightMode: boolean = false; rateUnitSub: Subscription; @@ -62,6 +63,7 @@ export class IncomingTransactionsGraphComponent implements OnInit, OnChanges, On return; } this.windowPreference = this.windowPreferenceOverride ? this.windowPreferenceOverride : this.storageService.getValue('graphWindowPreference'); + this.MA = this.calculateMA(this.data.series[0]); this.mountChart(); } @@ -72,7 +74,101 @@ export class IncomingTransactionsGraphComponent implements OnInit, OnChanges, On this.isLoading = false; } + /// calculate the moving average of maData + calculateMA(maData): number[][] { + //update const variables that are not changed + const ma: number[][] = []; + let sum = 0; + let i = 0; + const len = maData.length; + + //Adjust window length based on the length of the data + //5% appeared as a good amount from tests + //TODO: make this a text box in the UI + const maWindowLen = Math.ceil(len * 0.05); + + //calculate the center of the moving average window + const center = Math.floor(maWindowLen / 2); + + //calculate the centered moving average + for (i = center; i < len - center; i++) { + sum = 0; + //build out ma as we loop through the data + ma[i] = []; + ma[i].push(maData[i][0]); + for (let j = i - center; j <= i + center; j++) { + sum += maData[j][1]; + } + + ma[i].push(sum / maWindowLen); + } + + //return the moving average array + return ma; + } + mountChart(): void { + //create an array for the echart series + //similar to how it is done in mempool-graph.component.ts + const seriesGraph = []; + seriesGraph.push({ + zlevel: 0, + name: 'data', + data: this.data.series[0], + type: 'line', + smooth: false, + showSymbol: false, + symbol: 'none', + lineStyle: { + width: 3, + }, + markLine: { + silent: true, + symbol: 'none', + lineStyle: { + color: '#fff', + opacity: 1, + width: 2, + }, + data: [{ + yAxis: 1667, + label: { + show: false, + color: '#ffffff', + } + }], + } + }, + { + zlevel: 0, + name: 'MA', + data: this.MA, + type: 'line', + smooth: false, + showSymbol: false, + symbol: 'none', + lineStyle: { + width: 1, + color: "white", + }, + markLine: { + silent: true, + symbol: 'none', + lineStyle: { + color: '#fff', + opacity: 1, + width: 2, + }, + data: [{ + yAxis: 1667, + label: { + show: false, + color: '#ffffff', + } + }], + } + }); + this.mempoolStatsChartOption = { grid: { height: this.height, @@ -122,16 +218,20 @@ export class IncomingTransactionsGraphComponent implements OnInit, OnChanges, On type: 'line', }, formatter: (params: any) => { - const axisValueLabel: string = formatterXAxis(this.locale, this.windowPreference, params[0].axisValue); + const axisValueLabel: string = formatterXAxis(this.locale, this.windowPreference, params[0].axisValue); const colorSpan = (color: string) => ``; let itemFormatted = '
' + axisValueLabel + '
'; params.map((item: any, index: number) => { - if (index < 26) { - itemFormatted += `
-
${colorSpan(item.color)}
-
-
${formatNumber(this.weightMode ? item.value[1] * 4 : item.value[1], this.locale, '1.0-0')} ${this.weightMode ? 'WU' : 'vB'}/s
-
`; + + //Do no include MA in tooltip legend! + if (item.seriesName !== 'MA') { + if (index < 26) { + itemFormatted += `
+
${colorSpan(item.color)}
+
+
${formatNumber(item.value[1], this.locale, '1.0-0')}vB/s
+
`; + } } }); return `
${itemFormatted}
`; @@ -171,35 +271,7 @@ export class IncomingTransactionsGraphComponent implements OnInit, OnChanges, On } } }, - series: [ - { - zlevel: 0, - data: this.data.series[0], - type: 'line', - smooth: false, - showSymbol: false, - symbol: 'none', - lineStyle: { - width: 3, - }, - markLine: { - silent: true, - symbol: 'none', - lineStyle: { - color: '#fff', - opacity: 1, - width: 2, - }, - data: [{ - yAxis: 1667, - label: { - show: false, - color: '#ffffff', - } - }], - } - }, - ], + series: seriesGraph, visualMap: { show: false, top: 50,