machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into mononaut/hide-arrow-on-replace

Browse Source
This commit is contained in:
softsimon
2023-11-11 11:57:03 +09:00
committed by GitHub
parent 420fbf3d6f 6134376964
commit 874d2b371d
99 changed files with 2064 additions and 1312 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
backend/src/__fixtures__/mempool-config.template.json
View File

@@ -52,6 +52,8 @@
"REST_API_URL": "__ESPLORA_REST_API_URL__",
"UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__",
"RETRY_UNIX_SOCKET_AFTER": 888,
"REQUEST_TIMEOUT": 10000,
"FALLBACK_TIMEOUT": 5000,
"FALLBACK": []
},
"SECOND_CORE_RPC": {
@@ -69,6 +71,7 @@
"DATABASE": "__DATABASE_DATABASE__",
"USERNAME": "__DATABASE_USERNAME__",
"PASSWORD": "__DATABASE_PASSWORD__",
"PID_DIR": "__DATABASE_PID_FILE__",
"TIMEOUT": 3000
},
"SYSLOG": {

3
backend/src/__tests__/config.test.ts
View File

@@ -56,6 +56,8 @@ describe('Mempool Backend Config', () => {
REST_API_URL: 'http://127.0.0.1:3000',
UNIX_SOCKET_PATH: null,
RETRY_UNIX_SOCKET_AFTER: 30000,
REQUEST_TIMEOUT: 10000,
FALLBACK_TIMEOUT: 5000,
FALLBACK: [],
});
@@ -84,6 +86,7 @@ describe('Mempool Backend Config', () => {
USERNAME: 'mempool',
PASSWORD: 'mempool',
TIMEOUT: 180000,
PID_DIR: ''
});
expect(config.SYSLOG).toStrictEqual({

2
backend/src/api/bitcoin/bitcoin.routes.ts
View File

@@ -478,7 +478,7 @@ class BitcoinRoutes {
}
let nextHash = startFromHash;
for (let i = 0; i < 10 && nextHash; i++) {
for (let i = 0; i < 15 && nextHash; i++) {
const localBlock = blocks.getBlocks().find((b) => b.id === nextHash);
if (localBlock) {
returnBlocks.push(localBlock);

8
backend/src/api/bitcoin/esplora-api.ts
View File

@@ -75,9 +75,9 @@ class FailoverRouter {
const results = await Promise.allSettled(this.hosts.map(async (host) => {
if (host.socket) {
return this.pollConnection.get<number>('/blocks/tip/height', { socketPath: host.host, timeout: 5000 });
return this.pollConnection.get<number>('/blocks/tip/height', { socketPath: host.host, timeout: config.ESPLORA.FALLBACK_TIMEOUT });
} else {
return this.pollConnection.get<number>(host.host + '/blocks/tip/height', { timeout: 5000 });
return this.pollConnection.get<number>(host.host + '/blocks/tip/height', { timeout: config.ESPLORA.FALLBACK_TIMEOUT });
}
}));
const maxHeight = results.reduce((max, result) => Math.max(max, result.status === 'fulfilled' ? result.value?.data || 0 : 0), 0);
@@ -168,10 +168,10 @@ class FailoverRouter {
let axiosConfig;
let url;
if (host.socket) {
axiosConfig = { socketPath: host.host, timeout: 10000, responseType };
axiosConfig = { socketPath: host.host, timeout: config.ESPLORA.REQUEST_TIMEOUT, responseType };
url = path;
} else {
axiosConfig = { timeout: 10000, responseType };
axiosConfig = { timeout: config.ESPLORA.REQUEST_TIMEOUT, responseType };
url = host.host + path;
}
return (method === 'post'

7
backend/src/api/database-migration.ts
View File

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
import { RowDataPacket } from 'mysql2';
class DatabaseMigration {
private static currentVersion = 65;
private static currentVersion = 66;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -553,6 +553,11 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD accelerated_txs JSON DEFAULT "[]"');
await this.updateToSchemaVersion(65);
}
if (databaseSchemaVersion < 66) {
await this.$executeQuery('ALTER TABLE `statistics` ADD min_fee FLOAT UNSIGNED DEFAULT NULL');
await this.updateToSchemaVersion(66);
}
}
/**

29
backend/src/api/fee-api.ts
View File

@@ -3,21 +3,30 @@ import { Common } from './common';
import mempool from './mempool';
import projectedBlocks from './mempool-blocks';
interface RecommendedFees {
fastestFee: number,
halfHourFee: number,
hourFee: number,
economyFee: number,
minimumFee: number,
}
class FeeApi {
constructor() { }
defaultFee = Common.isLiquid() ? 0.1 : 1;
public getRecommendedFee() {
public getRecommendedFee(): RecommendedFees {
const pBlocks = projectedBlocks.getMempoolBlocks();
const mPool = mempool.getMempoolInfo();
const minimumFee = Math.ceil(mPool.mempoolminfee * 100000);
const defaultMinFee = Math.max(minimumFee, this.defaultFee);
if (!pBlocks.length) {
return {
'fastestFee': this.defaultFee,
'halfHourFee': this.defaultFee,
'hourFee': this.defaultFee,
'fastestFee': defaultMinFee,
'halfHourFee': defaultMinFee,
'hourFee': defaultMinFee,
'economyFee': minimumFee,
'minimumFee': minimumFee,
};
@@ -27,11 +36,15 @@ class FeeApi {
const secondMedianFee = pBlocks[1] ? this.optimizeMedianFee(pBlocks[1], pBlocks[2], firstMedianFee) : this.defaultFee;
const thirdMedianFee = pBlocks[2] ? this.optimizeMedianFee(pBlocks[2], pBlocks[3], secondMedianFee) : this.defaultFee;
// explicitly enforce a minimum of ceil(mempoolminfee) on all recommendations.
// simply rounding up recommended rates is insufficient, as the purging rate
// can exceed the median rate of projected blocks in some extreme scenarios
// (see https://bitcoin.stackexchange.com/a/120024)
return {
'fastestFee': firstMedianFee,
'halfHourFee': secondMedianFee,
'hourFee': thirdMedianFee,
'economyFee': Math.min(2 * minimumFee, thirdMedianFee),
'fastestFee': Math.max(minimumFee, firstMedianFee),
'halfHourFee': Math.max(minimumFee, secondMedianFee),
'hourFee': Math.max(minimumFee, thirdMedianFee),
'economyFee': Math.max(minimumFee, Math.min(2 * minimumFee, thirdMedianFee)),
'minimumFee': minimumFee,
};
}

2
backend/src/api/memory-cache.ts
View File

@@ -31,7 +31,7 @@ class MemoryCache {
}
private cleanup() {
this.cache = this.cache.filter((cache) => cache.expires < (new Date()));
this.cache = this.cache.filter((cache) => cache.expires > (new Date()));
}
}

10
backend/src/api/statistics/statistics-api.ts
View File

@@ -15,6 +15,7 @@ class StatisticsApi {
mempool_byte_weight,
fee_data,
total_fee,
min_fee,
vsize_1,
vsize_2,
vsize_3,
@@ -54,7 +55,7 @@ class StatisticsApi {
vsize_1800,
vsize_2000
)
VALUES (NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
VALUES (NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)`;
const [result]: any = await DB.query(query);
return result.insertId;
@@ -73,6 +74,7 @@ class StatisticsApi {
mempool_byte_weight,
fee_data,
total_fee,
min_fee,
vsize_1,
vsize_2,
vsize_3,
@@ -112,7 +114,7 @@ class StatisticsApi {
vsize_1800,
vsize_2000
)
VALUES (${statistics.added}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
VALUES (${statistics.added}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
const params: (string | number)[] = [
@@ -122,6 +124,7 @@ class StatisticsApi {
statistics.mempool_byte_weight,
statistics.fee_data,
statistics.total_fee,
statistics.min_fee,
statistics.vsize_1,
statistics.vsize_2,
statistics.vsize_3,
@@ -173,6 +176,7 @@ class StatisticsApi {
UNIX_TIMESTAMP(added) as added,
CAST(avg(unconfirmed_transactions) as DOUBLE) as unconfirmed_transactions,
CAST(avg(vbytes_per_second) as DOUBLE) as vbytes_per_second,
CAST(avg(min_fee) as DOUBLE) as min_fee,
CAST(avg(vsize_1) as DOUBLE) as vsize_1,
CAST(avg(vsize_2) as DOUBLE) as vsize_2,
CAST(avg(vsize_3) as DOUBLE) as vsize_3,
@@ -222,6 +226,7 @@ class StatisticsApi {
UNIX_TIMESTAMP(added) as added,
CAST(avg(unconfirmed_transactions) as DOUBLE) as unconfirmed_transactions,
CAST(avg(vbytes_per_second) as DOUBLE) as vbytes_per_second,
CAST(avg(min_fee) as DOUBLE) as min_fee,
vsize_1,
vsize_2,
vsize_3,
@@ -407,6 +412,7 @@ class StatisticsApi {
vbytes_per_second: s.vbytes_per_second,
mempool_byte_weight: s.mempool_byte_weight,
total_fee: s.total_fee,
min_fee: s.min_fee,
vsizes: [
s.vsize_1,
s.vsize_2,

4
backend/src/api/statistics/statistics.ts
View File

@@ -89,6 +89,9 @@ class Statistics {
}
});
// get minFee and convert to sats/vb
const minFee = memPool.getMempoolInfo().mempoolminfee * 100000;
try {
const insertId = await statisticsApi.$create({
added: 'NOW()',
@@ -98,6 +101,7 @@ class Statistics {
mempool_byte_weight: totalWeight,
total_fee: totalFee,
fee_data: '',
min_fee: minFee,
vsize_1: weightVsizeFees['1'] || 0,
vsize_2: weightVsizeFees['2'] || 0,
vsize_3: weightVsizeFees['3'] || 0,

6
backend/src/config.ts
View File

@@ -44,6 +44,8 @@ interface IConfig {
REST_API_URL: string;
UNIX_SOCKET_PATH: string | void | null;
RETRY_UNIX_SOCKET_AFTER: number;
REQUEST_TIMEOUT: number;
FALLBACK_TIMEOUT: number;
FALLBACK: string[];
};
LIGHTNING: {
@@ -93,6 +95,7 @@ interface IConfig {
USERNAME: string;
PASSWORD: string;
TIMEOUT: number;
PID_DIR: string;
};
SYSLOG: {
ENABLED: boolean;
@@ -189,6 +192,8 @@ const defaults: IConfig = {
'REST_API_URL': 'http://127.0.0.1:3000',
'UNIX_SOCKET_PATH': null,
'RETRY_UNIX_SOCKET_AFTER': 30000,
'REQUEST_TIMEOUT': 10000,
'FALLBACK_TIMEOUT': 5000,
'FALLBACK': [],
},
'ELECTRUM': {
@@ -219,6 +224,7 @@ const defaults: IConfig = {
'USERNAME': 'mempool',
'PASSWORD': 'mempool',
'TIMEOUT': 180000,
'PID_DIR': '',
},
'SYSLOG': {
'ENABLED': true,

29
backend/src/database.ts
View File

@@ -1,3 +1,5 @@
import * as fs from 'fs';
import path from 'path';
import config from './config';
import { createPool, Pool, PoolConnection } from 'mysql2/promise';
import logger from './logger';
@@ -101,6 +103,33 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr
}
}
public getPidLock(): boolean {
const filePath = path.join(config.DATABASE.PID_DIR || __dirname, `/mempool-${config.DATABASE.DATABASE}.pid`);
if (fs.existsSync(filePath)) {
const pid = fs.readFileSync(filePath).toString();
if (pid !== `${process.pid}`) {
const msg = `Already running on PID ${pid} (or pid file '${filePath}' is stale)`;
logger.err(msg);
throw new Error(msg);
} else {
return true;
}
} else {
fs.writeFileSync(filePath, `${process.pid}`);
return true;
}
}
public releasePidLock(): void {
const filePath = path.join(config.DATABASE.PID_DIR || __dirname, `/mempool-${config.DATABASE.DATABASE}.pid`);
if (fs.existsSync(filePath)) {
const pid = fs.readFileSync(filePath).toString();
if (pid === `${process.pid}`) {
fs.unlinkSync(filePath);
}
}
}
private async getPool(): Promise<Pool> {
if (this.pool === null) {
this.pool = createPool(this.poolConfig);

16
backend/src/index.ts
View File

@@ -91,11 +91,18 @@ class Server {
async startServer(worker = false): Promise<void> {
logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);
// Register cleanup listeners for exit events
['exit', 'SIGINT', 'SIGTERM', 'SIGUSR1', 'SIGUSR2', 'uncaughtException', 'unhandledRejection'].forEach(event => {
process.on(event, () => { this.onExit(event); });
});
if (config.MEMPOOL.BACKEND === 'esplora') {
bitcoinApi.startHealthChecks();
}
if (config.DATABASE.ENABLED) {
DB.getPidLock();
await DB.checkDbConnection();
try {
if (process.env.npm_config_reindex_blocks === 'true') { // Re-index requests
@@ -306,6 +313,15 @@ class Server {
this.lastHeapLogTime = now;
}
}
onExit(exitEvent): void {
if (config.DATABASE.ENABLED) {
DB.releasePidLock();
}
process.exit(0);
}
}
((): Server => new Server())();

2
backend/src/mempool.interfaces.ts
View File

@@ -300,6 +300,7 @@ export interface Statistic {
total_fee: number;
mempool_byte_weight: number;
fee_data: string;
min_fee: number;
vsize_1: number;
vsize_2: number;
@@ -346,6 +347,7 @@ export interface OptimizedStatistic {
vbytes_per_second: number;
total_fee: number;
mempool_byte_weight: number;
min_fee: number;
vsizes: number[];
}