machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into mononaut/failover-timeout-config

Browse Source
This commit is contained in:
softsimon
2023-11-10 14:09:43 +09:00
committed by GitHub
parent c1f0eac8f8 3580d060ce
commit a663cca3d0
118 changed files with 1935 additions and 919 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
backend/src/__fixtures__/mempool-config.template.json
View File

@@ -71,6 +71,7 @@
"DATABASE": "__DATABASE_DATABASE__",
"USERNAME": "__DATABASE_USERNAME__",
"PASSWORD": "__DATABASE_PASSWORD__",
"PID_DIR": "__DATABASE_PID_FILE__",
"TIMEOUT": 3000
},
"SYSLOG": {

1
backend/src/__tests__/config.test.ts
View File

@@ -86,6 +86,7 @@ describe('Mempool Backend Config', () => {
USERNAME: 'mempool',
PASSWORD: 'mempool',
TIMEOUT: 180000,
PID_DIR: ''
});
expect(config.SYSLOG).toStrictEqual({

2
backend/src/api/bitcoin/bitcoin.routes.ts
View File

@@ -478,7 +478,7 @@ class BitcoinRoutes {
}
let nextHash = startFromHash;
for (let i = 0; i < 10 && nextHash; i++) {
for (let i = 0; i < 15 && nextHash; i++) {
const localBlock = blocks.getBlocks().find((b) => b.id === nextHash);
if (localBlock) {
returnBlocks.push(localBlock);

7
backend/src/api/database-migration.ts
View File

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
import { RowDataPacket } from 'mysql2';
class DatabaseMigration {
private static currentVersion = 65;
private static currentVersion = 66;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -553,6 +553,11 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD accelerated_txs JSON DEFAULT "[]"');
await this.updateToSchemaVersion(65);
}
if (databaseSchemaVersion < 66) {
await this.$executeQuery('ALTER TABLE `statistics` ADD min_fee FLOAT UNSIGNED DEFAULT NULL');
await this.updateToSchemaVersion(66);
}
}
/**

18
backend/src/api/explorer/nodes.routes.ts
View File

@@ -42,6 +42,12 @@ class NodesRoutes {
switch (config.MEMPOOL.NETWORK) {
case 'testnet':
nodesList = [
'0259db43b4e4ac0ff12a805f2d81e521253ba2317f6739bc611d8e2fa156d64256',
'0352b9944b9a52bd2116c91f1ba70c4ef851ac5ba27e1b20f1d92da3ade010dd10',
'03424f5a7601eaa47482cb17100b31a84a04d14fb44b83a57eeceffd8e299878e3',
'032850492ee61a5f7006a2fda6925e4b4ec3782f2b6de2ff0e439ef5a38c3b2470',
'022c80bace98831c44c32fb69755f2b353434e0ee9e7fbda29507f7ef8abea1421',
'02c3559c833e6f99f9ca05fe503e0b4e7524dea9121344edfd3e811101e0c28680',
'032c7c7819276c4f706a04df1a0f1e10a5495994a7be4c1d3d28ca766e5a2b957b',
'025a7e38c2834dd843591a4d23d5f09cdeb77ddca85f673c2d944a14220ff14cf7',
'0395e2731a1673ef21d7a16a727c4fc4d4c35a861c428ce2c819c53d2b81c8bd55',
@@ -64,6 +70,12 @@ class NodesRoutes {
break;
case 'signet':
nodesList = [
'029fe3621fc0c6e08056a14b868f8fb9acca1aa28a129512f6cea0f0d7654d9f92',
'02f60cd7a3a4f1c953dd9554a6ebd51a34f8b10b8124b7fc43a0b381139b55c883',
'03cbbf581774700865eebd1be42d022bc004ba30881274ab304e088a25d70e773d',
'0243348cb3741cfe2d8485fa8375c29c7bc7cbb67577c363cb6987a5e5fd0052cc',
'02cb73e631af44bee600d80f8488a9194c9dc5c7590e575c421a070d1be05bc8e9',
'0306f55ee631aa1e2cd4d9b2bfcbc14404faec5c541cef8b2e6f779061029d09c4',
'03ddab321b760433cbf561b615ef62ac7d318630c5f51d523aaf5395b90b751956',
'033d92c7bfd213ef1b34c90e985fb5dc77f9ec2409d391492484e57a44c4aca1de',
'02ad010dda54253c1eb9efe38b0760657a3b43ecad62198c359c051c9d99d45781',
@@ -86,6 +98,12 @@ class NodesRoutes {
break;
default:
nodesList = [
'02b12b889fe3c943cb05645921040ef13d6d397a2e7a4ad000e28500c505ff26d6',
'0302240ac9d71b39617cbde2764837ec3d6198bd6074b15b75d2ff33108e89d2e1',
'03364a8ace313376e5e4b68c954e287c6388e16df9e9fdbaf0363ecac41105cbf6',
'03229ab4b7f692753e094b93df90530150680f86b535b5183b0cffd75b3df583fc',
'03a696eb7acde991c1be97a58a9daef416659539ae462b897f5e9ae361f990228e',
'0248bf26cf3a63ab8870f34dc0ec9e6c8c6288cdba96ba3f026f34ec0f13ac4055',
'03fbc17549ec667bccf397ababbcb4cdc0e3394345e4773079ab2774612ec9be61',
'03da9a8623241ccf95f19cd645c6cecd4019ac91570e976eb0a128bebbc4d8a437',
'03ca5340cf85cb2e7cf076e489f785410838de174e40be62723e8a60972ad75144',

29
backend/src/api/fee-api.ts
View File

@@ -3,21 +3,30 @@ import { Common } from './common';
import mempool from './mempool';
import projectedBlocks from './mempool-blocks';
interface RecommendedFees {
fastestFee: number,
halfHourFee: number,
hourFee: number,
economyFee: number,
minimumFee: number,
}
class FeeApi {
constructor() { }
defaultFee = Common.isLiquid() ? 0.1 : 1;
public getRecommendedFee() {
public getRecommendedFee(): RecommendedFees {
const pBlocks = projectedBlocks.getMempoolBlocks();
const mPool = mempool.getMempoolInfo();
const minimumFee = Math.ceil(mPool.mempoolminfee * 100000);
const defaultMinFee = Math.max(minimumFee, this.defaultFee);
if (!pBlocks.length) {
return {
'fastestFee': this.defaultFee,
'halfHourFee': this.defaultFee,
'hourFee': this.defaultFee,
'fastestFee': defaultMinFee,
'halfHourFee': defaultMinFee,
'hourFee': defaultMinFee,
'economyFee': minimumFee,
'minimumFee': minimumFee,
};
@@ -27,11 +36,15 @@ class FeeApi {
const secondMedianFee = pBlocks[1] ? this.optimizeMedianFee(pBlocks[1], pBlocks[2], firstMedianFee) : this.defaultFee;
const thirdMedianFee = pBlocks[2] ? this.optimizeMedianFee(pBlocks[2], pBlocks[3], secondMedianFee) : this.defaultFee;
// explicitly enforce a minimum of ceil(mempoolminfee) on all recommendations.
// simply rounding up recommended rates is insufficient, as the purging rate
// can exceed the median rate of projected blocks in some extreme scenarios
// (see https://bitcoin.stackexchange.com/a/120024)
return {
'fastestFee': firstMedianFee,
'halfHourFee': secondMedianFee,
'hourFee': thirdMedianFee,
'economyFee': Math.min(2 * minimumFee, thirdMedianFee),
'fastestFee': Math.max(minimumFee, firstMedianFee),
'halfHourFee': Math.max(minimumFee, secondMedianFee),
'hourFee': Math.max(minimumFee, thirdMedianFee),
'economyFee': Math.max(minimumFee, Math.min(2 * minimumFee, thirdMedianFee)),
'minimumFee': minimumFee,
};
}

2
backend/src/api/memory-cache.ts
View File

@@ -31,7 +31,7 @@ class MemoryCache {
}
private cleanup() {
this.cache = this.cache.filter((cache) => cache.expires < (new Date()));
this.cache = this.cache.filter((cache) => cache.expires > (new Date()));
}
}

13
backend/src/api/statistics/statistics-api.ts
View File

@@ -15,6 +15,7 @@ class StatisticsApi {
mempool_byte_weight,
fee_data,
total_fee,
min_fee,
vsize_1,
vsize_2,
vsize_3,
@@ -54,7 +55,7 @@ class StatisticsApi {
vsize_1800,
vsize_2000
)
VALUES (NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
VALUES (NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)`;
const [result]: any = await DB.query(query);
return result.insertId;
@@ -73,6 +74,7 @@ class StatisticsApi {
mempool_byte_weight,
fee_data,
total_fee,
min_fee,
vsize_1,
vsize_2,
vsize_3,
@@ -112,7 +114,7 @@ class StatisticsApi {
vsize_1800,
vsize_2000
)
VALUES (${statistics.added}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
VALUES (${statistics.added}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
const params: (string | number)[] = [
@@ -122,6 +124,7 @@ class StatisticsApi {
statistics.mempool_byte_weight,
statistics.fee_data,
statistics.total_fee,
statistics.min_fee,
statistics.vsize_1,
statistics.vsize_2,
statistics.vsize_3,
@@ -171,7 +174,9 @@ class StatisticsApi {
private getQueryForDaysAvg(div: number, interval: string) {
return `SELECT
UNIX_TIMESTAMP(added) as added,
CAST(avg(unconfirmed_transactions) as DOUBLE) as unconfirmed_transactions,
CAST(avg(vbytes_per_second) as DOUBLE) as vbytes_per_second,
CAST(avg(min_fee) as DOUBLE) as min_fee,
CAST(avg(vsize_1) as DOUBLE) as vsize_1,
CAST(avg(vsize_2) as DOUBLE) as vsize_2,
CAST(avg(vsize_3) as DOUBLE) as vsize_3,
@@ -219,7 +224,9 @@ class StatisticsApi {
private getQueryForDays(div: number, interval: string) {
return `SELECT
UNIX_TIMESTAMP(added) as added,
CAST(avg(unconfirmed_transactions) as DOUBLE) as unconfirmed_transactions,
CAST(avg(vbytes_per_second) as DOUBLE) as vbytes_per_second,
CAST(avg(min_fee) as DOUBLE) as min_fee,
vsize_1,
vsize_2,
vsize_3,
@@ -401,9 +408,11 @@ class StatisticsApi {
return statistic.map((s) => {
return {
added: s.added,
count: s.unconfirmed_transactions,
vbytes_per_second: s.vbytes_per_second,
mempool_byte_weight: s.mempool_byte_weight,
total_fee: s.total_fee,
min_fee: s.min_fee,
vsizes: [
s.vsize_1,
s.vsize_2,

4
backend/src/api/statistics/statistics.ts
View File

@@ -89,6 +89,9 @@ class Statistics {
}
});
// get minFee and convert to sats/vb
const minFee = memPool.getMempoolInfo().mempoolminfee * 100000;
try {
const insertId = await statisticsApi.$create({
added: 'NOW()',
@@ -98,6 +101,7 @@ class Statistics {
mempool_byte_weight: totalWeight,
total_fee: totalFee,
fee_data: '',
min_fee: minFee,
vsize_1: weightVsizeFees['1'] || 0,
vsize_2: weightVsizeFees['2'] || 0,
vsize_3: weightVsizeFees['3'] || 0,

9
backend/src/api/websocket-handler.ts
View File

@@ -486,6 +486,7 @@ class WebsocketHandler {
// pre-compute address transactions
const addressCache = this.makeAddressCache(newTransactions);
const removedAddressCache = this.makeAddressCache(deletedTransactions);
this.wss.clients.forEach(async (client) => {
if (client.readyState !== WebSocket.OPEN) {
@@ -526,11 +527,15 @@ class WebsocketHandler {
}
if (client['track-address']) {
const foundTransactions = Array.from(addressCache[client['track-address']]?.values() || []);
const newTransactions = Array.from(addressCache[client['track-address']]?.values() || []);
const removedTransactions = Array.from(removedAddressCache[client['track-address']]?.values() || []);
// txs may be missing prevouts in non-esplora backends
// so fetch the full transactions now
const fullTransactions = (config.MEMPOOL.BACKEND !== 'esplora') ? await this.getFullTransactions(foundTransactions) : foundTransactions;
const fullTransactions = (config.MEMPOOL.BACKEND !== 'esplora') ? await this.getFullTransactions(newTransactions) : newTransactions;
if (removedTransactions.length) {
response['address-removed-transactions'] = JSON.stringify(removedTransactions);
}
if (fullTransactions.length) {
response['address-transactions'] = JSON.stringify(fullTransactions);
}

2
backend/src/config.ts
View File

@@ -95,6 +95,7 @@ interface IConfig {
USERNAME: string;
PASSWORD: string;
TIMEOUT: number;
PID_DIR: string;
};
SYSLOG: {
ENABLED: boolean;
@@ -223,6 +224,7 @@ const defaults: IConfig = {
'USERNAME': 'mempool',
'PASSWORD': 'mempool',
'TIMEOUT': 180000,
'PID_DIR': '',
},
'SYSLOG': {
'ENABLED': true,

29
backend/src/database.ts
View File

@@ -1,3 +1,5 @@
import * as fs from 'fs';
import path from 'path';
import config from './config';
import { createPool, Pool, PoolConnection } from 'mysql2/promise';
import logger from './logger';
@@ -101,6 +103,33 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr
}
}
public getPidLock(): boolean {
const filePath = path.join(config.DATABASE.PID_DIR || __dirname, `/mempool-${config.DATABASE.DATABASE}.pid`);
if (fs.existsSync(filePath)) {
const pid = fs.readFileSync(filePath).toString();
if (pid !== `${process.pid}`) {
const msg = `Already running on PID ${pid} (or pid file '${filePath}' is stale)`;
logger.err(msg);
throw new Error(msg);
} else {
return true;
}
} else {
fs.writeFileSync(filePath, `${process.pid}`);
return true;
}
}
public releasePidLock(): void {
const filePath = path.join(config.DATABASE.PID_DIR || __dirname, `/mempool-${config.DATABASE.DATABASE}.pid`);
if (fs.existsSync(filePath)) {
const pid = fs.readFileSync(filePath).toString();
if (pid === `${process.pid}`) {
fs.unlinkSync(filePath);
}
}
}
private async getPool(): Promise<Pool> {
if (this.pool === null) {
this.pool = createPool(this.poolConfig);

16
backend/src/index.ts
View File

@@ -91,11 +91,18 @@ class Server {
async startServer(worker = false): Promise<void> {
logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);
// Register cleanup listeners for exit events
['exit', 'SIGINT', 'SIGTERM', 'SIGUSR1', 'SIGUSR2', 'uncaughtException', 'unhandledRejection'].forEach(event => {
process.on(event, () => { this.onExit(event); });
});
if (config.MEMPOOL.BACKEND === 'esplora') {
bitcoinApi.startHealthChecks();
}
if (config.DATABASE.ENABLED) {
DB.getPidLock();
await DB.checkDbConnection();
try {
if (process.env.npm_config_reindex_blocks === 'true') { // Re-index requests
@@ -306,6 +313,15 @@ class Server {
this.lastHeapLogTime = now;
}
}
onExit(exitEvent): void {
if (config.DATABASE.ENABLED) {
DB.releasePidLock();
}
process.exit(0);
}
}
((): Server => new Server())();

2
backend/src/mempool.interfaces.ts
View File

@@ -300,6 +300,7 @@ export interface Statistic {
total_fee: number;
mempool_byte_weight: number;
fee_data: string;
min_fee: number;
vsize_1: number;
vsize_2: number;
@@ -346,6 +347,7 @@ export interface OptimizedStatistic {
vbytes_per_second: number;
total_fee: number;
mempool_byte_weight: number;
min_fee: number;
vsizes: number[];
}