machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into natsoni/federation-utxos-expiry

Browse Source
This commit is contained in:
natsoni
2024-03-07 10:27:44 +01:00
committed by GitHub
parent 198939da81 1d877a746f
commit e8c5d478fa
63 changed files with 1202 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
backend/mempool-config.sample.json
View File

@@ -1,5 +1,6 @@
{
"MEMPOOL": {
"OFFICIAL": false,
"NETWORK": "mainnet",
"BACKEND": "electrum",
"ENABLED": true,

17
backend/package-lock.json generated
View File

@@ -9,6 +9,7 @@
"version": "3.0.0-dev",
"license": "GNU Affero General Public License v3.0",
"dependencies": {
"@babel/core": "^7.24.0",
"@mempool/electrum-client": "1.1.9",
"@types/node": "^18.15.3",
"axios": "~1.6.1",
@@ -1499,9 +1500,9 @@
}
},
"node_modules/@napi-rs/cli": {
"version": "2.16.1",
"resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.16.1.tgz",
"integrity": "sha512-L0Gr5iEQIDEbvWdDr1HUaBOxBSHL1VZhWSk1oryawoT8qJIY+KGfLFelU+Qma64ivCPbxYpkfPoKYVG3rcoGIA==",
"version": "2.18.0",
"resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.18.0.tgz",
"integrity": "sha512-lfSRT7cs3iC4L+kv9suGYQEezn5Nii7Kpu+THsYVI0tA1Vh59LH45p4QADaD7hvIkmOz79eEGtoKQ9nAkAPkzA==",
"bin": {
"napi": "scripts/index.js"
},
@@ -7669,7 +7670,7 @@
"version": "3.0.1",
"hasInstallScript": true,
"dependencies": {
"@napi-rs/cli": "2.16.1"
"@napi-rs/cli": "2.18.0"
},
"engines": {
"node": ">= 12"
@@ -8774,9 +8775,9 @@
"integrity": "sha512-mlvPiCzUlaETpYW3i6V87A24jjMYgsebaXtUo3WQyyLnYUuxs0KiXQ2mnKh3h15j8Xg/hfxeGIi+5OC9u0nftQ=="
},
"@napi-rs/cli": {
"version": "2.16.1",
"resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.16.1.tgz",
"integrity": "sha512-L0Gr5iEQIDEbvWdDr1HUaBOxBSHL1VZhWSk1oryawoT8qJIY+KGfLFelU+Qma64ivCPbxYpkfPoKYVG3rcoGIA=="
"version": "2.18.0",
"resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.18.0.tgz",
"integrity": "sha512-lfSRT7cs3iC4L+kv9suGYQEezn5Nii7Kpu+THsYVI0tA1Vh59LH45p4QADaD7hvIkmOz79eEGtoKQ9nAkAPkzA=="
},
"@noble/hashes": {
"version": "1.3.0",
@@ -12702,7 +12703,7 @@
"rust-gbt": {
"version": "file:rust-gbt",
"requires": {
"@napi-rs/cli": "2.16.1"
"@napi-rs/cli": "2.18.0"
}
},
"safe-buffer": {

10
backend/rust-gbt/Cargo.toml
View File

@@ -12,14 +12,14 @@ crate-type = ["cdylib"]
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
priority-queue = "1.3.2"
priority-queue = "2.0.2"
bytes = "1.4.0"
napi = { version = "2.13.2", features = ["napi8", "tokio_rt"] }
napi-derive = "2.13.0"
napi = { version = "2.16.0", features = ["napi8", "tokio_rt"] }
napi-derive = "2.16.0"
bytemuck = "1.13.1"
tracing = "0.1.36"
tracing-log = "0.1.3"
tracing-log = "0.2.0"
tracing-subscriber = { version = "0.3.15", features = ["env-filter"]}
[build-dependencies]
napi-build = "2.0.1"
napi-build = "2.1.2"

43
backend/rust-gbt/index.js
View File

@@ -237,6 +237,49 @@ switch (platform) {
loadError = e
}
break
case 'riscv64':
if (isMusl()) {
localFileExisted = existsSync(
join(__dirname, 'gbt.linux-riscv64-musl.node')
)
try {
if (localFileExisted) {
nativeBinding = require('./gbt.linux-riscv64-musl.node')
} else {
nativeBinding = require('gbt-linux-riscv64-musl')
}
} catch (e) {
loadError = e
}
} else {
localFileExisted = existsSync(
join(__dirname, 'gbt.linux-riscv64-gnu.node')
)
try {
if (localFileExisted) {
nativeBinding = require('./gbt.linux-riscv64-gnu.node')
} else {
nativeBinding = require('gbt-linux-riscv64-gnu')
}
} catch (e) {
loadError = e
}
}
break
case 's390x':
localFileExisted = existsSync(
join(__dirname, 'gbt.linux-s390x-gnu.node')
)
try {
if (localFileExisted) {
nativeBinding = require('./gbt.linux-s390x-gnu.node')
} else {
nativeBinding = require('gbt-linux-s390x-gnu')
}
} catch (e) {
loadError = e
}
break
default:
throw new Error(`Unsupported architecture on Linux: ${arch}`)
}

8
backend/rust-gbt/package-lock.json generated
View File

@@ -9,16 +9,16 @@
"version": "3.0.1",
"hasInstallScript": true,
"dependencies": {
"@napi-rs/cli": "2.16.1"
"@napi-rs/cli": "2.18.0"
},
"engines": {
"node": ">= 12"
}
},
"node_modules/@napi-rs/cli": {
"version": "2.16.1",
"resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.16.1.tgz",
"integrity": "sha512-L0Gr5iEQIDEbvWdDr1HUaBOxBSHL1VZhWSk1oryawoT8qJIY+KGfLFelU+Qma64ivCPbxYpkfPoKYVG3rcoGIA==",
"version": "2.18.0",
"resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.18.0.tgz",
"integrity": "sha512-lfSRT7cs3iC4L+kv9suGYQEezn5Nii7Kpu+THsYVI0tA1Vh59LH45p4QADaD7hvIkmOz79eEGtoKQ9nAkAPkzA==",
"bin": {
"napi": "scripts/index.js"
},

2
backend/rust-gbt/package.json
View File

@@ -25,7 +25,7 @@
}
},
"dependencies": {
"@napi-rs/cli": "2.16.1"
"@napi-rs/cli": "2.18.0"
},
"engines": {
"node": ">= 12"

4
backend/src/__fixtures__/mempool-config.template.json
View File

@@ -1,6 +1,7 @@
{
"MEMPOOL": {
"ENABLED": true,
"OFFICIAL": false,
"NETWORK": "__MEMPOOL_NETWORK__",
"BACKEND": "__MEMPOOL_BACKEND__",
"BLOCKS_SUMMARIES_INDEXING": true,
@@ -79,7 +80,8 @@
"USERNAME": "__DATABASE_USERNAME__",
"PASSWORD": "__DATABASE_PASSWORD__",
"PID_DIR": "__DATABASE_PID_FILE__",
"TIMEOUT": 3000
"TIMEOUT": 3000,
"POOL_SIZE": 100
},
"SYSLOG": {
"ENABLED": false,

4
backend/src/__tests__/config.test.ts
View File

@@ -14,6 +14,7 @@ describe('Mempool Backend Config', () => {
expect(config.MEMPOOL).toStrictEqual({
ENABLED: true,
OFFICIAL: false,
NETWORK: 'mainnet',
BACKEND: 'none',
BLOCKS_SUMMARIES_INDEXING: false,
@@ -93,7 +94,8 @@ describe('Mempool Backend Config', () => {
USERNAME: 'mempool',
PASSWORD: 'mempool',
TIMEOUT: 180000,
PID_DIR: ''
PID_DIR: '',
POOL_SIZE: 100,
});
expect(config.SYSLOG).toStrictEqual({

13
backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts
View File

@@ -29,6 +29,7 @@ export interface AbstractBitcoinApi {
$getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise<IEsploraApi.Outspend[]>;
startHealthChecks(): void;
getHealthStatus(): HealthCheckHost[];
}
export interface BitcoinRpcCredentials {
host: string;
@@ -38,3 +39,15 @@ export interface BitcoinRpcCredentials {
timeout: number;
cookie?: string;
}
export interface HealthCheckHost {
host: string;
active: boolean;
rtt: number;
latestHeight: number;
socket: boolean;
outOfSync: boolean;
unreachable: boolean;
checked: boolean;
lastChecked: number;
}

6
backend/src/api/bitcoin/bitcoin-api.ts
View File

@@ -1,5 +1,5 @@
import * as bitcoinjs from 'bitcoinjs-lib';
import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
import { AbstractBitcoinApi, HealthCheckHost } from './bitcoin-api-abstract-factory';
import { IBitcoinApi } from './bitcoin-api.interface';
import { IEsploraApi } from './esplora-api.interface';
import blocks from '../blocks';
@@ -382,6 +382,10 @@ class BitcoinApi implements AbstractBitcoinApi {
}
public startHealthChecks(): void {};
public getHealthStatus() {
return [];
}
}
export default BitcoinApi;

36
backend/src/api/bitcoin/esplora-api.ts
View File

@@ -1,7 +1,7 @@
import config from '../../config';
import axios, { AxiosResponse } from 'axios';
import axios, { AxiosResponse, isAxiosError } from 'axios';
import http from 'http';
import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
import { AbstractBitcoinApi, HealthCheckHost } from './bitcoin-api-abstract-factory';
import { IEsploraApi } from './esplora-api.interface';
import logger from '../../logger';
import { Common } from '../common';
@@ -10,6 +10,7 @@ interface FailoverHost {
host: string,
rtts: number[],
rtt: number,
timedOut?: boolean,
failures: number,
latestHeight?: number,
socket?: boolean,
@@ -17,6 +18,7 @@ interface FailoverHost {
unreachable?: boolean,
preferred?: boolean,
checked: boolean,
lastChecked?: number,
}
class FailoverRouter {
@@ -108,14 +110,20 @@ class FailoverRouter {
host.rtts = [];
host.rtt = Infinity;
}
host.timedOut = false;
} catch (e) {
host.outOfSync = true;
host.unreachable = true;
host.rtts = [];
host.rtt = Infinity;
if (isAxiosError(e) && (e.code === 'ECONNABORTED' || e.code === 'ETIMEDOUT')) {
host.timedOut = true;
} else {
host.timedOut = false;
}
}
host.checked = true;
host.lastChecked = Date.now();
// switch if the current host is out of sync or significantly slower than the next best alternative
const rankOrder = this.sortHosts();
@@ -143,7 +151,7 @@ class FailoverRouter {
private formatRanking(index: number, host: FailoverHost, active: FailoverHost, maxHeight: number): string {
const heightStatus = !host.checked ? '⏳' : (host.outOfSync ? '🚫' : (host.latestHeight && host.latestHeight < maxHeight ? '🟧' : '✅'));
return `${host === active ? '⭐️' : ' '} ${host.rtt < Infinity ? Math.round(host.rtt).toString().padStart(5, ' ') + 'ms' : ' - '} ${!host.checked ? '⏳' : (host.unreachable ? '🔥' : '✅')} | block: ${host.latestHeight || '??????'} ${heightStatus} | ${host.host} ${host === active ? '⭐️' : ' '}`;
return `${host === active ? '⭐️' : ' '} ${host.rtt < Infinity ? Math.round(host.rtt).toString().padStart(5, ' ') + 'ms' : (host.timedOut ? ' ⌛️💥 ' : ' - ')} ${!host.checked ? '⏳' : (host.unreachable ? '🔥' : '✅')} | block: ${host.latestHeight || '??????'} ${heightStatus} | ${host.host} ${host === active ? '⭐️' : ' '}`;
}
private updateFallback(): FailoverHost[] {
@@ -157,7 +165,7 @@ class FailoverRouter {
}
// sort hosts by connection quality, and update default fallback
private sortHosts(): FailoverHost[] {
public sortHosts(): FailoverHost[] {
// sort by connection quality
return this.hosts.slice().sort((a, b) => {
if ((a.unreachable || a.outOfSync) === (b.unreachable || b.outOfSync)) {
@@ -342,6 +350,24 @@ class ElectrsApi implements AbstractBitcoinApi {
public startHealthChecks(): void {
this.failoverRouter.startHealthChecks();
}
public getHealthStatus(): HealthCheckHost[] {
if (config.MEMPOOL.OFFICIAL) {
return this.failoverRouter.sortHosts().map(host => ({
host: host.host,
active: host === this.failoverRouter.activeHost,
rtt: host.rtt,
latestHeight: host.latestHeight || 0,
socket: !!host.socket,
outOfSync: !!host.outOfSync,
unreachable: !!host.unreachable,
checked: !!host.checked,
lastChecked: host.lastChecked || 0,
}));
} else {
return [];
}
}
}
export default ElectrsApi;

16
backend/src/api/websocket-handler.ts
View File

@@ -26,6 +26,7 @@ import mempool from './mempool';
import statistics from './statistics/statistics';
import accelerationCosts from './acceleration';
import accelerationRepository from '../repositories/AccelerationRepository';
import bitcoinApi from './bitcoin/bitcoin-api-factory';
interface AddressTransactions {
mempool: MempoolTransactionExtended[],
@@ -39,6 +40,7 @@ const wantable = [
'mempool-blocks',
'live-2h-chart',
'stats',
'tomahawk',
];
class WebsocketHandler {
@@ -123,7 +125,7 @@ class WebsocketHandler {
for (const sub of wantable) {
const key = `want-${sub}`;
const wants = parsedMessage.data.includes(sub);
if (wants && client['wants'] && !client[key]) {
if (wants && !client[key]) {
wantNow[key] = true;
}
client[key] = wants;
@@ -147,6 +149,10 @@ class WebsocketHandler {
response['da'] = this.socketData['da'];
}
if (wantNow['want-tomahawk']) {
response['tomahawk'] = JSON.stringify(bitcoinApi.getHealthStatus());
}
if (parsedMessage && parsedMessage['track-tx']) {
if (/^[a-fA-F0-9]{64}$/.test(parsedMessage['track-tx'])) {
client['track-tx'] = parsedMessage['track-tx'];
@@ -546,6 +552,10 @@ class WebsocketHandler {
response['mempool-blocks'] = getCachedResponse('mempool-blocks', mBlocks);
}
if (client['want-tomahawk']) {
response['tomahawk'] = getCachedResponse('tomahawk', bitcoinApi.getHealthStatus());
}
if (client['track-mempool-tx']) {
const tx = newTransactions.find((t) => t.txid === client['track-mempool-tx']);
if (tx) {
@@ -909,6 +919,10 @@ class WebsocketHandler {
response['mempool-blocks'] = getCachedResponse('mempool-blocks', mBlocks);
}
if (client['want-tomahawk']) {
response['tomahawk'] = getCachedResponse('tomahawk', bitcoinApi.getHealthStatus());
}
if (client['track-tx']) {
const trackTxid = client['track-tx'];
if (trackTxid && confirmedTxids[trackTxid]) {

4
backend/src/config.ts
View File

@@ -5,6 +5,7 @@ const configFromFile = require(
interface IConfig {
MEMPOOL: {
ENABLED: boolean;
OFFICIAL: boolean;
NETWORK: 'mainnet' | 'testnet' | 'signet' | 'liquid' | 'liquidtestnet';
BACKEND: 'esplora' | 'electrum' | 'none';
HTTP_PORT: number;
@@ -103,6 +104,7 @@ interface IConfig {
PASSWORD: string;
TIMEOUT: number;
PID_DIR: string;
POOL_SIZE: number;
};
SYSLOG: {
ENABLED: boolean;
@@ -161,6 +163,7 @@ interface IConfig {
const defaults: IConfig = {
'MEMPOOL': {
'ENABLED': true,
'OFFICIAL': false,
'NETWORK': 'mainnet',
'BACKEND': 'none',
'HTTP_PORT': 8999,
@@ -240,6 +243,7 @@ const defaults: IConfig = {
'PASSWORD': 'mempool',
'TIMEOUT': 180000,
'PID_DIR': '',
'POOL_SIZE': 100,
},
'SYSLOG': {
'ENABLED': true,

2
backend/src/database.ts
View File

@@ -21,7 +21,7 @@ import { execSync } from 'child_process';
database: config.DATABASE.DATABASE,
user: config.DATABASE.USERNAME,
password: config.DATABASE.PASSWORD,
connectionLimit: 10,
connectionLimit: config.DATABASE.POOL_SIZE,
supportBigNumbers: true,
timezone: '+00:00',
};

16
backend/src/index.ts
View File

@@ -155,11 +155,17 @@ class Server {
}
if (Common.isLiquid()) {
try {
icons.loadIcons();
} catch (e) {
logger.err('Cannot load liquid icons. Ignoring. Reason: ' + (e instanceof Error ? e.message : e));
}
const refreshIcons = () => {
try {
icons.loadIcons();
} catch (e) {
logger.err('Cannot load liquid icons. Ignoring. Reason: ' + (e instanceof Error ? e.message : e));
}
};
// Run once on startup.
refreshIcons();
// Matches crontab refresh interval for asset db.
setInterval(refreshIcons, 3600_000);
}
priceUpdater.$run();