machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into natsoni/add-block-fee-graph

Browse Source
This commit is contained in:
wiz
2024-05-10 04:59:17 +09:00
committed by GitHub
parent f16afe20be dcdb9361e8
commit 53a493c233
166 changed files with 2921 additions and 905 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts
View File

@@ -1,4 +1,4 @@
import { IBitcoinApi } from './bitcoin-api.interface';
import { IBitcoinApi, TestMempoolAcceptResult } from './bitcoin-api.interface';
import { IEsploraApi } from './esplora-api.interface';
export interface AbstractBitcoinApi {
@@ -22,6 +22,7 @@ export interface AbstractBitcoinApi {
$getScriptHash(scripthash: string): Promise<IEsploraApi.ScriptHash>;
$getScriptHashTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]>;
$sendRawTransaction(rawTransaction: string): Promise<string>;
$testMempoolAccept(rawTransactions: string[], maxfeerate?: number): Promise<TestMempoolAcceptResult[]>;
$getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend>;
$getOutspends(txId: string): Promise<IEsploraApi.Outspend[]>;
$getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]>;

13
backend/src/api/bitcoin/bitcoin-api.interface.ts
View File

@@ -205,3 +205,16 @@ export namespace IBitcoinApi {
"utxo_size_inc": number;
}
}
export interface TestMempoolAcceptResult {
txid: string,
wtxid: string,
allowed?: boolean,
vsize?: number,
fees?: {
base: number,
"effective-feerate": number,
"effective-includes": string[],
},
['reject-reason']?: string,
}

10
backend/src/api/bitcoin/bitcoin-api.ts
View File

@@ -1,6 +1,6 @@
import * as bitcoinjs from 'bitcoinjs-lib';
import { AbstractBitcoinApi, HealthCheckHost } from './bitcoin-api-abstract-factory';
import { IBitcoinApi } from './bitcoin-api.interface';
import { IBitcoinApi, TestMempoolAcceptResult } from './bitcoin-api.interface';
import { IEsploraApi } from './esplora-api.interface';
import blocks from '../blocks';
import mempool from '../mempool';
@@ -174,6 +174,14 @@ class BitcoinApi implements AbstractBitcoinApi {
return this.bitcoindClient.sendRawTransaction(rawTransaction);
}
async $testMempoolAccept(rawTransactions: string[], maxfeerate?: number): Promise<TestMempoolAcceptResult[]> {
if (rawTransactions.length) {
return this.bitcoindClient.testMempoolAccept(rawTransactions, maxfeerate ?? undefined);
} else {
return [];
}
}
async $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
const txOut = await this.bitcoindClient.getTxOut(txId, vout, false);
return {

14
backend/src/api/bitcoin/bitcoin.routes.ts
View File

@@ -55,6 +55,7 @@ class BitcoinRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'mempool/recent', this.getRecentMempoolTransactions)
.get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId', this.getTransaction)
.post(config.MEMPOOL.API_URL_PREFIX + 'tx', this.$postTransaction)
.post(config.MEMPOOL.API_URL_PREFIX + 'txs/test', this.$testTransactions)
.get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/hex', this.getRawTransaction)
.get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', this.getTransactionStatus)
.get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', this.getTransactionOutspends)
@@ -749,6 +750,19 @@ class BitcoinRoutes {
}
}
private async $testTransactions(req: Request, res: Response) {
try {
const rawTxs = Common.getTransactionsFromRequest(req);
const maxfeerate = parseFloat(req.query.maxfeerate as string);
const result = await bitcoinApi.$testMempoolAccept(rawTxs, maxfeerate);
res.send(result);
} catch (e: any) {
res.setHeader('content-type', 'text/plain');
res.status(400).send(e.message && e.code ? 'testmempoolaccept RPC error: ' + JSON.stringify({ code: e.code, message: e.message })
: (e.message || 'Error'));
}
}
}
export default new BitcoinRoutes();

5
backend/src/api/bitcoin/esplora-api.ts
View File

@@ -5,6 +5,7 @@ import { AbstractBitcoinApi, HealthCheckHost } from './bitcoin-api-abstract-fact
import { IEsploraApi } from './esplora-api.interface';
import logger from '../../logger';
import { Common } from '../common';
import { TestMempoolAcceptResult } from './bitcoin-api.interface';
interface FailoverHost {
host: string,
@@ -327,6 +328,10 @@ class ElectrsApi implements AbstractBitcoinApi {
throw new Error('Method not implemented.');
}
$testMempoolAccept(rawTransactions: string[], maxfeerate?: number): Promise<TestMempoolAcceptResult[]> {
throw new Error('Method not implemented.');
}
$getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
return this.failoverRouter.$get<IEsploraApi.Outspend>('/tx/' + txId + '/outspend/' + vout);
}

7
backend/src/api/blocks.ts
View File

@@ -839,8 +839,11 @@ class Blocks {
} else {
this.currentBlockHeight++;
logger.debug(`New block found (#${this.currentBlockHeight})!`);
this.updateTimerProgress(timer, `getting orphaned blocks for ${this.currentBlockHeight}`);
await chainTips.updateOrphanedBlocks();
// skip updating the orphan block cache if we've fallen behind the chain tip
if (this.currentBlockHeight >= blockHeightTip - 2) {
this.updateTimerProgress(timer, `getting orphaned blocks for ${this.currentBlockHeight}`);
await chainTips.updateOrphanedBlocks();
}
}
this.updateTimerProgress(timer, `getting block data for ${this.currentBlockHeight}`);

66
backend/src/api/chain-tips.ts
View File

@@ -12,32 +12,68 @@ export interface OrphanedBlock {
height: number;
hash: string;
status: 'valid-fork' | 'valid-headers' | 'headers-only';
prevhash: string;
}
class ChainTips {
private chainTips: ChainTip[] = [];
private orphanedBlocks: OrphanedBlock[] = [];
private orphanedBlocks: { [hash: string]: OrphanedBlock } = {};
private blockCache: { [hash: string]: OrphanedBlock } = {};
private orphansByHeight: { [height: number]: OrphanedBlock[] } = {};
public async updateOrphanedBlocks(): Promise<void> {
try {
this.chainTips = await bitcoinClient.getChainTips();
this.orphanedBlocks = [];
const start = Date.now();
const breakAt = start + 10000;
let newOrphans = 0;
this.orphanedBlocks = {};
for (const chain of this.chainTips) {
if (chain.status === 'valid-fork' || chain.status === 'valid-headers') {
let block = await bitcoinClient.getBlock(chain.hash);
while (block && block.confirmations === -1) {
this.orphanedBlocks.push({
height: block.height,
hash: block.hash,
status: chain.status
});
block = await bitcoinClient.getBlock(block.previousblockhash);
const orphans: OrphanedBlock[] = [];
let hash = chain.hash;
do {
let orphan = this.blockCache[hash];
if (!orphan) {
const block = await bitcoinClient.getBlock(hash);
if (block && block.confirmations === -1) {
newOrphans++;
orphan = {
height: block.height,
hash: block.hash,
status: chain.status,
prevhash: block.previousblockhash,
};
this.blockCache[hash] = orphan;
}
}
if (orphan) {
orphans.push(orphan);
}
hash = orphan?.prevhash;
} while (hash && (Date.now() < breakAt));
for (const orphan of orphans) {
this.orphanedBlocks[orphan.hash] = orphan;
}
}
if (Date.now() >= breakAt) {
logger.debug(`Breaking orphaned blocks updater after 10s, will continue next block`);
break;
}
}
logger.debug(`Updated orphaned blocks cache. Found ${this.orphanedBlocks.length} orphaned blocks`);
this.orphansByHeight = {};
const allOrphans = Object.values(this.orphanedBlocks);
for (const orphan of allOrphans) {
if (!this.orphansByHeight[orphan.height]) {
this.orphansByHeight[orphan.height] = [];
}
this.orphansByHeight[orphan.height].push(orphan);
}
logger.debug(`Updated orphaned blocks cache. Fetched ${newOrphans} new orphaned blocks. Total ${allOrphans.length}`);
} catch (e) {
logger.err(`Cannot get fetch orphaned blocks. Reason: ${e instanceof Error ? e.message : e}`);
}
@@ -48,13 +84,7 @@ class ChainTips {
return [];
}
const orphans: OrphanedBlock[] = [];
for (const block of this.orphanedBlocks) {
if (block.height === height) {
orphans.push(block);
}
}
return orphans;
return this.orphansByHeight[height] || [];
}
}

27
backend/src/api/common.ts
View File

@@ -946,6 +946,33 @@ export class Common {
return this.validateTransactionHex(matches[1].toLowerCase());
}
static getTransactionsFromRequest(req: Request, limit: number = 25): string[] {
if (!Array.isArray(req.body) || req.body.some(hex => typeof hex !== 'string')) {
throw Object.assign(new Error('Invalid request body (should be an array of hexadecimal strings)'), { code: -1 });
}
if (limit && req.body.length > limit) {
throw Object.assign(new Error('Exceeded maximum of 25 transactions'), { code: -1 });
}
const txs = req.body;
return txs.map(rawTx => {
// Support both upper and lower case hex
// Support both txHash= Form and direct API POST
const reg = /^((?:[a-fA-F0-9]{2})+)$/;
const matches = reg.exec(rawTx);
if (!matches || !matches[1]) {
throw Object.assign(new Error('Invalid hex string'), { code: -2 });
}
// Guaranteed to be a hex string of multiple of 2
// Guaranteed to be lower case
// Guaranteed to pass validation (see function below)
return this.validateTransactionHex(matches[1].toLowerCase());
});
}
private static validateTransactionHex(txhex: string): string {
// Do not mutate txhex

2
backend/src/api/explorer/channels.routes.ts
View File

@@ -54,9 +54,11 @@ class ChannelsRoutes {
if (index < -1) {
res.status(400).send('Invalid index');
return;
}
if (['open', 'active', 'closed'].includes(status) === false) {
res.status(400).send('Invalid status');
return;
}
const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, 10, status);

8
backend/src/api/explorer/nodes.api.ts
View File

@@ -666,7 +666,9 @@ class NodesApi {
node.last_update = null;
}
const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? '';
const uniqueAddr = [...new Set(node.addresses?.map(a => a.addr))];
const formattedSockets = (uniqueAddr.join(',')) ?? '';
const query = `INSERT INTO nodes(
public_key,
first_seen,
@@ -695,13 +697,13 @@ class NodesApi {
node.alias,
this.aliasToSearchText(node.alias),
node.color,
sockets,
formattedSockets,
JSON.stringify(node.features),
node.last_update,
node.alias,
this.aliasToSearchText(node.alias),
node.color,
sockets,
formattedSockets,
JSON.stringify(node.features),
]);
} catch (e) {

4
backend/src/api/mempool.ts
View File

@@ -404,6 +404,10 @@ class Mempool {
const newAccelerationMap: { [txid: string]: Acceleration } = {};
for (const acceleration of newAccelerations) {
// skip transactions we don't know about
if (!this.mempoolCache[acceleration.txid]) {
continue;
}
newAccelerationMap[acceleration.txid] = acceleration;
if (this.accelerations[acceleration.txid] == null) {
// new acceleration

109
backend/src/api/websocket-handler.ts
View File

@@ -3,6 +3,7 @@ import * as WebSocket from 'ws';
import {
BlockExtended, TransactionExtended, MempoolTransactionExtended, WebsocketResponse,
OptimizedStatistic, ILoadingIndicators, GbtCandidates, TxTrackingInfo,
MempoolBlockDelta, MempoolDelta, MempoolDeltaTxids
} from '../mempool.interfaces';
import blocks from './blocks';
import memPool from './mempool';
@@ -346,6 +347,17 @@ class WebsocketHandler {
}
}
if (parsedMessage && parsedMessage['track-accelerations'] != null) {
if (parsedMessage['track-accelerations']) {
client['track-accelerations'] = true;
response['accelerations'] = JSON.stringify({
accelerations: Object.values(memPool.getAccelerations()),
});
} else {
client['track-accelerations'] = false;
}
}
if (parsedMessage.action === 'init') {
if (!this.socketData['blocks']?.length || !this.socketData['da'] || !this.socketData['backendInfo'] || !this.socketData['conversions']) {
this.updateSocketData();
@@ -364,6 +376,18 @@ class WebsocketHandler {
client['track-donation'] = parsedMessage['track-donation'];
}
if (parsedMessage['track-mempool-txids'] === true) {
client['track-mempool-txids'] = true;
} else if (parsedMessage['track-mempool-txids'] === false) {
delete client['track-mempool-txids'];
}
if (parsedMessage['track-mempool'] === true) {
client['track-mempool'] = true;
} else if (parsedMessage['track-mempool'] === false) {
delete client['track-mempool'];
}
if (Object.keys(response).length) {
client.send(this.serializeResponse(response));
}
@@ -524,6 +548,7 @@ class WebsocketHandler {
const vBytesPerSecond = memPool.getVBytesPerSecond();
const rbfTransactions = Common.findRbfTransactions(newTransactions, deletedTransactions);
const da = difficultyAdjustment.getDifficultyAdjustment();
const accelerations = memPool.getAccelerations();
memPool.handleRbfTransactions(rbfTransactions);
const rbfChanges = rbfCache.getRbfChanges();
let rbfReplacements;
@@ -545,6 +570,33 @@ class WebsocketHandler {
const latestTransactions = memPool.getLatestTransactions();
if (memPool.isInSync()) {
this.mempoolSequence++;
}
const replacedTransactions: { replaced: string, by: TransactionExtended }[] = [];
for (const tx of newTransactions) {
if (rbfTransactions[tx.txid]) {
for (const replaced of rbfTransactions[tx.txid]) {
replacedTransactions.push({ replaced: replaced.txid, by: tx });
}
}
}
const mempoolDeltaTxids: MempoolDeltaTxids = {
sequence: this.mempoolSequence,
added: newTransactions.map(tx => tx.txid),
removed: deletedTransactions.map(tx => tx.txid),
mined: [],
replaced: replacedTransactions.map(replacement => ({ replaced: replacement.replaced, by: replacement.by.txid })),
};
const mempoolDelta: MempoolDelta = {
sequence: this.mempoolSequence,
added: newTransactions,
removed: deletedTransactions.map(tx => tx.txid),
mined: [],
replaced: replacedTransactions,
};
// update init data
const socketDataFields = {
'mempoolInfo': mempoolInfo,
@@ -604,9 +656,11 @@ class WebsocketHandler {
const addressCache = this.makeAddressCache(newTransactions);
const removedAddressCache = this.makeAddressCache(deletedTransactions);
if (memPool.isInSync()) {
this.mempoolSequence++;
}
// pre-compute acceleration delta
const accelerationUpdate = {
added: accelerationDelta.map(txid => accelerations[txid]).filter(acc => acc != null),
removed: accelerationDelta.filter(txid => !accelerations[txid]),
};
// TODO - Fix indentation after PR is merged
for (const server of this.webSocketServers) {
@@ -847,6 +901,18 @@ class WebsocketHandler {
response['rbfLatestSummary'] = getCachedResponse('rbfLatestSummary', rbfSummary);
}
if (client['track-mempool-txids']) {
response['mempool-txids'] = getCachedResponse('mempool-txids', mempoolDeltaTxids);
}
if (client['track-mempool']) {
response['mempool-transactions'] = getCachedResponse('mempool-transactions', mempoolDelta);
}
if (client['track-accelerations'] && (accelerationUpdate.added.length || accelerationUpdate.removed.length)) {
response['accelerations'] = getCachedResponse('accelerations', accelerationUpdate);
}
if (Object.keys(response).length) {
client.send(this.serializeResponse(response));
}
@@ -992,6 +1058,31 @@ class WebsocketHandler {
const mBlocksWithTransactions = mempoolBlocks.getMempoolBlocksWithTransactions();
if (memPool.isInSync()) {
this.mempoolSequence++;
}
const replacedTransactions: { replaced: string, by: TransactionExtended }[] = [];
for (const txid of Object.keys(rbfTransactions)) {
for (const replaced of rbfTransactions[txid].replaced) {
replacedTransactions.push({ replaced: replaced.txid, by: rbfTransactions[txid].replacedBy });
}
}
const mempoolDeltaTxids: MempoolDeltaTxids = {
sequence: this.mempoolSequence,
added: [],
removed: [],
mined: transactions.map(tx => tx.txid),
replaced: replacedTransactions.map(replacement => ({ replaced: replacement.replaced, by: replacement.by.txid })),
};
const mempoolDelta: MempoolDelta = {
sequence: this.mempoolSequence,
added: [],
removed: [],
mined: transactions.map(tx => tx.txid),
replaced: replacedTransactions,
};
const responseCache = { ...this.socketData };
function getCachedResponse(key, data): string {
if (!responseCache[key]) {
@@ -1000,10 +1091,6 @@ class WebsocketHandler {
return responseCache[key];
}
if (memPool.isInSync()) {
this.mempoolSequence++;
}
// TODO - Fix indentation after PR is merged
for (const server of this.webSocketServers) {
server.clients.forEach((client) => {
@@ -1185,6 +1272,14 @@ class WebsocketHandler {
}
}
if (client['track-mempool-txids']) {
response['mempool-txids'] = getCachedResponse('mempool-txids', mempoolDeltaTxids);
}
if (client['track-mempool']) {
response['mempool-transactions'] = getCachedResponse('mempool-transactions', mempoolDelta);
}
if (Object.keys(response).length) {
client.send(this.serializeResponse(response));
}

1
backend/src/index.ts
View File

@@ -131,6 +131,7 @@ class Server {
})
.use(express.urlencoded({ extended: true }))
.use(express.text({ type: ['text/plain', 'application/base64'] }))
.use(express.json())
;
if (config.DATABASE.ENABLED && config.FIAT_PRICE.ENABLED) {

16
backend/src/mempool.interfaces.ts
View File

@@ -71,6 +71,22 @@ export interface MempoolBlockDelta {
changed: MempoolDeltaChange[];
}
export interface MempoolDeltaTxids {
sequence: number,
added: string[];
removed: string[];
mined: string[];
replaced: { replaced: string, by: string }[];
}
export interface MempoolDelta {
sequence: number,
added: MempoolTransactionExtended[];
removed: string[];
mined: string[];
replaced: { replaced: string, by: TransactionExtended }[];
}
interface VinStrippedToScriptsig {
scriptsig: string;
}