machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use minfee second node to supply alt rbf policy mempool txs

Browse Source
This commit is contained in:
Mononaut
2023-01-17 17:50:15 -06:00
parent be27e3df52
commit cdfee53cf5
19 changed files with 294 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
backend/src/__fixtures__/mempool-config.template.json
View File

@@ -26,9 +26,10 @@
"INDEXING_BLOCKS_AMOUNT": 14,
"POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__",
"POOLS_JSON_URL": "__POOLS_JSON_URL__",
"ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__",
"ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__",
"CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__"
"ADVANCED_GBT_AUDIT": "__ADVANCED_GBT_AUDIT__",
"ADVANCED_GBT_MEMPOOL": "__ADVANCED_GBT_MEMPOOL__",
"CPFP_INDEXING": "__CPFP_INDEXING__",
"RBF_DUAL_NODE": "__RBF_DUAL_NODE__"
},
"CORE_RPC": {
"HOST": "__CORE_RPC_HOST__",

1
backend/src/__tests__/config.test.ts
View File

@@ -41,6 +41,7 @@ describe('Mempool Backend Config', () => {
ADVANCED_GBT_AUDIT: false,
ADVANCED_GBT_MEMPOOL: false,
CPFP_INDEXING: false,
RBF_DUAL_NODE: false,
});
expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });

175
backend/src/api/alt-mempool.ts Normal file
View File

@@ -0,0 +1,175 @@
import config from '../config';
import { TransactionExtended } from '../mempool.interfaces';
import logger from '../logger';
import { Common } from './common';
import { IBitcoinApi } from './bitcoin/bitcoin-api.interface';
import BitcoinApi from './bitcoin/bitcoin-api';
import bitcoinApi from './bitcoin/bitcoin-api-factory';
import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
import { IEsploraApi } from './bitcoin/esplora-api.interface';
import { Mempool } from './mempool';
class AltMempool extends Mempool {
private bitcoinSecondApi: BitcoinApi;
constructor() {
super();
this.bitcoinSecondApi = new BitcoinApi(bitcoinSecondClient, this, bitcoinApi);
}
protected init(): void {
// override
}
public setOutOfSync(): void {
this.inSync = false;
}
public setMempool(mempoolData: { [txId: string]: TransactionExtended }): void {
this.mempoolCache = mempoolData;
}
public getFirstSeenForTransactions(txIds: string[]): number[] {
const txTimes: number[] = [];
txIds.forEach((txId: string) => {
const tx = this.mempoolCache[txId];
if (tx && tx.firstSeen) {
txTimes.push(tx.firstSeen);
} else {
txTimes.push(0);
}
});
return txTimes;
}
public async $updateMempool(): Promise<void> {
logger.debug(`Updating alternative mempool...`);
const start = new Date().getTime();
const currentMempoolSize = Object.keys(this.mempoolCache).length;
const transactions = await this.bitcoinSecondApi.$getRawMempool();
const diff = transactions.length - currentMempoolSize;
this.mempoolCacheDelta = Math.abs(diff);
const loadingMempool = this.mempoolCacheDelta > 100;
for (const txid of transactions) {
if (!this.mempoolCache[txid]) {
try {
const transaction = await this.$fetchTransaction(txid);
this.mempoolCache[txid] = transaction;
if (loadingMempool && Object.keys(this.mempoolCache).length % 50 === 0) {
logger.info(`loaded ${Object.keys(this.mempoolCache).length}/${transactions.length} alternative mempool transactions`);
}
} catch (e) {
logger.debug(`Error finding transaction '${txid}' in the alternative mempool: ` + (e instanceof Error ? e.message : e));
}
}
if ((new Date().getTime()) - start > Mempool.WEBSOCKET_REFRESH_RATE_MS) {
break;
}
}
if (loadingMempool) {
logger.info(`loaded ${Object.keys(this.mempoolCache).length}/${transactions.length} alternative mempool transactions`);
}
// Prevent mempool from clear on bitcoind restart by delaying the deletion
if (this.mempoolProtection === 0
&& currentMempoolSize > 20000
&& transactions.length / currentMempoolSize <= 0.80
) {
this.mempoolProtection = 1;
setTimeout(() => {
this.mempoolProtection = 2;
}, 1000 * 60 * config.MEMPOOL.CLEAR_PROTECTION_MINUTES);
}
const deletedTransactions: string[] = [];
if (this.mempoolProtection !== 1) {
this.mempoolProtection = 0;
// Index object for faster search
const transactionsObject = {};
transactions.forEach((txId) => transactionsObject[txId] = true);
// Flag transactions for lazy deletion
for (const tx in this.mempoolCache) {
if (!transactionsObject[tx] && !this.mempoolCache[tx].deleteAfter) {
deletedTransactions.push(this.mempoolCache[tx].txid);
}
}
for (const txid of deletedTransactions) {
delete this.mempoolCache[txid];
}
}
this.mempoolCacheDelta = Math.abs(transactions.length - Object.keys(this.mempoolCache).length);
const end = new Date().getTime();
const time = end - start;
logger.debug(`Alt mempool updated in ${time / 1000} seconds. New size: ${Object.keys(this.mempoolCache).length} (${diff > 0 ? '+' + diff : diff})`);
}
public getTransaction(txid: string): TransactionExtended {
return this.mempoolCache[txid] || null;
}
protected async $fetchTransaction(txid: string): Promise<TransactionExtended> {
const rawTx = await this.bitcoinSecondApi.$getRawTransaction(txid, false, true, false);
return this.extendTransaction(rawTx);
}
protected extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended {
// @ts-ignore
if (transaction.vsize) {
// @ts-ignore
return transaction;
}
const feePerVbytes = Math.max(Common.isLiquid() ? 0.1 : 1,
(transaction.fee || 0) / (transaction.weight / 4));
const transactionExtended: TransactionExtended = Object.assign({
vsize: Math.round(transaction.weight / 4),
feePerVsize: feePerVbytes,
effectiveFeePerVsize: feePerVbytes,
}, transaction);
if (!transaction.status.confirmed) {
transactionExtended.firstSeen = Math.round((new Date().getTime() / 1000));
}
return transactionExtended;
}
public async $updateMemPoolInfo(): Promise<void> {
this.mempoolInfo = await this.$getMempoolInfo();
}
public getMempoolInfo(): IBitcoinApi.MempoolInfo {
return this.mempoolInfo;
}
public getTxPerSecond(): number {
return this.txPerSecond;
}
public getVBytesPerSecond(): number {
return this.vBytesPerSecond;
}
public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended; }): void {
for (const rbfTransaction in rbfTransactions) {
if (this.mempoolCache[rbfTransaction]) {
// Erase the replaced transactions from the local mempool
delete this.mempoolCache[rbfTransaction];
}
}
}
protected updateTxPerSecond(): void {}
protected deleteExpiredTransactions(): void {}
protected $getMempoolInfo(): any {
return bitcoinSecondClient.getMempoolInfo();
}
}
export default new AltMempool();

7
backend/src/api/bitcoin/bitcoin-api-factory.ts
View File

@@ -4,19 +4,20 @@ import EsploraApi from './esplora-api';
import BitcoinApi from './bitcoin-api';
import ElectrumApi from './electrum-api';
import bitcoinClient from './bitcoin-client';
import mempool from '../mempool';
function bitcoinApiFactory(): AbstractBitcoinApi {
switch (config.MEMPOOL.BACKEND) {
case 'esplora':
return new EsploraApi();
case 'electrum':
return new ElectrumApi(bitcoinClient);
return new ElectrumApi(bitcoinClient, mempool);
case 'none':
default:
return new BitcoinApi(bitcoinClient);
return new BitcoinApi(bitcoinClient, mempool);
}
}
export const bitcoinCoreApi = new BitcoinApi(bitcoinClient);
export const bitcoinCoreApi = new BitcoinApi(bitcoinClient, mempool);
export default bitcoinApiFactory();

36
backend/src/api/bitcoin/bitcoin-api.ts
View File

@@ -3,15 +3,18 @@ import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
import { IBitcoinApi } from './bitcoin-api.interface';
import { IEsploraApi } from './esplora-api.interface';
import blocks from '../blocks';
import mempool from '../mempool';
import { TransactionExtended } from '../../mempool.interfaces';
import { MempoolBlock, TransactionExtended } from '../../mempool.interfaces';
class BitcoinApi implements AbstractBitcoinApi {
private mempool: any = null;
private rawMempoolCache: IBitcoinApi.RawMempool | null = null;
protected bitcoindClient: any;
protected backupBitcoinApi: BitcoinApi;
constructor(bitcoinClient: any) {
constructor(bitcoinClient: any, mempool: any, backupBitcoinApi: any = null) {
this.bitcoindClient = bitcoinClient;
this.mempool = mempool;
this.backupBitcoinApi = backupBitcoinApi;
}
static convertBlock(block: IBitcoinApi.Block): IEsploraApi.Block {
@@ -34,7 +37,7 @@ class BitcoinApi implements AbstractBitcoinApi {
$getRawTransaction(txId: string, skipConversion = false, addPrevout = false, lazyPrevouts = false): Promise<IEsploraApi.Transaction> {
// If the transaction is in the mempool we already converted and fetched the fee. Only prevouts are missing
const txInMempool = mempool.getMempool()[txId];
const txInMempool = this.mempool.getMempool()[txId];
if (txInMempool && addPrevout) {
return this.$addPrevouts(txInMempool);
}
@@ -118,7 +121,7 @@ class BitcoinApi implements AbstractBitcoinApi {
$getAddressPrefix(prefix: string): string[] {
const found: { [address: string]: string } = {};
const mp = mempool.getMempool();
const mp = this.mempool.getMempool();
for (const tx in mp) {
for (const vout of mp[tx].vout) {
if (vout.scriptpubkey_address.indexOf(prefix) === 0) {
@@ -260,7 +263,7 @@ class BitcoinApi implements AbstractBitcoinApi {
return transaction;
}
let mempoolEntry: IBitcoinApi.MempoolEntry;
if (!mempool.isInSync() && !this.rawMempoolCache) {
if (!this.mempool.isInSync() && !this.rawMempoolCache) {
this.rawMempoolCache = await this.$getRawMempoolVerbose();
}
if (this.rawMempoolCache && this.rawMempoolCache[transaction.txid]) {
@@ -316,10 +319,23 @@ class BitcoinApi implements AbstractBitcoinApi {
transaction.vin[i].lazy = true;
continue;
}
const innerTx = await this.$getRawTransaction(transaction.vin[i].txid, false, false);
transaction.vin[i].prevout = innerTx.vout[transaction.vin[i].vout];
this.addInnerScriptsToVin(transaction.vin[i]);
totalIn += innerTx.vout[transaction.vin[i].vout].value;
let innerTx;
try {
innerTx = await this.$getRawTransaction(transaction.vin[i].txid, false, false);
} catch (e) {
if (this.backupBitcoinApi) {
// input tx is confirmed, but the preferred client has txindex=0, so fetch from the backup client instead.
const backupTx = await this.backupBitcoinApi.$getRawTransaction(transaction.vin[i].txid);
innerTx = JSON.parse(JSON.stringify(backupTx));
} else {
throw e;
}
}
if (innerTx) {
transaction.vin[i].prevout = innerTx.vout[transaction.vin[i].vout];
this.addInnerScriptsToVin(transaction.vin[i]);
totalIn += innerTx.vout[transaction.vin[i].vout].value;
}
}
if (lazyPrevouts && transaction.vin.length > 12) {
transaction.fee = -1;

19
backend/src/api/bitcoin/bitcoin.routes.ts
View File

@@ -19,6 +19,7 @@ import bitcoinClient from './bitcoin-client';
import difficultyAdjustment from '../difficulty-adjustment';
import transactionRepository from '../../repositories/TransactionRepository';
import rbfCache from '../rbf-cache';
import altMempool from '../alt-mempool';
class BitcoinRoutes {
public initRoutes(app: Application) {
@@ -32,6 +33,7 @@ class BitcoinRoutes {
.get(config.MEMPOOL.API_URL_PREFIX + 'backend-info', this.getBackendInfo)
.get(config.MEMPOOL.API_URL_PREFIX + 'init-data', this.getInitData)
.get(config.MEMPOOL.API_URL_PREFIX + 'validate-address/:address', this.validateAddress)
.get(config.MEMPOOL.API_URL_PREFIX + 'tx/alt/:txId', this.getAltTransaction)
.get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/replaces', this.getRbfHistory)
.get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/cached', this.getCachedTx)
.post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', this.$postTransactionForm)
@@ -241,6 +243,23 @@ class BitcoinRoutes {
}
}
private async getAltTransaction(req: Request, res: Response) {
try {
const transaction = altMempool.getTransaction(req.params.txId);
if (transaction) {
res.json(transaction);
} else {
res.status(404).send('No such transaction in the alternate mempool');
}
} catch (e) {
let statusCode = 500;
if (e instanceof Error && e instanceof Error && e.message && e.message.indexOf('No such mempool or blockchain transaction') > -1) {
statusCode = 404;
}
res.status(statusCode).send(e instanceof Error ? e.message : e);
}
}
private async getRawTransaction(req: Request, res: Response) {
try {
const transaction: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(req.params.txId, true);

4
backend/src/api/bitcoin/electrum-api.ts
View File

@@ -12,8 +12,8 @@ import memoryCache from '../memory-cache';
class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
private electrumClient: any;
constructor(bitcoinClient: any) {
super(bitcoinClient);
constructor(bitcoinClient: any, mempool: any) {
super(bitcoinClient, mempool);
const electrumConfig = { client: 'mempool-v2', version: '1.4' };
const electrumPersistencePolicy = { retryPeriod: 10000, maxRetry: 1000, callback: null };

2
backend/src/api/mempool-blocks.ts
View File

@@ -249,7 +249,7 @@ class MempoolBlocks {
cluster.forEach(txid => {
if (txid === tx.txid) {
matched = true;
} else {
} else if (mempool[txid]) {
const relative = {
txid: txid,
fee: mempool[txid].fee,

40
backend/src/api/mempool.ts
View File

@@ -10,28 +10,32 @@ import bitcoinClient from './bitcoin/bitcoin-client';
import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
import rbfCache from './rbf-cache';
class Mempool {
private static WEBSOCKET_REFRESH_RATE_MS = 10000;
private static LAZY_DELETE_AFTER_SECONDS = 30;
private inSync: boolean = false;
private mempoolCacheDelta: number = -1;
private mempoolCache: { [txId: string]: TransactionExtended } = {};
private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
export class Mempool {
protected static WEBSOCKET_REFRESH_RATE_MS = 10000;
protected static LAZY_DELETE_AFTER_SECONDS = 30;
protected inSync: boolean = false;
protected mempoolCacheDelta: number = -1;
protected mempoolCache: { [txId: string]: TransactionExtended } = {};
protected mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 };
private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
protected mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
deletedTransactions: TransactionExtended[]) => void) | undefined;
private asyncMempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
protected asyncMempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
deletedTransactions: TransactionExtended[]) => Promise<void>) | undefined;
private txPerSecondArray: number[] = [];
private txPerSecond: number = 0;
protected txPerSecondArray: number[] = [];
protected txPerSecond: number = 0;
private vBytesPerSecondArray: VbytesPerSecond[] = [];
private vBytesPerSecond: number = 0;
private mempoolProtection = 0;
private latestTransactions: any[] = [];
protected vBytesPerSecondArray: VbytesPerSecond[] = [];
protected vBytesPerSecond: number = 0;
protected mempoolProtection = 0;
protected latestTransactions: any[] = [];
constructor() {
this.init();
}
protected init(): void {
setInterval(this.updateTxPerSecond.bind(this), 1000);
setInterval(this.deleteExpiredTransactions.bind(this), 20000);
}
@@ -217,7 +221,7 @@ class Mempool {
}
}
private updateTxPerSecond() {
protected updateTxPerSecond() {
const nowMinusTimeSpan = new Date().getTime() - (1000 * config.STATISTICS.TX_PER_SECOND_SAMPLE_PERIOD);
this.txPerSecondArray = this.txPerSecondArray.filter((unixTime) => unixTime > nowMinusTimeSpan);
this.txPerSecond = this.txPerSecondArray.length / config.STATISTICS.TX_PER_SECOND_SAMPLE_PERIOD || 0;
@@ -230,7 +234,7 @@ class Mempool {
}
}
private deleteExpiredTransactions() {
protected deleteExpiredTransactions() {
const now = new Date().getTime();
for (const tx in this.mempoolCache) {
const lazyDeleteAt = this.mempoolCache[tx].deleteAfter;
@@ -241,7 +245,7 @@ class Mempool {
}
}
private $getMempoolInfo() {
protected $getMempoolInfo() {
if (config.MEMPOOL.USE_SECOND_NODE_FOR_MINFEE) {
return Promise.all([
bitcoinClient.getMempoolInfo(),

2
backend/src/config.ts
View File

@@ -32,6 +32,7 @@ interface IConfig {
ADVANCED_GBT_AUDIT: boolean;
ADVANCED_GBT_MEMPOOL: boolean;
CPFP_INDEXING: boolean;
RBF_DUAL_NODE: boolean;
};
ESPLORA: {
REST_API_URL: string;
@@ -153,6 +154,7 @@ const defaults: IConfig = {
'ADVANCED_GBT_AUDIT': false,
'ADVANCED_GBT_MEMPOOL': false,
'CPFP_INDEXING': false,
'RBF_DUAL_NODE': false,
},
'ESPLORA': {
'REST_API_URL': 'http://127.0.0.1:3000',

4
backend/src/index.ts
View File

@@ -17,6 +17,7 @@ import logger from './logger';
import backendInfo from './api/backend-info';
import loadingIndicators from './api/loading-indicators';
import mempool from './api/mempool';
import altMempool from './api/alt-mempool';
import elementsParser from './api/liquid/elements-parser';
import databaseMigration from './api/database-migration';
import syncAssets from './sync-assets';
@@ -170,6 +171,9 @@ class Server {
await poolsUpdater.updatePoolsJson();
await blocks.$updateBlocks();
await memPool.$updateMempool();
if (config.MEMPOOL.RBF_DUAL_NODE) {
await altMempool.$updateMempool();
}
indexer.$run();
setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);