From a393f42b5eada9cf7cd7e01e79d9d9c13e8887a0 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 25 Jul 2023 16:35:21 +0900 Subject: [PATCH] strip non-essential data from redis cache txs --- backend/src/api/bitcoin/bitcoin-api.ts | 138 +++---------------------- backend/src/api/mempool.ts | 8 +- backend/src/api/redis-cache.ts | 39 ++++++- backend/src/api/transaction-utils.ts | 116 +++++++++++++++++++++ 4 files changed, 172 insertions(+), 129 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index a1cf767d9..132cda91a 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -5,6 +5,7 @@ import { IEsploraApi } from './esplora-api.interface'; import blocks from '../blocks'; import mempool from '../mempool'; import { TransactionExtended } from '../../mempool.interfaces'; +import transactionUtils from '../transaction-utils'; class BitcoinApi implements AbstractBitcoinApi { private rawMempoolCache: IBitcoinApi.RawMempool | null = null; @@ -63,9 +64,16 @@ class BitcoinApi implements AbstractBitcoinApi { return Promise.resolve([]); } - $getTransactionHex(txId: string): Promise { - return this.$getRawTransaction(txId, true) - .then((tx) => tx.hex || ''); + async $getTransactionHex(txId: string): Promise { + const txInMempool = mempool.getMempool()[txId]; + if (txInMempool && txInMempool.hex) { + return txInMempool.hex; + } + + return this.bitcoindClient.getRawTransaction(txId, true) + .then((transaction: IBitcoinApi.Transaction) => { + return transaction.hex; + }); } $getBlockHeightTip(): Promise { @@ -209,7 +217,7 @@ class BitcoinApi implements AbstractBitcoinApi { scriptpubkey: vout.scriptPubKey.hex, scriptpubkey_address: vout.scriptPubKey && vout.scriptPubKey.address ? vout.scriptPubKey.address : vout.scriptPubKey.addresses ? vout.scriptPubKey.addresses[0] : '', - scriptpubkey_asm: vout.scriptPubKey.asm ? this.convertScriptSigAsm(vout.scriptPubKey.hex) : '', + scriptpubkey_asm: vout.scriptPubKey.asm ? transactionUtils.convertScriptSigAsm(vout.scriptPubKey.hex) : '', scriptpubkey_type: this.translateScriptPubKeyType(vout.scriptPubKey.type), }; }); @@ -219,7 +227,7 @@ class BitcoinApi implements AbstractBitcoinApi { is_coinbase: !!vin.coinbase, prevout: null, scriptsig: vin.scriptSig && vin.scriptSig.hex || vin.coinbase || '', - scriptsig_asm: vin.scriptSig && this.convertScriptSigAsm(vin.scriptSig.hex) || '', + scriptsig_asm: vin.scriptSig && transactionUtils.convertScriptSigAsm(vin.scriptSig.hex) || '', sequence: vin.sequence, txid: vin.txid || '', vout: vin.vout || 0, @@ -291,7 +299,7 @@ class BitcoinApi implements AbstractBitcoinApi { } const innerTx = await this.$getRawTransaction(vin.txid, false, false); vin.prevout = innerTx.vout[vin.vout]; - this.addInnerScriptsToVin(vin); + transactionUtils.addInnerScriptsToVin(vin); } return transaction; } @@ -330,7 +338,7 @@ class BitcoinApi implements AbstractBitcoinApi { } const innerTx = await this.$getRawTransaction(transaction.vin[i].txid, false, false); transaction.vin[i].prevout = innerTx.vout[transaction.vin[i].vout]; - this.addInnerScriptsToVin(transaction.vin[i]); + transactionUtils.addInnerScriptsToVin(transaction.vin[i]); totalIn += innerTx.vout[transaction.vin[i].vout].value; } if (lazyPrevouts && transaction.vin.length > 12) { @@ -342,122 +350,6 @@ class BitcoinApi implements AbstractBitcoinApi { return transaction; } - private convertScriptSigAsm(hex: string): string { - const buf = Buffer.from(hex, 'hex'); - - const b: string[] = []; - - let i = 0; - while (i < buf.length) { - const op = buf[i]; - if (op >= 0x01 && op <= 0x4e) { - i++; - let push: number; - if (op === 0x4c) { - push = buf.readUInt8(i); - b.push('OP_PUSHDATA1'); - i += 1; - } else if (op === 0x4d) { - push = buf.readUInt16LE(i); - b.push('OP_PUSHDATA2'); - i += 2; - } else if (op === 0x4e) { - push = buf.readUInt32LE(i); - b.push('OP_PUSHDATA4'); - i += 4; - } else { - push = op; - b.push('OP_PUSHBYTES_' + push); - } - - const data = buf.slice(i, i + push); - if (data.length !== push) { - break; - } - - b.push(data.toString('hex')); - i += data.length; - } else { - if (op === 0x00) { - b.push('OP_0'); - } else if (op === 0x4f) { - b.push('OP_PUSHNUM_NEG1'); - } else if (op === 0xb1) { - b.push('OP_CLTV'); - } else if (op === 0xb2) { - b.push('OP_CSV'); - } else if (op === 0xba) { - b.push('OP_CHECKSIGADD'); - } else { - const opcode = bitcoinjs.script.toASM([ op ]); - if (opcode && op < 0xfd) { - if (/^OP_(\d+)$/.test(opcode)) { - b.push(opcode.replace(/^OP_(\d+)$/, 'OP_PUSHNUM_$1')); - } else { - b.push(opcode); - } - } else { - b.push('OP_RETURN_' + op); - } - } - i += 1; - } - } - - return b.join(' '); - } - - private addInnerScriptsToVin(vin: IEsploraApi.Vin): void { - if (!vin.prevout) { - return; - } - - if (vin.prevout.scriptpubkey_type === 'p2sh') { - const redeemScript = vin.scriptsig_asm.split(' ').reverse()[0]; - vin.inner_redeemscript_asm = this.convertScriptSigAsm(redeemScript); - if (vin.witness && vin.witness.length > 2) { - const witnessScript = vin.witness[vin.witness.length - 1]; - vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript); - } - } - - if (vin.prevout.scriptpubkey_type === 'v0_p2wsh' && vin.witness) { - const witnessScript = vin.witness[vin.witness.length - 1]; - vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript); - } - - if (vin.prevout.scriptpubkey_type === 'v1_p2tr' && vin.witness) { - const witnessScript = this.witnessToP2TRScript(vin.witness); - if (witnessScript !== null) { - vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript); - } - } - } - - /** - * This function must only be called when we know the witness we are parsing - * is a taproot witness. - * @param witness An array of hex strings that represents the witness stack of - * the input. - * @returns null if the witness is not a script spend, and the hex string of - * the script item if it is a script spend. - */ - private witnessToP2TRScript(witness: string[]): string | null { - if (witness.length < 2) return null; - // Note: see BIP341 for parsing details of witness stack - - // If there are at least two witness elements, and the first byte of the - // last element is 0x50, this last element is called annex a and - // is removed from the witness stack. - const hasAnnex = witness[witness.length - 1].substring(0, 2) === '50'; - // If there are at least two witness elements left, script path spending is used. - // Call the second-to-last stack element s, the script. - // (Note: this phrasing from BIP341 assumes we've *removed* the annex from the stack) - if (hasAnnex && witness.length < 3) return null; - const positionOfScript = hasAnnex ? witness.length - 3 : witness.length - 2; - return witness[positionOfScript]; - } - } export default BitcoinApi; diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 73a6fdfeb..d5214de5d 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -149,8 +149,8 @@ class Mempool { logger.err('failed to fetch bulk mempool transactions from esplora'); } } - return newTransactions; logger.info(`Done inserting loaded mempool transactions into local cache`); + return newTransactions; } public async $updateMemPoolInfo() { @@ -219,7 +219,11 @@ class Mempool { logger.info(`Missing ${transactions.length - currentMempoolSize} mempool transactions, attempting to reload in bulk from esplora`); try { newTransactions = await this.$reloadMempool(transactions.length); - redisCache.$addTransactions(newTransactions); + if (config.REDIS.ENABLED) { + for (const tx of newTransactions) { + await redisCache.$addTransaction(tx); + } + } loaded = true; } catch (e) { logger.err('failed to load mempool in bulk from esplora, falling back to fetching individual transactions'); diff --git a/backend/src/api/redis-cache.ts b/backend/src/api/redis-cache.ts index 8f7d54606..540467caf 100644 --- a/backend/src/api/redis-cache.ts +++ b/backend/src/api/redis-cache.ts @@ -5,6 +5,7 @@ import logger from '../logger'; import config from '../config'; import { BlockExtended, BlockSummary, MempoolTransactionExtended } from '../mempool.interfaces'; import rbfCache from './rbf-cache'; +import transactionUtils from './transaction-utils'; enum NetworkDB { mainnet = 0, @@ -20,7 +21,7 @@ class RedisCache { private schemaVersion = 1; private cacheQueue: MempoolTransactionExtended[] = []; - private txFlushLimit: number = 1000; + private txFlushLimit: number = 10000; constructor() { if (config.REDIS.ENABLED) { @@ -81,7 +82,7 @@ class RedisCache { async $addTransaction(tx: MempoolTransactionExtended) { this.cacheQueue.push(tx); - if (this.cacheQueue.length > this.txFlushLimit) { + if (this.cacheQueue.length >= this.txFlushLimit) { await this.$flushTransactions(); } } @@ -89,15 +90,28 @@ class RedisCache { async $flushTransactions() { const success = await this.$addTransactions(this.cacheQueue); if (success) { + logger.info(`Flushed ${this.cacheQueue.length} transactions to Redis cache`); this.cacheQueue = []; + } else { + logger.err(`Failed to flush ${this.cacheQueue.length} transactions to Redis cache`); } } - async $addTransactions(newTransactions: MempoolTransactionExtended[]): Promise { + private async $addTransactions(newTransactions: MempoolTransactionExtended[]): Promise { try { await this.$ensureConnected(); await Promise.all(newTransactions.map(tx => { - return this.client.json.set(`mempool:${tx.txid.slice(0,1)}`, tx.txid, tx); + const minified: any = { ...tx }; + delete minified.hex; + for (const vin of minified.vin) { + delete vin.inner_redeemscript_asm; + delete vin.inner_witnessscript_asm; + delete vin.scriptsig_asm; + } + for (const vout of minified.vout) { + delete vout.scriptpubkey_asm; + } + return this.client.json.set(`mempool:${tx.txid.slice(0,1)}`, tx.txid, minified); })); return true; } catch (e) { @@ -201,6 +215,7 @@ class RedisCache { const loadedBlockSummaries = await this.$getBlockSummaries(); // Load mempool const loadedMempool = await this.$getMempool(); + this.inflateLoadedTxs(loadedMempool); // Load rbf data const rbfTxs = await this.$getRbfEntries('tx'); const rbfTrees = await this.$getRbfEntries('tree'); @@ -217,6 +232,22 @@ class RedisCache { }); } + private inflateLoadedTxs(mempool: { [txid: string]: MempoolTransactionExtended }) { + for (const tx of Object.values(mempool)) { + for (const vin of tx.vin) { + if (vin.scriptsig) { + vin.scriptsig_asm = transactionUtils.convertScriptSigAsm(vin.scriptsig); + transactionUtils.addInnerScriptsToVin(vin); + } + } + for (const vout of tx.vout) { + if (vout.scriptpubkey) { + vout.scriptpubkey_asm = transactionUtils.convertScriptSigAsm(vout.scriptpubkey); + } + } + } + } + } export default new RedisCache(); diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index 009fe1dde..e141a6076 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -188,6 +188,122 @@ class TransactionUtils { 16 ); } + + public addInnerScriptsToVin(vin: IEsploraApi.Vin): void { + if (!vin.prevout) { + return; + } + + if (vin.prevout.scriptpubkey_type === 'p2sh') { + const redeemScript = vin.scriptsig_asm.split(' ').reverse()[0]; + vin.inner_redeemscript_asm = this.convertScriptSigAsm(redeemScript); + if (vin.witness && vin.witness.length > 2) { + const witnessScript = vin.witness[vin.witness.length - 1]; + vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript); + } + } + + if (vin.prevout.scriptpubkey_type === 'v0_p2wsh' && vin.witness) { + const witnessScript = vin.witness[vin.witness.length - 1]; + vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript); + } + + if (vin.prevout.scriptpubkey_type === 'v1_p2tr' && vin.witness) { + const witnessScript = this.witnessToP2TRScript(vin.witness); + if (witnessScript !== null) { + vin.inner_witnessscript_asm = this.convertScriptSigAsm(witnessScript); + } + } + } + + public convertScriptSigAsm(hex: string): string { + const buf = Buffer.from(hex, 'hex'); + + const b: string[] = []; + + let i = 0; + while (i < buf.length) { + const op = buf[i]; + if (op >= 0x01 && op <= 0x4e) { + i++; + let push: number; + if (op === 0x4c) { + push = buf.readUInt8(i); + b.push('OP_PUSHDATA1'); + i += 1; + } else if (op === 0x4d) { + push = buf.readUInt16LE(i); + b.push('OP_PUSHDATA2'); + i += 2; + } else if (op === 0x4e) { + push = buf.readUInt32LE(i); + b.push('OP_PUSHDATA4'); + i += 4; + } else { + push = op; + b.push('OP_PUSHBYTES_' + push); + } + + const data = buf.slice(i, i + push); + if (data.length !== push) { + break; + } + + b.push(data.toString('hex')); + i += data.length; + } else { + if (op === 0x00) { + b.push('OP_0'); + } else if (op === 0x4f) { + b.push('OP_PUSHNUM_NEG1'); + } else if (op === 0xb1) { + b.push('OP_CLTV'); + } else if (op === 0xb2) { + b.push('OP_CSV'); + } else if (op === 0xba) { + b.push('OP_CHECKSIGADD'); + } else { + const opcode = bitcoinjs.script.toASM([ op ]); + if (opcode && op < 0xfd) { + if (/^OP_(\d+)$/.test(opcode)) { + b.push(opcode.replace(/^OP_(\d+)$/, 'OP_PUSHNUM_$1')); + } else { + b.push(opcode); + } + } else { + b.push('OP_RETURN_' + op); + } + } + i += 1; + } + } + + return b.join(' '); + } + + /** + * This function must only be called when we know the witness we are parsing + * is a taproot witness. + * @param witness An array of hex strings that represents the witness stack of + * the input. + * @returns null if the witness is not a script spend, and the hex string of + * the script item if it is a script spend. + */ + public witnessToP2TRScript(witness: string[]): string | null { + if (witness.length < 2) return null; + // Note: see BIP341 for parsing details of witness stack + + // If there are at least two witness elements, and the first byte of the + // last element is 0x50, this last element is called annex a and + // is removed from the witness stack. + const hasAnnex = witness[witness.length - 1].substring(0, 2) === '50'; + // If there are at least two witness elements left, script path spending is used. + // Call the second-to-last stack element s, the script. + // (Note: this phrasing from BIP341 assumes we've *removed* the annex from the stack) + if (hasAnnex && witness.length < 3) return null; + const positionOfScript = hasAnnex ? witness.length - 3 : witness.length - 2; + return witness[positionOfScript]; + } } export default new TransactionUtils();