diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 99b69c208..32becd00d 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -44,7 +44,8 @@ }, "ESPLORA": { "REST_API_URL": "http://127.0.0.1:3000", - "UNIX_SOCKET_PATH": "/tmp/esplora-bitcoin-mainnet" + "UNIX_SOCKET_PATH": "/tmp/esplora-bitcoin-mainnet", + "RETRY_UNIX_SOCKET_AFTER": 30000 }, "SECOND_CORE_RPC": { "HOST": "127.0.0.1", @@ -60,7 +61,8 @@ "SOCKET": "/var/run/mysql/mysql.sock", "DATABASE": "mempool", "USERNAME": "mempool", - "PASSWORD": "mempool" + "PASSWORD": "mempool", + "TIMEOUT": 180000 }, "SYSLOG": { "ENABLED": true, diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 625f3d8a0..eb082d89f 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -45,7 +45,8 @@ }, "ESPLORA": { "REST_API_URL": "__ESPLORA_REST_API_URL__", - "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__" + "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__", + "RETRY_UNIX_SOCKET_AFTER": "__ESPLORA_RETRY_UNIX_SOCKET_AFTER__" }, "SECOND_CORE_RPC": { "HOST": "__SECOND_CORE_RPC_HOST__", @@ -61,7 +62,8 @@ "PORT": 18, "DATABASE": "__DATABASE_DATABASE__", "USERNAME": "__DATABASE_USERNAME__", - "PASSWORD": "__DATABASE_PASSWORD__" + "PASSWORD": "__DATABASE_PASSWORD__", + "TIMEOUT": "__DATABASE_TIMEOUT__" }, "SYSLOG": { "ENABLED": false, diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 75b87c6cf..aa287308b 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -47,7 +47,7 @@ describe('Mempool Backend Config', () => { expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true }); - expect(config.ESPLORA).toStrictEqual({ REST_API_URL: 'http://127.0.0.1:3000', UNIX_SOCKET_PATH: null }); + expect(config.ESPLORA).toStrictEqual({ REST_API_URL: 'http://127.0.0.1:3000', UNIX_SOCKET_PATH: null, RETRY_UNIX_SOCKET_AFTER: 30000 }); expect(config.CORE_RPC).toStrictEqual({ HOST: '127.0.0.1', @@ -72,7 +72,8 @@ describe('Mempool Backend Config', () => { PORT: 3306, DATABASE: 'mempool', USERNAME: 'mempool', - PASSWORD: 'mempool' + PASSWORD: 'mempool', + TIMEOUT: 180000, }); expect(config.SYSLOG).toStrictEqual({ diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index 6e1cb3787..7435e3b99 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -93,17 +93,7 @@ class Audit { } else { if (!isDisplaced[tx.txid]) { added.push(tx.txid); - } else { } - let blockIndex = -1; - let index = -1; - projectedBlocks.forEach((block, bi) => { - const i = block.transactionIds.indexOf(tx.txid); - if (i >= 0) { - blockIndex = bi; - index = i; - } - }); overflowWeight += tx.weight; } totalWeight += tx.weight; diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index c6323d041..18d688e9b 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -32,8 +32,10 @@ class BitcoinRoutes { .get(config.MEMPOOL.API_URL_PREFIX + 'backend-info', this.getBackendInfo) .get(config.MEMPOOL.API_URL_PREFIX + 'init-data', this.getInitData) .get(config.MEMPOOL.API_URL_PREFIX + 'validate-address/:address', this.validateAddress) - .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/replaces', this.getRbfHistory) + .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/rbf', this.getRbfHistory) .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/cached', this.getCachedTx) + .get(config.MEMPOOL.API_URL_PREFIX + 'replacements', this.getRbfReplacements) + .get(config.MEMPOOL.API_URL_PREFIX + 'fullrbf/replacements', this.getFullRbfReplacements) .post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', this.$postTransactionForm) .get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => { try { @@ -94,6 +96,7 @@ class BitcoinRoutes { .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', this.getBlock) .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions) .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary) + .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', this.getBlockTipHeight) .post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion) .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this)) .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this)) @@ -110,7 +113,6 @@ class BitcoinRoutes { .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', this.getTransactionStatus) .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', this.getTransactionOutspends) .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', this.getBlockHeader) - .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', this.getBlockTipHeight) .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/hash', this.getBlockTipHash) .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/raw', this.getRawBlock) .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txids', this.getTxIdsForBlock) @@ -589,10 +591,14 @@ class BitcoinRoutes { } } - private async getBlockTipHeight(req: Request, res: Response) { + private getBlockTipHeight(req: Request, res: Response) { try { - const result = await bitcoinApi.$getBlockHeightTip(); - res.json(result); + const result = blocks.getCurrentBlockHeight(); + if (!result) { + return res.status(503).send(`Service Temporarily Unavailable`); + } + res.setHeader('content-type', 'text/plain'); + res.send(result.toString()); } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); } @@ -638,8 +644,30 @@ class BitcoinRoutes { private async getRbfHistory(req: Request, res: Response) { try { - const result = rbfCache.getReplaces(req.params.txId); - res.json(result || []); + const replacements = rbfCache.getRbfTree(req.params.txId) || null; + const replaces = rbfCache.getReplaces(req.params.txId) || null; + res.json({ + replacements, + replaces + }); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async getRbfReplacements(req: Request, res: Response) { + try { + const result = rbfCache.getRbfTrees(false); + res.json(result); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + private async getFullRbfReplacements(req: Request, res: Response) { + try { + const result = rbfCache.getRbfTrees(true); + res.json(result); } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); } diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index ff6219587..ee7fa4765 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -3,68 +3,102 @@ import axios, { AxiosRequestConfig } from 'axios'; import http from 'http'; import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; import { IEsploraApi } from './esplora-api.interface'; +import logger from '../../logger'; const axiosConnection = axios.create({ httpAgent: new http.Agent({ keepAlive: true, }) }); class ElectrsApi implements AbstractBitcoinApi { - axiosConfig: AxiosRequestConfig = config.ESPLORA.UNIX_SOCKET_PATH ? { + private axiosConfigWithUnixSocket: AxiosRequestConfig = config.ESPLORA.UNIX_SOCKET_PATH ? { socketPath: config.ESPLORA.UNIX_SOCKET_PATH, timeout: 10000, } : { timeout: 10000, }; + private axiosConfigTcpSocketOnly: AxiosRequestConfig = { + timeout: 10000, + }; - constructor() { } + unixSocketRetryTimeout; + activeAxiosConfig; + + constructor() { + this.activeAxiosConfig = this.axiosConfigWithUnixSocket; + } + + fallbackToTcpSocket() { + if (!this.unixSocketRetryTimeout) { + logger.err(`Unable to connect to esplora unix socket. Falling back to tcp socket. Retrying unix socket in ${config.ESPLORA.RETRY_UNIX_SOCKET_AFTER / 1000} seconds`); + // Retry the unix socket after a few seconds + this.unixSocketRetryTimeout = setTimeout(() => { + logger.info(`Retrying to use unix socket for esplora now (applied for the next query)`); + this.activeAxiosConfig = this.axiosConfigWithUnixSocket; + this.unixSocketRetryTimeout = undefined; + }, config.ESPLORA.RETRY_UNIX_SOCKET_AFTER); + } + + // Use the TCP socket (reach a different esplora instance through nginx) + this.activeAxiosConfig = this.axiosConfigTcpSocketOnly; + } + + $queryWrapper(url, responseType = 'json'): Promise { + return axiosConnection.get(url, { ...this.activeAxiosConfig, responseType: responseType }) + .then((response) => response.data) + .catch((e) => { + if (e?.code === 'ECONNREFUSED') { + this.fallbackToTcpSocket(); + // Retry immediately + return axiosConnection.get(url, this.activeAxiosConfig) + .then((response) => response.data) + .catch((e) => { + logger.warn(`Cannot query esplora through the unix socket nor the tcp socket. Exception ${e}`); + throw e; + }); + } else { + throw e; + } + }); + } $getRawMempool(): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig) - .then((response) => response.data); + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/mempool/txids'); } $getRawTransaction(txId: string): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig) - .then((response) => response.data); + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/tx/' + txId); } $getTransactionHex(txId: string): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex', this.axiosConfig) - .then((response) => response.data); + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex'); } $getBlockHeightTip(): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig) - .then((response) => response.data); + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/blocks/tip/height'); } $getBlockHashTip(): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig) - .then((response) => response.data); + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/blocks/tip/hash'); } $getTxIdsForBlock(hash: string): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig) - .then((response) => response.data); + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids'); } $getBlockHash(height: number): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig) - .then((response) => response.data); + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/block-height/' + height); } $getBlockHeader(hash: string): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig) - .then((response) => response.data); + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header'); } $getBlock(hash: string): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig) - .then((response) => response.data); + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/block/' + hash); } $getRawBlock(hash: string): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' }) + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", 'arraybuffer') .then((response) => { return Buffer.from(response.data); }); } @@ -85,13 +119,11 @@ class ElectrsApi implements AbstractBitcoinApi { } $getOutspend(txId: string, vout: number): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig) - .then((response) => response.data); + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout); } $getOutspends(txId: string): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig) - .then((response) => response.data); + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends'); } async $getBatchedOutspends(txId: string[]): Promise { diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 13232cc8e..2837d40a0 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -36,6 +36,8 @@ class Blocks { private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = []; private newAsyncBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => Promise)[] = []; + private mainLoopTimeout: number = 120000; + constructor() { } public getBlocks(): BlockExtended[] { @@ -528,8 +530,12 @@ class Blocks { } public async $updateBlocks() { + // warn if this run stalls the main loop for more than 2 minutes + const timer = this.startTimer(); + let fastForwarded = false; const blockHeightTip = await bitcoinApi.$getBlockHeightTip(); + this.updateTimerProgress(timer, 'got block height tip'); if (this.blocks.length === 0) { this.currentBlockHeight = Math.max(blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT, -1); @@ -547,16 +553,21 @@ class Blocks { if (!this.lastDifficultyAdjustmentTime) { const blockchainInfo = await bitcoinClient.getBlockchainInfo(); + this.updateTimerProgress(timer, 'got blockchain info for initial difficulty adjustment'); if (blockchainInfo.blocks === blockchainInfo.headers) { const heightDiff = blockHeightTip % 2016; const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff); + this.updateTimerProgress(timer, 'got block hash for initial difficulty adjustment'); const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); + this.updateTimerProgress(timer, 'got block for initial difficulty adjustment'); this.lastDifficultyAdjustmentTime = block.timestamp; this.currentDifficulty = block.difficulty; if (blockHeightTip >= 2016) { const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016); + this.updateTimerProgress(timer, 'got previous block hash for initial difficulty adjustment'); const previousPeriodBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(previousPeriodBlockHash); + this.updateTimerProgress(timer, 'got previous block for initial difficulty adjustment'); this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100; logger.debug(`Initial difficulty adjustment data set.`); } @@ -571,9 +582,11 @@ class Blocks { } else { this.currentBlockHeight++; logger.debug(`New block found (#${this.currentBlockHeight})!`); + this.updateTimerProgress(timer, `getting orphaned blocks for ${this.currentBlockHeight}`); await chainTips.updateOrphanedBlocks(); } + this.updateTimerProgress(timer, `getting block data for ${this.currentBlockHeight}`); const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight); const verboseBlock = await bitcoinClient.getBlock(blockHash, 2); const block = BitcoinApi.convertBlock(verboseBlock); @@ -582,39 +595,51 @@ class Blocks { const cpfpSummary: CpfpSummary = Common.calculateCpfp(block.height, transactions); const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions); const blockSummary: BlockSummary = this.summarizeBlock(verboseBlock); + this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`); // start async callbacks + this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`); const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions)); if (Common.indexingEnabled()) { if (!fastForwarded) { const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1); + this.updateTimerProgress(timer, `got block by height for ${this.currentBlockHeight}`); if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock.id) { logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`, logger.tags.mining); // We assume there won't be a reorg with more than 10 block depth + this.updateTimerProgress(timer, `rolling back diverged chain from ${this.currentBlockHeight}`); await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10); await HashratesRepository.$deleteLastEntries(); await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10); + this.updateTimerProgress(timer, `rolled back chain divergence from ${this.currentBlockHeight}`); for (let i = 10; i >= 0; --i) { const newBlock = await this.$indexBlock(lastBlock.height - i); + this.updateTimerProgress(timer, `reindexed block`); await this.$getStrippedBlockTransactions(newBlock.id, true, true); + this.updateTimerProgress(timer, `reindexed block summary`); if (config.MEMPOOL.CPFP_INDEXING) { await this.$indexCPFP(newBlock.id, lastBlock.height - i); + this.updateTimerProgress(timer, `reindexed block cpfp`); } } await mining.$indexDifficultyAdjustments(); await DifficultyAdjustmentsRepository.$deleteLastAdjustment(); + this.updateTimerProgress(timer, `reindexed difficulty adjustments`); logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`, logger.tags.mining); indexer.reindex(); } await blocksRepository.$saveBlockInDatabase(blockExtended); + this.updateTimerProgress(timer, `saved ${this.currentBlockHeight} to database`); const lastestPriceId = await PricesRepository.$getLatestPriceId(); + this.updateTimerProgress(timer, `got latest price id ${this.currentBlockHeight}`); if (priceUpdater.historyInserted === true && lastestPriceId !== null) { await blocksRepository.$saveBlockPrices([{ height: blockExtended.height, priceId: lastestPriceId, }]); + this.updateTimerProgress(timer, `saved prices for ${this.currentBlockHeight}`); } else { logger.debug(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`, logger.tags.mining); setTimeout(() => { @@ -625,9 +650,11 @@ class Blocks { // Save blocks summary for visualization if it's enabled if (Common.blocksSummariesIndexingEnabled() === true) { await this.$getStrippedBlockTransactions(blockExtended.id, true); + this.updateTimerProgress(timer, `saved block summary for ${this.currentBlockHeight}`); } if (config.MEMPOOL.CPFP_INDEXING) { this.$saveCpfp(blockExtended.id, this.currentBlockHeight, cpfpSummary); + this.updateTimerProgress(timer, `saved cpfp for ${this.currentBlockHeight}`); } } } @@ -640,6 +667,7 @@ class Blocks { difficulty: block.difficulty, adjustment: Math.round((block.difficulty / this.currentDifficulty) * 1000000) / 1000000, // Remove float point noise }); + this.updateTimerProgress(timer, `saved difficulty adjustment for ${this.currentBlockHeight}`); } this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100; @@ -664,7 +692,33 @@ class Blocks { } // wait for pending async callbacks to finish + this.updateTimerProgress(timer, `waiting for async callbacks to complete for ${this.currentBlockHeight}`); await Promise.all(callbackPromises); + this.updateTimerProgress(timer, `async callbacks completed for ${this.currentBlockHeight}`); + } + + this.clearTimer(timer); + } + + private startTimer() { + const state: any = { + start: Date.now(), + progress: 'begin $updateBlocks', + timer: null, + }; + state.timer = setTimeout(() => { + logger.err(`$updateBlocks stalled at "${state.progress}"`); + }, this.mainLoopTimeout); + return state; + } + + private updateTimerProgress(state, msg) { + state.progress = msg; + } + + private clearTimer(state) { + if (state.timer) { + clearTimeout(state.timer); } } @@ -857,6 +911,7 @@ class Blocks { } if (cleanBlock.fee_amt_percentiles !== null) { cleanBlock.median_fee_amt = cleanBlock.fee_amt_percentiles[3]; + await blocksRepository.$updateFeeAmounts(cleanBlock.hash, cleanBlock.fee_amt_percentiles, cleanBlock.median_fee_amt); } } diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index fed3f4fa2..b044e2866 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -57,11 +57,11 @@ export class Common { return arr; } - static findRbfTransactions(added: TransactionExtended[], deleted: TransactionExtended[]): { [txid: string]: TransactionExtended } { - const matches: { [txid: string]: TransactionExtended } = {}; - deleted - .forEach((deletedTx) => { - const foundMatches = added.find((addedTx) => { + static findRbfTransactions(added: TransactionExtended[], deleted: TransactionExtended[]): { [txid: string]: TransactionExtended[] } { + const matches: { [txid: string]: TransactionExtended[] } = {}; + added + .forEach((addedTx) => { + const foundMatches = deleted.filter((deletedTx) => { // The new tx must, absolutely speaking, pay at least as much fee as the replaced tx. return addedTx.fee > deletedTx.fee // The new transaction must pay more fee per kB than the replaced tx. @@ -70,8 +70,8 @@ export class Common { && deletedTx.vin.some((deletedVin) => addedTx.vin.some((vin) => vin.txid === deletedVin.txid && vin.vout === deletedVin.vout)); }); - if (foundMatches) { - matches[deletedTx.txid] = foundMatches; + if (foundMatches?.length) { + matches[addedTx.txid] = foundMatches; } }); return matches; diff --git a/backend/src/api/disk-cache.ts b/backend/src/api/disk-cache.ts index c50d3cef8..7a38e7da0 100644 --- a/backend/src/api/disk-cache.ts +++ b/backend/src/api/disk-cache.ts @@ -7,14 +7,18 @@ import logger from '../logger'; import config from '../config'; import { TransactionExtended } from '../mempool.interfaces'; import { Common } from './common'; +import rbfCache from './rbf-cache'; class DiskCache { private cacheSchemaVersion = 3; + private rbfCacheSchemaVersion = 1; private static TMP_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/tmp-cache.json'; private static TMP_FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/tmp-cache{number}.json'; private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json'; private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json'; + private static TMP_RBF_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/tmp-rbfcache.json'; + private static RBF_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/rbfcache.json'; private static CHUNK_FILES = 25; private isWritingCache = false; @@ -100,6 +104,32 @@ class DiskCache { logger.warn('Error writing to cache file: ' + (e instanceof Error ? e.message : e)); this.isWritingCache = false; } + + try { + logger.debug('Writing rbf data to disk cache (async)...'); + this.isWritingCache = true; + const rbfData = rbfCache.dump(); + if (sync) { + fs.writeFileSync(DiskCache.TMP_RBF_FILE_NAME, JSON.stringify({ + network: config.MEMPOOL.NETWORK, + rbfCacheSchemaVersion: this.rbfCacheSchemaVersion, + rbf: rbfData, + }), { flag: 'w' }); + fs.renameSync(DiskCache.TMP_RBF_FILE_NAME, DiskCache.RBF_FILE_NAME); + } else { + await fsPromises.writeFile(DiskCache.TMP_RBF_FILE_NAME, JSON.stringify({ + network: config.MEMPOOL.NETWORK, + rbfCacheSchemaVersion: this.rbfCacheSchemaVersion, + rbf: rbfData, + }), { flag: 'w' }); + await fsPromises.rename(DiskCache.TMP_RBF_FILE_NAME, DiskCache.RBF_FILE_NAME); + } + logger.debug('Rbf data saved to disk cache'); + this.isWritingCache = false; + } catch (e) { + logger.warn('Error writing rbf data to cache file: ' + (e instanceof Error ? e.message : e)); + this.isWritingCache = false; + } } wipeCache(): void { @@ -124,7 +154,19 @@ class DiskCache { } } - loadMempoolCache(): void { + wipeRbfCache() { + logger.notice(`Wipping nodejs backend cache/rbfcache.json file`); + + try { + fs.unlinkSync(DiskCache.RBF_FILE_NAME); + } catch (e: any) { + if (e?.code !== 'ENOENT') { + logger.err(`Cannot wipe cache file ${DiskCache.RBF_FILE_NAME}. Exception ${JSON.stringify(e)}`); + } + } + } + + async $loadMempoolCache(): Promise { if (!fs.existsSync(DiskCache.FILE_NAME)) { return; } @@ -168,12 +210,35 @@ class DiskCache { } } - memPool.setMempool(data.mempool); + await memPool.$setMempool(data.mempool); blocks.setBlocks(data.blocks); blocks.setBlockSummaries(data.blockSummaries || []); } catch (e) { logger.warn('Failed to parse mempoool and blocks cache. Skipping. Reason: ' + (e instanceof Error ? e.message : e)); } + + try { + let rbfData: any = {}; + const rbfCacheData = fs.readFileSync(DiskCache.RBF_FILE_NAME, 'utf8'); + if (rbfCacheData) { + logger.info('Restoring rbf data from disk cache'); + rbfData = JSON.parse(rbfCacheData); + if (rbfData.rbfCacheSchemaVersion === undefined || rbfData.rbfCacheSchemaVersion !== this.rbfCacheSchemaVersion) { + logger.notice('Rbf disk cache contains an outdated schema version. Clearing it and skipping the cache loading.'); + return this.wipeRbfCache(); + } + if (rbfData.network && rbfData.network !== config.MEMPOOL.NETWORK) { + logger.notice('Rbf disk cache contains data from a different network. Clearing it and skipping the cache loading.'); + return this.wipeRbfCache(); + } + } + + if (rbfData?.rbf) { + rbfCache.load(rbfData.rbf); + } + } catch (e) { + logger.warn('Failed to parse rbf cache. Skipping. Reason: ' + (e instanceof Error ? e.message : e)); + } } } diff --git a/backend/src/api/liquid/icons.ts b/backend/src/api/liquid/icons.ts index ee08757d0..a963c703c 100644 --- a/backend/src/api/liquid/icons.ts +++ b/backend/src/api/liquid/icons.ts @@ -2,7 +2,7 @@ import * as fs from 'fs'; import logger from '../../logger'; class Icons { - private static FILE_NAME = './icons.json'; + private static FILE_NAME = '/elements/asset_registry_db/icons.json'; private iconIds: string[] = []; private icons: { [assetId: string]: string; } = {}; diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index bf5c16955..49fda543b 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -54,7 +54,14 @@ class MempoolBlocks { }); // First sort - memPoolArray.sort((a, b) => b.feePerVsize - a.feePerVsize); + memPoolArray.sort((a, b) => { + if (a.feePerVsize === b.feePerVsize) { + // tie-break by lexicographic txid order for stability + return a.txid < b.txid ? -1 : 1; + } else { + return b.feePerVsize - a.feePerVsize; + } + }); // Loop through and traverse all ancestors and sum up all the sizes + fees // Pass down size + fee to all unconfirmed children @@ -68,7 +75,14 @@ class MempoolBlocks { }); // Final sort, by effective fee - memPoolArray.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize); + memPoolArray.sort((a, b) => { + if (a.effectiveFeePerVsize === b.effectiveFeePerVsize) { + // tie-break by lexicographic txid order for stability + return a.txid < b.txid ? -1 : 1; + } else { + return b.effectiveFeePerVsize - a.effectiveFeePerVsize; + } + }); const end = new Date().getTime(); const time = end - start; @@ -88,14 +102,26 @@ class MempoolBlocks { private calculateMempoolBlocks(transactionsSorted: TransactionExtended[]): MempoolBlockWithTransactions[] { const mempoolBlocks: MempoolBlockWithTransactions[] = []; let blockWeight = 0; + let blockVsize = 0; let transactions: TransactionExtended[] = []; transactionsSorted.forEach((tx) => { if (blockWeight + tx.weight <= config.MEMPOOL.BLOCK_WEIGHT_UNITS || mempoolBlocks.length === config.MEMPOOL.MEMPOOL_BLOCKS_AMOUNT - 1) { + tx.position = { + block: mempoolBlocks.length, + vsize: blockVsize + (tx.vsize / 2), + }; blockWeight += tx.weight; + blockVsize += tx.vsize; transactions.push(tx); } else { mempoolBlocks.push(this.dataToMempoolBlocks(transactions)); + blockVsize = 0; + tx.position = { + block: mempoolBlocks.length, + vsize: blockVsize + (tx.vsize / 2), + }; + blockVsize += tx.vsize; blockWeight = tx.weight; transactions = [tx]; } @@ -148,7 +174,7 @@ class MempoolBlocks { return mempoolBlockDeltas; } - public async makeBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, saveResults: boolean = false): Promise { + public async $makeBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, saveResults: boolean = false): Promise { // prepare a stripped down version of the mempool with only the minimum necessary data // to reduce the overhead of passing this data to the worker thread const strippedMempool: { [txid: string]: ThreadTransaction } = {}; @@ -206,10 +232,10 @@ class MempoolBlocks { return this.mempoolBlocks; } - public async updateBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, added: TransactionExtended[], removed: string[], saveResults: boolean = false): Promise { + public async $updateBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, added: TransactionExtended[], removed: string[], saveResults: boolean = false): Promise { if (!this.txSelectionWorker) { // need to reset the worker - this.makeBlockTemplates(newMempool, saveResults); + await this.$makeBlockTemplates(newMempool, saveResults); return; } // prepare a stripped down version of the mempool with only the minimum necessary data @@ -256,9 +282,17 @@ class MempoolBlocks { private processBlockTemplates(mempool, blocks, clusters, saveResults): MempoolBlockWithTransactions[] { // update this thread's mempool with the results - blocks.forEach(block => { + blocks.forEach((block, blockIndex) => { + let runningVsize = 0; block.forEach(tx => { if (tx.txid && tx.txid in mempool) { + // save position in projected blocks + mempool[tx.txid].position = { + block: blockIndex, + vsize: runningVsize + (mempool[tx.txid].vsize / 2), + }; + runningVsize += mempool[tx.txid].vsize; + if (tx.effectiveFeePerVsize != null) { mempool[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize; } diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 1be1faceb..d476d6bca 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -20,7 +20,7 @@ class Mempool { maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 }; private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) => void) | undefined; - private asyncMempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[], + private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) => Promise) | undefined; private txPerSecondArray: number[] = []; @@ -36,6 +36,8 @@ class Mempool { private timer = new Date().getTime(); private missingTxCount = 0; + private mainLoopTimeout: number = 120000; + constructor() { setInterval(this.updateTxPerSecond.bind(this), 1000); } @@ -71,20 +73,20 @@ class Mempool { public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) => Promise) { - this.asyncMempoolChangedCallback = fn; + this.$asyncMempoolChangedCallback = fn; } public getMempool(): { [txid: string]: TransactionExtended } { return this.mempoolCache; } - public setMempool(mempoolData: { [txId: string]: TransactionExtended }) { + public async $setMempool(mempoolData: { [txId: string]: TransactionExtended }) { this.mempoolCache = mempoolData; if (this.mempoolChangedCallback) { this.mempoolChangedCallback(this.mempoolCache, [], []); } - if (this.asyncMempoolChangedCallback) { - this.asyncMempoolChangedCallback(this.mempoolCache, [], []); + if (this.$asyncMempoolChangedCallback) { + await this.$asyncMempoolChangedCallback(this.mempoolCache, [], []); } } @@ -119,10 +121,15 @@ class Mempool { public async $updateMempool(): Promise { logger.debug(`Updating mempool...`); + + // warn if this run stalls the main loop for more than 2 minutes + const timer = this.startTimer(); + const start = new Date().getTime(); let hasChange: boolean = false; const currentMempoolSize = Object.keys(this.mempoolCache).length; const transactions = await bitcoinApi.$getRawMempool(); + this.updateTimerProgress(timer, 'got raw mempool'); const diff = transactions.length - currentMempoolSize; const newTransactions: TransactionExtended[] = []; @@ -146,6 +153,7 @@ class Mempool { if (!this.mempoolCache[txid]) { try { const transaction = await transactionUtils.$getTransactionExtended(txid); + this.updateTimerProgress(timer, 'fetched new transaction'); this.mempoolCache[txid] = transaction; if (this.inSync) { this.txPerSecondArray.push(new Date().getTime()); @@ -222,22 +230,50 @@ class Mempool { if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) { this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions); } - if (this.asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) { - await this.asyncMempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions); + if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) { + this.updateTimerProgress(timer, 'running async mempool callback'); + await this.$asyncMempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions); + this.updateTimerProgress(timer, 'completed async mempool callback'); } const end = new Date().getTime(); const time = end - start; logger.debug(`Mempool updated in ${time / 1000} seconds. New size: ${Object.keys(this.mempoolCache).length} (${diff > 0 ? '+' + diff : diff})`); + + this.clearTimer(timer); } - public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended; }) { + private startTimer() { + const state: any = { + start: Date.now(), + progress: 'begin $updateMempool', + timer: null, + }; + state.timer = setTimeout(() => { + logger.err(`$updateMempool stalled at "${state.progress}"`); + }, this.mainLoopTimeout); + return state; + } + + private updateTimerProgress(state, msg) { + state.progress = msg; + } + + private clearTimer(state) { + if (state.timer) { + clearTimeout(state.timer); + } + } + + public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended[]; }): void { for (const rbfTransaction in rbfTransactions) { - if (this.mempoolCache[rbfTransaction]) { + if (this.mempoolCache[rbfTransaction] && rbfTransactions[rbfTransaction]?.length) { // Store replaced transactions - rbfCache.add(this.mempoolCache[rbfTransaction], rbfTransactions[rbfTransaction].txid); + rbfCache.add(rbfTransactions[rbfTransaction], this.mempoolCache[rbfTransaction]); // Erase the replaced transactions from the local mempool - delete this.mempoolCache[rbfTransaction]; + for (const replaced of rbfTransactions[rbfTransaction]) { + delete this.mempoolCache[replaced.txid]; + } } } } diff --git a/backend/src/api/rbf-cache.ts b/backend/src/api/rbf-cache.ts index 410239e73..7f910e0f7 100644 --- a/backend/src/api/rbf-cache.ts +++ b/backend/src/api/rbf-cache.ts @@ -1,46 +1,173 @@ -import { TransactionExtended } from "../mempool.interfaces"; +import { TransactionExtended, TransactionStripped } from "../mempool.interfaces"; +import bitcoinApi from './bitcoin/bitcoin-api-factory'; +import { Common } from "./common"; + +interface RbfTransaction extends TransactionStripped { + rbf?: boolean; + mined?: boolean; +} + +interface RbfTree { + tx: RbfTransaction; + time: number; + interval?: number; + mined?: boolean; + fullRbf: boolean; + replaces: RbfTree[]; +} class RbfCache { - private replacedBy: { [txid: string]: string; } = {}; - private replaces: { [txid: string]: string[] } = {}; - private txs: { [txid: string]: TransactionExtended } = {}; - private expiring: { [txid: string]: Date } = {}; + private replacedBy: Map = new Map(); + private replaces: Map = new Map(); + private rbfTrees: Map = new Map(); // sequences of consecutive replacements + private dirtyTrees: Set = new Set(); + private treeMap: Map = new Map(); // map of txids to sequence ids + private txs: Map = new Map(); + private expiring: Map = new Map(); constructor() { setInterval(this.cleanup.bind(this), 1000 * 60 * 60); } - public add(replacedTx: TransactionExtended, newTxId: string): void { - this.replacedBy[replacedTx.txid] = newTxId; - this.txs[replacedTx.txid] = replacedTx; - if (!this.replaces[newTxId]) { - this.replaces[newTxId] = []; + public add(replaced: TransactionExtended[], newTxExtended: TransactionExtended): void { + if (!newTxExtended || !replaced?.length) { + return; } - this.replaces[newTxId].push(replacedTx.txid); + + const newTx = Common.stripTransaction(newTxExtended) as RbfTransaction; + const newTime = newTxExtended.firstSeen || Date.now(); + newTx.rbf = newTxExtended.vin.some((v) => v.sequence < 0xfffffffe); + this.txs.set(newTx.txid, newTxExtended); + + // maintain rbf trees + let fullRbf = false; + const replacedTrees: RbfTree[] = []; + for (const replacedTxExtended of replaced) { + const replacedTx = Common.stripTransaction(replacedTxExtended) as RbfTransaction; + replacedTx.rbf = replacedTxExtended.vin.some((v) => v.sequence < 0xfffffffe); + this.replacedBy.set(replacedTx.txid, newTx.txid); + if (this.treeMap.has(replacedTx.txid)) { + const treeId = this.treeMap.get(replacedTx.txid); + if (treeId) { + const tree = this.rbfTrees.get(treeId); + this.rbfTrees.delete(treeId); + if (tree) { + tree.interval = newTime - tree?.time; + replacedTrees.push(tree); + fullRbf = fullRbf || tree.fullRbf; + } + } + } else { + const replacedTime = replacedTxExtended.firstSeen || Date.now(); + replacedTrees.push({ + tx: replacedTx, + time: replacedTime, + interval: newTime - replacedTime, + fullRbf: !replacedTx.rbf, + replaces: [], + }); + fullRbf = fullRbf || !replacedTx.rbf; + this.txs.set(replacedTx.txid, replacedTxExtended); + } + } + const treeId = replacedTrees[0].tx.txid; + const newTree = { + tx: newTx, + time: newTxExtended.firstSeen || Date.now(), + fullRbf, + replaces: replacedTrees + }; + this.rbfTrees.set(treeId, newTree); + this.updateTreeMap(treeId, newTree); + this.replaces.set(newTx.txid, replacedTrees.map(tree => tree.tx.txid)); + this.dirtyTrees.add(treeId); } public getReplacedBy(txId: string): string | undefined { - return this.replacedBy[txId]; + return this.replacedBy.get(txId); } public getReplaces(txId: string): string[] | undefined { - return this.replaces[txId]; + return this.replaces.get(txId); } public getTx(txId: string): TransactionExtended | undefined { - return this.txs[txId]; + return this.txs.get(txId); + } + + public getRbfTree(txId: string): RbfTree | void { + return this.rbfTrees.get(this.treeMap.get(txId) || ''); + } + + // get a paginated list of RbfTrees + // ordered by most recent replacement time + public getRbfTrees(onlyFullRbf: boolean, after?: string): RbfTree[] { + const limit = 25; + const trees: RbfTree[] = []; + const used = new Set(); + const replacements: string[][] = Array.from(this.replacedBy).reverse(); + const afterTree = after ? this.treeMap.get(after) : null; + let ready = !afterTree; + for (let i = 0; i < replacements.length && trees.length <= limit - 1; i++) { + const txid = replacements[i][1]; + const treeId = this.treeMap.get(txid) || ''; + if (treeId === afterTree) { + ready = true; + } else if (ready) { + if (!used.has(treeId)) { + const tree = this.rbfTrees.get(treeId); + used.add(treeId); + if (tree && (!onlyFullRbf || tree.fullRbf)) { + trees.push(tree); + } + } + } + } + return trees; + } + + // get map of rbf trees that have been updated since the last call + public getRbfChanges(): { trees: {[id: string]: RbfTree }, map: { [txid: string]: string }} { + const changes: { trees: {[id: string]: RbfTree }, map: { [txid: string]: string }} = { + trees: {}, + map: {}, + }; + this.dirtyTrees.forEach(id => { + const tree = this.rbfTrees.get(id); + if (tree) { + changes.trees[id] = tree; + this.getTransactionsInTree(tree).forEach(tx => { + changes.map[tx.txid] = id; + }); + } + }); + this.dirtyTrees = new Set(); + return changes; + } + + public mined(txid): void { + const treeId = this.treeMap.get(txid); + if (treeId && this.rbfTrees.has(treeId)) { + const tree = this.rbfTrees.get(treeId); + if (tree) { + this.setTreeMined(tree, txid); + tree.mined = true; + this.dirtyTrees.add(treeId); + } + } + this.evict(txid); } // flag a transaction as removed from the mempool public evict(txid): void { - this.expiring[txid] = new Date(Date.now() + 1000 * 86400); // 24 hours + this.expiring.set(txid, new Date(Date.now() + 1000 * 86400)); // 24 hours } private cleanup(): void { const currentDate = new Date(); for (const txid in this.expiring) { - if (this.expiring[txid] < currentDate) { - delete this.expiring[txid]; + if ((this.expiring.get(txid) || 0) < currentDate) { + this.expiring.delete(txid); this.remove(txid); } } @@ -48,18 +175,147 @@ class RbfCache { // remove a transaction & all previous versions from the cache private remove(txid): void { - // don't remove a transaction while a newer version remains in the mempool - if (this.replaces[txid] && !this.replacedBy[txid]) { - const replaces = this.replaces[txid]; - delete this.replaces[txid]; - for (const tx of replaces) { + // don't remove a transaction if a newer version remains in the mempool + if (!this.replacedBy.has(txid)) { + const replaces = this.replaces.get(txid); + this.replaces.delete(txid); + this.treeMap.delete(txid); + this.txs.delete(txid); + this.expiring.delete(txid); + for (const tx of (replaces || [])) { // recursively remove prior versions from the cache - delete this.replacedBy[tx]; - delete this.txs[tx]; + this.replacedBy.delete(tx); + // if this is the id of a tree, remove that too + if (this.treeMap.get(tx) === tx) { + this.rbfTrees.delete(tx); + } this.remove(tx); } } } + + private updateTreeMap(newId: string, tree: RbfTree): void { + this.treeMap.set(tree.tx.txid, newId); + tree.replaces.forEach(subtree => { + this.updateTreeMap(newId, subtree); + }); + } + + private getTransactionsInTree(tree: RbfTree, txs: RbfTransaction[] = []): RbfTransaction[] { + txs.push(tree.tx); + tree.replaces.forEach(subtree => { + this.getTransactionsInTree(subtree, txs); + }); + return txs; + } + + private setTreeMined(tree: RbfTree, txid: string): void { + if (tree.tx.txid === txid) { + tree.tx.mined = true; + } else { + tree.replaces.forEach(subtree => { + this.setTreeMined(subtree, txid); + }); + } + } + + public dump(): any { + const trees = Array.from(this.rbfTrees.values()).map((tree: RbfTree) => { return this.exportTree(tree); }); + + return { + txs: Array.from(this.txs.entries()), + trees, + expiring: Array.from(this.expiring.entries()), + }; + } + + public async load({ txs, trees, expiring }): Promise { + txs.forEach(txEntry => { + this.txs.set(txEntry[0], txEntry[1]); + }); + for (const deflatedTree of trees) { + await this.importTree(deflatedTree.root, deflatedTree.root, deflatedTree, this.txs); + } + expiring.forEach(expiringEntry => { + this.expiring.set(expiringEntry[0], expiringEntry[1]); + }); + this.cleanup(); + } + + exportTree(tree: RbfTree, deflated: any = null) { + if (!deflated) { + deflated = { + root: tree.tx.txid, + }; + } + deflated[tree.tx.txid] = { + tx: tree.tx.txid, + txMined: tree.tx.mined, + time: tree.time, + interval: tree.interval, + mined: tree.mined, + fullRbf: tree.fullRbf, + replaces: tree.replaces.map(child => child.tx.txid), + }; + tree.replaces.forEach(child => { + this.exportTree(child, deflated); + }); + return deflated; + } + + async importTree(root, txid, deflated, txs: Map, mined: boolean = false): Promise { + const treeInfo = deflated[txid]; + const replaces: RbfTree[] = []; + + // check if any transactions in this tree have already been confirmed + mined = mined || treeInfo.mined; + if (!mined) { + try { + const apiTx = await bitcoinApi.$getRawTransaction(txid); + if (apiTx?.status?.confirmed) { + mined = true; + this.evict(txid); + } + } catch (e) { + // most transactions do not exist + } + } + + // recursively reconstruct child trees + for (const childId of treeInfo.replaces) { + const replaced = await this.importTree(root, childId, deflated, txs, mined); + if (replaced) { + this.replacedBy.set(replaced.tx.txid, txid); + replaces.push(replaced); + if (replaced.mined) { + mined = true; + } + } + } + this.replaces.set(txid, replaces.map(t => t.tx.txid)); + + const tx = txs.get(txid); + if (!tx) { + return; + } + const strippedTx = Common.stripTransaction(tx) as RbfTransaction; + strippedTx.rbf = tx.vin.some((v) => v.sequence < 0xfffffffe); + strippedTx.mined = treeInfo.txMined; + const tree = { + tx: strippedTx, + time: treeInfo.time, + interval: treeInfo.interval, + mined: mined, + fullRbf: treeInfo.fullRbf, + replaces, + }; + this.treeMap.set(txid, root); + if (root === txid) { + this.rbfTrees.set(root, tree); + this.dirtyTrees.add(root); + } + return tree; + } } export default new RbfCache(); diff --git a/backend/src/api/tx-selection-worker.ts b/backend/src/api/tx-selection-worker.ts index 7297cbe88..c035099a3 100644 --- a/backend/src/api/tx-selection-worker.ts +++ b/backend/src/api/tx-selection-worker.ts @@ -2,7 +2,6 @@ import config from '../config'; import logger from '../logger'; import { ThreadTransaction, MempoolBlockWithTransactions, AuditTransaction } from '../mempool.interfaces'; import { PairingHeap } from '../utils/pairing-heap'; -import { Common } from './common'; import { parentPort } from 'worker_threads'; let mempool: { [txid: string]: ThreadTransaction } = {}; @@ -72,7 +71,14 @@ function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction }) } // Sort by descending ancestor score - mempoolArray.sort((a, b) => (b.score || 0) - (a.score || 0)); + mempoolArray.sort((a, b) => { + if (b.score === a.score) { + // tie-break by lexicographic txid order for stability + return a.txid < b.txid ? -1 : 1; + } else { + return (b.score || 0) - (a.score || 0); + } + }); // Build blocks by greedily choosing the highest feerate package // (i.e. the package rooted in the transaction with the best ancestor score) @@ -80,7 +86,14 @@ function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction }) let blockWeight = 4000; let blockSize = 0; let transactions: AuditTransaction[] = []; - const modified: PairingHeap = new PairingHeap((a, b): boolean => (a.score || 0) > (b.score || 0)); + const modified: PairingHeap = new PairingHeap((a, b): boolean => { + if (a.score === b.score) { + // tie-break by lexicographic txid order for stability + return a.txid > b.txid; + } else { + return (a.score || 0) > (b.score || 0); + } + }); let overflow: AuditTransaction[] = []; let failures = 0; let top = 0; @@ -107,7 +120,7 @@ function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction }) if (nextTx && !nextTx?.used) { // Check if the package fits into this block - if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) { + if (blocks.length >= 7 || (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS)) { const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values()); // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count) const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx]; @@ -175,34 +188,14 @@ function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction }) overflow = []; } } - // pack any leftover transactions into the last block - for (const tx of overflow) { - if (!tx || tx?.used) { - continue; - } - blockWeight += tx.weight; - const mempoolTx = mempool[tx.txid]; - // update original copy of this tx with effective fee rate & relatives data - mempoolTx.effectiveFeePerVsize = tx.score; - if (tx.ancestorMap.size > 0) { - cpfpClusters[tx.txid] = Array.from(tx.ancestorMap?.values()).map(a => a.txid); - mempoolTx.cpfpRoot = tx.txid; - } - mempoolTx.cpfpChecked = true; - transactions.push(tx); - tx.used = true; + + if (overflow.length > 0) { + logger.warn('GBT overflow list unexpectedly non-empty after final block constructed'); } - const blockTransactions = transactions.map(t => mempool[t.txid]); - restOfArray.forEach(tx => { - blockWeight += tx.weight; - tx.effectiveFeePerVsize = tx.feePerVsize; - tx.cpfpChecked = false; - blockTransactions.push(tx); - }); - if (blockTransactions.length) { - blocks.push(blockTransactions); + // add the final unbounded block if it contains any transactions + if (transactions.length > 0) { + blocks.push(transactions.map(t => mempool[t.txid])); } - transactions = []; const end = Date.now(); const time = end - start; diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 7dbd48c46..f28f284c7 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -26,6 +26,10 @@ class WebsocketHandler { private wss: WebSocket.Server | undefined; private extraInitProperties = {}; + private numClients = 0; + private numConnected = 0; + private numDisconnected = 0; + constructor() { } setWebsocketServer(wss: WebSocket.Server) { @@ -42,7 +46,11 @@ class WebsocketHandler { } this.wss.on('connection', (client: WebSocket) => { + this.numConnected++; client.on('error', logger.info); + client.on('close', () => { + this.numDisconnected++; + }); client.on('message', async (message: string) => { try { const parsedMessage: WebsocketResponse = JSON.parse(message); @@ -58,9 +66,10 @@ class WebsocketHandler { if (parsedMessage && parsedMessage['track-tx']) { if (/^[a-fA-F0-9]{64}$/.test(parsedMessage['track-tx'])) { client['track-tx'] = parsedMessage['track-tx']; + const trackTxid = client['track-tx']; // Client is telling the transaction wasn't found if (parsedMessage['watch-mempool']) { - const rbfCacheTxid = rbfCache.getReplacedBy(client['track-tx']); + const rbfCacheTxid = rbfCache.getReplacedBy(trackTxid); if (rbfCacheTxid) { response['txReplaced'] = { txid: rbfCacheTxid, @@ -68,7 +77,7 @@ class WebsocketHandler { client['track-tx'] = null; } else { // It might have appeared before we had the time to start watching for it - const tx = memPool.getMempool()[client['track-tx']]; + const tx = memPool.getMempool()[trackTxid]; if (tx) { if (config.MEMPOOL.BACKEND === 'esplora') { response['tx'] = tx; @@ -92,6 +101,13 @@ class WebsocketHandler { } } } + const tx = memPool.getMempool()[trackTxid]; + if (tx && tx.position) { + response['txPosition'] = { + txid: trackTxid, + position: tx.position, + }; + } } else { client['track-tx'] = null; } @@ -132,6 +148,14 @@ class WebsocketHandler { } } + if (parsedMessage && parsedMessage['track-rbf'] !== undefined) { + if (['all', 'fullRbf'].includes(parsedMessage['track-rbf'])) { + client['track-rbf'] = parsedMessage['track-rbf']; + } else { + client['track-rbf'] = false; + } + } + if (parsedMessage.action === 'init') { const _blocks = blocks.getBlocks().slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT); if (!_blocks) { @@ -232,6 +256,8 @@ class WebsocketHandler { throw new Error('WebSocket.Server is not set'); } + this.printLogs(); + this.wss.clients.forEach((client) => { if (client.readyState !== WebSocket.OPEN) { return; @@ -247,14 +273,16 @@ class WebsocketHandler { }); } - async handleMempoolChange(newMempool: { [txid: string]: TransactionExtended }, + async $handleMempoolChange(newMempool: { [txid: string]: TransactionExtended }, newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]): Promise { if (!this.wss) { throw new Error('WebSocket.Server is not set'); } + this.printLogs(); + if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { - await mempoolBlocks.updateBlockTemplates(newMempool, newTransactions, deletedTransactions.map(tx => tx.txid), true); + await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions.map(tx => tx.txid), true); } else { mempoolBlocks.updateMempoolBlocks(newMempool, true); } @@ -266,6 +294,13 @@ class WebsocketHandler { const rbfTransactions = Common.findRbfTransactions(newTransactions, deletedTransactions); const da = difficultyAdjustment.getDifficultyAdjustment(); memPool.handleRbfTransactions(rbfTransactions); + const rbfChanges = rbfCache.getRbfChanges(); + let rbfReplacements; + let fullRbfReplacements; + if (Object.keys(rbfChanges.trees).length) { + rbfReplacements = rbfCache.getRbfTrees(false); + fullRbfReplacements = rbfCache.getRbfTrees(true); + } const recommendedFees = feeApi.getRecommendedFee(); this.wss.clients.forEach(async (client) => { @@ -374,9 +409,10 @@ class WebsocketHandler { } if (client['track-tx']) { + const trackTxid = client['track-tx']; const outspends: object = {}; newTransactions.forEach((tx) => tx.vin.forEach((vin, i) => { - if (vin.txid === client['track-tx']) { + if (vin.txid === trackTxid) { outspends[vin.vout] = { vin: i, txid: tx.txid, @@ -388,16 +424,25 @@ class WebsocketHandler { response['utxoSpent'] = outspends; } - if (rbfTransactions[client['track-tx']]) { - for (const rbfTransaction in rbfTransactions) { - if (client['track-tx'] === rbfTransaction) { - response['rbfTransaction'] = { - txid: rbfTransactions[rbfTransaction].txid, - }; - break; - } + const rbfReplacedBy = rbfCache.getReplacedBy(client['track-tx']); + if (rbfReplacedBy) { + response['rbfTransaction'] = { + txid: rbfReplacedBy, } } + + const rbfChange = rbfChanges.map[client['track-tx']]; + if (rbfChange) { + response['rbfInfo'] = rbfChanges.trees[rbfChange]; + } + + const mempoolTx = newMempool[trackTxid]; + if (mempoolTx && mempoolTx.position) { + response['txPosition'] = { + txid: trackTxid, + position: mempoolTx.position, + }; + } } if (client['track-mempool-block'] >= 0) { @@ -410,6 +455,12 @@ class WebsocketHandler { } } + if (client['track-rbf'] === 'all' && rbfReplacements) { + response['rbfLatest'] = rbfReplacements; + } else if (client['track-rbf'] === 'fullRbf' && fullRbfReplacements) { + response['rbfLatest'] = fullRbfReplacements; + } + if (Object.keys(response).length) { client.send(JSON.stringify(response)); } @@ -421,17 +472,25 @@ class WebsocketHandler { throw new Error('WebSocket.Server is not set'); } + this.printLogs(); + const _memPool = memPool.getMempool(); if (config.MEMPOOL.AUDIT) { let projectedBlocks; + let auditMempool = _memPool; // template calculation functions have mempool side effects, so calculate audits using // a cloned copy of the mempool if we're running a different algorithm for mempool updates - const auditMempool = (config.MEMPOOL.ADVANCED_GBT_AUDIT === config.MEMPOOL.ADVANCED_GBT_MEMPOOL) ? _memPool : deepClone(_memPool); - if (config.MEMPOOL.ADVANCED_GBT_AUDIT) { - projectedBlocks = await mempoolBlocks.makeBlockTemplates(auditMempool, false); + const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL; + if (separateAudit) { + auditMempool = deepClone(_memPool); + if (config.MEMPOOL.ADVANCED_GBT_AUDIT) { + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false); + } else { + projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false); + } } else { - projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false); + projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); } if (Common.indexingEnabled() && memPool.isInSync()) { @@ -477,16 +536,14 @@ class WebsocketHandler { } } - const removed: string[] = []; // Update mempool to remove transactions included in the new block for (const txId of txIds) { delete _memPool[txId]; - removed.push(txId); - rbfCache.evict(txId); + rbfCache.mined(txId); } if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { - await mempoolBlocks.updateBlockTemplates(_memPool, [], removed, true); + await mempoolBlocks.$makeBlockTemplates(_memPool, true); } else { mempoolBlocks.updateMempoolBlocks(_memPool, true); } @@ -516,8 +573,19 @@ class WebsocketHandler { response['mempool-blocks'] = mBlocks; } - if (client['track-tx'] && txIds.indexOf(client['track-tx']) > -1) { - response['txConfirmed'] = true; + if (client['track-tx']) { + const trackTxid = client['track-tx']; + if (txIds.indexOf(trackTxid) > -1) { + response['txConfirmed'] = true; + } else { + const mempoolTx = _memPool[trackTxid]; + if (mempoolTx && mempoolTx.position) { + response['txPosition'] = { + txid: trackTxid, + position: mempoolTx.position, + }; + } + } } if (client['track-address']) { @@ -597,6 +665,17 @@ class WebsocketHandler { client.send(JSON.stringify(response)); }); } + + private printLogs(): void { + if (this.wss) { + const count = this.wss?.clients?.size || 0; + const diff = count - this.numClients; + this.numClients = count; + logger.debug(`${count} websocket clients | ${this.numConnected} connected | ${this.numDisconnected} disconnected | (${diff >= 0 ? '+' : ''}${diff})`); + this.numConnected = 0; + this.numDisconnected = 0; + } + } } export default new WebsocketHandler(); diff --git a/backend/src/config.ts b/backend/src/config.ts index 81db16081..ff5ea4f9f 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -38,6 +38,7 @@ interface IConfig { ESPLORA: { REST_API_URL: string; UNIX_SOCKET_PATH: string | void | null; + RETRY_UNIX_SOCKET_AFTER: number; }; LIGHTNING: { ENABLED: boolean; @@ -85,6 +86,7 @@ interface IConfig { DATABASE: string; USERNAME: string; PASSWORD: string; + TIMEOUT: number; }; SYSLOG: { ENABLED: boolean; @@ -165,6 +167,7 @@ const defaults: IConfig = { 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', 'UNIX_SOCKET_PATH': null, + 'RETRY_UNIX_SOCKET_AFTER': 30000, }, 'ELECTRUM': { 'HOST': '127.0.0.1', @@ -192,7 +195,8 @@ const defaults: IConfig = { 'PORT': 3306, 'DATABASE': 'mempool', 'USERNAME': 'mempool', - 'PASSWORD': 'mempool' + 'PASSWORD': 'mempool', + 'TIMEOUT': 180000, }, 'SYSLOG': { 'ENABLED': true, diff --git a/backend/src/database.ts b/backend/src/database.ts index a504eb0fa..070774c92 100644 --- a/backend/src/database.ts +++ b/backend/src/database.ts @@ -33,8 +33,32 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr OkPacket[] | ResultSetHeader>(query, params?): Promise<[T, FieldPacket[]]> { this.checkDBFlag(); - const pool = await this.getPool(); - return pool.query(query, params); + let hardTimeout; + if (query?.timeout != null) { + hardTimeout = Math.floor(query.timeout * 1.1); + } else { + hardTimeout = config.DATABASE.TIMEOUT; + } + if (hardTimeout > 0) { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + reject(new Error(`DB query failed to return, reject or time out within ${hardTimeout / 1000}s - ${query?.sql?.slice(0, 160) || (typeof(query) === 'string' || query instanceof String ? query?.slice(0, 160) : 'unknown query')}`)); + }, hardTimeout); + + this.getPool().then(pool => { + return pool.query(query, params) as Promise<[T, FieldPacket[]]>; + }).then(result => { + resolve(result); + }).catch(error => { + reject(error); + }).finally(() => { + clearTimeout(timer); + }); + }); + } else { + const pool = await this.getPool(); + return pool.query(query, params); + } } public async checkDbConnection() { diff --git a/backend/src/index.ts b/backend/src/index.ts index a7f805313..384f96c38 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -45,7 +45,8 @@ class Server { private wss: WebSocket.Server | undefined; private server: http.Server | undefined; private app: Application; - private currentBackendRetryInterval = 5; + private currentBackendRetryInterval = 1; + private backendRetryCount = 0; private maxHeapSize: number = 0; private heapLogInterval: number = 60; @@ -120,7 +121,7 @@ class Server { await poolsUpdater.updatePoolsJson(); // Needs to be done before loading the disk cache because we sometimes wipe it await syncAssets.syncAssets$(); if (config.MEMPOOL.ENABLED) { - diskCache.loadMempoolCache(); + await diskCache.$loadMempoolCache(); } if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isPrimary) { @@ -178,23 +179,23 @@ class Server { logger.debug(msg); } } - memPool.deleteExpiredTransactions(); await blocks.$updateBlocks(); + memPool.deleteExpiredTransactions(); await memPool.$updateMempool(); indexer.$run(); setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS); - this.currentBackendRetryInterval = 5; + this.backendRetryCount = 0; } catch (e: any) { - let loggerMsg = `Exception in runMainUpdateLoop(). Retrying in ${this.currentBackendRetryInterval} sec.`; + this.backendRetryCount++; + let loggerMsg = `Exception in runMainUpdateLoop() (count: ${this.backendRetryCount}). Retrying in ${this.currentBackendRetryInterval} sec.`; loggerMsg += ` Reason: ${(e instanceof Error ? e.message : e)}.`; if (e?.stack) { loggerMsg += ` Stack trace: ${e.stack}`; } // When we get a first Exception, only `logger.debug` it and retry after 5 seconds // From the second Exception, `logger.warn` the Exception and increase the retry delay - // Maximum retry delay is 60 seconds - if (this.currentBackendRetryInterval > 5) { + if (this.backendRetryCount >= 5) { logger.warn(loggerMsg); mempool.setOutOfSync(); } else { @@ -204,8 +205,6 @@ class Server { logger.debug(`AxiosError: ${e?.message}`); } setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.currentBackendRetryInterval); - this.currentBackendRetryInterval *= 2; - this.currentBackendRetryInterval = Math.min(this.currentBackendRetryInterval, 60); } } @@ -238,7 +237,7 @@ class Server { websocketHandler.setupConnectionHandling(); if (config.MEMPOOL.ENABLED) { statistics.setNewStatisticsEntryCallback(websocketHandler.handleNewStatistic.bind(websocketHandler)); - memPool.setAsyncMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler)); + memPool.setAsyncMempoolChangedCallback(websocketHandler.$handleMempoolChange.bind(websocketHandler)); blocks.setNewAsyncBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler)); } priceUpdater.setRatesChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler)); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index b43d05c13..25cd8b95d 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -81,6 +81,10 @@ export interface TransactionExtended extends IEsploraApi.Transaction { bestDescendant?: BestDescendant | null; cpfpChecked?: boolean; deleteAfter?: number; + position?: { + block: number, + vsize: number, + }; } export interface AuditTransaction { diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 69d597e1f..a014e317e 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -13,6 +13,48 @@ import chainTips from '../api/chain-tips'; import blocks from '../api/blocks'; import BlocksAuditsRepository from './BlocksAuditsRepository'; +interface DatabaseBlock { + id: string; + height: number; + version: number; + timestamp: number; + bits: number; + nonce: number; + difficulty: number; + merkle_root: string; + tx_count: number; + size: number; + weight: number; + previousblockhash: string; + mediantime: number; + totalFees: number; + medianFee: number; + feeRange: string; + reward: number; + poolId: number; + poolName: string; + poolSlug: string; + avgFee: number; + avgFeeRate: number; + coinbaseRaw: string; + coinbaseAddress: string; + coinbaseSignature: string; + coinbaseSignatureAscii: string; + avgTxSize: number; + totalInputs: number; + totalOutputs: number; + totalOutputAmt: number; + medianFeeAmt: number; + feePercentiles: string; + segwitTotalTxs: number; + segwitTotalSize: number; + segwitTotalWeight: number; + header: string; + utxoSetChange: number; + utxoSetSize: number; + totalInputAmt: number; +} + const BLOCK_DB_FIELDS = ` blocks.hash AS id, blocks.height, @@ -52,7 +94,7 @@ const BLOCK_DB_FIELDS = ` blocks.header, blocks.utxoset_change AS utxoSetChange, blocks.utxoset_size AS utxoSetSize, - blocks.total_input_amt AS totalInputAmts + blocks.total_input_amt AS totalInputAmt `; class BlocksRepository { @@ -171,6 +213,32 @@ class BlocksRepository { } } + /** + * Update missing fee amounts fields + * + * @param blockHash + * @param feeAmtPercentiles + * @param medianFeeAmt + */ + public async $updateFeeAmounts(blockHash: string, feeAmtPercentiles, medianFeeAmt) : Promise { + try { + const query = ` + UPDATE blocks + SET fee_percentiles = ?, median_fee_amt = ? + WHERE hash = ? + `; + const params: any[] = [ + JSON.stringify(feeAmtPercentiles), + medianFeeAmt, + blockHash + ]; + await DB.query(query, params); + } catch (e: any) { + logger.err(`Cannot update fee amounts for block ${blockHash}. Reason: ' + ${e instanceof Error ? e.message : e}`); + throw e; + } + } + /** * Get all block height that have not been indexed between [startHeight, endHeight] */ @@ -432,7 +500,7 @@ class BlocksRepository { const blocks: BlockExtended[] = []; for (const block of rows) { - blocks.push(await this.formatDbBlockIntoExtendedBlock(block)); + blocks.push(await this.formatDbBlockIntoExtendedBlock(block as DatabaseBlock)); } return blocks; @@ -459,7 +527,7 @@ class BlocksRepository { return null; } - return await this.formatDbBlockIntoExtendedBlock(rows[0]); + return await this.formatDbBlockIntoExtendedBlock(rows[0] as DatabaseBlock); } catch (e) { logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e)); throw e; @@ -908,7 +976,7 @@ class BlocksRepository { * * @param dbBlk */ - private async formatDbBlockIntoExtendedBlock(dbBlk: any): Promise { + private async formatDbBlockIntoExtendedBlock(dbBlk: DatabaseBlock): Promise { const blk: Partial = {}; const extras: Partial = {}; @@ -980,11 +1048,12 @@ class BlocksRepository { if (extras.feePercentiles === null) { const block = await bitcoinClient.getBlock(dbBlk.id, 2); const summary = blocks.summarizeBlock(block); - await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.hash, summary.transactions); + await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.id, summary.transactions); extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id); } if (extras.feePercentiles !== null) { extras.medianFeeAmt = extras.feePercentiles[3]; + await this.$updateFeeAmounts(dbBlk.id, extras.feePercentiles, extras.medianFeeAmt); } } diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts index de70322bd..f2560fbe7 100644 --- a/backend/src/repositories/BlocksSummariesRepository.ts +++ b/backend/src/repositories/BlocksSummariesRepository.ts @@ -17,26 +17,6 @@ class BlocksSummariesRepository { return undefined; } - public async $saveSummary(params: { height: number, mined?: BlockSummary}): Promise { - const blockId = params.mined?.id; - try { - const transactions = JSON.stringify(params.mined?.transactions || []); - await DB.query(` - INSERT INTO blocks_summaries (height, id, transactions, template) - VALUE (?, ?, ?, ?) - ON DUPLICATE KEY UPDATE - transactions = ? - `, [params.height, blockId, transactions, '[]', transactions]); - } catch (e: any) { - if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart - logger.debug(`Cannot save block summary for ${blockId} because it has already been indexed, ignoring`); - } else { - logger.debug(`Cannot save block summary for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`); - throw e; - } - } - } - public async $saveTransactions(blockHeight: number, blockId: string, transactions: TransactionStripped[]): Promise { try { const transactionsStr = JSON.stringify(transactions); diff --git a/backend/src/tasks/lightning/forensics.service.ts b/backend/src/tasks/lightning/forensics.service.ts index 7837cb4d5..65ea61dc1 100644 --- a/backend/src/tasks/lightning/forensics.service.ts +++ b/backend/src/tasks/lightning/forensics.service.ts @@ -152,7 +152,7 @@ class ForensicsService { ++progress; const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); if (elapsedSeconds > 10) { - logger.info(`Updating channel closed channel forensics ${progress}/${channels.length}`); + logger.debug(`Updating channel closed channel forensics ${progress}/${channels.length}`); this.loggerTimer = new Date().getTime() / 1000; } } @@ -257,7 +257,7 @@ class ForensicsService { ++progress; const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); if (elapsedSeconds > 10) { - logger.info(`Updating opened channel forensics ${progress}/${channels?.length}`); + logger.debug(`Updating opened channel forensics ${progress}/${channels?.length}`); this.loggerTimer = new Date().getTime() / 1000; this.truncateTempCache(); } diff --git a/backend/src/tasks/lightning/network-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts index 28f60bbf9..aca3dbef8 100644 --- a/backend/src/tasks/lightning/network-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -300,7 +300,7 @@ class NetworkSyncService { ++progress; const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); if (elapsedSeconds > config.LIGHTNING.LOGGER_UPDATE_INTERVAL) { - logger.info(`Checking if channel has been closed ${progress}/${channels.length}`, logger.tags.ln); + logger.debug(`Checking if channel has been closed ${progress}/${channels.length}`, logger.tags.ln); this.loggerTimer = new Date().getTime() / 1000; } } diff --git a/docker/README.md b/docker/README.md index e81e45b42..b669b37c8 100644 --- a/docker/README.md +++ b/docker/README.md @@ -205,7 +205,8 @@ Corresponding `docker-compose.yml` overrides: ```json "ESPLORA": { "REST_API_URL": "http://127.0.0.1:3000", - "UNIX_SOCKET_PATH": "/tmp/esplora-socket" + "UNIX_SOCKET_PATH": "/tmp/esplora-socket", + "RETRY_UNIX_SOCKET_AFTER": 30000 }, ``` @@ -215,6 +216,7 @@ Corresponding `docker-compose.yml` overrides: environment: ESPLORA_REST_API_URL: "" ESPLORA_UNIX_SOCKET_PATH: "" + ESPLORA_RETRY_UNIX_SOCKET_AFTER: "" ... ``` @@ -267,6 +269,7 @@ Corresponding `docker-compose.yml` overrides: DATABASE_DATABASE: "" DATABASE_USERNAME: "" DATABASE_PASSWORD: "" + DATABASE_TIMEOUT: "" ... ``` diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 7d2737e9c..fd8abaf02 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -43,7 +43,8 @@ }, "ESPLORA": { "REST_API_URL": "__ESPLORA_REST_API_URL__", - "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__" + "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__", + "RETRY_UNIX_SOCKET_AFTER": __ESPLORA_RETRY_UNIX_SOCKET_AFTER__ }, "SECOND_CORE_RPC": { "HOST": "__SECOND_CORE_RPC_HOST__", @@ -59,7 +60,8 @@ "PORT": __DATABASE_PORT__, "DATABASE": "__DATABASE_DATABASE__", "USERNAME": "__DATABASE_USERNAME__", - "PASSWORD": "__DATABASE_PASSWORD__" + "PASSWORD": "__DATABASE_PASSWORD__", + "TIMEOUT": "__DATABASE_TIMEOUT__" }, "SYSLOG": { "ENABLED": __SYSLOG_ENABLED__, diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 55da48464..a54f16ec6 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -47,6 +47,7 @@ __ELECTRUM_TLS_ENABLED__=${ELECTRUM_TLS_ENABLED:=false} # ESPLORA __ESPLORA_REST_API_URL__=${ESPLORA_REST_API_URL:=http://127.0.0.1:3000} __ESPLORA_UNIX_SOCKET_PATH__=${ESPLORA_UNIX_SOCKET_PATH:=null} +__ESPLORA_RETRY_UNIX_SOCKET_AFTER__=${ESPLORA_RETRY_UNIX_SOCKET_AFTER:=30000} # SECOND_CORE_RPC __SECOND_CORE_RPC_HOST__=${SECOND_CORE_RPC_HOST:=127.0.0.1} @@ -63,6 +64,7 @@ __DATABASE_PORT__=${DATABASE_PORT:=3306} __DATABASE_DATABASE__=${DATABASE_DATABASE:=mempool} __DATABASE_USERNAME__=${DATABASE_USERNAME:=mempool} __DATABASE_PASSWORD__=${DATABASE_PASSWORD:=mempool} +__DATABASE_TIMEOUT__=${DATABASE_TIMEOUT:=180000} # SYSLOG __SYSLOG_ENABLED__=${SYSLOG_ENABLED:=false} @@ -168,6 +170,7 @@ sed -i "s/__ELECTRUM_TLS_ENABLED__/${__ELECTRUM_TLS_ENABLED__}/g" mempool-config sed -i "s!__ESPLORA_REST_API_URL__!${__ESPLORA_REST_API_URL__}!g" mempool-config.json sed -i "s!__ESPLORA_UNIX_SOCKET_PATH__!${__ESPLORA_UNIX_SOCKET_PATH__}!g" mempool-config.json +sed -i "s!__ESPLORA_RETRY_UNIX_SOCKET_AFTER__!${__ESPLORA_RETRY_UNIX_SOCKET_AFTER__}!g" mempool-config.json sed -i "s/__SECOND_CORE_RPC_HOST__/${__SECOND_CORE_RPC_HOST__}/g" mempool-config.json sed -i "s/__SECOND_CORE_RPC_PORT__/${__SECOND_CORE_RPC_PORT__}/g" mempool-config.json diff --git a/docker/frontend/entrypoint.sh b/docker/frontend/entrypoint.sh index b6946578b..013b1ce53 100644 --- a/docker/frontend/entrypoint.sh +++ b/docker/frontend/entrypoint.sh @@ -39,6 +39,7 @@ __AUDIT__=${AUDIT:=false} __MAINNET_BLOCK_AUDIT_START_HEIGHT__=${MAINNET_BLOCK_AUDIT_START_HEIGHT:=0} __TESTNET_BLOCK_AUDIT_START_HEIGHT__=${TESTNET_BLOCK_AUDIT_START_HEIGHT:=0} __SIGNET_BLOCK_AUDIT_START_HEIGHT__=${SIGNET_BLOCK_AUDIT_START_HEIGHT:=0} +__FULL_RBF_ENABLED__=${FULL_RBF_ENABLED:=false} __HISTORICAL_PRICE__=${HISTORICAL_PRICE:=true} # Export as environment variables to be used by envsubst @@ -65,6 +66,7 @@ export __AUDIT__ export __MAINNET_BLOCK_AUDIT_START_HEIGHT__ export __TESTNET_BLOCK_AUDIT_START_HEIGHT__ export __SIGNET_BLOCK_AUDIT_START_HEIGHT__ +export __FULL_RBF_ENABLED__ export __HISTORICAL_PRICE__ folder=$(find /var/www/mempool -name "config.js" | xargs dirname) diff --git a/frontend/mempool-frontend-config.sample.json b/frontend/mempool-frontend-config.sample.json index 084cbd0ef..c45425612 100644 --- a/frontend/mempool-frontend-config.sample.json +++ b/frontend/mempool-frontend-config.sample.json @@ -22,5 +22,6 @@ "TESTNET_BLOCK_AUDIT_START_HEIGHT": 0, "SIGNET_BLOCK_AUDIT_START_HEIGHT": 0, "LIGHTNING": false, + "FULL_RBF_ENABLED": false, "HISTORICAL_PRICE": true } diff --git a/frontend/src/app/app-routing.module.ts b/frontend/src/app/app-routing.module.ts index 90ea84a82..06334c5b5 100644 --- a/frontend/src/app/app-routing.module.ts +++ b/frontend/src/app/app-routing.module.ts @@ -14,6 +14,7 @@ import { TrademarkPolicyComponent } from './components/trademark-policy/trademar import { BisqMasterPageComponent } from './components/bisq-master-page/bisq-master-page.component'; import { PushTransactionComponent } from './components/push-transaction/push-transaction.component'; import { BlocksList } from './components/blocks-list/blocks-list.component'; +import { RbfList } from './components/rbf-list/rbf-list.component'; import { LiquidMasterPageComponent } from './components/liquid-master-page/liquid-master-page.component'; import { AssetGroupComponent } from './components/assets/asset-group/asset-group.component'; import { AssetsFeaturedComponent } from './components/assets/assets-featured/assets-featured.component'; @@ -56,6 +57,10 @@ let routes: Routes = [ path: 'blocks', component: BlocksList, }, + { + path: 'rbf', + component: RbfList, + }, { path: 'terms-of-service', component: TermsOfServiceComponent @@ -162,6 +167,10 @@ let routes: Routes = [ path: 'blocks', component: BlocksList, }, + { + path: 'rbf', + component: RbfList, + }, { path: 'terms-of-service', component: TermsOfServiceComponent @@ -264,6 +273,10 @@ let routes: Routes = [ path: 'blocks', component: BlocksList, }, + { + path: 'rbf', + component: RbfList, + }, { path: 'terms-of-service', component: TermsOfServiceComponent diff --git a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.html b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.html index 58d555657..0c0564603 100644 --- a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.html +++ b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.html @@ -2,7 +2,7 @@
-
+
 
diff --git a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts index d48d1f299..91273169c 100644 --- a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts +++ b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts @@ -1,4 +1,4 @@ -import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, ChangeDetectorRef } from '@angular/core'; +import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, ChangeDetectorRef, HostListener } from '@angular/core'; import { Subscription, Observable, fromEvent, merge, of, combineLatest } from 'rxjs'; import { MempoolBlock } from '../../interfaces/websocket.interface'; import { StateService } from '../../services/state.service'; @@ -8,7 +8,7 @@ import { feeLevels, mempoolFeeColors } from '../../app.constants'; import { specialBlocks } from '../../app.constants'; import { RelativeUrlPipe } from '../../shared/pipes/relative-url/relative-url.pipe'; import { Location } from '@angular/common'; -import { DifficultyAdjustment } from '../../interfaces/node-api.interface'; +import { DifficultyAdjustment, MempoolPosition } from '../../interfaces/node-api.interface'; import { animate, style, transition, trigger } from '@angular/animations'; @Component({ @@ -58,6 +58,7 @@ export class MempoolBlocksComponent implements OnInit, OnDestroy { transition = 'background 2s, right 2s, transform 1s'; markIndex: number; + txPosition: MempoolPosition; txFeePerVSize: number; resetTransitionTimeout: number; @@ -152,10 +153,14 @@ export class MempoolBlocksComponent implements OnInit, OnDestroy { this.markBlocksSubscription = this.stateService.markBlock$ .subscribe((state) => { this.markIndex = undefined; + this.txPosition = undefined; this.txFeePerVSize = undefined; if (state.mempoolBlockIndex !== undefined) { this.markIndex = state.mempoolBlockIndex; } + if (state.mempoolPosition) { + this.txPosition = state.mempoolPosition; + } if (state.txFeePerVSize) { this.txFeePerVSize = state.txFeePerVSize; } @@ -222,8 +227,13 @@ export class MempoolBlocksComponent implements OnInit, OnDestroy { clearTimeout(this.resetTransitionTimeout); } + @HostListener('window:resize', ['$event']) + onResize(): void { + this.animateEntry = false; + } + trackByFn(index: number, block: MempoolBlock) { - return (block.isStack) ? 'stack' : block.index; + return (block.isStack) ? `stack-${block.index}` : block.index; } reduceMempoolBlocksToFitScreen(blocks: MempoolBlock[]): MempoolBlock[] { @@ -297,7 +307,7 @@ export class MempoolBlocksComponent implements OnInit, OnDestroy { } calculateTransactionPosition() { - if ((!this.txFeePerVSize && (this.markIndex === undefined || this.markIndex === -1)) || !this.mempoolBlocks) { + if ((!this.txPosition && !this.txFeePerVSize && (this.markIndex === undefined || this.markIndex === -1)) || !this.mempoolBlocks) { this.arrowVisible = false; return; } else if (this.markIndex > -1) { @@ -315,33 +325,43 @@ export class MempoolBlocksComponent implements OnInit, OnDestroy { this.arrowVisible = true; - let found = false; - for (let txInBlockIndex = 0; txInBlockIndex < this.mempoolBlocks.length && !found; txInBlockIndex++) { - const block = this.mempoolBlocks[txInBlockIndex]; - for (let i = 0; i < block.feeRange.length - 1 && !found; i++) { - if (this.txFeePerVSize < block.feeRange[i + 1] && this.txFeePerVSize >= block.feeRange[i]) { - const feeRangeIndex = i; - const feeRangeChunkSize = 1 / (block.feeRange.length - 1); + if (this.txPosition) { + if (this.txPosition.block >= this.mempoolBlocks.length) { + this.rightPosition = ((this.mempoolBlocks.length - 1) * (this.blockWidth + this.blockPadding)) + this.blockWidth; + } else { + const positionInBlock = Math.min(1, this.txPosition.vsize / this.stateService.blockVSize) * this.blockWidth; + const positionOfBlock = this.txPosition.block * (this.blockWidth + this.blockPadding); + this.rightPosition = positionOfBlock + positionInBlock; + } + } else { + let found = false; + for (let txInBlockIndex = 0; txInBlockIndex < this.mempoolBlocks.length && !found; txInBlockIndex++) { + const block = this.mempoolBlocks[txInBlockIndex]; + for (let i = 0; i < block.feeRange.length - 1 && !found; i++) { + if (this.txFeePerVSize < block.feeRange[i + 1] && this.txFeePerVSize >= block.feeRange[i]) { + const feeRangeIndex = i; + const feeRangeChunkSize = 1 / (block.feeRange.length - 1); - const txFee = this.txFeePerVSize - block.feeRange[i]; - const max = block.feeRange[i + 1] - block.feeRange[i]; - const blockLocation = txFee / max; + const txFee = this.txFeePerVSize - block.feeRange[i]; + const max = block.feeRange[i + 1] - block.feeRange[i]; + const blockLocation = txFee / max; - const chunkPositionOffset = blockLocation * feeRangeChunkSize; - const feePosition = feeRangeChunkSize * feeRangeIndex + chunkPositionOffset; + const chunkPositionOffset = blockLocation * feeRangeChunkSize; + const feePosition = feeRangeChunkSize * feeRangeIndex + chunkPositionOffset; - const blockedFilledPercentage = (block.blockVSize > this.stateService.blockVSize ? this.stateService.blockVSize : block.blockVSize) / this.stateService.blockVSize; - const arrowRightPosition = txInBlockIndex * (this.blockWidth + this.blockPadding) - + ((1 - feePosition) * blockedFilledPercentage * this.blockWidth); + const blockedFilledPercentage = (block.blockVSize > this.stateService.blockVSize ? this.stateService.blockVSize : block.blockVSize) / this.stateService.blockVSize; + const arrowRightPosition = txInBlockIndex * (this.blockWidth + this.blockPadding) + + ((1 - feePosition) * blockedFilledPercentage * this.blockWidth); - this.rightPosition = arrowRightPosition; + this.rightPosition = arrowRightPosition; + found = true; + } + } + if (this.txFeePerVSize >= block.feeRange[block.feeRange.length - 1]) { + this.rightPosition = txInBlockIndex * (this.blockWidth + this.blockPadding); found = true; } } - if (this.txFeePerVSize >= block.feeRange[block.feeRange.length - 1]) { - this.rightPosition = txInBlockIndex * (this.blockWidth + this.blockPadding); - found = true; - } } } diff --git a/frontend/src/app/components/rbf-list/rbf-list.component.html b/frontend/src/app/components/rbf-list/rbf-list.component.html new file mode 100644 index 000000000..eebb7e152 --- /dev/null +++ b/frontend/src/app/components/rbf-list/rbf-list.component.html @@ -0,0 +1,46 @@ +
+

RBF Replacements

+ + +
+
+
+ + +
+
+
+ +
+ +
+ +
+

+ + Mined + Full RBF + + +

+
+ +
+
+ +
+

there are no replacements in the mempool yet!

+
+ + + +
+ +
diff --git a/frontend/src/app/components/rbf-list/rbf-list.component.scss b/frontend/src/app/components/rbf-list/rbf-list.component.scss new file mode 100644 index 000000000..792bb8836 --- /dev/null +++ b/frontend/src/app/components/rbf-list/rbf-list.component.scss @@ -0,0 +1,35 @@ +.spinner-border { + height: 25px; + width: 25px; + margin-top: 13px; +} + +.rbf-trees { + .info { + display: flex; + flex-direction: row; + justify-content: space-between; + align-items: baseline; + margin: 0; + margin-bottom: 0.5em; + + .type { + .badge { + margin-left: .5em; + } + } + } + + .tree { + margin-bottom: 1em; + } + + .timeline-wrapper.mined { + border: solid 4px #1a9436; + } + + .no-replacements { + margin: 1em; + text-align: center; + } +} \ No newline at end of file diff --git a/frontend/src/app/components/rbf-list/rbf-list.component.ts b/frontend/src/app/components/rbf-list/rbf-list.component.ts new file mode 100644 index 000000000..3fe04ba6e --- /dev/null +++ b/frontend/src/app/components/rbf-list/rbf-list.component.ts @@ -0,0 +1,81 @@ +import { Component, OnInit, ChangeDetectionStrategy, OnDestroy } from '@angular/core'; +import { ActivatedRoute, Router } from '@angular/router'; +import { BehaviorSubject, EMPTY, merge, Observable, Subscription } from 'rxjs'; +import { catchError, switchMap, tap } from 'rxjs/operators'; +import { WebsocketService } from '../../services/websocket.service'; +import { RbfTree } from '../../interfaces/node-api.interface'; +import { ApiService } from '../../services/api.service'; +import { StateService } from '../../services/state.service'; + +@Component({ + selector: 'app-rbf-list', + templateUrl: './rbf-list.component.html', + styleUrls: ['./rbf-list.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class RbfList implements OnInit, OnDestroy { + rbfTrees$: Observable; + nextRbfSubject = new BehaviorSubject(null); + urlFragmentSubscription: Subscription; + fullRbfEnabled: boolean; + fullRbf: boolean; + isLoading = true; + + constructor( + private route: ActivatedRoute, + private router: Router, + private apiService: ApiService, + public stateService: StateService, + private websocketService: WebsocketService, + ) { + this.fullRbfEnabled = stateService.env.FULL_RBF_ENABLED; + } + + ngOnInit(): void { + this.urlFragmentSubscription = this.route.fragment.subscribe((fragment) => { + this.fullRbf = (fragment === 'fullrbf'); + this.websocketService.startTrackRbf(this.fullRbf ? 'fullRbf' : 'all'); + this.nextRbfSubject.next(null); + }); + + this.rbfTrees$ = merge( + this.nextRbfSubject.pipe( + switchMap(() => { + return this.apiService.getRbfList$(this.fullRbf); + }), + catchError((e) => { + return EMPTY; + }) + ), + this.stateService.rbfLatest$ + ) + .pipe( + tap(() => { + this.isLoading = false; + }) + ); + } + + toggleFullRbf(event) { + this.router.navigate([], { + relativeTo: this.route, + fragment: this.fullRbf ? null : 'fullrbf' + }); + } + + isFullRbf(tree: RbfTree): boolean { + return tree.fullRbf; + } + + isMined(tree: RbfTree): boolean { + return tree.mined; + } + + // pageChange(page: number) { + // this.fromTreeSubject.next(this.lastTreeId); + // } + + ngOnDestroy(): void { + this.websocketService.stopTrackRbf(); + } +} \ No newline at end of file diff --git a/frontend/src/app/components/rbf-timeline/rbf-timeline-tooltip.component.html b/frontend/src/app/components/rbf-timeline/rbf-timeline-tooltip.component.html new file mode 100644 index 000000000..1ce224760 --- /dev/null +++ b/frontend/src/app/components/rbf-timeline/rbf-timeline-tooltip.component.html @@ -0,0 +1,38 @@ +
+ + + + + + + + + + + + + + + + + + + + + + + +
Transaction + {{ rbfInfo.tx.txid | shortenString : 16}} +
First seen
Fee{{ rbfInfo.tx.fee | number }} sat
Virtual size
Status + RBF + RBF + Mined +
+
diff --git a/frontend/src/app/components/rbf-timeline/rbf-timeline-tooltip.component.scss b/frontend/src/app/components/rbf-timeline/rbf-timeline-tooltip.component.scss new file mode 100644 index 000000000..cd31aa562 --- /dev/null +++ b/frontend/src/app/components/rbf-timeline/rbf-timeline-tooltip.component.scss @@ -0,0 +1,25 @@ +.rbf-tooltip { + position: fixed; + z-index: 3; + background: rgba(#11131f, 0.95); + border-radius: 4px; + box-shadow: 1px 1px 10px rgba(0,0,0,0.5); + color: #b1b1b1; + display: flex; + flex-direction: column; + justify-content: space-between; + padding: 10px 15px; + text-align: left; + pointer-events: none; + + .badge { + margin-right: 1em; + &:last-child { + margin-right: 0; + } + } +} + +.td-width { + padding-right: 10px; +} \ No newline at end of file diff --git a/frontend/src/app/components/rbf-timeline/rbf-timeline-tooltip.component.ts b/frontend/src/app/components/rbf-timeline/rbf-timeline-tooltip.component.ts new file mode 100644 index 000000000..b9da63c86 --- /dev/null +++ b/frontend/src/app/components/rbf-timeline/rbf-timeline-tooltip.component.ts @@ -0,0 +1,35 @@ +import { Component, ElementRef, ViewChild, Input, OnChanges } from '@angular/core'; +import { RbfInfo } from '../../interfaces/node-api.interface'; + +@Component({ + selector: 'app-rbf-timeline-tooltip', + templateUrl: './rbf-timeline-tooltip.component.html', + styleUrls: ['./rbf-timeline-tooltip.component.scss'], +}) +export class RbfTimelineTooltipComponent implements OnChanges { + @Input() rbfInfo: RbfInfo | void; + @Input() cursorPosition: { x: number, y: number }; + + tooltipPosition = null; + + @ViewChild('tooltip') tooltipElement: ElementRef; + + constructor() {} + + ngOnChanges(changes): void { + if (changes.cursorPosition && changes.cursorPosition.currentValue) { + let x = Math.max(10, changes.cursorPosition.currentValue.x - 50); + let y = changes.cursorPosition.currentValue.y + 20; + if (this.tooltipElement) { + const elementBounds = this.tooltipElement.nativeElement.getBoundingClientRect(); + if ((x + elementBounds.width) > (window.innerWidth - 10)) { + x = Math.max(0, window.innerWidth - elementBounds.width - 10); + } + if (y + elementBounds.height > (window.innerHeight - 20)) { + y = y - elementBounds.height - 20; + } + } + this.tooltipPosition = { x, y }; + } + } +} diff --git a/frontend/src/app/components/rbf-timeline/rbf-timeline.component.html b/frontend/src/app/components/rbf-timeline/rbf-timeline.component.html new file mode 100644 index 000000000..e9d531cb8 --- /dev/null +++ b/frontend/src/app/components/rbf-timeline/rbf-timeline.component.html @@ -0,0 +1,73 @@ +
+
+
+
+ +
+ +
+
+ +
+
+ + +
+
+ + +
+
+ +
+ + {{ cell.replacement.tx.fee / (cell.replacement.tx.vsize) | feeRounding }} sat/vB +
+ + + +
+
+
+ + + +
+
+
+ + +
+
+
+ + +
+ + + +
+ + + + + +
diff --git a/frontend/src/app/components/rbf-timeline/rbf-timeline.component.scss b/frontend/src/app/components/rbf-timeline/rbf-timeline.component.scss new file mode 100644 index 000000000..97388b98e --- /dev/null +++ b/frontend/src/app/components/rbf-timeline/rbf-timeline.component.scss @@ -0,0 +1,193 @@ +.rbf-timeline { + position: relative; + width: 100%; + padding: 1em 0; + + &::after, &::before { + content: ''; + display: block; + position: absolute; + top: 0; + bottom: 0; + width: 2em; + z-index: 2; + } + + &::before { + left: 0; + background: linear-gradient(to right, #24273e, #24273e, transparent); + } + + &::after { + right: 0; + background: linear-gradient(to left, #24273e, #24273e, transparent); + } + + .timeline-wrapper { + position: relative; + width: calc(100% - 2em); + margin: auto; + overflow-x: auto; + -ms-overflow-style: none; + scrollbar-width: none; + + &::-webkit-scrollbar { + display: none; + } + } + + .intervals, .nodes { + min-width: 100%; + display: flex; + flex-direction: row; + justify-content: space-between; + align-items: flex-start; + text-align: center; + + .node, .node-spacer, .connector { + width: 6em; + min-width: 6em; + flex-grow: 1; + } + + .interval, .interval-spacer { + width: 8em; + min-width: 5em; + max-width: 8em; + height: 32px; + display: flex; + flex-direction: row; + justify-content: center; + align-items: flex-end; + } + + .interval { + overflow: visible; + } + + .interval-time { + font-size: 12px; + line-height: 16px; + white-space: nowrap; + } + } + + .node, .interval-spacer { + position: relative; + .track { + position: absolute; + height: 10px; + left: -5px; + right: -5px; + top: 0; + transform: translateY(-50%); + background: #105fb0; + border-radius: 5px; + } + &.first-node { + .track { + left: 50%; + } + } + &:last-child { + .track { + right: 50%; + } + } + } + + .nodes { + position: relative; + margin-top: 1em; + .node { + .shape-border { + display: block; + margin: auto; + height: calc(1em + 8px); + width: calc(1em + 8px); + margin-bottom: -8px; + transform: translateY(-50%); + border-radius: 10%; + cursor: pointer; + padding: 4px; + background: transparent; + transition: background-color 300ms, padding 300ms; + + .shape { + width: 100%; + height: 100%; + border-radius: 10%; + background: white; + transition: background-color 300ms, border 300ms; + } + + &.rbf, &.rbf .shape { + border-radius: 50%; + } + } + + .symbol::ng-deep { + display: block; + margin-top: -0.5em; + } + + &.selected { + .shape-border { + background: #9339f4; + } + } + + &.mined { + .shape-border { + background: #1a9436; + } + } + + .shape-border:hover { + padding: 0px; + .shape { + background: #1bd8f4; + } + } + + &.selected.mined { + .shape-border { + background: #1a9436; + height: calc(1em + 16px); + width: calc(1em + 16px); + + .shape { + border: solid 4px #9339f4; + } + + &:hover { + padding: 4px; + .shape { + border-width: 1px; + border-color: #1bd8f4 + } + } + } + } + } + + .connector { + position: relative; + height: 10px; + + .corner, .pipe { + position: absolute; + left: -10px; + width: 20px; + height: 108px; + bottom: 50%; + border-right: solid 10px #105fb0; + } + + .corner { + border-bottom: solid 10px #105fb0; + border-bottom-right-radius: 10px; + } + } + } +} \ No newline at end of file diff --git a/frontend/src/app/components/rbf-timeline/rbf-timeline.component.ts b/frontend/src/app/components/rbf-timeline/rbf-timeline.component.ts new file mode 100644 index 000000000..f02e8ca35 --- /dev/null +++ b/frontend/src/app/components/rbf-timeline/rbf-timeline.component.ts @@ -0,0 +1,191 @@ +import { Component, Input, OnInit, OnChanges, Inject, LOCALE_ID, HostListener } from '@angular/core'; +import { Router } from '@angular/router'; +import { RbfInfo, RbfTree } from '../../interfaces/node-api.interface'; +import { StateService } from '../../services/state.service'; +import { ApiService } from '../../services/api.service'; + +type Connector = 'pipe' | 'corner'; + +interface TimelineCell { + replacement?: RbfInfo, + connector?: Connector, + first?: boolean, +} + +@Component({ + selector: 'app-rbf-timeline', + templateUrl: './rbf-timeline.component.html', + styleUrls: ['./rbf-timeline.component.scss'], +}) +export class RbfTimelineComponent implements OnInit, OnChanges { + @Input() replacements: RbfTree; + @Input() txid: string; + rows: TimelineCell[][] = []; + + hoverInfo: RbfInfo | void = null; + tooltipPosition = null; + + dir: 'rtl' | 'ltr' = 'ltr'; + + constructor( + private router: Router, + private stateService: StateService, + private apiService: ApiService, + @Inject(LOCALE_ID) private locale: string, + ) { + if (this.locale.startsWith('ar') || this.locale.startsWith('fa') || this.locale.startsWith('he')) { + this.dir = 'rtl'; + } + } + + ngOnInit(): void { + this.rows = this.buildTimelines(this.replacements); + } + + ngOnChanges(changes): void { + this.rows = this.buildTimelines(this.replacements); + if (changes.txid) { + setTimeout(() => { this.scrollToSelected(); }); + } + } + + // converts a tree of RBF events into a format that can be more easily rendered in HTML + buildTimelines(tree: RbfTree): TimelineCell[][] { + if (!tree) return []; + + const split = this.splitTimelines(tree); + const timelines = this.prepareTimelines(split); + return this.connectTimelines(timelines); + } + + // splits a tree into N leaf-to-root paths + splitTimelines(tree: RbfTree, tail: RbfInfo[] = []): RbfInfo[][] { + const replacements = [...tail, tree]; + if (tree.replaces.length) { + return [].concat(...tree.replaces.map(subtree => this.splitTimelines(subtree, replacements))); + } else { + return [[...replacements]]; + } + } + + // merges separate leaf-to-root paths into a coherent forking timeline + // represented as a 2D array of Rbf events + prepareTimelines(lines: RbfInfo[][]): RbfInfo[][] { + lines.sort((a, b) => b.length - a.length); + + const rows = lines.map(() => []); + let lineGroups = [lines]; + let done = false; + let column = 0; // sanity check for while loop stopping condition + while (!done && column < 100) { + // iterate over timelines element-by-element + // at each step, group lines which share a common transaction at their head + // (i.e. lines terminating in the same replacement event) + let index = 0; + let emptyCount = 0; + const nextGroups = []; + for (const group of lineGroups) { + const toMerge: { [txid: string]: RbfInfo[][] } = {}; + let emptyInGroup = 0; + let first = true; + for (const line of group) { + const head = line.shift() || null; + if (first) { + // only insert the first instance of the replacement node + rows[index].unshift(head); + first = false; + } else { + // substitute duplicates with empty cells + // (we'll fill these in with connecting lines later) + rows[index].unshift(null); + } + // group the tails of the remaining lines for the next iteration + if (line.length) { + const nextId = line[0].tx.txid; + if (!toMerge[nextId]) { + toMerge[nextId] = []; + } + toMerge[nextId].push(line); + } else { + emptyInGroup++; + } + index++; + } + for (const merged of Object.values(toMerge).sort((a, b) => b.length - a.length)) { + nextGroups.push(merged); + } + for (let i = 0; i < emptyInGroup; i++) { + nextGroups.push([[]]); + } + emptyCount += emptyInGroup; + lineGroups = nextGroups; + done = (emptyCount >= rows.length); + } + column++; + } + return rows; + } + + // annotates a 2D timeline array with info needed to draw connecting lines for multi-replacements + connectTimelines(timelines: RbfInfo[][]): TimelineCell[][] { + const rows: TimelineCell[][] = []; + timelines.forEach((lines, row) => { + rows.push([]); + let started = false; + let finished = false; + lines.forEach((replacement, column) => { + const cell: TimelineCell = {}; + if (replacement) { + cell.replacement = replacement; + } + rows[row].push(cell); + if (replacement) { + if (!started) { + cell.first = true; + started = true; + } + } else if (started && !finished) { + if (column < timelines[row].length) { + let matched = false; + for (let i = row; i >= 0 && !matched; i--) { + const nextCell = rows[i][column]; + if (nextCell.replacement) { + matched = true; + } else if (i === row) { + rows[i][column] = { + connector: 'corner' + }; + } else if (nextCell.connector !== 'corner') { + rows[i][column] = { + connector: 'pipe' + }; + } + } + } + finished = true; + } + }); + }); + return rows; + } + + scrollToSelected() { + const node = document.getElementById('node-' + this.txid); + if (node) { + node.scrollIntoView({ block: 'nearest', inline: 'center' }); + } + } + + @HostListener('pointermove', ['$event']) + onPointerMove(event) { + this.tooltipPosition = { x: event.clientX, y: event.clientY }; + } + + onHover(event, replacement): void { + this.hoverInfo = replacement; + } + + onBlur(event): void { + this.hoverInfo = null; + } +} diff --git a/frontend/src/app/components/start/start.component.ts b/frontend/src/app/components/start/start.component.ts index dde7bc234..7e83fb516 100644 --- a/frontend/src/app/components/start/start.component.ts +++ b/frontend/src/app/components/start/start.component.ts @@ -137,9 +137,11 @@ export class StartComponent implements OnInit, OnDestroy { } onMouseDown(event: MouseEvent) { - this.mouseDragStartX = event.clientX; - this.resetMomentum(event.clientX); - this.blockchainScrollLeftInit = this.blockchainContainer.nativeElement.scrollLeft; + if (!(event.which > 1 || event.button > 0)) { + this.mouseDragStartX = event.clientX; + this.resetMomentum(event.clientX); + this.blockchainScrollLeftInit = this.blockchainContainer.nativeElement.scrollLeft; + } } onPointerDown(event: PointerEvent) { if (this.isiOS) { diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index 04d13b07a..0b5e12001 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -1,18 +1,11 @@
-