From c675d1c498d4e20b2645d20fc8fdb40cb9fb9e4e Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Tue, 28 Mar 2023 23:07:50 +0900 Subject: [PATCH 01/12] Make sure to scan closed channels even if config.MEMPOOL.ENABLE = false --- backend/src/tasks/lightning/network-sync.service.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/src/tasks/lightning/network-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts index 3e5ae1366..fc04f5023 100644 --- a/backend/src/tasks/lightning/network-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -269,7 +269,11 @@ class NetworkSyncService { } private async $scanForClosedChannels(): Promise { - if (this.closedChannelsScanBlock === blocks.getCurrentBlockHeight()) { + let currentBlockHeight = blocks.getCurrentBlockHeight(); + if (config.MEMPOOL.ENABLED === false) { // https://github.com/mempool/mempool/issues/3582 + currentBlockHeight = await bitcoinApi.$getBlockHeightTip(); + } + if (this.closedChannelsScanBlock === currentBlockHeight) { logger.debug(`We've already scan closed channels for this block, skipping.`); return; } @@ -305,7 +309,7 @@ class NetworkSyncService { } } - this.closedChannelsScanBlock = blocks.getCurrentBlockHeight(); + this.closedChannelsScanBlock = currentBlockHeight; logger.debug(`Closed channels scan completed at block ${this.closedChannelsScanBlock}`, logger.tags.ln); } catch (e) { logger.err(`$scanForClosedChannels() error: ${e instanceof Error ? e.message : e}`, logger.tags.ln); From 22ee9916ddbe8f65130d2f63bae7aab739798e57 Mon Sep 17 00:00:00 2001 From: TechMiX Date: Wed, 19 Apr 2023 12:14:21 +0200 Subject: [PATCH 02/12] fix change component and audit button position in RTL mode --- frontend/src/app/components/change/change.component.html | 2 +- frontend/src/styles.scss | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/components/change/change.component.html b/frontend/src/app/components/change/change.component.html index 117a0c534..ffc00bf5f 100644 --- a/frontend/src/app/components/change/change.component.html +++ b/frontend/src/app/components/change/change.component.html @@ -1,3 +1,3 @@ - {{ change >= 0 ? '+' : '' }}{{ change | amountShortener }}% + ‎{{ change >= 0 ? '+' : '' }}{{ change | amountShortener }}% diff --git a/frontend/src/styles.scss b/frontend/src/styles.scss index d9ea867dc..d64450b93 100644 --- a/frontend/src/styles.scss +++ b/frontend/src/styles.scss @@ -988,6 +988,10 @@ th { margin-right: 10px; } } + + .btn-audit { + margin-left: .5em; + } } .scriptmessage { From 6e83bee23fc93234a2f81666e290a5fd93ddff54 Mon Sep 17 00:00:00 2001 From: wiz Date: Thu, 11 May 2023 11:08:09 -0500 Subject: [PATCH 03/12] ops: Disable mempool loop for lightning backends --- production/mempool-config.mainnet-lightning.json | 1 + production/mempool-config.signet-lightning.json | 1 + production/mempool-config.testnet-lightning.json | 1 + 3 files changed, 3 insertions(+) diff --git a/production/mempool-config.mainnet-lightning.json b/production/mempool-config.mainnet-lightning.json index 21e7109e9..41e42a5bd 100644 --- a/production/mempool-config.mainnet-lightning.json +++ b/production/mempool-config.mainnet-lightning.json @@ -1,5 +1,6 @@ { "MEMPOOL": { + "ENABLED": false, "NETWORK": "mainnet", "BACKEND": "esplora", "HTTP_PORT": 8993, diff --git a/production/mempool-config.signet-lightning.json b/production/mempool-config.signet-lightning.json index 7751d8f0e..9971729e2 100644 --- a/production/mempool-config.signet-lightning.json +++ b/production/mempool-config.signet-lightning.json @@ -1,5 +1,6 @@ { "MEMPOOL": { + "ENABLED": false, "NETWORK": "signet", "BACKEND": "esplora", "HTTP_PORT": 8991, diff --git a/production/mempool-config.testnet-lightning.json b/production/mempool-config.testnet-lightning.json index d8283b779..ff7d4766f 100644 --- a/production/mempool-config.testnet-lightning.json +++ b/production/mempool-config.testnet-lightning.json @@ -1,5 +1,6 @@ { "MEMPOOL": { + "ENABLED": false, "NETWORK": "testnet", "BACKEND": "esplora", "HTTP_PORT": 8992, From 22a491717a8466d700960ab54a6c1a915b647962 Mon Sep 17 00:00:00 2001 From: secondl1ght <85003930+secondl1ght@users.noreply.github.com> Date: Wed, 14 Jun 2023 21:48:36 -0600 Subject: [PATCH 04/12] update frontend local instructions --- frontend/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/README.md b/frontend/README.md index b626a23b9..8fc77a2b4 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -17,7 +17,7 @@ Get the latest Mempool code: ``` git clone https://github.com/mempool/mempool -cd mempool +cd mempool/frontend ``` ### 2. Specify Website From fe6da62daba8fec27ca6caf3ef199443d61d8db5 Mon Sep 17 00:00:00 2001 From: secondl1ght <85003930+secondl1ght@users.noreply.github.com> Date: Wed, 14 Jun 2023 21:55:17 -0600 Subject: [PATCH 05/12] create secondl1ght.txt contributor file --- contributors/secondl1ght.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 contributors/secondl1ght.txt diff --git a/contributors/secondl1ght.txt b/contributors/secondl1ght.txt new file mode 100644 index 000000000..a386a103a --- /dev/null +++ b/contributors/secondl1ght.txt @@ -0,0 +1,3 @@ +I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of June 14, 2023. + +Signed: secondl1ght From 5f3fd85834f6a0e3c47942e44dc593c38352c2d2 Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 15 Jun 2023 21:39:51 -0400 Subject: [PATCH 06/12] the last two docker-compose overrides need MEMPOOL_ in front --- docker/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/README.md b/docker/README.md index b669b37c8..d95bc7aee 100644 --- a/docker/README.md +++ b/docker/README.md @@ -144,8 +144,8 @@ Corresponding `docker-compose.yml` overrides: MEMPOOL_ADVANCED_GBT_AUDIT: "" MEMPOOL_ADVANCED_GBT_MEMPOOL: "" MEMPOOL_CPFP_INDEXING: "" - MAX_BLOCKS_BULK_QUERY: "" - DISK_CACHE_BLOCK_INTERVAL: "" + MEMPOOL_MAX_BLOCKS_BULK_QUERY: "" + MEMPOOL_DISK_CACHE_BLOCK_INTERVAL: "" ... ``` From e8aea3832074956690c7df4d60f0668d733568f4 Mon Sep 17 00:00:00 2001 From: Peter Date: Thu, 15 Jun 2023 21:46:09 -0400 Subject: [PATCH 07/12] contributer license --- contributors/pfoytik.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 contributors/pfoytik.txt diff --git a/contributors/pfoytik.txt b/contributors/pfoytik.txt new file mode 100644 index 000000000..f15f7cb33 --- /dev/null +++ b/contributors/pfoytik.txt @@ -0,0 +1,3 @@ +I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of June 15, 2023. + +Signed pfoytik From f1966768a7e767ee8f611b8e821793883ce73acf Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 19 Jun 2023 18:14:09 -0400 Subject: [PATCH 08/12] exclude fullrbf txs from audit and label in visualization --- backend/src/api/audit.ts | 15 +++++++++++---- backend/src/api/database-migration.ts | 6 +++++- backend/src/api/rbf-cache.ts | 13 +++++++++++++ backend/src/api/websocket-handler.ts | 11 ++++++----- backend/src/mempool.interfaces.ts | 1 + .../src/repositories/BlocksAuditsRepository.ts | 8 +++++--- .../components/block-overview-graph/tx-view.ts | 3 ++- .../block-overview-tooltip.component.html | 1 + .../src/app/components/block/block.component.ts | 16 +++++++++++++++- .../src/app/interfaces/node-api.interface.ts | 3 ++- .../src/app/interfaces/websocket.interface.ts | 2 +- 11 files changed, 62 insertions(+), 17 deletions(-) diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index 6c5f96988..e79196a7a 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -1,19 +1,21 @@ import config from '../config'; import logger from '../logger'; import { TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces'; +import rbfCache from './rbf-cache'; const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners class Audit { auditBlock(transactions: TransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended }) - : { censored: string[], added: string[], fresh: string[], sigop: string[], score: number, similarity: number } { + : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], score: number, similarity: number } { if (!projectedBlocks?.[0]?.transactionIds || !mempool) { - return { censored: [], added: [], fresh: [], sigop: [], score: 0, similarity: 1 }; + return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], score: 0, similarity: 1 }; } const matches: string[] = []; // present in both mined block and template const added: string[] = []; // present in mined block, not in template const fresh: string[] = []; // missing, but firstSeen within PROPAGATION_MARGIN + const fullrbf: string[] = []; // either missing or present, and part of a fullrbf replacement const isCensored = {}; // missing, without excuse const isDisplaced = {}; let displacedWeight = 0; @@ -35,7 +37,9 @@ class Audit { for (const txid of projectedBlocks[0].transactionIds) { if (!inBlock[txid]) { // tx is recent, may have reached the miner too late for inclusion - if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) { + if (rbfCache.isFullRbf(txid)) { + fullrbf.push(txid); + } else if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) { fresh.push(txid); } else { isCensored[txid] = true; @@ -91,7 +95,9 @@ class Audit { if (inTemplate[tx.txid]) { matches.push(tx.txid); } else { - if (!isDisplaced[tx.txid]) { + if (rbfCache.isFullRbf(tx.txid)) { + fullrbf.push(tx.txid); + } else if (!isDisplaced[tx.txid]) { added.push(tx.txid); } overflowWeight += tx.weight; @@ -138,6 +144,7 @@ class Audit { added, fresh, sigop: [], + fullrbf, score, similarity, }; diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 22b42dac7..a9266a016 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 62; + private static currentVersion = 63; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -539,6 +539,10 @@ class DatabaseMigration { await this.updateToSchemaVersion(62); } + if (databaseSchemaVersion < 63 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fullrbf_txs JSON DEFAULT "[]"'); + await this.updateToSchemaVersion(63); + } } /** diff --git a/backend/src/api/rbf-cache.ts b/backend/src/api/rbf-cache.ts index f0a916c8c..79d5ff2d1 100644 --- a/backend/src/api/rbf-cache.ts +++ b/backend/src/api/rbf-cache.ts @@ -169,6 +169,19 @@ class RbfCache { } } + // is the transaction involved in a full rbf replacement? + public isFullRbf(txid: string): boolean { + const treeId = this.treeMap.get(txid); + if (!treeId) { + return false; + } + const tree = this.rbfTrees.get(treeId); + if (!tree) { + return false; + } + return tree?.fullRbf; + } + private cleanup(): void { const now = Date.now(); for (const txid of this.expiring.keys()) { diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 97092d2b1..ae536b72e 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -583,6 +583,10 @@ class WebsocketHandler { const _memPool = memPool.getMempool(); + const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap()); + memPool.handleMinedRbfTransactions(rbfTransactions); + memPool.removeFromSpendMap(transactions); + if (config.MEMPOOL.AUDIT) { let projectedBlocks; let auditMempool = _memPool; @@ -605,7 +609,7 @@ class WebsocketHandler { } if (Common.indexingEnabled() && memPool.isInSync()) { - const { censored, added, fresh, sigop, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool); + const { censored, added, fresh, sigop, fullrbf, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool); const matchRate = Math.round(score * 100 * 100) / 100; const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : []; @@ -633,6 +637,7 @@ class WebsocketHandler { missingTxs: censored, freshTxs: fresh, sigopTxs: sigop, + fullrbfTxs: fullrbf, matchRate: matchRate, expectedFees: totalFees, expectedWeight: totalWeight, @@ -652,10 +657,6 @@ class WebsocketHandler { } } - const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap()); - memPool.handleMinedRbfTransactions(rbfTransactions); - memPool.removeFromSpendMap(transactions); - // Update mempool to remove transactions included in the new block for (const txId of txIds) { delete _memPool[txId]; diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index cc038ecfd..adcb9645d 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -34,6 +34,7 @@ export interface BlockAudit { missingTxs: string[], freshTxs: string[], sigopTxs: string[], + fullrbfTxs: string[], addedTxs: string[], matchRate: number, expectedFees?: number, diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index 8ad035f32..16a4e733d 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -6,9 +6,9 @@ import { BlockAudit, AuditScore } from '../mempool.interfaces'; class BlocksAuditRepositories { public async $saveAudit(audit: BlockAudit): Promise { try { - await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, match_rate, expected_fees, expected_weight) - VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs), - JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]); + await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, fullrbf_txs, match_rate, expected_fees, expected_weight) + VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs), + JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]); } catch (e: any) { if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`); @@ -69,6 +69,7 @@ class BlocksAuditRepositories { added_txs as addedTxs, fresh_txs as freshTxs, sigop_txs as sigopTxs, + fullrbf_txs as fullrbfTxs, match_rate as matchRate, expected_fees as expectedFees, expected_weight as expectedWeight @@ -83,6 +84,7 @@ class BlocksAuditRepositories { rows[0].addedTxs = JSON.parse(rows[0].addedTxs); rows[0].freshTxs = JSON.parse(rows[0].freshTxs); rows[0].sigopTxs = JSON.parse(rows[0].sigopTxs); + rows[0].fullrbfTxs = JSON.parse(rows[0].fullrbfTxs); rows[0].template = JSON.parse(rows[0].template); return rows[0]; diff --git a/frontend/src/app/components/block-overview-graph/tx-view.ts b/frontend/src/app/components/block-overview-graph/tx-view.ts index bb3d9563c..7d3e0ee13 100644 --- a/frontend/src/app/components/block-overview-graph/tx-view.ts +++ b/frontend/src/app/components/block-overview-graph/tx-view.ts @@ -37,7 +37,7 @@ export default class TxView implements TransactionStripped { value: number; feerate: number; rate?: number; - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf'; context?: 'projected' | 'actual'; scene?: BlockScene; @@ -172,6 +172,7 @@ export default class TxView implements TransactionStripped { return auditColors.censored; case 'missing': case 'sigop': + case 'fullrbf': return marginalFeeColors[feeLevelIndex] || marginalFeeColors[mempoolFeeColors.length - 1]; case 'fresh': return auditColors.missing; diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html index 795958fe3..eb6d97e40 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html @@ -48,6 +48,7 @@ Recently broadcasted Added Marginal fee rate + Full RBF diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 927222dbc..ad008089d 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -317,6 +317,7 @@ export class BlockComponent implements OnInit, OnDestroy { const isSelected = {}; const isFresh = {}; const isSigop = {}; + const isFullRbf = {}; this.numMissing = 0; this.numUnexpected = 0; @@ -339,6 +340,9 @@ export class BlockComponent implements OnInit, OnDestroy { for (const txid of blockAudit.sigopTxs || []) { isSigop[txid] = true; } + for (const txid of blockAudit.fullrbfTxs || []) { + isFullRbf[txid] = true; + } // set transaction statuses for (const tx of blockAudit.template) { tx.context = 'projected'; @@ -347,7 +351,15 @@ export class BlockComponent implements OnInit, OnDestroy { } else if (inBlock[tx.txid]) { tx.status = 'found'; } else { - tx.status = isFresh[tx.txid] ? 'fresh' : (isSigop[tx.txid] ? 'sigop' : 'missing'); + if (isFresh[tx.txid]) { + tx.status = 'fresh'; + } else if (isSigop[tx.txid]) { + tx.status = 'sigop'; + } else if (isFullRbf[tx.txid]) { + tx.status = 'fullrbf'; + } else { + tx.status = 'missing'; + } isMissing[tx.txid] = true; this.numMissing++; } @@ -360,6 +372,8 @@ export class BlockComponent implements OnInit, OnDestroy { tx.status = 'added'; } else if (inTemplate[tx.txid]) { tx.status = 'found'; + } else if (isFullRbf[tx.txid]) { + tx.status = 'fullrbf'; } else { tx.status = 'selected'; isSelected[tx.txid] = true; diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 68c45b3b2..82e1ae50d 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -156,6 +156,7 @@ export interface BlockAudit extends BlockExtended { addedTxs: string[], freshTxs: string[], sigopTxs: string[], + fullrbfTxs: string[], matchRate: number, expectedFees: number, expectedWeight: number, @@ -171,7 +172,7 @@ export interface TransactionStripped { fee: number; vsize: number; value: number; - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf'; context?: 'projected' | 'actual'; } diff --git a/frontend/src/app/interfaces/websocket.interface.ts b/frontend/src/app/interfaces/websocket.interface.ts index 41643fb73..20a114c72 100644 --- a/frontend/src/app/interfaces/websocket.interface.ts +++ b/frontend/src/app/interfaces/websocket.interface.ts @@ -77,7 +77,7 @@ export interface TransactionStripped { vsize: number; value: number; rate?: number; // effective fee rate - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf'; context?: 'projected' | 'actual'; } From 385cb087d3f4f98b63fab3b210a7471800eb89a3 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 8 Jul 2023 00:33:14 -0400 Subject: [PATCH 09/12] Replace cached blocks on reorg, serve stale blocks --- backend/src/api/bitcoin/bitcoin-api.ts | 1 + .../src/api/bitcoin/esplora-api.interface.ts | 1 + backend/src/api/blocks.ts | 26 +++++++++++++++---- .../repositories/BlocksAuditsRepository.ts | 4 +-- .../src/app/interfaces/electrs.interface.ts | 1 + 5 files changed, 25 insertions(+), 8 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 307736737..a0cc41770 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -29,6 +29,7 @@ class BitcoinApi implements AbstractBitcoinApi { weight: block.weight, previousblockhash: block.previousblockhash, mediantime: block.mediantime, + stale: block.confirmations === -1, }; } diff --git a/backend/src/api/bitcoin/esplora-api.interface.ts b/backend/src/api/bitcoin/esplora-api.interface.ts index 6d50bddfd..5b86952b0 100644 --- a/backend/src/api/bitcoin/esplora-api.interface.ts +++ b/backend/src/api/bitcoin/esplora-api.interface.ts @@ -89,6 +89,7 @@ export namespace IEsploraApi { weight: number; previousblockhash: string; mediantime: number; + stale: boolean; } export interface Address { diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 16335c36a..4d218ed54 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -656,10 +656,6 @@ class Blocks { const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, cpfpSummary.transactions); this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`); - // start async callbacks - this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`); - const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions)); - if (Common.indexingEnabled()) { if (!fastForwarded) { const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1); @@ -671,9 +667,11 @@ class Blocks { await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10); await HashratesRepository.$deleteLastEntries(); await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10); + this.blocks = this.blocks.slice(0, -10); this.updateTimerProgress(timer, `rolled back chain divergence from ${this.currentBlockHeight}`); for (let i = 10; i >= 0; --i) { const newBlock = await this.$indexBlock(lastBlock.height - i); + this.blocks.push(newBlock); this.updateTimerProgress(timer, `reindexed block`); let cpfpSummary; if (config.MEMPOOL.CPFP_INDEXING) { @@ -722,6 +720,10 @@ class Blocks { } } + // start async callbacks + this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`); + const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions)); + if (block.height % 2016 === 0) { if (Common.indexingEnabled()) { await DifficultyAdjustmentsRepository.$saveAdjustments({ @@ -814,6 +816,16 @@ class Blocks { return blockExtended; } + public async $indexStaleBlock(hash: string): Promise { + const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash); + const transactions = await this.$getTransactionsExtended(hash, block.height, true); + const blockExtended = await this.$getBlockExtended(block, transactions); + + blockExtended.canonical = await bitcoinApi.$getBlockHash(block.height); + + return blockExtended; + } + /** * Get one block by its hash */ @@ -831,7 +843,11 @@ class Blocks { // Bitcoin network, add our custom data on top const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash); - return await this.$indexBlock(block.height); + if (block.stale) { + return await this.$indexStaleBlock(hash); + } else { + return await this.$indexBlock(block.height); + } } public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false, diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index 8ad035f32..31107db75 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -62,8 +62,7 @@ class BlocksAuditRepositories { public async $getBlockAudit(hash: string): Promise { try { const [rows]: any[] = await DB.query( - `SELECT blocks.height, blocks.hash as id, UNIX_TIMESTAMP(blocks.blockTimestamp) as timestamp, blocks.size, - blocks.weight, blocks.tx_count, + `SELECT blocks_audits.height, blocks_audits.hash as id, UNIX_TIMESTAMP(blocks_audits.time) as timestamp, template, missing_txs as missingTxs, added_txs as addedTxs, @@ -73,7 +72,6 @@ class BlocksAuditRepositories { expected_fees as expectedFees, expected_weight as expectedWeight FROM blocks_audits - JOIN blocks ON blocks.hash = blocks_audits.hash JOIN blocks_templates ON blocks_templates.id = blocks_audits.hash WHERE blocks_audits.hash = "${hash}" `); diff --git a/frontend/src/app/interfaces/electrs.interface.ts b/frontend/src/app/interfaces/electrs.interface.ts index dcccfb67c..4cb6b41db 100644 --- a/frontend/src/app/interfaces/electrs.interface.ts +++ b/frontend/src/app/interfaces/electrs.interface.ts @@ -120,6 +120,7 @@ export interface Block { size: number; weight: number; previousblockhash: string; + stale?: boolean; } export interface Address { From 6942a6fd6a4b53493d4fc27bf7afb6914fe605f0 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 8 Jul 2023 00:34:57 -0400 Subject: [PATCH 10/12] Show alert banner on stale blocks --- backend/src/mempool.interfaces.ts | 1 + .../app/components/block/block.component.html | 6 +++++ .../app/components/block/block.component.scss | 23 +++++++++++++++++++ .../src/app/interfaces/electrs.interface.ts | 1 + 4 files changed, 31 insertions(+) diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index cc038ecfd..27577ad0c 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -227,6 +227,7 @@ export interface BlockExtension { */ export interface BlockExtended extends IEsploraApi.Block { extras: BlockExtension; + canonical?: string; } export interface BlockSummary { diff --git a/frontend/src/app/components/block/block.component.html b/frontend/src/app/components/block/block.component.html index aa441ad9a..f77ddeb6f 100644 --- a/frontend/src/app/components/block/block.component.html +++ b/frontend/src/app/components/block/block.component.html @@ -1,6 +1,10 @@
+

Block Genesis @@ -23,6 +27,8 @@
+ +

diff --git a/frontend/src/app/components/block/block.component.scss b/frontend/src/app/components/block/block.component.scss index 08091cb86..a15c876e6 100644 --- a/frontend/src/app/components/block/block.component.scss +++ b/frontend/src/app/components/block/block.component.scss @@ -1,3 +1,26 @@ +.title-block { + flex-wrap: wrap; + align-items: baseline; + @media (min-width: 650px) { + flex-direction: row; + } + h1 { + margin: 0rem; + margin-right: 15px; + line-height: 1; + } + + .alert-mempool { + flex-direction: row; + flex-wrap: wrap; + } + + .container-button { + align-self: center; + margin-right: 1em; + } +} + .qr-wrapper { background-color: #FFF; padding: 10px; diff --git a/frontend/src/app/interfaces/electrs.interface.ts b/frontend/src/app/interfaces/electrs.interface.ts index 4cb6b41db..2739d2b06 100644 --- a/frontend/src/app/interfaces/electrs.interface.ts +++ b/frontend/src/app/interfaces/electrs.interface.ts @@ -121,6 +121,7 @@ export interface Block { weight: number; previousblockhash: string; stale?: boolean; + canonical?: string; } export interface Address { From 11d1a68f78c8324cb1efdc9c9fd45ac2ceb7dd84 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 11 Jul 2023 14:36:42 +0900 Subject: [PATCH 11/12] Upgrade bitcoin core to v25.0 --- production/install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/install b/production/install index 5ee445a21..0297d4b4f 100755 --- a/production/install +++ b/production/install @@ -332,7 +332,7 @@ BITCOIN_REPO_URL=https://github.com/bitcoin/bitcoin BITCOIN_REPO_NAME=bitcoin BITCOIN_REPO_BRANCH=master #BITCOIN_LATEST_RELEASE=$(curl -s https://api.github.com/repos/bitcoin/bitcoin/releases/latest|grep tag_name|head -1|cut -d '"' -f4) -BITCOIN_LATEST_RELEASE=v23.0 +BITCOIN_LATEST_RELEASE=v25.0 echo -n '.' BISQ_REPO_URL=https://github.com/bisq-network/bisq From ca2830d6d8c4e223c9400fcad52acc92e5597ced Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 11 Jul 2023 16:03:44 +0900 Subject: [PATCH 12/12] fix price updater loop on testnet/signet --- backend/src/indexer.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 4b120867f..88f44d587 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -6,6 +6,7 @@ import logger from './logger'; import bitcoinClient from './api/bitcoin/bitcoin-client'; import priceUpdater from './tasks/price-updater'; import PricesRepository from './repositories/PricesRepository'; +import config from './config'; export interface CoreIndex { name: string; @@ -72,7 +73,7 @@ class Indexer { return; } - if (task === 'blocksPrices' && !this.tasksRunning.includes(task)) { + if (task === 'blocksPrices' && !this.tasksRunning.includes(task) && !['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { this.tasksRunning.push(task); const lastestPriceId = await PricesRepository.$getLatestPriceId(); if (priceUpdater.historyInserted === false || lastestPriceId === null) {