From befec362de44c0bfe72871440932b43a2af0fed5 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Mon, 12 Jun 2023 07:07:15 -0400 Subject: [PATCH 01/73] Replace disclaimer text regarding tx acceleration --- frontend/src/app/docs/api-docs/api-docs.component.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/docs/api-docs/api-docs.component.html b/frontend/src/app/docs/api-docs/api-docs.component.html index 392cda19e..b788d2739 100644 --- a/frontend/src/app/docs/api-docs/api-docs.component.html +++ b/frontend/src/app/docs/api-docs/api-docs.component.html @@ -10,8 +10,8 @@
-

mempool.space merely provides data about the Bitcoin network. It cannot help you with retrieving funds, confirming your transaction quicker, etc.

For any such requests, you need to get in touch with the entity that helped make the transaction (wallet software, exchange company, etc).

-

mempool.space merely provides data about the Bitcoin network. It cannot help you with retrieving funds, confirming your transaction quicker, etc.

For any such requests, you need to get in touch with the entity that helped make the transaction (wallet software, exchange company, etc).

+

mempool.space merely provides data about the Bitcoin network. It cannot help you with retrieving funds, wallet issues, etc.

For any such requests, you need to get in touch with the entity that helped make the transaction (wallet software, exchange company, etc).

+

mempool.space merely provides data about the Bitcoin network. It cannot help you with retrieving funds, wallet issues, etc.

For any such requests, you need to get in touch with the entity that helped make the transaction (wallet software, exchange company, etc).

From be0742b9332f937067f34c292560fa8cb4909fa8 Mon Sep 17 00:00:00 2001 From: wiz Date: Sun, 16 Jul 2023 17:50:36 +0900 Subject: [PATCH 02/73] ops: Move electrs scripts to mempool/electrs repo --- production/electrs-start-liquid | 24 ------------------ production/electrs-start-liquidtestnet | 24 ------------------ production/electrs-start-mainnet | 22 ---------------- production/electrs-start-signet | 23 ----------------- production/electrs-start-testnet | 23 ----------------- production/install | 35 +++----------------------- 6 files changed, 3 insertions(+), 148 deletions(-) delete mode 100755 production/electrs-start-liquid delete mode 100755 production/electrs-start-liquidtestnet delete mode 100755 production/electrs-start-mainnet delete mode 100755 production/electrs-start-signet delete mode 100755 production/electrs-start-testnet diff --git a/production/electrs-start-liquid b/production/electrs-start-liquid deleted file mode 100755 index a28135836..000000000 --- a/production/electrs-start-liquid +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env zsh -cd "${HOME}/electrs" -#source "${HOME}/.cargo/env" -#export PATH="${HOME}/.cargo/bin:${PATH}" - -until false -do - cargo run \ - --release \ - --features liquid \ - --bin electrs \ - -- \ - -vvv \ - --asset-db-path "${HOME}/asset_registry_db" \ - --address-search \ - --cors '*' \ - --db-dir __ELECTRS_DATA_ROOT__ \ - --network liquid \ - --daemon-dir "${HOME}" \ - --http-socket-file '/elements/socket/esplora-liquid-mainnet' \ - --cookie '__ELEMENTS_RPC_USER__:__ELEMENTS_RPC_PASS__' \ - --precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt" - sleep 1 -done diff --git a/production/electrs-start-liquidtestnet b/production/electrs-start-liquidtestnet deleted file mode 100755 index 828e96533..000000000 --- a/production/electrs-start-liquidtestnet +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/local/bin/zsh -cd "${HOME}/electrs" -#source "${HOME}/.cargo/env" -#export PATH="${HOME}/.cargo/bin:${PATH}" - -until false -do - cargo run \ - --release \ - --features liquid \ - --bin electrs \ - -- \ - -vv \ - --asset-db-path "${HOME}/asset_registry_testnet_db" \ - --address-search \ - --cors '*' \ - --db-dir __ELECTRS_DATA_ROOT__ \ - --network liquidtestnet \ - --daemon-dir "${HOME}" \ - --http-socket-file '/elements/socket/esplora-liquid-testnet' \ - --cookie '__ELEMENTS_RPC_USER__:__ELEMENTS_RPC_PASS__' \ - --precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt" - sleep 1 -done diff --git a/production/electrs-start-mainnet b/production/electrs-start-mainnet deleted file mode 100755 index c6a8c4d54..000000000 --- a/production/electrs-start-mainnet +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env zsh -cd "${HOME}/electrs" -#source "${HOME}/.cargo/env" -#export PATH="${HOME}/.cargo/bin:${PATH}" - -until false -do - cargo run \ - --release \ - --bin electrs \ - -- \ - -vvvv \ - --address-search \ - --cors '*' \ - --db-dir __ELECTRS_DATA_ROOT__ \ - --daemon-dir "${HOME}" \ - --http-socket-file '/bitcoin/socket/esplora-bitcoin-mainnet' \ - --cookie '__BITCOIN_RPC_USER__:__BITCOIN_RPC_PASS__' \ - --precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt" - - sleep 3 -done diff --git a/production/electrs-start-signet b/production/electrs-start-signet deleted file mode 100755 index 40e1d1115..000000000 --- a/production/electrs-start-signet +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env zsh -cd "${HOME}/electrs" -#source "${HOME}/.cargo/env" -#export PATH="${HOME}/.cargo/bin:${PATH}" - -until false -do - cargo run \ - --release \ - --bin electrs \ - -- \ - -vv \ - --network signet \ - --address-search \ - --cors '*' \ - --db-dir __ELECTRS_DATA_ROOT__ \ - --daemon-rpc-addr '127.0.0.1:38332' \ - --daemon-dir "${HOME}" \ - --http-socket-file '/bitcoin/socket/esplora-bitcoin-signet' \ - --cookie '__BITCOIN_RPC_USER__:__BITCOIN_RPC_PASS__' \ - --precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt" - sleep 1 -done diff --git a/production/electrs-start-testnet b/production/electrs-start-testnet deleted file mode 100755 index ce05de2de..000000000 --- a/production/electrs-start-testnet +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env zsh -cd "${HOME}/electrs" -#source $HOME/.cargo/env -#export PATH=$HOME/.cargo/bin:$PATH - -until false -do - cargo run \ - --release \ - --bin electrs \ - -- \ - -vvvv \ - --network testnet \ - --address-search \ - --cors '*' \ - --db-dir __ELECTRS_DATA_ROOT__ \ - --daemon-dir "${HOME}" \ - --http-socket-file '/bitcoin/socket/esplora-bitcoin-testnet' \ - --cookie '__BITCOIN_RPC_USER__:__BITCOIN_RPC_PASS__' \ - --precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt" - - sleep 3 -done diff --git a/production/install b/production/install index 1121f5b4f..05b868ba0 100755 --- a/production/install +++ b/production/install @@ -1600,15 +1600,8 @@ fi ######################################## # Electrs instance for Bitcoin Mainnet # ######################################## - if [ "${BITCOIN_MAINNET_ENABLE}" = ON ];then - echo "[*] Installing Bitcoin Mainnet electrs start script" - osSudo "${ROOT_USER}" install -c -o "${BITCOIN_USER}" -g "${BITCOIN_GROUP}" -m 755 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/electrs-start-mainnet" "${BITCOIN_ELECTRS_HOME}" - - echo "[*] Configuring Bitcoin Mainnet RPC credentials in electrs start script" - osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_USER__/${BITCOIN_RPC_USER}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-mainnet" - osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_PASS__/${BITCOIN_RPC_PASS}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-mainnet" - osSudo "${ROOT_USER}" sed -i.orig "s!__ELECTRS_DATA_ROOT__!${ELECTRS_DATA_ROOT}!" "${BITCOIN_ELECTRS_HOME}/electrs-start-mainnet" + echo "[*] FIXME: must only crontab enabled daemons" fi ######################################## @@ -1616,13 +1609,7 @@ fi ######################################## if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then - echo "[*] Installing Bitcoin Testnet electrs start script" - osSudo "${ROOT_USER}" install -c -o "${BITCOIN_USER}" -g "${BITCOIN_GROUP}" -m 755 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/electrs-start-testnet" "${BITCOIN_ELECTRS_HOME}" - - echo "[*] Configuring Bitcoin Testnet RPC credentials in electrs start script" - osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_USER__/${BITCOIN_RPC_USER}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-testnet" - osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_PASS__/${BITCOIN_RPC_PASS}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-testnet" - osSudo "${ROOT_USER}" sed -i.orig "s!__ELECTRS_DATA_ROOT__!${ELECTRS_DATA_ROOT}!" "${BITCOIN_ELECTRS_HOME}/electrs-start-testnet" + echo "[*] FIXME: must only crontab enabled daemons" fi ####################################### @@ -1630,13 +1617,7 @@ fi ####################################### if [ "${BITCOIN_SIGNET_ENABLE}" = ON ];then - echo "[*] Installing Bitcoin Signet electrs start script" - osSudo "${ROOT_USER}" install -c -o "${BITCOIN_USER}" -g "${BITCOIN_GROUP}" -m 755 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/electrs-start-signet" "${BITCOIN_ELECTRS_HOME}" - - echo "[*] Configuring Bitcoin Signet RPC credentials in electrs start script" - osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_USER__/${BITCOIN_RPC_USER}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-signet" - osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_PASS__/${BITCOIN_RPC_PASS}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-signet" - osSudo "${ROOT_USER}" sed -i.orig "s!__ELECTRS_DATA_ROOT__!${ELECTRS_DATA_ROOT}!" "${BITCOIN_ELECTRS_HOME}/electrs-start-signet" + echo "[*] FIXME: must only crontab enabled daemons" fi ######################################## @@ -1644,21 +1625,12 @@ fi ######################################## if [ "${ELEMENTS_LIQUID_ENABLE}" = ON ];then - echo "[*] Installing Elements Liquid electrs start script" - osSudo "${ROOT_USER}" install -c -o "${ELEMENTS_USER}" -g "${ELEMENTS_GROUP}" -m 755 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/electrs-start-liquid" "${ELEMENTS_ELECTRS_HOME}" - echo "[*] Installing Elements crontab" case $OS in FreeBSD) - echo "[*] FIXME: must only crontab enabled daemons" osSudo "${ROOT_USER}" crontab -u "${ELEMENTS_USER}" "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/elements.crontab" ;; esac - - echo "[*] Configuring Elements Liquid RPC credentials in electrs start script" - osSudo "${ROOT_USER}" sed -i.orig "s/__ELEMENTS_RPC_USER__/${ELEMENTS_RPC_USER}/" "${ELEMENTS_ELECTRS_HOME}/electrs-start-liquid" - osSudo "${ROOT_USER}" sed -i.orig "s/__ELEMENTS_RPC_PASS__/${ELEMENTS_RPC_PASS}/" "${ELEMENTS_ELECTRS_HOME}/electrs-start-liquid" - osSudo "${ROOT_USER}" sed -i.orig "s!__ELECTRS_DATA_ROOT__!${ELECTRS_DATA_ROOT}!" "${ELEMENTS_ELECTRS_HOME}/electrs-start-liquid" fi ################################################ @@ -1687,7 +1659,6 @@ fi echo "[*] Installing crontabs" case $OS in FreeBSD) - echo "[*] FIXME: must only crontab enabled daemons" osSudo "${ROOT_USER}" crontab -u "${BITCOIN_USER}" "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/bitcoin.crontab" osSudo "${ROOT_USER}" crontab -u "${MINFEE_USER}" "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/minfee.crontab" ;; From 6f25e25711ec7a6d333e330307cd1f3b509709ba Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 22 May 2023 18:16:58 -0400 Subject: [PATCH 03/73] include accelerated tx data in block audits --- backend/src/api/audit.ts | 11 ++++++++--- backend/src/api/database-migration.ts | 7 ++++++- backend/src/api/mempool-blocks.ts | 2 +- backend/src/api/websocket-handler.ts | 3 ++- backend/src/mempool.interfaces.ts | 1 + backend/src/repositories/BlocksAuditsRepository.ts | 9 ++++++--- .../app/components/block-overview-graph/tx-view.ts | 5 ++++- .../block-overview-tooltip.component.html | 1 + frontend/src/app/components/block/block.component.ts | 10 ++++++++++ frontend/src/app/interfaces/node-api.interface.ts | 3 ++- frontend/src/app/interfaces/websocket.interface.ts | 2 +- 11 files changed, 42 insertions(+), 12 deletions(-) diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index a909fc2b6..63da288a1 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -6,16 +6,17 @@ import rbfCache from './rbf-cache'; const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners class Audit { - auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }) - : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], score: number, similarity: number } { + auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }, accelerations: { [txid: string]: number }) + : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } { if (!projectedBlocks?.[0]?.transactionIds || !mempool) { - return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], score: 0, similarity: 1 }; + return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 0, similarity: 1 }; } const matches: string[] = []; // present in both mined block and template const added: string[] = []; // present in mined block, not in template const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN const rbf: string[] = []; // either missing or present, and either part of a full-rbf replacement, or a conflict with the mined block + const accelerated: string[] = []; // prioritized by the mempool accelerator const isCensored = {}; // missing, without excuse const isDisplaced = {}; let displacedWeight = 0; @@ -28,6 +29,9 @@ class Audit { const now = Math.round((Date.now() / 1000)); for (const tx of transactions) { inBlock[tx.txid] = tx; + if (accelerations[tx.txid]) { + accelerated.push(tx.txid); + } } // coinbase is always expected if (transactions[0]) { @@ -149,6 +153,7 @@ class Audit { fresh, sigop: [], fullrbf: rbf, + accelerated, score, similarity, }; diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 7c7608aff..b7dc39493 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 64; + private static currentVersion = 65; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -548,6 +548,11 @@ class DatabaseMigration { await this.$executeQuery('ALTER TABLE `nodes` ADD features text NULL'); await this.updateToSchemaVersion(64); } + + if (databaseSchemaVersion < 65 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `blocks_audits` ADD accelerated_txs JSON DEFAULT "[]"'); + await this.updateToSchemaVersion(65); + } } /** diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 08508310d..5ca5cff09 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -206,7 +206,7 @@ class MempoolBlocks { return mempoolBlockDeltas; } - public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): Promise { + public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, accelerations: { [txid: string]: number } = {}): Promise { const start = Date.now(); // reset mempool short ids diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 56c8513cd..91004e292 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -666,7 +666,7 @@ class WebsocketHandler { } if (Common.indexingEnabled()) { - const { censored, added, fresh, sigop, fullrbf, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool); + const { censored, added, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool, accelerations); const matchRate = Math.round(score * 100 * 100) / 100; const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : []; @@ -695,6 +695,7 @@ class WebsocketHandler { freshTxs: fresh, sigopTxs: sigop, fullrbfTxs: fullrbf, + acceleratedTxs: accelerated, matchRate: matchRate, expectedFees: totalFees, expectedWeight: totalWeight, diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 25e7f0387..d0549cd97 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -36,6 +36,7 @@ export interface BlockAudit { sigopTxs: string[], fullrbfTxs: string[], addedTxs: string[], + acceleratedTxs: string[], matchRate: number, expectedFees?: number, expectedWeight?: number, diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index f7a2a59b5..9c7568567 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -6,9 +6,9 @@ import { BlockAudit, AuditScore } from '../mempool.interfaces'; class BlocksAuditRepositories { public async $saveAudit(audit: BlockAudit): Promise { try { - await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, fullrbf_txs, match_rate, expected_fees, expected_weight) - VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs), - JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]); + await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, fullrbf_txs, accelerated_txs, match_rate, expected_fees, expected_weight) + VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs), + JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), JSON.stringify(audit.acceleratedTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]); } catch (e: any) { if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`); @@ -69,6 +69,7 @@ class BlocksAuditRepositories { fresh_txs as freshTxs, sigop_txs as sigopTxs, fullrbf_txs as fullrbfTxs, + accelerated_txs as acceleratedTxs, match_rate as matchRate, expected_fees as expectedFees, expected_weight as expectedWeight @@ -83,6 +84,8 @@ class BlocksAuditRepositories { rows[0].freshTxs = JSON.parse(rows[0].freshTxs); rows[0].sigopTxs = JSON.parse(rows[0].sigopTxs); rows[0].fullrbfTxs = JSON.parse(rows[0].fullrbfTxs); + rows[0].acceleratedTxs = JSON.parse(rows[0].acceleratedTxs); + rows[0].transactions = JSON.parse(rows[0].transactions); rows[0].template = JSON.parse(rows[0].template); return rows[0]; diff --git a/frontend/src/app/components/block-overview-graph/tx-view.ts b/frontend/src/app/components/block-overview-graph/tx-view.ts index 1b8c88704..ac93eee0c 100644 --- a/frontend/src/app/components/block-overview-graph/tx-view.ts +++ b/frontend/src/app/components/block-overview-graph/tx-view.ts @@ -17,6 +17,7 @@ const auditColors = { missing: darken(desaturate(hexToColor('f344df'), 0.3), 0.7), added: hexToColor('0099ff'), selected: darken(desaturate(hexToColor('0099ff'), 0.3), 0.7), + accelerated: hexToColor('8F5FF6'), }; // convert from this class's update format to TxSprite's update format @@ -38,7 +39,7 @@ export default class TxView implements TransactionStripped { value: number; feerate: number; rate?: number; - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; scene?: BlockScene; @@ -216,6 +217,8 @@ export default class TxView implements TransactionStripped { return auditColors.added; case 'selected': return marginalFeeColors[feeLevelIndex] || marginalFeeColors[mempoolFeeColors.length - 1]; + case 'accelerated': + return auditColors.accelerated; case 'found': if (this.context === 'projected') { return auditFeeColors[feeLevelIndex] || auditFeeColors[mempoolFeeColors.length - 1]; diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html index c62779b69..8a410f3df 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html @@ -54,6 +54,7 @@ Added Marginal fee rate Conflicting + Accelerated diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index ec9a49504..1e94b1a9c 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -340,6 +340,7 @@ export class BlockComponent implements OnInit, OnDestroy { const isFresh = {}; const isSigop = {}; const isRbf = {}; + const isAccelerated = {}; this.numMissing = 0; this.numUnexpected = 0; @@ -365,6 +366,9 @@ export class BlockComponent implements OnInit, OnDestroy { for (const txid of blockAudit.fullrbfTxs || []) { isRbf[txid] = true; } + for (const txid of blockAudit.acceleratedTxs || []) { + isAccelerated[txid] = true; + } // set transaction statuses for (const tx of blockAudit.template) { tx.context = 'projected'; @@ -389,6 +393,9 @@ export class BlockComponent implements OnInit, OnDestroy { isMissing[tx.txid] = true; this.numMissing++; } + if (isAccelerated[tx.txid]) { + tx.status = 'accelerated'; + } } for (const [index, tx] of transactions.entries()) { tx.context = 'actual'; @@ -405,6 +412,9 @@ export class BlockComponent implements OnInit, OnDestroy { isSelected[tx.txid] = true; this.numUnexpected++; } + if (isAccelerated[tx.txid]) { + tx.status = 'accelerated'; + } } for (const tx of transactions) { inBlock[tx.txid] = true; diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 4249fd9db..7e7acfcf3 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -158,6 +158,7 @@ export interface BlockAudit extends BlockExtended { freshTxs: string[], sigopTxs: string[], fullrbfTxs: string[], + acceleratedTxs: string[], matchRate: number, expectedFees: number, expectedWeight: number, @@ -174,7 +175,7 @@ export interface TransactionStripped { vsize: number; value: number; rate?: number; // effective fee rate - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; } diff --git a/frontend/src/app/interfaces/websocket.interface.ts b/frontend/src/app/interfaces/websocket.interface.ts index e0ecdfeda..fb3c6d0c2 100644 --- a/frontend/src/app/interfaces/websocket.interface.ts +++ b/frontend/src/app/interfaces/websocket.interface.ts @@ -89,7 +89,7 @@ export interface TransactionStripped { vsize: number; value: number; rate?: number; // effective fee rate - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; } From 8d70d3bd727c7060c15a522a7d4c5771c8db0240 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 26 May 2023 21:10:32 -0400 Subject: [PATCH 04/73] use accelerated rates for block templates & show in viz --- backend/src/api/audit.ts | 4 +-- backend/src/api/bitcoin/bitcoin.routes.ts | 1 + backend/src/api/common.ts | 1 + backend/src/api/mempool-blocks.ts | 15 +++++----- backend/src/api/mempool.ts | 29 +++++++++++++++++++ backend/src/api/websocket-handler.ts | 5 +++- backend/src/mempool.interfaces.ts | 2 ++ .../block-overview-graph.component.ts | 2 +- .../block-overview-graph/block-scene.ts | 3 +- .../block-overview-graph/tx-view.ts | 13 ++++++++- .../block-overview-tooltip.component.html | 3 +- .../block-overview-tooltip.component.ts | 2 ++ .../mempool-block-overview.component.ts | 1 - .../transaction/transaction.component.html | 3 +- .../transaction/transaction.component.ts | 3 ++ .../src/app/interfaces/electrs.interface.ts | 1 + .../src/app/interfaces/node-api.interface.ts | 1 + .../src/app/interfaces/websocket.interface.ts | 3 +- 18 files changed, 75 insertions(+), 17 deletions(-) diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index 63da288a1..9710d0362 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -6,7 +6,7 @@ import rbfCache from './rbf-cache'; const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners class Audit { - auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }, accelerations: { [txid: string]: number }) + auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }) : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } { if (!projectedBlocks?.[0]?.transactionIds || !mempool) { return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 0, similarity: 1 }; @@ -29,7 +29,7 @@ class Audit { const now = Math.round((Date.now() / 1000)); for (const tx of transactions) { inBlock[tx.txid] = tx; - if (accelerations[tx.txid]) { + if (tx.acceleration) { accelerated.push(tx.txid); } } diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index ffdb2e629..e2887b706 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -214,6 +214,7 @@ class BitcoinRoutes { effectiveFeePerVsize: tx.effectiveFeePerVsize || null, sigops: tx.sigops, adjustedVsize: tx.adjustedVsize, + acceleration: tx.acceleration }); return; } diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index cd9da3d2a..775da2643 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -111,6 +111,7 @@ export class Common { fee: tx.fee, vsize: tx.weight / 4, value: tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0), + acc: tx.acceleration || undefined, rate: tx.effectiveFeePerVsize, }; } diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 5ca5cff09..81f2092a0 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -170,7 +170,7 @@ class MempoolBlocks { for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) { let added: TransactionStripped[] = []; let removed: string[] = []; - const changed: { txid: string, rate: number | undefined }[] = []; + const changed: { txid: string, rate: number | undefined, acc: number | undefined }[] = []; if (mempoolBlocks[i] && !prevBlocks[i]) { added = mempoolBlocks[i].transactions; } else if (!mempoolBlocks[i] && prevBlocks[i]) { @@ -192,8 +192,8 @@ class MempoolBlocks { mempoolBlocks[i].transactions.forEach(tx => { if (!prevIds[tx.txid]) { added.push(tx); - } else if (tx.rate !== prevIds[tx.txid].rate) { - changed.push({ txid: tx.txid, rate: tx.rate }); + } else if (tx.rate !== prevIds[tx.txid].rate || tx.acc !== prevIds[tx.txid].acc) { + changed.push({ txid: tx.txid, rate: tx.rate, acc: tx.acc }); } }); } @@ -206,7 +206,7 @@ class MempoolBlocks { return mempoolBlockDeltas; } - public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, accelerations: { [txid: string]: number } = {}): Promise { + public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): Promise { const start = Date.now(); // reset mempool short ids @@ -222,7 +222,7 @@ class MempoolBlocks { if (entry.uid !== null && entry.uid !== undefined) { const stripped = { uid: entry.uid, - fee: entry.fee, + fee: entry.fee + (entry.acceleration || 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, @@ -285,13 +285,14 @@ class MempoolBlocks { for (const tx of Object.values(added)) { this.setUid(tx, true); } - const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => (uid !== null && uid !== undefined)) as number[]; + + const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => uid != null) as number[]; // prepare a stripped down version of the mempool with only the minimum necessary data // to reduce the overhead of passing this data to the worker thread const addedStripped: CompactThreadTransaction[] = added.filter(entry => (entry.uid !== null && entry.uid !== undefined)).map(entry => { return { uid: entry.uid || 0, - fee: entry.fee, + fee: entry.fee + (entry.acceleration || 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 945b78738..8e350c4fe 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -23,6 +23,8 @@ class Mempool { private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]) => Promise) | undefined; + private accelerations: { [txId: string]: number } = {}; + private txPerSecondArray: number[] = []; private txPerSecond: number = 0; @@ -301,6 +303,17 @@ class Mempool { const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx)); this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6); + const newAccelerations: { txid: string, delta: number }[] = []; + newTransactions.forEach(tx => { + if (tx.txid.startsWith('00')) { + const delta = Math.floor(Math.random() * 100000) + 100000; + newAccelerations.push({ txid: tx.txid, delta }); + tx.acceleration = delta; + } + }); + this.addAccelerations(newAccelerations); + this.removeAccelerations(deletedTransactions.map(tx => tx.txid)); + this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize); if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) { @@ -325,6 +338,22 @@ class Mempool { this.clearTimer(timer); } + public getAccelerations(): { [txid: string]: number } { + return this.accelerations; + } + + public addAccelerations(newAccelerations: { txid: string, delta: number }[]): void { + for (const acceleration of newAccelerations) { + this.accelerations[acceleration.txid] = acceleration.delta; + } + } + + public removeAccelerations(txids: string[]): void { + for (const txid of txids) { + delete this.accelerations[txid]; + } + } + private startTimer() { const state: any = { start: Date.now(), diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 91004e292..a33a0f0fa 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -21,6 +21,7 @@ import Audit from './audit'; import { deepClone } from '../utils/clone'; import priceUpdater from '../tasks/price-updater'; import { ApiPrice } from '../repositories/PricesRepository'; +import mempool from './mempool'; // valid 'want' subscriptions const wantable = [ @@ -666,7 +667,7 @@ class WebsocketHandler { } if (Common.indexingEnabled()) { - const { censored, added, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool, accelerations); + const { censored, added, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool); const matchRate = Math.round(score * 100 * 100) / 100; const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : []; @@ -737,6 +738,8 @@ class WebsocketHandler { const fees = feeApi.getRecommendedFee(); const mempoolInfo = memPool.getMempoolInfo(); + memPool.removeAccelerations(txIds); + // update init data this.updateSocketDataFields({ 'mempoolInfo': mempoolInfo, diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index d0549cd97..185256619 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -92,6 +92,7 @@ export interface TransactionExtended extends IEsploraApi.Transaction { block: number, vsize: number, }; + acceleration?: number; uid?: number; } @@ -183,6 +184,7 @@ export interface TransactionStripped { fee: number; vsize: number; value: number; + acc?: number; rate?: number; // effective fee rate } diff --git a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts index 49da16d55..fe847103f 100644 --- a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts +++ b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts @@ -147,7 +147,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On } } - update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { + update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { if (this.scene) { this.scene.update(add, remove, change, direction, resetLayout); this.start(); diff --git a/frontend/src/app/components/block-overview-graph/block-scene.ts b/frontend/src/app/components/block-overview-graph/block-scene.ts index 510803f03..94984bae2 100644 --- a/frontend/src/app/components/block-overview-graph/block-scene.ts +++ b/frontend/src/app/components/block-overview-graph/block-scene.ts @@ -150,7 +150,7 @@ export default class BlockScene { this.updateAll(startTime, 200, direction); } - update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { + update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { const startTime = performance.now(); const removed = this.removeBatch(remove, startTime, direction); @@ -175,6 +175,7 @@ export default class BlockScene { // update effective rates change.forEach(tx => { if (this.txs[tx.txid]) { + this.txs[tx.txid].acc = tx.acc; this.txs[tx.txid].feerate = tx.rate || (this.txs[tx.txid].fee / this.txs[tx.txid].vsize); this.txs[tx.txid].rate = tx.rate; this.txs[tx.txid].dirty = true; diff --git a/frontend/src/app/components/block-overview-graph/tx-view.ts b/frontend/src/app/components/block-overview-graph/tx-view.ts index ac93eee0c..690b974e3 100644 --- a/frontend/src/app/components/block-overview-graph/tx-view.ts +++ b/frontend/src/app/components/block-overview-graph/tx-view.ts @@ -38,6 +38,7 @@ export default class TxView implements TransactionStripped { vsize: number; value: number; feerate: number; + acc?: number; rate?: number; status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; @@ -64,6 +65,7 @@ export default class TxView implements TransactionStripped { this.vsize = tx.vsize; this.value = tx.value; this.feerate = tx.rate || (tx.fee / tx.vsize); // sort by effective fee rate where available + this.acc = tx.acc; this.rate = tx.rate; this.status = tx.status; this.initialised = false; @@ -200,6 +202,11 @@ export default class TxView implements TransactionStripped { const feeLevelColor = feeColors[feeLevelIndex] || feeColors[mempoolFeeColors.length - 1]; // Normal mode if (!this.scene?.highlightingEnabled) { + if (this.acc) { + return auditColors.accelerated; + } else { + return feeLevelColor; + } return feeLevelColor; } // Block audit @@ -226,7 +233,11 @@ export default class TxView implements TransactionStripped { return feeLevelColor; } default: - return feeLevelColor; + if (this.acc) { + return auditColors.accelerated; + } else { + return feeLevelColor; + } } } } diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html index 8a410f3df..a53cfdc9c 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html @@ -29,7 +29,8 @@ - Effective fee rate + Effective fee rate + Accelerated fee rate diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts index 61c294263..65d0f984c 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts @@ -21,6 +21,7 @@ export class BlockOverviewTooltipComponent implements OnChanges { vsize = 1; feeRate = 0; effectiveRate; + acceleration; tooltipPosition: Position = { x: 0, y: 0 }; @@ -53,6 +54,7 @@ export class BlockOverviewTooltipComponent implements OnChanges { this.vsize = tx.vsize || 1; this.feeRate = this.fee / this.vsize; this.effectiveRate = tx.rate; + this.acceleration = tx.acc; } } } diff --git a/frontend/src/app/components/mempool-block-overview/mempool-block-overview.component.ts b/frontend/src/app/components/mempool-block-overview/mempool-block-overview.component.ts index 30632a862..226be5210 100644 --- a/frontend/src/app/components/mempool-block-overview/mempool-block-overview.component.ts +++ b/frontend/src/app/components/mempool-block-overview/mempool-block-overview.component.ts @@ -94,7 +94,6 @@ export class MempoolBlockOverviewComponent implements OnInit, OnDestroy, OnChang updateBlock(delta: MempoolBlockDelta): void { const blockMined = (this.stateService.latestBlockHeight > this.lastBlockHeight); - if (this.blockIndex !== this.index) { const direction = (this.blockIndex == null || this.index < this.blockIndex) ? this.poolDirection : this.chainDirection; this.blockGraph.replace(delta.added, direction); diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index d4cd6913d..81a6106db 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -488,7 +488,8 @@ - Effective fee rate + Accelerated fee rate + Effective fee rate
diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index e856f34eb..f1d218a79 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -183,6 +183,9 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { } else { this.tx.effectiveFeePerVsize = cpfpInfo.effectiveFeePerVsize; } + if (cpfpInfo.acceleration) { + this.tx.acceleration = cpfpInfo.acceleration; + } this.cpfpInfo = cpfpInfo; this.hasEffectiveFeeRate = hasRelatives || (this.tx.effectiveFeePerVsize && (Math.abs(this.tx.effectiveFeePerVsize - this.tx.feePerVsize) > 0.01)); diff --git a/frontend/src/app/interfaces/electrs.interface.ts b/frontend/src/app/interfaces/electrs.interface.ts index df19f7491..5c15b0ae4 100644 --- a/frontend/src/app/interfaces/electrs.interface.ts +++ b/frontend/src/app/interfaces/electrs.interface.ts @@ -19,6 +19,7 @@ export interface Transaction { ancestors?: Ancestor[]; bestDescendant?: BestDescendant | null; cpfpChecked?: boolean; + acceleration?: number; deleteAfter?: number; _unblinded?: any; _deduced?: boolean; diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 7e7acfcf3..fe6233866 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -27,6 +27,7 @@ export interface CpfpInfo { effectiveFeePerVsize?: number; sigops?: number; adjustedVsize?: number; + acceleration?: number; } export interface RbfInfo { diff --git a/frontend/src/app/interfaces/websocket.interface.ts b/frontend/src/app/interfaces/websocket.interface.ts index fb3c6d0c2..f8686042b 100644 --- a/frontend/src/app/interfaces/websocket.interface.ts +++ b/frontend/src/app/interfaces/websocket.interface.ts @@ -70,7 +70,7 @@ export interface MempoolBlockWithTransactions extends MempoolBlock { export interface MempoolBlockDelta { added: TransactionStripped[], removed: string[], - changed?: { txid: string, rate: number | undefined }[]; + changed?: { txid: string, rate: number | undefined, acc: number | undefined }[]; } export interface MempoolInfo { @@ -88,6 +88,7 @@ export interface TransactionStripped { fee: number; vsize: number; value: number; + acc?: number; // acceleration delta rate?: number; // effective fee rate status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; From 8e295969a37ef4d5c69b73b4f8be07909519cc7a Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 30 May 2023 19:35:39 -0400 Subject: [PATCH 05/73] Refactor acceleration tracking --- backend/src/api/mempool-blocks.ts | 20 ++++--- backend/src/api/mempool.ts | 72 +++++++++++++++--------- backend/src/api/services/acceleration.ts | 20 +++++++ backend/src/api/websocket-handler.ts | 6 +- 4 files changed, 79 insertions(+), 39 deletions(-) create mode 100644 backend/src/api/services/acceleration.ts diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 81f2092a0..bc0def7a8 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -5,6 +5,7 @@ import { Common, OnlineFeeStatsCalculator } from './common'; import config from '../config'; import { Worker } from 'worker_threads'; import path from 'path'; +import mempool from './mempool'; const MAX_UINT32 = Math.pow(2, 32) - 1; @@ -212,9 +213,11 @@ class MempoolBlocks { // reset mempool short ids this.resetUids(); for (const tx of Object.values(newMempool)) { - this.setUid(tx); + this.setUid(tx, true); } + const accelerations = mempool.getAccelerations(); + // prepare a stripped down version of the mempool with only the minimum necessary data // to reduce the overhead of passing this data to the worker thread const strippedMempool: Map = new Map(); @@ -222,7 +225,7 @@ class MempoolBlocks { if (entry.uid !== null && entry.uid !== undefined) { const stripped = { uid: entry.uid, - fee: entry.fee + (entry.acceleration || 0), + fee: entry.fee + (accelerations[entry.txid] || 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, @@ -273,7 +276,7 @@ class MempoolBlocks { return this.mempoolBlocks; } - public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], saveResults: boolean = false): Promise { + public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], accelerationDelta: string[] = [], saveResults: boolean = false): Promise { if (!this.txSelectionWorker) { // need to reset the worker await this.$makeBlockTemplates(newMempool, saveResults); @@ -282,17 +285,20 @@ class MempoolBlocks { const start = Date.now(); - for (const tx of Object.values(added)) { + const accelerations = mempool.getAccelerations(); + const addedAndChanged: MempoolTransactionExtended[] = accelerationDelta.map(txid => newMempool[txid]).filter(tx => tx != null).concat(added); + + for (const tx of addedAndChanged) { this.setUid(tx, true); } - const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => uid != null) as number[]; + // prepare a stripped down version of the mempool with only the minimum necessary data // to reduce the overhead of passing this data to the worker thread - const addedStripped: CompactThreadTransaction[] = added.filter(entry => (entry.uid !== null && entry.uid !== undefined)).map(entry => { + const addedStripped: CompactThreadTransaction[] = addedAndChanged.filter(entry => entry.uid != null).map(entry => { return { uid: entry.uid || 0, - fee: entry.fee + (entry.acceleration || 0), + fee: entry.fee + (accelerations[entry.txid] || 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 8e350c4fe..7bde83df9 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -1,5 +1,5 @@ import config from '../config'; -import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; +import bitcoinApi from './bitcoin/bitcoin-api-factory'; import { MempoolTransactionExtended, TransactionExtended, VbytesPerSecond } from '../mempool.interfaces'; import logger from '../logger'; import { Common } from './common'; @@ -9,7 +9,7 @@ import loadingIndicators from './loading-indicators'; import bitcoinClient from './bitcoin/bitcoin-client'; import bitcoinSecondClient from './bitcoin/bitcoin-second-client'; import rbfCache from './rbf-cache'; -import { IEsploraApi } from './bitcoin/esplora-api.interface'; +import accelerationApi from './services/acceleration'; class Mempool { private inSync: boolean = false; @@ -19,9 +19,9 @@ class Mempool { private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0, maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 }; private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[], - deletedTransactions: MempoolTransactionExtended[]) => void) | undefined; + deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => void) | undefined; private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[], - deletedTransactions: MempoolTransactionExtended[]) => Promise) | undefined; + deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise) | undefined; private accelerations: { [txId: string]: number } = {}; @@ -68,12 +68,12 @@ class Mempool { } public setMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; }, - newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]) => void): void { + newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => void): void { this.mempoolChangedCallback = fn; } public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; }, mempoolSize: number, - newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]) => Promise): void { + newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise): void { this.$asyncMempoolChangedCallback = fn; } @@ -98,10 +98,10 @@ class Mempool { count++; } if (this.mempoolChangedCallback) { - this.mempoolChangedCallback(this.mempoolCache, [], []); + this.mempoolChangedCallback(this.mempoolCache, [], [], []); } if (this.$asyncMempoolChangedCallback) { - await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], []); + await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], [], []); } this.addToSpendMap(Object.values(this.mempoolCache)); } @@ -303,25 +303,19 @@ class Mempool { const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx)); this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6); - const newAccelerations: { txid: string, delta: number }[] = []; - newTransactions.forEach(tx => { - if (tx.txid.startsWith('00')) { - const delta = Math.floor(Math.random() * 100000) + 100000; - newAccelerations.push({ txid: tx.txid, delta }); - tx.acceleration = delta; - } - }); - this.addAccelerations(newAccelerations); - this.removeAccelerations(deletedTransactions.map(tx => tx.txid)); + const accelerationDelta = await this.$updateAccelerations(); + if (accelerationDelta.length) { + hasChange = true; + } this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize); if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) { - this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions); + this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions, accelerationDelta); } if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) { this.updateTimerProgress(timer, 'running async mempool callback'); - await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions); + await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions, accelerationDelta); this.updateTimerProgress(timer, 'completed async mempool callback'); } @@ -342,15 +336,37 @@ class Mempool { return this.accelerations; } - public addAccelerations(newAccelerations: { txid: string, delta: number }[]): void { - for (const acceleration of newAccelerations) { - this.accelerations[acceleration.txid] = acceleration.delta; - } - } + public async $updateAccelerations(): Promise { + try { + const newAccelerations = await accelerationApi.fetchAccelerations$(); - public removeAccelerations(txids: string[]): void { - for (const txid of txids) { - delete this.accelerations[txid]; + const changed: string[] = []; + + const newAccelerationMap: { [txid: string]: number } = {}; + for (const acceleration of newAccelerations) { + newAccelerationMap[acceleration.txid] = acceleration.feeDelta; + if (this.accelerations[acceleration.txid] == null) { + // new acceleration + changed.push(acceleration.txid); + } else if (this.accelerations[acceleration.txid] !== acceleration.feeDelta) { + // feeDelta changed + changed.push(acceleration.txid); + } + } + + for (const oldTxid of Object.keys(this.accelerations)) { + if (!newAccelerationMap[oldTxid]) { + // removed + changed.push(oldTxid); + } + } + + this.accelerations = newAccelerationMap; + + return changed; + } catch (e: any) { + logger.debug(`Failed to update accelerations: ` + (e instanceof Error ? e.message : e)); + return []; } } diff --git a/backend/src/api/services/acceleration.ts b/backend/src/api/services/acceleration.ts new file mode 100644 index 000000000..efbf32f5d --- /dev/null +++ b/backend/src/api/services/acceleration.ts @@ -0,0 +1,20 @@ +import { query } from '../../utils/axios-query'; +import config from '../../config'; + +export interface Acceleration { + txid: string, + feeDelta: number, +} + +class AccelerationApi { + public async fetchAccelerations$(): Promise { + if (config.MEMPOOL_SERVICES.ACCELERATIONS) { + const response = await query(`${config.MEMPOOL_SERVICES.API}/accelerations`); + return (response as Acceleration[]) || []; + } else { + return []; + } + } +} + +export default new AccelerationApi(); \ No newline at end of file diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index a33a0f0fa..3a90b29f4 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -381,7 +381,7 @@ class WebsocketHandler { } async $handleMempoolChange(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, - newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]): Promise { + newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]): Promise { if (!this.wss) { throw new Error('WebSocket.Server is not set'); } @@ -392,7 +392,7 @@ class WebsocketHandler { if (config.MEMPOOL.RUST_GBT) { await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions); } else { - await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, true); + await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true); } } else { mempoolBlocks.updateMempoolBlocks(newMempool, true); @@ -738,8 +738,6 @@ class WebsocketHandler { const fees = feeApi.getRecommendedFee(); const mempoolInfo = memPool.getMempoolInfo(); - memPool.removeAccelerations(txIds); - // update init data this.updateSocketDataFields({ 'mempoolInfo': mempoolInfo, From 8ade3be8cbfe22b2730f8c8acd22b022423fa32d Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 30 May 2023 19:51:11 -0400 Subject: [PATCH 06/73] Implement accelerations API & config setting --- backend/mempool-config.sample.json | 4 ++++ .../src/__fixtures__/mempool-config.template.json | 4 ++++ backend/src/__tests__/config.test.ts | 7 +++++++ backend/src/api/mempool.ts | 4 ++++ backend/src/config.ts | 12 +++++++++++- docker/backend/mempool-config.json | 4 ++++ docker/backend/start.sh | 9 +++++++++ 7 files changed, 43 insertions(+), 1 deletion(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index e3df7d2fe..8f8b82475 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -136,5 +136,9 @@ "trusted", "servers" ] + }, + "MEMPOOL_SERVICES": { + "API": "https://mempool.space/api", + "ACCELERATIONS": false } } diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 4213f0ffb..d6754f966 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -127,5 +127,9 @@ "AUDIT": false, "AUDIT_START_HEIGHT": 774000, "SERVERS": [] + }, + "MEMPOOl_SERVICES": { + "API": "__MEMPOOL_SERVICES_API__", + "ACCELERATIONS": "__MEMPOOL_SERVICES_ACCELERATIONS__" } } diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index dc1beaa46..2c83d1af5 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -127,6 +127,11 @@ describe('Mempool Backend Config', () => { AUDIT_START_HEIGHT: 774000, SERVERS: [] }); + + expect(config.MEMPOOL_SERVICES).toStrictEqual({ + API: "", + ACCELERATIONS: false, + }); }); }); @@ -160,6 +165,8 @@ describe('Mempool Backend Config', () => { expect(config.PRICE_DATA_SERVER).toStrictEqual(fixture.PRICE_DATA_SERVER); expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER); + + expect(config.MEMPOOL_SERVICES).toStrictEqual(fixture.MEMPOOL_SERVICES); }); }); diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 7bde83df9..b114c89fc 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -337,6 +337,10 @@ class Mempool { } public async $updateAccelerations(): Promise { + if (!config.MEMPOOL_SERVICES.ACCELERATIONS) { + return []; + } + try { const newAccelerations = await accelerationApi.fetchAccelerations$(); diff --git a/backend/src/config.ts b/backend/src/config.ts index 09d279537..ceb569e06 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -137,7 +137,11 @@ interface IConfig { AUDIT: boolean; AUDIT_START_HEIGHT: number; SERVERS: string[]; - } + }, + MEMPOOL_SERVICES: { + API: string; + ACCELERATIONS: boolean; + }, } const defaults: IConfig = { @@ -275,6 +279,10 @@ const defaults: IConfig = { 'AUDIT': false, 'AUDIT_START_HEIGHT': 774000, 'SERVERS': [], + }, + 'MEMPOOL_SERVICES': { + 'API': '', + 'ACCELERATIONS': false, } }; @@ -296,6 +304,7 @@ class Config implements IConfig { EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER']; MAXMIND: IConfig['MAXMIND']; REPLICATION: IConfig['REPLICATION']; + MEMPOOL_SERVICES: IConfig['MEMPOOL_SERVICES']; constructor() { const configs = this.merge(configFromFile, defaults); @@ -316,6 +325,7 @@ class Config implements IConfig { this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER; this.MAXMIND = configs.MAXMIND; this.REPLICATION = configs.REPLICATION; + this.MEMPOOL_SERVICES = configs.MEMPOOL_SERVICES; } merge = (...objects: object[]): IConfig => { diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 2ff76d5dd..71fe3bd65 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -133,5 +133,9 @@ "AUDIT": __REPLICATION_AUDIT__, "AUDIT_START_HEIGHT": __REPLICATION_AUDIT_START_HEIGHT__, "SERVERS": __REPLICATION_SERVERS__ + }, + "MEMPOOL_SERVICES": { + "API": "__MEMPOOL_SERVICES_API__", + "ACCELERATIONS": __MEMPOOL_SERVICES_ACCELERATIONS__ } } diff --git a/docker/backend/start.sh b/docker/backend/start.sh index c34d804b4..66f873605 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -136,6 +136,10 @@ __REPLICATION_AUDIT__=${REPLICATION_AUDIT:=true} __REPLICATION_AUDIT_START_HEIGHT__=${REPLICATION_AUDIT_START_HEIGHT:=774000} __REPLICATION_SERVERS__=${REPLICATION_SERVERS:=[]} +# MEMPOOL_SERVICES +__MEMPOOL_SERVICES_API__==${MEMPOOL_SERVICES_API:=""} +__MEMPOOL_SERVICES_ACCELERATIONS__==${MEMPOOL_SERVICES_ACCELERATIONS:=false} + mkdir -p "${__MEMPOOL_CACHE_DIR__}" @@ -262,4 +266,9 @@ sed -i "s!__REPLICATION_AUDIT__!${__REPLICATION_AUDIT__}!g" mempool-config.json sed -i "s!__REPLICATION_AUDIT_START_HEIGHT__!${__REPLICATION_AUDIT_START_HEIGHT__}!g" mempool-config.json sed -i "s!__REPLICATION_SERVERS__!${__REPLICATION_SERVERS__}!g" mempool-config.json +# MEMPOOL_SERVICES +sed -i "s!__MEMPOOL_SERVICES_API__!${__MEMPOOL_SERVICES_API__}!g" mempool-config.json +sed -i "s!__MEMPOOL_SERVICES_ACCELERATIONS__!${__MEMPOOL_SERVICES_ACCELERATIONS__}!g" mempool-config.json + + node /backend/package/index.js From 53a564b3a8ec5aa6f7771e36cebd68c5859dcc1f Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 3 Jun 2023 16:54:12 -0400 Subject: [PATCH 07/73] Refactor accelerated audits --- backend/src/api/audit.ts | 4 ++-- backend/src/api/mempool-blocks.ts | 28 +++++++++++++----------- backend/src/api/mempool.ts | 2 +- backend/src/api/services/acceleration.ts | 20 ++++++++++++++++- backend/src/api/websocket-handler.ts | 17 ++++++++++---- 5 files changed, 50 insertions(+), 21 deletions(-) diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index 9710d0362..e78b2796f 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -6,7 +6,7 @@ import rbfCache from './rbf-cache'; const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners class Audit { - auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }) + auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }, useAccelerations: boolean = false) : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } { if (!projectedBlocks?.[0]?.transactionIds || !mempool) { return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 0, similarity: 1 }; @@ -29,7 +29,7 @@ class Audit { const now = Math.round((Date.now() / 1000)); for (const tx of transactions) { inBlock[tx.txid] = tx; - if (tx.acceleration) { + if (mempool[tx.txid] && mempool[tx.txid].acceleration) { accelerated.push(tx.txid); } } diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index bc0def7a8..0ce0ea522 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -207,7 +207,7 @@ class MempoolBlocks { return mempoolBlockDeltas; } - public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): Promise { + public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false): Promise { const start = Date.now(); // reset mempool short ids @@ -216,7 +216,7 @@ class MempoolBlocks { this.setUid(tx, true); } - const accelerations = mempool.getAccelerations(); + const accelerations = useAccelerations ? mempool.getAccelerations() : {}; // prepare a stripped down version of the mempool with only the minimum necessary data // to reduce the overhead of passing this data to the worker thread @@ -225,7 +225,7 @@ class MempoolBlocks { if (entry.uid !== null && entry.uid !== undefined) { const stripped = { uid: entry.uid, - fee: entry.fee + (accelerations[entry.txid] || 0), + fee: entry.fee + (useAccelerations ? (accelerations[entry.txid] || 0) : 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, @@ -265,7 +265,7 @@ class MempoolBlocks { // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); - const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), saveResults); + const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, saveResults); logger.debug(`makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); @@ -276,17 +276,17 @@ class MempoolBlocks { return this.mempoolBlocks; } - public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], accelerationDelta: string[] = [], saveResults: boolean = false): Promise { + public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], accelerationDelta: string[] = [], saveResults: boolean = false, useAccelerations: boolean = false): Promise { if (!this.txSelectionWorker) { // need to reset the worker - await this.$makeBlockTemplates(newMempool, saveResults); + await this.$makeBlockTemplates(newMempool, saveResults, useAccelerations); return; } const start = Date.now(); - const accelerations = mempool.getAccelerations(); - const addedAndChanged: MempoolTransactionExtended[] = accelerationDelta.map(txid => newMempool[txid]).filter(tx => tx != null).concat(added); + const accelerations = useAccelerations ? mempool.getAccelerations() : {}; + const addedAndChanged: MempoolTransactionExtended[] = useAccelerations ? accelerationDelta.map(txid => newMempool[txid]).filter(tx => tx != null).concat(added) : added; for (const tx of addedAndChanged) { this.setUid(tx, true); @@ -298,7 +298,7 @@ class MempoolBlocks { const addedStripped: CompactThreadTransaction[] = addedAndChanged.filter(entry => entry.uid != null).map(entry => { return { uid: entry.uid || 0, - fee: entry.fee + (accelerations[entry.txid] || 0), + fee: entry.fee + (useAccelerations ? (accelerations[entry.txid] || 0) : 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, @@ -325,7 +325,7 @@ class MempoolBlocks { // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); - this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), saveResults); + this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, saveResults); logger.debug(`updateBlockTemplates completed in ${(Date.now() - start) / 1000} seconds`); } catch (e) { logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e)); @@ -362,7 +362,7 @@ class MempoolBlocks { if (saveResults) { this.rustInitialized = true; } - const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, saveResults); + const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, {}, saveResults); logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); return processed; } catch (e) { @@ -414,7 +414,7 @@ class MempoolBlocks { if (mempoolSize !== resultMempoolSize) { throw new Error('GBT returned wrong number of transactions, cache is probably out of sync'); } else { - this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, true); + this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, {}, true); } this.removeUids(removedUids); logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); @@ -424,7 +424,7 @@ class MempoolBlocks { } } - private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], saveResults): MempoolBlockWithTransactions[] { + private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], accelerations, saveResults): MempoolBlockWithTransactions[] { for (const [txid, rate] of rates) { if (txid in mempool) { mempool[txid].effectiveFeePerVsize = rate; @@ -503,6 +503,8 @@ class MempoolBlocks { mempoolTx.cpfpChecked = true; } + mempoolTx.acceleration = accelerations[txid]; + // online calculation of stack-of-blocks fee stats if (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) { feeStatsCalculator.processNext(mempoolTx); diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index b114c89fc..4efe17731 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -342,7 +342,7 @@ class Mempool { } try { - const newAccelerations = await accelerationApi.fetchAccelerations$(); + const newAccelerations = await accelerationApi.$fetchAccelerations(); const changed: string[] = []; diff --git a/backend/src/api/services/acceleration.ts b/backend/src/api/services/acceleration.ts index efbf32f5d..8c849cdd8 100644 --- a/backend/src/api/services/acceleration.ts +++ b/backend/src/api/services/acceleration.ts @@ -1,5 +1,6 @@ import { query } from '../../utils/axios-query'; import config from '../../config'; +import { BlockExtended, PoolTag } from '../../mempool.interfaces'; export interface Acceleration { txid: string, @@ -7,7 +8,7 @@ export interface Acceleration { } class AccelerationApi { - public async fetchAccelerations$(): Promise { + public async $fetchAccelerations(): Promise { if (config.MEMPOOL_SERVICES.ACCELERATIONS) { const response = await query(`${config.MEMPOOL_SERVICES.API}/accelerations`); return (response as Acceleration[]) || []; @@ -15,6 +16,23 @@ class AccelerationApi { return []; } } + + public async $fetchPools(): Promise { + if (config.MEMPOOL_SERVICES.ACCELERATIONS) { + const response = await query(`${config.MEMPOOL_SERVICES.API}/partners`); + return (response as PoolTag[]) || []; + } else { + return []; + } + } + + public async $isAcceleratedBlock(block: BlockExtended): Promise { + const pools = await this.$fetchPools(); + if (block?.extras?.pool?.id == null) { + return false; + } + return pools.reduce((match, tag) => match || tag.uniqueId === block.extras.pool.id, false); + } } export default new AccelerationApi(); \ No newline at end of file diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 3a90b29f4..51fccbcbb 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -21,7 +21,7 @@ import Audit from './audit'; import { deepClone } from '../utils/clone'; import priceUpdater from '../tasks/price-updater'; import { ApiPrice } from '../repositories/PricesRepository'; -import mempool from './mempool'; +import accelerationApi from './services/acceleration'; // valid 'want' subscriptions const wantable = [ @@ -392,7 +392,7 @@ class WebsocketHandler { if (config.MEMPOOL.RUST_GBT) { await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions); } else { - await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true); + await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS); } } else { mempoolBlocks.updateMempoolBlocks(newMempool, true); @@ -648,6 +648,7 @@ class WebsocketHandler { if (config.MEMPOOL.AUDIT && memPool.isInSync()) { let projectedBlocks; let auditMempool = _memPool; + const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && await accelerationApi.$isAcceleratedBlock(block); // template calculation functions have mempool side effects, so calculate audits using // a cloned copy of the mempool if we're running a different algorithm for mempool updates const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL; @@ -657,13 +658,17 @@ class WebsocketHandler { if (config.MEMPOOL.RUST_GBT) { projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool); } else { - projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false); + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated); } } else { projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false); } } else { - projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); + if ((config.MEMPOOL_SERVICES.ACCELERATIONS && !isAccelerated)) { + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated); + } else { + projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); + } } if (Common.indexingEnabled()) { @@ -723,11 +728,15 @@ class WebsocketHandler { } if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { +<<<<<<< HEAD if (config.MEMPOOL.RUST_GBT) { await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions); } else { await mempoolBlocks.$makeBlockTemplates(_memPool, true); } +======= + await mempoolBlocks.$makeBlockTemplates(_memPool, true, config.MEMPOOL_SERVICES.ACCELERATIONS); +>>>>>>> 77b0a8ecc (Refactor accelerated audits) } else { mempoolBlocks.updateMempoolBlocks(_memPool, true); } From 31988ae6eb6771eb56a3ff12f0eece376f4b776c Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 13 Jun 2023 13:35:25 -0400 Subject: [PATCH 08/73] support for acceleration mempool blocks animation --- backend/src/api/websocket-handler.ts | 24 ++++++++++----- .../mempool-blocks.component.html | 2 +- .../mempool-blocks.component.scss | 30 +++++++++++++++++++ .../mempool-blocks.component.ts | 1 + .../transaction/transaction.component.html | 4 +-- .../src/app/interfaces/node-api.interface.ts | 1 + 6 files changed, 51 insertions(+), 11 deletions(-) diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 51fccbcbb..7716ed7b7 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -173,9 +173,15 @@ class WebsocketHandler { } const tx = memPool.getMempool()[trackTxid]; if (tx && tx.position) { + const position: { block: number, vsize: number, accelerated?: number } = { + ...tx.position + }; + if (tx.acceleration) { + position.accelerated = tx.acceleration; + } response['txPosition'] = JSON.stringify({ txid: trackTxid, - position: tx.position, + position }); } } else { @@ -600,7 +606,10 @@ class WebsocketHandler { if (mempoolTx && mempoolTx.position) { response['txPosition'] = JSON.stringify({ txid: trackTxid, - position: mempoolTx.position, + position: { + ...mempoolTx.position, + accelerated: mempoolTx.acceleration || undefined, + } }); } } @@ -728,15 +737,11 @@ class WebsocketHandler { } if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { -<<<<<<< HEAD if (config.MEMPOOL.RUST_GBT) { await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions); } else { - await mempoolBlocks.$makeBlockTemplates(_memPool, true); + await mempoolBlocks.$makeBlockTemplates(_memPool, true, config.MEMPOOL_SERVICES.ACCELERATIONS); } -======= - await mempoolBlocks.$makeBlockTemplates(_memPool, true, config.MEMPOOL_SERVICES.ACCELERATIONS); ->>>>>>> 77b0a8ecc (Refactor accelerated audits) } else { mempoolBlocks.updateMempoolBlocks(_memPool, true); } @@ -799,7 +804,10 @@ class WebsocketHandler { if (mempoolTx && mempoolTx.position) { response['txPosition'] = JSON.stringify({ txid: trackTxid, - position: mempoolTx.position, + position: { + ...mempoolTx.position, + accelerated: mempoolTx.acceleration || undefined, + } }); } } diff --git a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.html b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.html index 9c5c338c0..59d35c91e 100644 --- a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.html +++ b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.html @@ -49,7 +49,7 @@
-
+
diff --git a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.scss b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.scss index 40f43a015..606699d93 100644 --- a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.scss +++ b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.scss @@ -169,4 +169,34 @@ transform: translate(calc(-0.2 * var(--block-size)), calc(1.1 * var(--block-size))); border-radius: 2px; z-index: -1; +} + +.blink{ + width:400px; + height:400px; + border-bottom: 35px solid #FFF; + animation: blink 0.2s infinite; +} +@keyframes blink{ + 0% { + border-bottom: 35px solid green; + } + 50% { + border-bottom: 35px solid yellow; + } + 100% { + border-bottom: 35px solid orange; + } +} + +@-webkit-keyframes blink{ + 0% { + border-bottom: 35px solid green; + } + 50% { + border-bottom: 35px solid yellow; + } + 100% { + border-bottom: 35px solid orange; + } } \ No newline at end of file diff --git a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts index 71075b261..33db897a5 100644 --- a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts +++ b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts @@ -26,6 +26,7 @@ import { animate, style, transition, trigger } from '@angular/animations'; export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy { @Input() minimal: boolean = false; @Input() blockWidth: number = 125; + @Input() containerWidth: number = null; @Input() count: number = null; @Input() spotlight: number = 0; @Input() getHref?: (index) => string = (index) => `/mempool-block/${index}`; diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index 81a6106db..7422c6894 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -488,8 +488,8 @@ - Accelerated fee rate - Effective fee rate + Accelerated fee rate + Effective fee rate
diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index fe6233866..a0a3da5e6 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -188,6 +188,7 @@ export interface RbfTransaction extends TransactionStripped { export interface MempoolPosition { block: number, vsize: number, + accelerated?: boolean } export interface RewardStats { From e6bb46892640a688a930f6ec5f20c0ec2981022f Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 13 Jun 2023 17:03:36 -0400 Subject: [PATCH 09/73] include per-tx pools in /accelerations endpoint --- backend/src/api/mempool-blocks.ts | 8 +++--- backend/src/api/mempool.ts | 36 +++++++++++++++++++----- backend/src/api/services/acceleration.ts | 20 ++++--------- backend/src/api/websocket-handler.ts | 5 ++-- 4 files changed, 42 insertions(+), 27 deletions(-) diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 0ce0ea522..606c98c83 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,6 +1,6 @@ import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction } from '../../rust-gbt'; import logger from '../logger'; -import { MempoolBlock, MempoolTransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats } from '../mempool.interfaces'; +import { MempoolBlock, MempoolTransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, PoolTag } from '../mempool.interfaces'; import { Common, OnlineFeeStatsCalculator } from './common'; import config from '../config'; import { Worker } from 'worker_threads'; @@ -207,7 +207,7 @@ class MempoolBlocks { return mempoolBlockDeltas; } - public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false): Promise { + public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { const start = Date.now(); // reset mempool short ids @@ -225,7 +225,7 @@ class MempoolBlocks { if (entry.uid !== null && entry.uid !== undefined) { const stripped = { uid: entry.uid, - fee: entry.fee + (useAccelerations ? (accelerations[entry.txid] || 0) : 0), + fee: entry.fee + (useAccelerations && (!accelerationPool || accelerations[entry.txid]?.pools?.includes(accelerationPool)) ? (accelerations[entry.txid]?.feeDelta || 0) : 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, @@ -298,7 +298,7 @@ class MempoolBlocks { const addedStripped: CompactThreadTransaction[] = addedAndChanged.filter(entry => entry.uid != null).map(entry => { return { uid: entry.uid || 0, - fee: entry.fee + (useAccelerations ? (accelerations[entry.txid] || 0) : 0), + fee: entry.fee + (useAccelerations ? (accelerations[entry.txid]?.feeDelta || 0) : 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 4efe17731..107aa41cb 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -23,7 +23,7 @@ class Mempool { private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise) | undefined; - private accelerations: { [txId: string]: number } = {}; + private accelerations: { [txId: string]: Acceleration } = {}; private txPerSecondArray: number[] = []; private txPerSecond: number = 0; @@ -332,7 +332,7 @@ class Mempool { this.clearTimer(timer); } - public getAccelerations(): { [txid: string]: number } { + public getAccelerations(): { [txid: string]: Acceleration } { return this.accelerations; } @@ -346,15 +346,37 @@ class Mempool { const changed: string[] = []; - const newAccelerationMap: { [txid: string]: number } = {}; + const newAccelerationMap: { [txid: string]: Acceleration } = {}; for (const acceleration of newAccelerations) { - newAccelerationMap[acceleration.txid] = acceleration.feeDelta; + newAccelerationMap[acceleration.txid] = acceleration; if (this.accelerations[acceleration.txid] == null) { // new acceleration changed.push(acceleration.txid); - } else if (this.accelerations[acceleration.txid] !== acceleration.feeDelta) { - // feeDelta changed - changed.push(acceleration.txid); + } else { + if (this.accelerations[acceleration.txid].feeDelta !== acceleration.feeDelta) { + // feeDelta changed + changed.push(acceleration.txid); + } else if (this.accelerations[acceleration.txid].pools?.length) { + let poolsChanged = false; + const pools = new Set(); + this.accelerations[acceleration.txid].pools.forEach(pool => { + pools.add(pool); + }); + acceleration.pools.forEach(pool => { + if (!pools.has(pool)) { + poolsChanged = true; + } else { + pools.delete(pool); + } + }); + if (pools.size > 0) { + poolsChanged = true; + } + if (poolsChanged) { + // pools changed + changed.push(acceleration.txid); + } + } } } diff --git a/backend/src/api/services/acceleration.ts b/backend/src/api/services/acceleration.ts index 8c849cdd8..1c2fff7c6 100644 --- a/backend/src/api/services/acceleration.ts +++ b/backend/src/api/services/acceleration.ts @@ -5,6 +5,7 @@ import { BlockExtended, PoolTag } from '../../mempool.interfaces'; export interface Acceleration { txid: string, feeDelta: number, + pools: number[], } class AccelerationApi { @@ -17,21 +18,12 @@ class AccelerationApi { } } - public async $fetchPools(): Promise { - if (config.MEMPOOL_SERVICES.ACCELERATIONS) { - const response = await query(`${config.MEMPOOL_SERVICES.API}/partners`); - return (response as PoolTag[]) || []; - } else { - return []; + public isAcceleratedBlock(block: BlockExtended, accelerations: Acceleration[]): boolean { + let anyAccelerated = false; + for (let i = 0; i < accelerations.length && !anyAccelerated; i++) { + anyAccelerated = anyAccelerated || accelerations[i].pools?.includes(block.extras.pool.id); } - } - - public async $isAcceleratedBlock(block: BlockExtended): Promise { - const pools = await this.$fetchPools(); - if (block?.extras?.pool?.id == null) { - return false; - } - return pools.reduce((match, tag) => match || tag.uniqueId === block.extras.pool.id, false); + return anyAccelerated; } } diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 7716ed7b7..ced6642d8 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -22,6 +22,7 @@ import { deepClone } from '../utils/clone'; import priceUpdater from '../tasks/price-updater'; import { ApiPrice } from '../repositories/PricesRepository'; import accelerationApi from './services/acceleration'; +import mempool from './mempool'; // valid 'want' subscriptions const wantable = [ @@ -657,7 +658,7 @@ class WebsocketHandler { if (config.MEMPOOL.AUDIT && memPool.isInSync()) { let projectedBlocks; let auditMempool = _memPool; - const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && await accelerationApi.$isAcceleratedBlock(block); + const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations())); // template calculation functions have mempool side effects, so calculate audits using // a cloned copy of the mempool if we're running a different algorithm for mempool updates const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL; @@ -667,7 +668,7 @@ class WebsocketHandler { if (config.MEMPOOL.RUST_GBT) { projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool); } else { - projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated); + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id); } } else { projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false); From 017d3f70e4d0f3d0a3f6558aad96e4b09016165a Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 18 Jul 2023 15:05:44 +0900 Subject: [PATCH 10/73] Add acceleration support to rust gbt --- backend/rust-gbt/index.d.ts | 8 ++- backend/rust-gbt/src/audit_transaction.rs | 27 +++++---- backend/rust-gbt/src/gbt.rs | 13 ++++- backend/rust-gbt/src/lib.rs | 10 +++- backend/src/api/mempool-blocks.ts | 56 ++++++++++++++----- backend/src/api/mempool.ts | 2 +- backend/src/api/websocket-handler.ts | 8 +-- backend/src/mempool.interfaces.ts | 4 +- backend/src/replication/AuditReplication.ts | 1 + .../block-overview-graph.component.ts | 2 +- .../block-overview-graph/block-scene.ts | 2 +- .../block-overview-graph/tx-view.ts | 2 +- .../fee-distribution-graph.component.ts | 2 +- .../src/app/interfaces/websocket.interface.ts | 4 +- 14 files changed, 96 insertions(+), 45 deletions(-) diff --git a/backend/rust-gbt/index.d.ts b/backend/rust-gbt/index.d.ts index 33ae32bdf..2bd8a620a 100644 --- a/backend/rust-gbt/index.d.ts +++ b/backend/rust-gbt/index.d.ts @@ -12,6 +12,10 @@ export interface ThreadTransaction { effectiveFeePerVsize: number inputs: Array } +export interface ThreadAcceleration { + uid: number + delta: number +} export class GbtGenerator { constructor() /** @@ -19,13 +23,13 @@ export class GbtGenerator { * * Rejects if the thread panics or if the Mutex is poisoned. */ - make(mempool: Array, maxUid: number): Promise + make(mempool: Array, accelerations: Array, maxUid: number): Promise /** * # Errors * * Rejects if the thread panics or if the Mutex is poisoned. */ - update(newTxs: Array, removeTxs: Array, maxUid: number): Promise + update(newTxs: Array, removeTxs: Array, accelerations: Array, maxUid: number): Promise } /** * The result from calling the gbt function. diff --git a/backend/rust-gbt/src/audit_transaction.rs b/backend/rust-gbt/src/audit_transaction.rs index 3e25a18a0..9b7472c30 100644 --- a/backend/rust-gbt/src/audit_transaction.rs +++ b/backend/rust-gbt/src/audit_transaction.rs @@ -1,6 +1,6 @@ use crate::{ u32_hasher_types::{u32hashset_new, U32HasherState}, - ThreadTransaction, + ThreadTransaction, thread_acceleration::ThreadAcceleration, }; use std::{ cmp::Ordering, @@ -88,37 +88,42 @@ impl Ord for AuditTransaction { } #[inline] -fn calc_fee_rate(fee: f64, vsize: f64) -> f64 { - fee / (if vsize == 0.0 { 1.0 } else { vsize }) +fn calc_fee_rate(fee: u64, vsize: f64) -> f64 { + (fee as f64) / (if vsize == 0.0 { 1.0 } else { vsize }) } impl AuditTransaction { - pub fn from_thread_transaction(tx: &ThreadTransaction) -> Self { + pub fn from_thread_transaction(tx: &ThreadTransaction, maybe_acceleration: Option>) -> Self { + let fee_delta = match maybe_acceleration { + Some(Some(acceleration)) => acceleration.delta, + _ => 0.0 + }; + let fee = (tx.fee as u64) + (fee_delta as u64); // rounded up to the nearest integer let is_adjusted = tx.weight < (tx.sigops * 20); let sigop_adjusted_vsize = ((tx.weight + 3) / 4).max(tx.sigops * 5); let sigop_adjusted_weight = tx.weight.max(tx.sigops * 20); - let effective_fee_per_vsize = if is_adjusted { - calc_fee_rate(tx.fee, f64::from(sigop_adjusted_weight) / 4.0) + let effective_fee_per_vsize = if is_adjusted || fee_delta > 0.0 { + calc_fee_rate(fee, f64::from(sigop_adjusted_weight) / 4.0) } else { tx.effective_fee_per_vsize }; Self { uid: tx.uid, order: tx.order, - fee: tx.fee as u64, + fee, weight: tx.weight, sigop_adjusted_weight, sigop_adjusted_vsize, sigops: tx.sigops, - adjusted_fee_per_vsize: calc_fee_rate(tx.fee, f64::from(sigop_adjusted_vsize)), + adjusted_fee_per_vsize: calc_fee_rate(fee, f64::from(sigop_adjusted_vsize)), effective_fee_per_vsize, dependency_rate: f64::INFINITY, inputs: tx.inputs.clone(), relatives_set_flag: false, ancestors: u32hashset_new(), children: u32hashset_new(), - ancestor_fee: tx.fee as u64, + ancestor_fee: fee, ancestor_sigop_adjusted_weight: sigop_adjusted_weight, ancestor_sigop_adjusted_vsize: sigop_adjusted_vsize, ancestor_sigops: tx.sigops, @@ -156,7 +161,7 @@ impl AuditTransaction { // grows, so if we think of 0 as "grew infinitely" then dependency_rate would be // the smaller of the two. If either side is NaN, the other side is returned. self.dependency_rate.min(calc_fee_rate( - self.ancestor_fee as f64, + self.ancestor_fee, f64::from(self.ancestor_sigop_adjusted_weight) / 4.0, )) } @@ -172,7 +177,7 @@ impl AuditTransaction { #[inline] fn calc_new_score(&mut self) { self.score = self.adjusted_fee_per_vsize.min(calc_fee_rate( - self.ancestor_fee as f64, + self.ancestor_fee, f64::from(self.ancestor_sigop_adjusted_vsize), )); } diff --git a/backend/rust-gbt/src/gbt.rs b/backend/rust-gbt/src/gbt.rs index 09b6377e6..0bf7f9999 100644 --- a/backend/rust-gbt/src/gbt.rs +++ b/backend/rust-gbt/src/gbt.rs @@ -5,7 +5,7 @@ use tracing::{info, trace}; use crate::{ audit_transaction::{partial_cmp_uid_score, AuditTransaction}, u32_hasher_types::{u32hashset_new, u32priority_queue_with_capacity, U32HasherState}, - GbtResult, ThreadTransactionsMap, + GbtResult, ThreadTransactionsMap, thread_acceleration::ThreadAcceleration, }; const MAX_BLOCK_WEIGHT_UNITS: u32 = 4_000_000 - 4_000; @@ -53,7 +53,13 @@ impl Ord for TxPriority { // TODO: Make gbt smaller to fix these lints. #[allow(clippy::too_many_lines)] #[allow(clippy::cognitive_complexity)] -pub fn gbt(mempool: &mut ThreadTransactionsMap, max_uid: usize) -> GbtResult { +pub fn gbt(mempool: &mut ThreadTransactionsMap, accelerations: &[ThreadAcceleration], max_uid: usize) -> GbtResult { + let mut indexed_accelerations = Vec::with_capacity(max_uid + 1); + indexed_accelerations.resize(max_uid + 1, None); + for acceleration in accelerations { + indexed_accelerations[acceleration.uid as usize] = Some(acceleration); + } + let mempool_len = mempool.len(); let mut audit_pool: AuditPool = Vec::with_capacity(max_uid + 1); audit_pool.resize(max_uid + 1, None); @@ -63,7 +69,8 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap, max_uid: usize) -> GbtResult { info!("Initializing working structs"); for (uid, tx) in &mut *mempool { - let audit_tx = AuditTransaction::from_thread_transaction(tx); + let acceleration = indexed_accelerations.get(*uid as usize); + let audit_tx = AuditTransaction::from_thread_transaction(tx, acceleration.copied()); // Safety: audit_pool and mempool_stack must always contain the same transactions audit_pool[*uid as usize] = Some(ManuallyDrop::new(audit_tx)); mempool_stack.push(*uid); diff --git a/backend/rust-gbt/src/lib.rs b/backend/rust-gbt/src/lib.rs index 516a26402..53db0ba21 100644 --- a/backend/rust-gbt/src/lib.rs +++ b/backend/rust-gbt/src/lib.rs @@ -9,6 +9,7 @@ use napi::bindgen_prelude::Result; use napi_derive::napi; use thread_transaction::ThreadTransaction; +use thread_acceleration::ThreadAcceleration; use tracing::{debug, info, trace}; use tracing_log::LogTracer; use tracing_subscriber::{EnvFilter, FmtSubscriber}; @@ -19,6 +20,7 @@ use std::sync::{Arc, Mutex}; mod audit_transaction; mod gbt; mod thread_transaction; +mod thread_acceleration; mod u32_hasher_types; use u32_hasher_types::{u32hashmap_with_capacity, U32HasherState}; @@ -74,10 +76,11 @@ impl GbtGenerator { /// /// Rejects if the thread panics or if the Mutex is poisoned. #[napi] - pub async fn make(&self, mempool: Vec, max_uid: u32) -> Result { + pub async fn make(&self, mempool: Vec, accelerations: Vec, max_uid: u32) -> Result { trace!("make: Current State {:#?}", self.thread_transactions); run_task( Arc::clone(&self.thread_transactions), + accelerations, max_uid as usize, move |map| { for tx in mempool { @@ -96,11 +99,13 @@ impl GbtGenerator { &self, new_txs: Vec, remove_txs: Vec, + accelerations: Vec, max_uid: u32, ) -> Result { trace!("update: Current State {:#?}", self.thread_transactions); run_task( Arc::clone(&self.thread_transactions), + accelerations, max_uid as usize, move |map| { for tx in new_txs { @@ -141,6 +146,7 @@ pub struct GbtResult { /// to the `HashMap` as the only argument. (A move closure is recommended to meet the bounds) async fn run_task( thread_transactions: Arc>, + accelerations: Vec, max_uid: usize, callback: F, ) -> Result @@ -159,7 +165,7 @@ where callback(&mut map); info!("Starting gbt algorithm for {} elements...", map.len()); - let result = gbt::gbt(&mut map, max_uid); + let result = gbt::gbt(&mut map, &accelerations, max_uid); info!("Finished gbt algorithm for {} elements...", map.len()); debug!( diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 606c98c83..16772b7cd 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,4 +1,4 @@ -import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction } from '../../rust-gbt'; +import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from '../../rust-gbt'; import logger from '../logger'; import { MempoolBlock, MempoolTransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, PoolTag } from '../mempool.interfaces'; import { Common, OnlineFeeStatsCalculator } from './common'; @@ -171,7 +171,7 @@ class MempoolBlocks { for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) { let added: TransactionStripped[] = []; let removed: string[] = []; - const changed: { txid: string, rate: number | undefined, acc: number | undefined }[] = []; + const changed: { txid: string, rate: number | undefined, acc: boolean | undefined }[] = []; if (mempoolBlocks[i] && !prevBlocks[i]) { added = mempoolBlocks[i].transactions; } else if (!mempoolBlocks[i] && prevBlocks[i]) { @@ -265,7 +265,7 @@ class MempoolBlocks { // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); - const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, saveResults); + const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, accelerationPool, saveResults); logger.debug(`makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); @@ -325,7 +325,7 @@ class MempoolBlocks { // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); - this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, saveResults); + this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, null, saveResults); logger.debug(`updateBlockTemplates completed in ${(Date.now() - start) / 1000} seconds`); } catch (e) { logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e)); @@ -337,7 +337,7 @@ class MempoolBlocks { this.rustGbtGenerator = new GbtGenerator(); } - private async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): Promise { + private async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { const start = Date.now(); // reset mempool short ids @@ -353,16 +353,25 @@ class MempoolBlocks { tx.inputs = tx.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[]; } + const accelerations = useAccelerations ? mempool.getAccelerations() : {}; + const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]); + const convertedAccelerations = acceleratedList.map(acc => { + return { + uid: this.getUid(newMempool[acc.txid]), + delta: acc.feeDelta, + }; + }); + // run the block construction algorithm in a separate thread, and wait for a result const rustGbt = saveResults ? this.rustGbtGenerator : new GbtGenerator(); try { const { blocks, blockWeights, rates, clusters } = this.convertNapiResultTxids( - await rustGbt.make(Object.values(newMempool) as RustThreadTransaction[], this.nextUid), + await rustGbt.make(Object.values(newMempool) as RustThreadTransaction[], convertedAccelerations as RustThreadAcceleration[], this.nextUid), ); if (saveResults) { this.rustInitialized = true; } - const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, {}, saveResults); + const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, saveResults); logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); return processed; } catch (e) { @@ -374,19 +383,20 @@ class MempoolBlocks { return this.mempoolBlocks; } - public async $oneOffRustBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }): Promise { - return this.$rustMakeBlockTemplates(newMempool, false); + public async $oneOffRustBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, useAccelerations: boolean, accelerationPool?: number): Promise { + return this.$rustMakeBlockTemplates(newMempool, false, useAccelerations, accelerationPool); } - public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[]): Promise { + public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], useAccelerations: boolean, accelerationPool?: number): Promise { // GBT optimization requires that uids never get too sparse // as a sanity check, we should also explicitly prevent uint32 uid overflow if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * mempoolSize), MAX_UINT32)) { this.resetRustGbt(); } + if (!this.rustInitialized) { // need to reset the worker - await this.$rustMakeBlockTemplates(newMempool, true); + await this.$rustMakeBlockTemplates(newMempool, true, useAccelerations, accelerationPool); return; } @@ -401,12 +411,22 @@ class MempoolBlocks { } const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => (uid !== null && uid !== undefined)) as number[]; + const accelerations = useAccelerations ? mempool.getAccelerations() : {}; + const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]); + const convertedAccelerations = acceleratedList.map(acc => { + return { + uid: this.getUid(newMempool[acc.txid]), + delta: acc.feeDelta, + }; + }); + // run the block construction algorithm in a separate thread, and wait for a result try { const { blocks, blockWeights, rates, clusters } = this.convertNapiResultTxids( await this.rustGbtGenerator.update( added as RustThreadTransaction[], removedUids, + convertedAccelerations as RustThreadAcceleration[], this.nextUid, ), ); @@ -414,7 +434,7 @@ class MempoolBlocks { if (mempoolSize !== resultMempoolSize) { throw new Error('GBT returned wrong number of transactions, cache is probably out of sync'); } else { - this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, {}, true); + this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, true); } this.removeUids(removedUids); logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); @@ -424,7 +444,7 @@ class MempoolBlocks { } } - private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], accelerations, saveResults): MempoolBlockWithTransactions[] { + private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], accelerations, accelerationPool, saveResults): MempoolBlockWithTransactions[] { for (const [txid, rate] of rates) { if (txid in mempool) { mempool[txid].effectiveFeePerVsize = rate; @@ -503,7 +523,15 @@ class MempoolBlocks { mempoolTx.cpfpChecked = true; } - mempoolTx.acceleration = accelerations[txid]; + const acceleration = accelerations[txid]; + if (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool))) { + mempoolTx.acceleration = true; + for (const ancestor of mempoolTx.ancestors || []) { + mempool[ancestor.txid].acceleration = true; + } + } else { + delete mempoolTx.acceleration; + } // online calculation of stack-of-blocks fee stats if (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) { diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 107aa41cb..d1a2d70a9 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -9,7 +9,7 @@ import loadingIndicators from './loading-indicators'; import bitcoinClient from './bitcoin/bitcoin-client'; import bitcoinSecondClient from './bitcoin/bitcoin-second-client'; import rbfCache from './rbf-cache'; -import accelerationApi from './services/acceleration'; +import accelerationApi, { Acceleration } from './services/acceleration'; class Mempool { private inSync: boolean = false; diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index ced6642d8..e42e87acd 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -174,7 +174,7 @@ class WebsocketHandler { } const tx = memPool.getMempool()[trackTxid]; if (tx && tx.position) { - const position: { block: number, vsize: number, accelerated?: number } = { + const position: { block: number, vsize: number, accelerated?: boolean } = { ...tx.position }; if (tx.acceleration) { @@ -397,7 +397,7 @@ class WebsocketHandler { if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { if (config.MEMPOOL.RUST_GBT) { - await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions); + await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions, true,); } else { await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS); } @@ -666,7 +666,7 @@ class WebsocketHandler { auditMempool = deepClone(_memPool); if (config.MEMPOOL.ADVANCED_GBT_AUDIT) { if (config.MEMPOOL.RUST_GBT) { - projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool); + projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool, isAccelerated, block.extras.pool.id); } else { projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id); } @@ -739,7 +739,7 @@ class WebsocketHandler { if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { if (config.MEMPOOL.RUST_GBT) { - await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions); + await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions, true); } else { await mempoolBlocks.$makeBlockTemplates(_memPool, true, config.MEMPOOL_SERVICES.ACCELERATIONS); } diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 185256619..4715440e4 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -92,7 +92,7 @@ export interface TransactionExtended extends IEsploraApi.Transaction { block: number, vsize: number, }; - acceleration?: number; + acceleration?: boolean; uid?: number; } @@ -184,7 +184,7 @@ export interface TransactionStripped { fee: number; vsize: number; value: number; - acc?: number; + acc?: boolean; rate?: number; // effective fee rate } diff --git a/backend/src/replication/AuditReplication.ts b/backend/src/replication/AuditReplication.ts index 26bf6dad7..5de9de0da 100644 --- a/backend/src/replication/AuditReplication.ts +++ b/backend/src/replication/AuditReplication.ts @@ -116,6 +116,7 @@ class AuditReplication { freshTxs: auditSummary.freshTxs || [], sigopTxs: auditSummary.sigopTxs || [], fullrbfTxs: auditSummary.fullrbfTxs || [], + acceleratedTxs: auditSummary.acceleratedTxs || [], matchRate: auditSummary.matchRate, expectedFees: auditSummary.expectedFees, expectedWeight: auditSummary.expectedWeight, diff --git a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts index fe847103f..634d0f524 100644 --- a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts +++ b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts @@ -147,7 +147,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On } } - update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { + update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: boolean | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { if (this.scene) { this.scene.update(add, remove, change, direction, resetLayout); this.start(); diff --git a/frontend/src/app/components/block-overview-graph/block-scene.ts b/frontend/src/app/components/block-overview-graph/block-scene.ts index 94984bae2..cb0537e2a 100644 --- a/frontend/src/app/components/block-overview-graph/block-scene.ts +++ b/frontend/src/app/components/block-overview-graph/block-scene.ts @@ -150,7 +150,7 @@ export default class BlockScene { this.updateAll(startTime, 200, direction); } - update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { + update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: boolean | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { const startTime = performance.now(); const removed = this.removeBatch(remove, startTime, direction); diff --git a/frontend/src/app/components/block-overview-graph/tx-view.ts b/frontend/src/app/components/block-overview-graph/tx-view.ts index 690b974e3..db2c4f6ae 100644 --- a/frontend/src/app/components/block-overview-graph/tx-view.ts +++ b/frontend/src/app/components/block-overview-graph/tx-view.ts @@ -38,7 +38,7 @@ export default class TxView implements TransactionStripped { vsize: number; value: number; feerate: number; - acc?: number; + acc?: boolean; rate?: number; status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; diff --git a/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts b/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts index f275588a1..1d9c289d8 100644 --- a/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts +++ b/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts @@ -64,7 +64,7 @@ export class FeeDistributionGraphComponent implements OnInit, OnChanges, OnDestr return; } const samples = []; - const txs = this.transactions.map(tx => { return { vsize: tx.vsize, rate: tx.rate || (tx.fee / tx.vsize) }; }).sort((a, b) => { return b.rate - a.rate; }); + const txs = this.transactions.filter(tx => !tx.acc).map(tx => { return { vsize: tx.vsize, rate: tx.rate || (tx.fee / tx.vsize) }; }).sort((a, b) => { return b.rate - a.rate; }); const maxBlockVSize = this.stateService.env.BLOCK_WEIGHT_UNITS / 4; const sampleInterval = maxBlockVSize / this.numSamples; let cumVSize = 0; diff --git a/frontend/src/app/interfaces/websocket.interface.ts b/frontend/src/app/interfaces/websocket.interface.ts index f8686042b..43ab1e5f4 100644 --- a/frontend/src/app/interfaces/websocket.interface.ts +++ b/frontend/src/app/interfaces/websocket.interface.ts @@ -70,7 +70,7 @@ export interface MempoolBlockWithTransactions extends MempoolBlock { export interface MempoolBlockDelta { added: TransactionStripped[], removed: string[], - changed?: { txid: string, rate: number | undefined, acc: number | undefined }[]; + changed?: { txid: string, rate: number | undefined, acc: boolean | undefined }[]; } export interface MempoolInfo { @@ -88,7 +88,7 @@ export interface TransactionStripped { fee: number; vsize: number; value: number; - acc?: number; // acceleration delta + acc?: boolean; // is accelerated? rate?: number; // effective fee rate status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; From 765a531b84a64a5d4d66108f01ad9e9453196bfa Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 18 Jul 2023 15:14:33 +0900 Subject: [PATCH 11/73] fix tests --- backend/src/__fixtures__/mempool-config.template.json | 6 +++--- backend/src/__tests__/gbt/gbt-tests.ts | 2 +- docker/backend/start.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index d6754f966..21c8e51c0 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -128,8 +128,8 @@ "AUDIT_START_HEIGHT": 774000, "SERVERS": [] }, - "MEMPOOl_SERVICES": { - "API": "__MEMPOOL_SERVICES_API__", - "ACCELERATIONS": "__MEMPOOL_SERVICES_ACCELERATIONS__" + "MEMPOOL_SERVICES": { + "API": "", + "ACCELERATIONS": false } } diff --git a/backend/src/__tests__/gbt/gbt-tests.ts b/backend/src/__tests__/gbt/gbt-tests.ts index 0651faac4..0c2eb0176 100644 --- a/backend/src/__tests__/gbt/gbt-tests.ts +++ b/backend/src/__tests__/gbt/gbt-tests.ts @@ -15,7 +15,7 @@ describe('Rust GBT', () => { test('should produce the same template as getBlockTemplate from Bitcoin Core', async () => { const rustGbt = new GbtGenerator(); const { mempool, maxUid } = mempoolFromArrayBuffer(vectorBuffer.buffer); - const result = await rustGbt.make(mempool, maxUid); + const result = await rustGbt.make(mempool, [], maxUid); const blocks: [string, number][][] = result.blocks.map(block => { return block.map(uid => [vectorUidMap.get(uid) || 'missing', uid]); diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 66f873605..c9088c0a6 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -137,8 +137,8 @@ __REPLICATION_AUDIT_START_HEIGHT__=${REPLICATION_AUDIT_START_HEIGHT:=774000} __REPLICATION_SERVERS__=${REPLICATION_SERVERS:=[]} # MEMPOOL_SERVICES -__MEMPOOL_SERVICES_API__==${MEMPOOL_SERVICES_API:=""} -__MEMPOOL_SERVICES_ACCELERATIONS__==${MEMPOOL_SERVICES_ACCELERATIONS:=false} +__MEMPOOL_SERVICES_API__=${MEMPOOL_SERVICES_API:=""} +__MEMPOOL_SERVICES_ACCELERATIONS__=${MEMPOOL_SERVICES_ACCELERATIONS:=false} mkdir -p "${__MEMPOOL_CACHE_DIR__}" From 67bd3856ecf7259c36cc9f817b9903e24de7f546 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 18 Jul 2023 16:08:25 +0900 Subject: [PATCH 12/73] fix audit highlightning and fee ranges --- backend/src/api/common.ts | 2 +- backend/src/api/mempool-blocks.ts | 5 +++-- backend/src/api/websocket-handler.ts | 6 +++--- backend/src/repositories/BlocksAuditsRepository.ts | 1 - frontend/src/app/components/block/block.component.ts | 3 +++ frontend/src/app/interfaces/node-api.interface.ts | 1 + 6 files changed, 11 insertions(+), 7 deletions(-) diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 775da2643..b2f476da3 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -461,7 +461,7 @@ export class Common { }; } - static calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string }[]): EffectiveFeeStats { + static calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string, acceleration?: boolean }[]): EffectiveFeeStats { const sortedTxs = transactions.map(tx => { return { txid: tx.txid, weight: tx.weight, rate: tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4)) }; }).sort((a, b) => a.rate - b.rate); let weightCount = 0; diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 16772b7cd..599a53c23 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -337,7 +337,8 @@ class MempoolBlocks { this.rustGbtGenerator = new GbtGenerator(); } - private async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { + public async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { + console.log('$rustMakeBlockTemplates'); const start = Date.now(); // reset mempool short ids @@ -569,7 +570,7 @@ class MempoolBlocks { private dataToMempoolBlocks(transactionIds: string[], transactions: MempoolTransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, feeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions { if (!feeStats) { - feeStats = Common.calcEffectiveFeeStatistics(transactions); + feeStats = Common.calcEffectiveFeeStatistics(transactions.filter(tx => !tx.acceleration)); } return { blockSize: totalSize, diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index e42e87acd..9eb624b2e 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -397,7 +397,7 @@ class WebsocketHandler { if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { if (config.MEMPOOL.RUST_GBT) { - await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions, true,); + await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions, config.MEMPOOL_SERVICES.ACCELERATIONS); } else { await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS); } @@ -661,7 +661,7 @@ class WebsocketHandler { const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations())); // template calculation functions have mempool side effects, so calculate audits using // a cloned copy of the mempool if we're running a different algorithm for mempool updates - const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL; + const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL || isAccelerated; if (separateAudit) { auditMempool = deepClone(_memPool); if (config.MEMPOOL.ADVANCED_GBT_AUDIT) { @@ -675,7 +675,7 @@ class WebsocketHandler { } } else { if ((config.MEMPOOL_SERVICES.ACCELERATIONS && !isAccelerated)) { - projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated); + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, false); } else { projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); } diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index 9c7568567..c17958d2b 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -85,7 +85,6 @@ class BlocksAuditRepositories { rows[0].sigopTxs = JSON.parse(rows[0].sigopTxs); rows[0].fullrbfTxs = JSON.parse(rows[0].fullrbfTxs); rows[0].acceleratedTxs = JSON.parse(rows[0].acceleratedTxs); - rows[0].transactions = JSON.parse(rows[0].transactions); rows[0].template = JSON.parse(rows[0].template); return rows[0]; diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 1e94b1a9c..1345717bd 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -347,6 +347,9 @@ export class BlockComponent implements OnInit, OnDestroy { if (blockAudit?.template) { for (const tx of blockAudit.template) { inTemplate[tx.txid] = true; + if (tx.acc) { + isAccelerated[tx.txid] = true; + } } for (const tx of transactions) { inBlock[tx.txid] = true; diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index a0a3da5e6..cb42eaed3 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -176,6 +176,7 @@ export interface TransactionStripped { vsize: number; value: number; rate?: number; // effective fee rate + acc?: boolean; status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; } From 7247a616e51d3115f0b70e9a2cb67a36308ef808 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 18 Jul 2023 16:32:52 +0900 Subject: [PATCH 13/73] check in missing rust-gbt file --- backend/rust-gbt/src/thread_acceleration.rs | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 backend/rust-gbt/src/thread_acceleration.rs diff --git a/backend/rust-gbt/src/thread_acceleration.rs b/backend/rust-gbt/src/thread_acceleration.rs new file mode 100644 index 000000000..618cac3db --- /dev/null +++ b/backend/rust-gbt/src/thread_acceleration.rs @@ -0,0 +1,8 @@ +use napi_derive::napi; + +#[derive(Debug)] +#[napi(object)] +pub struct ThreadAcceleration { + pub uid: u32, + pub delta: f64, // fee delta +} From a4bdf4abe44cc8ac9e80782fcff0efdf6796bb11 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 18 Jul 2023 17:30:51 +0900 Subject: [PATCH 14/73] fix pool-dependent accelerated audit handling --- backend/rust-gbt/src/audit_transaction.rs | 2 +- backend/src/api/mempool-blocks.ts | 13 +++++++------ backend/src/api/websocket-handler.ts | 6 +++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/backend/rust-gbt/src/audit_transaction.rs b/backend/rust-gbt/src/audit_transaction.rs index 9b7472c30..fe20e5a14 100644 --- a/backend/rust-gbt/src/audit_transaction.rs +++ b/backend/rust-gbt/src/audit_transaction.rs @@ -130,7 +130,7 @@ impl AuditTransaction { score: 0.0, used: false, modified: false, - dirty: effective_fee_per_vsize != tx.effective_fee_per_vsize, + dirty: effective_fee_per_vsize != tx.effective_fee_per_vsize || fee_delta > 0.0, } } diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 599a53c23..6d3fa0036 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -388,7 +388,7 @@ class MempoolBlocks { return this.$rustMakeBlockTemplates(newMempool, false, useAccelerations, accelerationPool); } - public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], useAccelerations: boolean, accelerationPool?: number): Promise { + public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], useAccelerations: boolean, accelerationPool?: number): Promise { // GBT optimization requires that uids never get too sparse // as a sanity check, we should also explicitly prevent uint32 uid overflow if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * mempoolSize), MAX_UINT32)) { @@ -397,8 +397,7 @@ class MempoolBlocks { if (!this.rustInitialized) { // need to reset the worker - await this.$rustMakeBlockTemplates(newMempool, true, useAccelerations, accelerationPool); - return; + return this.$rustMakeBlockTemplates(newMempool, true, useAccelerations, accelerationPool); } const start = Date.now(); @@ -435,13 +434,15 @@ class MempoolBlocks { if (mempoolSize !== resultMempoolSize) { throw new Error('GBT returned wrong number of transactions, cache is probably out of sync'); } else { - this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, true); + const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, true); + this.removeUids(removedUids); + logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); + return processed; } - this.removeUids(removedUids); - logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); } catch (e) { logger.err('RUST updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e)); this.resetRustGbt(); + return this.mempoolBlocks; } } diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 9eb624b2e..bed059ad2 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -661,7 +661,7 @@ class WebsocketHandler { const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations())); // template calculation functions have mempool side effects, so calculate audits using // a cloned copy of the mempool if we're running a different algorithm for mempool updates - const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL || isAccelerated; + const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL; if (separateAudit) { auditMempool = deepClone(_memPool); if (config.MEMPOOL.ADVANCED_GBT_AUDIT) { @@ -674,8 +674,8 @@ class WebsocketHandler { projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false); } } else { - if ((config.MEMPOOL_SERVICES.ACCELERATIONS && !isAccelerated)) { - projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, false); + if ((config.MEMPOOL_SERVICES.ACCELERATIONS)) { + projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(auditMempool, Object.keys(auditMempool).length, [], [], isAccelerated, block.extras.pool.id); } else { projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); } From 12fc82ab29b1f9f3771328151842749b972134d4 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 19 Jul 2023 11:18:04 +0900 Subject: [PATCH 15/73] fix mismatched use of gbt implementations --- backend/src/api/mempool-blocks.ts | 12 +++++++++--- backend/src/api/websocket-handler.ts | 6 +++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 6d3fa0036..1ea4d2af8 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -211,9 +211,12 @@ class MempoolBlocks { const start = Date.now(); // reset mempool short ids - this.resetUids(); + if (saveResults) { + this.resetUids(); + } + // set missing short ids for (const tx of Object.values(newMempool)) { - this.setUid(tx, true); + this.setUid(tx, !saveResults); } const accelerations = useAccelerations ? mempool.getAccelerations() : {}; @@ -497,6 +500,8 @@ class MempoolBlocks { } } + const isAccelerated : { [txid: string]: boolean } = {}; + const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2; // update this thread's mempool with the results let mempoolTx: MempoolTransactionExtended; @@ -526,10 +531,11 @@ class MempoolBlocks { } const acceleration = accelerations[txid]; - if (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool))) { + if (isAccelerated[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) { mempoolTx.acceleration = true; for (const ancestor of mempoolTx.ancestors || []) { mempool[ancestor.txid].acceleration = true; + isAccelerated[ancestor.txid] = true; } } else { delete mempoolTx.acceleration; diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index bed059ad2..f5c940218 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -675,7 +675,11 @@ class WebsocketHandler { } } else { if ((config.MEMPOOL_SERVICES.ACCELERATIONS)) { - projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(auditMempool, Object.keys(auditMempool).length, [], [], isAccelerated, block.extras.pool.id); + if (config.MEMPOOL.RUST_GBT) { + projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(auditMempool, Object.keys(auditMempool).length, [], [], isAccelerated, block.extras.pool.id); + } else { + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id); + } } else { projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); } From 6d9ba1ebcdd97b0fe5e2dc00c9482545257c8b47 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 22 Jul 2023 09:46:35 +0900 Subject: [PATCH 16/73] /accelerations -> /accelerator/accelerations --- backend/src/api/services/acceleration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/services/acceleration.ts b/backend/src/api/services/acceleration.ts index 1c2fff7c6..635dc8300 100644 --- a/backend/src/api/services/acceleration.ts +++ b/backend/src/api/services/acceleration.ts @@ -11,7 +11,7 @@ export interface Acceleration { class AccelerationApi { public async $fetchAccelerations(): Promise { if (config.MEMPOOL_SERVICES.ACCELERATIONS) { - const response = await query(`${config.MEMPOOL_SERVICES.API}/accelerations`); + const response = await query(`${config.MEMPOOL_SERVICES.API}/accelerator/accelerations`); return (response as Acceleration[]) || []; } else { return []; From c9630a4a8c96a875196cdfa0726e3059b3589e3a Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 30 Jul 2023 18:56:57 +0900 Subject: [PATCH 17/73] Use log10 scale for projected block fee graph --- .../fee-distribution-graph.component.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts b/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts index f275588a1..212510e71 100644 --- a/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts +++ b/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts @@ -74,14 +74,14 @@ export class FeeDistributionGraphComponent implements OnInit, OnChanges, OnDestr this.labelInterval = this.numSamples / this.numLabels; while (nextSample <= maxBlockVSize) { if (txIndex >= txs.length) { - samples.push([(1 - (sampleIndex / this.numSamples)) * 100, 0]); + samples.push([(1 - (sampleIndex / this.numSamples)) * 100, 0.000001]); nextSample += sampleInterval; sampleIndex++; continue; } while (txs[txIndex] && nextSample < cumVSize + txs[txIndex].vsize) { - samples.push([(1 - (sampleIndex / this.numSamples)) * 100, txs[txIndex].rate]); + samples.push([(1 - (sampleIndex / this.numSamples)) * 100, txs[txIndex].rate || 0.000001]); nextSample += sampleInterval; sampleIndex++; } @@ -118,7 +118,9 @@ export class FeeDistributionGraphComponent implements OnInit, OnChanges, OnDestr }, }, yAxis: { - type: 'value', + type: 'log', + min: 1, + max: this.data.reduce((min, val) => Math.max(min, val[1]), 1), // name: 'Effective Fee Rate s/vb', // nameLocation: 'middle', splitLine: { @@ -129,12 +131,16 @@ export class FeeDistributionGraphComponent implements OnInit, OnChanges, OnDestr } }, axisLabel: { + show: true, formatter: (value: number): string => { const unitValue = this.weightMode ? value / 4 : value; const selectedPowerOfTen = selectPowerOfTen(unitValue); const newVal = Math.round(unitValue / selectedPowerOfTen.divider); return `${newVal}${selectedPowerOfTen.unit}`; }, + }, + axisTick: { + show: true, } }, series: [{ From 3035619b61e466583e5b2be9b8f6fbe23db922e2 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 31 Jul 2023 15:03:54 +0900 Subject: [PATCH 18/73] Get blocks from electrs again --- backend/src/api/bitcoin/bitcoin.routes.ts | 4 ++-- backend/src/api/blocks.ts | 12 ++++++------ backend/src/api/mempool.ts | 2 +- backend/src/api/mining/mining.ts | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index ffdb2e629..b24c83932 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -6,7 +6,7 @@ import websocketHandler from '../websocket-handler'; import mempool from '../mempool'; import feeApi from '../fee-api'; import mempoolBlocks from '../mempool-blocks'; -import bitcoinApi, { bitcoinCoreApi } from './bitcoin-api-factory'; +import bitcoinApi from './bitcoin-api-factory'; import { Common } from '../common'; import backendInfo from '../backend-info'; import transactionUtils from '../transaction-utils'; @@ -483,7 +483,7 @@ class BitcoinRoutes { returnBlocks.push(localBlock); nextHash = localBlock.previousblockhash; } else { - const block = await bitcoinCoreApi.$getBlock(nextHash); + const block = await bitcoinApi.$getBlock(nextHash); returnBlocks.push(block); nextHash = block.previousblockhash; } diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index f86bc53e9..88f85ba6e 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -599,7 +599,7 @@ class Blocks { loadingIndicators.setProgress('block-indexing', progress, false); } const blockHash = await bitcoinApi.$getBlockHash(blockHeight); - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, null, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -656,7 +656,7 @@ class Blocks { const heightDiff = blockHeightTip % 2016; const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff); this.updateTimerProgress(timer, 'got block hash for initial difficulty adjustment'); - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); this.updateTimerProgress(timer, 'got block for initial difficulty adjustment'); this.lastDifficultyAdjustmentTime = block.timestamp; this.currentDifficulty = block.difficulty; @@ -664,7 +664,7 @@ class Blocks { if (blockHeightTip >= 2016) { const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016); this.updateTimerProgress(timer, 'got previous block hash for initial difficulty adjustment'); - const previousPeriodBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(previousPeriodBlockHash); + const previousPeriodBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(previousPeriodBlockHash); this.updateTimerProgress(timer, 'got previous block for initial difficulty adjustment'); this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100; logger.debug(`Initial difficulty adjustment data set.`); @@ -862,7 +862,7 @@ class Blocks { } const blockHash = await bitcoinApi.$getBlockHash(height); - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -874,7 +874,7 @@ class Blocks { } public async $indexStaleBlock(hash: string): Promise { - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash); const transactions = await this.$getTransactionsExtended(hash, block.height, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -899,7 +899,7 @@ class Blocks { } // Bitcoin network, add our custom data on top - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash); if (block.stale) { return await this.$indexStaleBlock(hash); } else { diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index d5214de5d..b9b648012 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -1,5 +1,5 @@ import config from '../config'; -import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; +import bitcoinApi from './bitcoin/bitcoin-api-factory'; import { MempoolTransactionExtended, TransactionExtended, VbytesPerSecond } from '../mempool.interfaces'; import logger from '../logger'; import { Common } from './common'; diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index e190492b8..beda0bc48 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -11,7 +11,7 @@ import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjust import config from '../../config'; import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository'; import PricesRepository from '../../repositories/PricesRepository'; -import { bitcoinCoreApi } from '../bitcoin/bitcoin-api-factory'; +import bitcoinApi from '../bitcoin/bitcoin-api-factory'; import { IEsploraApi } from '../bitcoin/esplora-api.interface'; import database from '../../database'; @@ -201,7 +201,7 @@ class Mining { try { const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp; - const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); const genesisTimestamp = genesisBlock.timestamp * 1000; const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps(); @@ -312,7 +312,7 @@ class Mining { const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp; try { - const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); const genesisTimestamp = genesisBlock.timestamp * 1000; const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp); const lastMidnight = this.getDateMidnight(new Date()); @@ -421,7 +421,7 @@ class Mining { } const blocks: any = await BlocksRepository.$getBlocksDifficulty(); - const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); let currentDifficulty = genesisBlock.difficulty; let totalIndexed = 0; From 3dbafc271bd3d34db1e2fbce09d4b1a62046c791 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 31 Jul 2023 16:03:18 +0900 Subject: [PATCH 19/73] fix indexing log prints --- backend/src/api/blocks.ts | 42 +++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 88f85ba6e..2e792bdba 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -419,8 +419,8 @@ class Blocks { let newlyIndexed = 0; let totalIndexed = indexedBlockSummariesHashesArray.length; let indexedThisRun = 0; - let timer = new Date().getTime() / 1000; - const startedAt = new Date().getTime() / 1000; + let timer = Date.now() / 1000; + const startedAt = Date.now() / 1000; for (const block of indexedBlocks) { if (indexedBlockSummariesHashes[block.hash] === true) { @@ -428,13 +428,13 @@ class Blocks { } // Logging - const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer)); + const elapsedSeconds = (Date.now() / 1000) - timer; if (elapsedSeconds > 5) { - const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds); + const runningFor = (Date.now() / 1000) - startedAt; + const blockPerSeconds = indexedThisRun / elapsedSeconds; const progress = Math.round(totalIndexed / indexedBlocks.length * 10000) / 100; - logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining); - timer = new Date().getTime() / 1000; + logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor.toFixed(2)} seconds`, logger.tags.mining); + timer = Date.now() / 1000; indexedThisRun = 0; } @@ -477,18 +477,18 @@ class Blocks { // Logging let count = 0; let countThisRun = 0; - let timer = new Date().getTime() / 1000; - const startedAt = new Date().getTime() / 1000; + let timer = Date.now() / 1000; + const startedAt = Date.now() / 1000; for (const height of unindexedBlockHeights) { // Logging const hash = await bitcoinApi.$getBlockHash(height); - const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer); + const elapsedSeconds = (Date.now() / 1000) - timer; if (elapsedSeconds > 5) { - const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - const blockPerSeconds = (countThisRun / elapsedSeconds); + const runningFor = (Date.now() / 1000) - startedAt; + const blockPerSeconds = countThisRun / elapsedSeconds; const progress = Math.round(count / unindexedBlockHeights.length * 10000) / 100; - logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor} seconds`); - timer = new Date().getTime() / 1000; + logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor.toFixed(2)} seconds`); + timer = Date.now() / 1000; countThisRun = 0; } @@ -567,8 +567,8 @@ class Blocks { let totalIndexed = await blocksRepository.$blockCountBetweenHeight(currentBlockHeight, lastBlockToIndex); let indexedThisRun = 0; let newlyIndexed = 0; - const startedAt = new Date().getTime() / 1000; - let timer = new Date().getTime() / 1000; + const startedAt = Date.now() / 1000; + let timer = Date.now() / 1000; while (currentBlockHeight >= lastBlockToIndex) { const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1); @@ -588,13 +588,13 @@ class Blocks { } ++indexedThisRun; ++totalIndexed; - const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer); + const elapsedSeconds = (Date.now() / 1000) - timer; if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) { - const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds); + const runningFor = (Date.now() / 1000) - startedAt; + const blockPerSeconds = indexedThisRun / elapsedSeconds; const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100; - logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining); - timer = new Date().getTime() / 1000; + logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress.toFixed(2)}%) | elapsed: ${runningFor.toFixed(2)} seconds`, logger.tags.mining); + timer = Date.now() / 1000; indexedThisRun = 0; loadingIndicators.setProgress('block-indexing', progress, false); } From 34e5da56e3b919b5124d4d6aff44defbdb09ab32 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 31 Jul 2023 16:41:36 +0900 Subject: [PATCH 20/73] reduce mempool poll rate while indexing --- backend/src/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index 51d407f6f..222537d0b 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -195,7 +195,8 @@ class Server { // rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS const elapsed = Date.now() - start; - const remainingTime = Math.max(0, config.MEMPOOL.POLL_RATE_MS - elapsed) + const pollRate = config.MEMPOOL.POLL_RATE_MS * (indexer.indexerRunning ? 10 : 1); + const remainingTime = Math.max(0, pollRate - elapsed); setTimeout(this.runMainUpdateLoop.bind(this), numHandledBlocks > 0 ? 0 : remainingTime); this.backendRetryCount = 0; } catch (e: any) { From ca1b079209379a450a90ea63b970e6933d0122d4 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 31 Jul 2023 18:02:18 +0900 Subject: [PATCH 21/73] Get block txs from esplora, index CPFP together with summaries --- backend/src/api/blocks.ts | 59 +++++++++++++++----- backend/src/repositories/BlocksRepository.ts | 15 ++++- 2 files changed, 58 insertions(+), 16 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 2e792bdba..20c015f44 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -438,7 +438,14 @@ class Blocks { indexedThisRun = 0; } - await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary + + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(block.hash)).map(tx => transactionUtils.extendTransaction(tx)); + const cpfpSummary = await this.$indexCPFP(block.hash, block.height, txs); + await this.$getStrippedBlockTransactions(block.hash, true, true, cpfpSummary, block.height); // This will index the block summary + } else { + await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary + } // Logging indexedThisRun++; @@ -942,10 +949,15 @@ class Blocks { }), }; } else { - // Call Core RPC - const block = await bitcoinClient.getBlock(hash, 2); - summary = this.summarizeBlock(block); - height = block.height; + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx)); + summary = this.summarizeBlockTransactions(hash, txs); + } else { + // Call Core RPC + const block = await bitcoinClient.getBlock(hash, 2); + summary = this.summarizeBlock(block); + height = block.height; + } } if (height == null) { const block = await bitcoinApi.$getBlock(hash); @@ -1068,8 +1080,17 @@ class Blocks { if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) { cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash); if (cleanBlock.fee_amt_percentiles === null) { - const block = await bitcoinClient.getBlock(cleanBlock.hash, 2); - const summary = this.summarizeBlock(block); + + let summary; + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(cleanBlock.hash)).map(tx => transactionUtils.extendTransaction(tx)); + summary = this.summarizeBlockTransactions(cleanBlock.hash, txs); + } else { + // Call Core RPC + const block = await bitcoinClient.getBlock(cleanBlock.hash, 2); + summary = this.summarizeBlock(block); + } + await BlocksSummariesRepository.$saveTransactions(cleanBlock.height, cleanBlock.hash, summary.transactions); cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash); } @@ -1129,19 +1150,29 @@ class Blocks { return this.currentBlockHeight; } - public async $indexCPFP(hash: string, height: number): Promise { - const block = await bitcoinClient.getBlock(hash, 2); - const transactions = block.tx.map(tx => { - tx.fee *= 100_000_000; - return tx; - }); + public async $indexCPFP(hash: string, height: number, txs?: TransactionExtended[]): Promise { + let transactions = txs; + if (!transactions) { + if (config.MEMPOOL.BACKEND === 'esplora') { + transactions = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx)); + } + if (!transactions) { + const block = await bitcoinClient.getBlock(hash, 2); + transactions = block.tx.map(tx => { + tx.fee *= 100_000_000; + return tx; + }); + } + } - const summary = Common.calculateCpfp(height, transactions); + const summary = Common.calculateCpfp(height, transactions as TransactionExtended[]); await this.$saveCpfp(hash, height, summary); const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions); await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats); + + return summary; } public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise { diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 078b85a03..5f20f5abd 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -1,3 +1,4 @@ +import bitcoinApi from '../api/bitcoin/bitcoin-api-factory'; import { BlockExtended, BlockExtension, BlockPrice, EffectiveFeeStats } from '../mempool.interfaces'; import DB from '../database'; import logger from '../logger'; @@ -12,6 +13,7 @@ import config from '../config'; import chainTips from '../api/chain-tips'; import blocks from '../api/blocks'; import BlocksAuditsRepository from './BlocksAuditsRepository'; +import transactionUtils from '../api/transaction-utils'; interface DatabaseBlock { id: string; @@ -1036,8 +1038,17 @@ class BlocksRepository { { extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id); if (extras.feePercentiles === null) { - const block = await bitcoinClient.getBlock(dbBlk.id, 2); - const summary = blocks.summarizeBlock(block); + + let summary; + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(dbBlk.id)).map(tx => transactionUtils.extendTransaction(tx)); + summary = blocks.summarizeBlockTransactions(dbBlk.id, txs); + } else { + // Call Core RPC + const block = await bitcoinClient.getBlock(dbBlk.id, 2); + summary = blocks.summarizeBlock(block); + } + await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.id, summary.transactions); extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id); } From 4c4737a118a9db2820149b81980b1a10ef03ac19 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Mon, 31 Jul 2023 14:10:49 -0700 Subject: [PATCH 22/73] Fix config unit test that was returning early --- backend/src/__tests__/config.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 0c06b03e1..edfcc7f47 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -181,12 +181,12 @@ describe('Mempool Backend Config', () => { // We have a few cases where we can't follow the pattern if (root === 'MEMPOOL' && key === 'HTTP_PORT') { console.log('skipping check for MEMPOOL_HTTP_PORT'); - return; + continue; } switch (typeof value) { case 'object': { if (Array.isArray(value)) { - return; + continue; } else { parseJson(value, key); } From 301b4214a1c180d32616ee6450000a739259afd0 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Mon, 31 Jul 2023 14:11:31 -0700 Subject: [PATCH 23/73] Fix RUST GBT Docker override --- docker/backend/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 7071493fa..e05c73710 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -170,7 +170,7 @@ sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-co sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json -sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_GBT__}!g" mempool-config.json +sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_RUST_GBT__}!g" mempool-config.json sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json From fb78bcd2a01de55e59705656371927991191cc67 Mon Sep 17 00:00:00 2001 From: Bastien Guillaumat Date: Tue, 1 Aug 2023 03:13:58 +0200 Subject: [PATCH 24/73] Add bguillaumat.txt to contributors --- contributors/bguillaumat.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 contributors/bguillaumat.txt diff --git a/contributors/bguillaumat.txt b/contributors/bguillaumat.txt new file mode 100644 index 000000000..ac14a07c7 --- /dev/null +++ b/contributors/bguillaumat.txt @@ -0,0 +1,3 @@ +I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022. + +Signed: bguillaumat From 0705c352bc31070d48c97f89f3a5350b57d589cc Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 31 Jul 2023 15:03:54 +0900 Subject: [PATCH 25/73] Get blocks from electrs again --- backend/src/api/bitcoin/bitcoin.routes.ts | 4 ++-- backend/src/api/blocks.ts | 12 ++++++------ backend/src/api/mempool.ts | 2 +- backend/src/api/mining/mining.ts | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index d2a77c995..f27bb7797 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -6,7 +6,7 @@ import websocketHandler from '../websocket-handler'; import mempool from '../mempool'; import feeApi from '../fee-api'; import mempoolBlocks from '../mempool-blocks'; -import bitcoinApi, { bitcoinCoreApi } from './bitcoin-api-factory'; +import bitcoinApi from './bitcoin-api-factory'; import { Common } from '../common'; import backendInfo from '../backend-info'; import transactionUtils from '../transaction-utils'; @@ -483,7 +483,7 @@ class BitcoinRoutes { returnBlocks.push(localBlock); nextHash = localBlock.previousblockhash; } else { - const block = await bitcoinCoreApi.$getBlock(nextHash); + const block = await bitcoinApi.$getBlock(nextHash); returnBlocks.push(block); nextHash = block.previousblockhash; } diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index f86bc53e9..88f85ba6e 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -599,7 +599,7 @@ class Blocks { loadingIndicators.setProgress('block-indexing', progress, false); } const blockHash = await bitcoinApi.$getBlockHash(blockHeight); - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, null, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -656,7 +656,7 @@ class Blocks { const heightDiff = blockHeightTip % 2016; const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff); this.updateTimerProgress(timer, 'got block hash for initial difficulty adjustment'); - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); this.updateTimerProgress(timer, 'got block for initial difficulty adjustment'); this.lastDifficultyAdjustmentTime = block.timestamp; this.currentDifficulty = block.difficulty; @@ -664,7 +664,7 @@ class Blocks { if (blockHeightTip >= 2016) { const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016); this.updateTimerProgress(timer, 'got previous block hash for initial difficulty adjustment'); - const previousPeriodBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(previousPeriodBlockHash); + const previousPeriodBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(previousPeriodBlockHash); this.updateTimerProgress(timer, 'got previous block for initial difficulty adjustment'); this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100; logger.debug(`Initial difficulty adjustment data set.`); @@ -862,7 +862,7 @@ class Blocks { } const blockHash = await bitcoinApi.$getBlockHash(height); - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -874,7 +874,7 @@ class Blocks { } public async $indexStaleBlock(hash: string): Promise { - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash); const transactions = await this.$getTransactionsExtended(hash, block.height, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -899,7 +899,7 @@ class Blocks { } // Bitcoin network, add our custom data on top - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash); if (block.stale) { return await this.$indexStaleBlock(hash); } else { diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index d5214de5d..b9b648012 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -1,5 +1,5 @@ import config from '../config'; -import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; +import bitcoinApi from './bitcoin/bitcoin-api-factory'; import { MempoolTransactionExtended, TransactionExtended, VbytesPerSecond } from '../mempool.interfaces'; import logger from '../logger'; import { Common } from './common'; diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index e190492b8..beda0bc48 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -11,7 +11,7 @@ import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjust import config from '../../config'; import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository'; import PricesRepository from '../../repositories/PricesRepository'; -import { bitcoinCoreApi } from '../bitcoin/bitcoin-api-factory'; +import bitcoinApi from '../bitcoin/bitcoin-api-factory'; import { IEsploraApi } from '../bitcoin/esplora-api.interface'; import database from '../../database'; @@ -201,7 +201,7 @@ class Mining { try { const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp; - const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); const genesisTimestamp = genesisBlock.timestamp * 1000; const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps(); @@ -312,7 +312,7 @@ class Mining { const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp; try { - const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); const genesisTimestamp = genesisBlock.timestamp * 1000; const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp); const lastMidnight = this.getDateMidnight(new Date()); @@ -421,7 +421,7 @@ class Mining { } const blocks: any = await BlocksRepository.$getBlocksDifficulty(); - const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); let currentDifficulty = genesisBlock.difficulty; let totalIndexed = 0; From e9ffd1192c49f8de3fde84fa8f598b29b89adbca Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 31 Jul 2023 18:28:08 +0900 Subject: [PATCH 26/73] Use bits to detect difficulty adjustments, not difficulty --- backend/src/api/mining/mining.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index beda0bc48..3eeb3e9e5 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -423,6 +423,7 @@ class Mining { const blocks: any = await BlocksRepository.$getBlocksDifficulty(); const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); let currentDifficulty = genesisBlock.difficulty; + let currentBits = genesisBlock.bits; let totalIndexed = 0; if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && indexedHeights[0] !== true) { @@ -436,17 +437,18 @@ class Mining { const oldestConsecutiveBlock = await BlocksRepository.$getOldestConsecutiveBlock(); if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== -1) { - currentDifficulty = oldestConsecutiveBlock.difficulty; + currentBits = oldestConsecutiveBlock.bits; } let totalBlockChecked = 0; let timer = new Date().getTime() / 1000; for (const block of blocks) { - if (block.difficulty !== currentDifficulty) { + if (block.bits !== currentBits) { if (indexedHeights[block.height] === true) { // Already indexed if (block.height >= oldestConsecutiveBlock.height) { currentDifficulty = block.difficulty; + currentBits = block.bits; } continue; } @@ -464,6 +466,7 @@ class Mining { totalIndexed++; if (block.height >= oldestConsecutiveBlock.height) { currentDifficulty = block.difficulty; + currentBits = block.bits; } } From fefbc4adf94deeebff06dd16dac5bbc3df16392c Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 1 Aug 2023 11:16:35 +0900 Subject: [PATCH 27/73] Fix difficulty indexing db queries to return bits --- backend/src/repositories/BlocksRepository.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 078b85a03..ca54fc1b7 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -539,7 +539,7 @@ class BlocksRepository { */ public async $getBlocksDifficulty(): Promise { try { - const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty FROM blocks`); + const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty, bits FROM blocks`); return rows; } catch (e) { logger.err('Cannot get blocks difficulty list from the db. Reason: ' + (e instanceof Error ? e.message : e)); @@ -848,7 +848,7 @@ class BlocksRepository { */ public async $getOldestConsecutiveBlock(): Promise { try { - const [rows]: any = await DB.query(`SELECT height, UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty FROM blocks ORDER BY height DESC`); + const [rows]: any = await DB.query(`SELECT height, UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty, bits FROM blocks ORDER BY height DESC`); for (let i = 0; i < rows.length - 1; ++i) { if (rows[i].height - rows[i + 1].height > 1) { return rows[i]; From 93c5b059b753d5664d12ff596475a10915836d00 Mon Sep 17 00:00:00 2001 From: softsimon Date: Tue, 1 Aug 2023 15:55:03 +0900 Subject: [PATCH 28/73] Base mempool break limit of current poll rate --- backend/src/api/mempool.ts | 6 +++--- backend/src/index.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index b9b648012..85b6e6101 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -182,7 +182,7 @@ class Mempool { return txTimes; } - public async $updateMempool(transactions: string[]): Promise { + public async $updateMempool(transactions: string[], pollRate: number): Promise { logger.debug(`Updating mempool...`); // warn if this run stalls the main loop for more than 2 minutes @@ -258,7 +258,7 @@ class Mempool { } } - if (Date.now() - intervalTimer > 5_000) { + if (Date.now() - intervalTimer > Math.max(pollRate * 2, 5_000)) { if (this.inSync) { // Break and restart mempool loop if we spend too much time processing // new transactions that may lead to falling behind on block height @@ -270,7 +270,7 @@ class Mempool { if (Math.floor(progress) < 100) { loadingIndicators.setProgress('mempool', progress); } - intervalTimer = Date.now() + intervalTimer = Date.now(); } } } diff --git a/backend/src/index.ts b/backend/src/index.ts index 222537d0b..185a47067 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -188,14 +188,14 @@ class Server { } const newMempool = await bitcoinApi.$getRawMempool(); const numHandledBlocks = await blocks.$updateBlocks(); + const pollRate = config.MEMPOOL.POLL_RATE_MS * (indexer.indexerRunning ? 10 : 1); if (numHandledBlocks === 0) { - await memPool.$updateMempool(newMempool); + await memPool.$updateMempool(newMempool, pollRate); } indexer.$run(); // rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS const elapsed = Date.now() - start; - const pollRate = config.MEMPOOL.POLL_RATE_MS * (indexer.indexerRunning ? 10 : 1); const remainingTime = Math.max(0, pollRate - elapsed); setTimeout(this.runMainUpdateLoop.bind(this), numHandledBlocks > 0 ? 0 : remainingTime); this.backendRetryCount = 0; From b72b87952ef1b7d054a32816a78f502c1574bba6 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Tue, 1 Aug 2023 16:00:16 +0900 Subject: [PATCH 29/73] Implement wiz footer suggestions --- .../global-footer/global-footer.component.html | 10 ++++++---- .../global-footer/global-footer.component.scss | 5 +++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index b1ba84109..58b7c5b1e 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -2,7 +2,10 @@
-
The Mempool Open Source Project ®
+

Explore the full Bitcoin ecosystem ™

@@ -17,9 +20,6 @@

Sign In

-

Broadcast Transaction

-

Connect to our Nodes

-

About The Mempool Open Source Project®

@@ -28,6 +28,8 @@

Mining Dashboard

Lightning Dashboard

Recent Blocks

+

Broadcast Transaction

+

Connect to our Nodes

API Documentation

diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.scss b/frontend/src/app/shared/components/global-footer/global-footer.component.scss index defbe937e..cec41f25b 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.scss +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.scss @@ -89,6 +89,11 @@ footer .row.version p a { color: #09a3ba; } +.main-logo { + max-width: 220px; + margin: 0 auto 20px auto; +} + @media (max-width: 992px) { footer .row.main .links.outer { From e0814277d18817a9426ee1654981c7795432196b Mon Sep 17 00:00:00 2001 From: junderw Date: Mon, 31 Jul 2023 18:22:13 -0700 Subject: [PATCH 30/73] Fix: Use bits to calculate difficulty instead of floating points --- .../api/difficulty-adjustment.test.ts | 48 +++++++++++++- backend/src/api/blocks.ts | 17 +++-- backend/src/api/difficulty-adjustment.ts | 62 +++++++++++++++++++ 3 files changed, 120 insertions(+), 7 deletions(-) diff --git a/backend/src/__tests__/api/difficulty-adjustment.test.ts b/backend/src/__tests__/api/difficulty-adjustment.test.ts index da1ac0d2c..c3e8e1a88 100644 --- a/backend/src/__tests__/api/difficulty-adjustment.test.ts +++ b/backend/src/__tests__/api/difficulty-adjustment.test.ts @@ -1,4 +1,8 @@ -import { calcDifficultyAdjustment, DifficultyAdjustment } from '../../api/difficulty-adjustment'; +import { + calcBitsDifference, + calcDifficultyAdjustment, + DifficultyAdjustment, +} from '../../api/difficulty-adjustment'; describe('Mempool Difficulty Adjustment', () => { test('should calculate Difficulty Adjustments properly', () => { @@ -86,4 +90,46 @@ describe('Mempool Difficulty Adjustment', () => { expect(result).toStrictEqual(vector[1]); } }); + + test('should calculate Difficulty change from bits fields of two blocks', () => { + // Check same exponent + check min max for output + expect(calcBitsDifference(0x1d000200, 0x1d000100)).toEqual(100); + expect(calcBitsDifference(0x1d000400, 0x1d000100)).toEqual(300); + expect(calcBitsDifference(0x1d000800, 0x1d000100)).toEqual(300); // Actually 700 + expect(calcBitsDifference(0x1d000100, 0x1d000200)).toEqual(-50); + expect(calcBitsDifference(0x1d000100, 0x1d000400)).toEqual(-75); + expect(calcBitsDifference(0x1d000100, 0x1d000800)).toEqual(-75); // Actually -87.5 + // Check new higher exponent + expect(calcBitsDifference(0x1c000200, 0x1d000001)).toEqual(100); + expect(calcBitsDifference(0x1c000400, 0x1d000001)).toEqual(300); + expect(calcBitsDifference(0x1c000800, 0x1d000001)).toEqual(300); + expect(calcBitsDifference(0x1c000100, 0x1d000002)).toEqual(-50); + expect(calcBitsDifference(0x1c000100, 0x1d000004)).toEqual(-75); + expect(calcBitsDifference(0x1c000100, 0x1d000008)).toEqual(-75); + // Check new lower exponent + expect(calcBitsDifference(0x1d000002, 0x1c000100)).toEqual(100); + expect(calcBitsDifference(0x1d000004, 0x1c000100)).toEqual(300); + expect(calcBitsDifference(0x1d000008, 0x1c000100)).toEqual(300); + expect(calcBitsDifference(0x1d000001, 0x1c000200)).toEqual(-50); + expect(calcBitsDifference(0x1d000001, 0x1c000400)).toEqual(-75); + expect(calcBitsDifference(0x1d000001, 0x1c000800)).toEqual(-75); + // Check error when exponents are too far apart + expect(() => calcBitsDifference(0x1d000001, 0x1a000800)).toThrow( + /Impossible exponent difference/ + ); + // Check invalid inputs + expect(() => calcBitsDifference(0x7f000001, 0x1a000800)).toThrow( + /Invalid bits/ + ); + expect(() => calcBitsDifference(0, 0x1a000800)).toThrow(/Invalid bits/); + expect(() => calcBitsDifference(100.2783, 0x1a000800)).toThrow( + /Invalid bits/ + ); + expect(() => calcBitsDifference(0x00800000, 0x1a000800)).toThrow( + /Invalid bits/ + ); + expect(() => calcBitsDifference(0x1c000000, 0x1a000800)).toThrow( + /Invalid bits/ + ); + }); }); diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 20c015f44..4919e71a6 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -28,12 +28,13 @@ import chainTips from './chain-tips'; import websocketHandler from './websocket-handler'; import redisCache from './redis-cache'; import rbfCache from './rbf-cache'; +import { calcBitsDifference } from './difficulty-adjustment'; class Blocks { private blocks: BlockExtended[] = []; private blockSummaries: BlockSummary[] = []; private currentBlockHeight = 0; - private currentDifficulty = 0; + private currentBits = 0; private lastDifficultyAdjustmentTime = 0; private previousDifficultyRetarget = 0; private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = []; @@ -666,14 +667,14 @@ class Blocks { const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); this.updateTimerProgress(timer, 'got block for initial difficulty adjustment'); this.lastDifficultyAdjustmentTime = block.timestamp; - this.currentDifficulty = block.difficulty; + this.currentBits = block.bits; if (blockHeightTip >= 2016) { const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016); this.updateTimerProgress(timer, 'got previous block hash for initial difficulty adjustment'); const previousPeriodBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(previousPeriodBlockHash); this.updateTimerProgress(timer, 'got previous block for initial difficulty adjustment'); - this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100; + this.previousDifficultyRetarget = calcBitsDifference(previousPeriodBlock.bits, block.bits); logger.debug(`Initial difficulty adjustment data set.`); } } else { @@ -786,14 +787,18 @@ class Blocks { time: block.timestamp, height: block.height, difficulty: block.difficulty, - adjustment: Math.round((block.difficulty / this.currentDifficulty) * 1000000) / 1000000, // Remove float point noise + adjustment: Math.round( + // calcBitsDifference returns +- percentage, +100 returns to positive, /100 returns to ratio. + // Instead of actually doing /100, just reduce the multiplier. + (calcBitsDifference(this.currentBits, block.bits) + 100) * 10000 + ) / 1000000, // Remove float point noise }); this.updateTimerProgress(timer, `saved difficulty adjustment for ${this.currentBlockHeight}`); } - this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100; + this.previousDifficultyRetarget = calcBitsDifference(this.currentBits, block.bits); this.lastDifficultyAdjustmentTime = block.timestamp; - this.currentDifficulty = block.difficulty; + this.currentBits = block.bits; } // wait for pending async callbacks to finish diff --git a/backend/src/api/difficulty-adjustment.ts b/backend/src/api/difficulty-adjustment.ts index 1f37d8be9..23d0c33de 100644 --- a/backend/src/api/difficulty-adjustment.ts +++ b/backend/src/api/difficulty-adjustment.ts @@ -16,6 +16,68 @@ export interface DifficultyAdjustment { expectedBlocks: number; // Block count } +/** + * Calculate the difficulty increase/decrease by using the `bits` integer contained in two + * block headers. + * + * Warning: Only compare `bits` from blocks in two adjacent difficulty periods. This code + * assumes the maximum difference is x4 or /4 (as per the protocol) and will throw an + * error if an exponent difference of 2 or more is seen. + * + * @param {number} oldBits The 32 bit `bits` integer from a block header. + * @param {number} newBits The 32 bit `bits` integer from a block header in the next difficulty period. + * @returns {number} A floating point decimal of the difficulty change from old to new. + * (ie. 21.3 means 21.3% increase in difficulty, -21.3 is a 21.3% decrease in difficulty) + */ +export function calcBitsDifference(oldBits: number, newBits: number): number { + // Must be + // - integer + // - highest exponent is 0x1f, so max value (as integer) is 0x1f0000ff + // - min value is 1 (exponent = 0) + // - highest bit of the number-part is +- sign, it must not be 1 + const verifyBits = (bits: number): void => { + if ( + Math.floor(bits) !== bits || + bits > 0x1f0000ff || + bits < 1 || + (bits & 0x00800000) !== 0 || + (bits & 0x007fffff) === 0 + ) { + throw new Error('Invalid bits'); + } + }; + verifyBits(oldBits); + verifyBits(newBits); + + // No need to mask exponents because we checked the bounds above + const oldExp = oldBits >> 24; + const newExp = newBits >> 24; + const oldNum = oldBits & 0x007fffff; + const newNum = newBits & 0x007fffff; + // The diff can only possibly be 1, 0, -1 + // (because maximum difficulty change is x4 or /4 (2 bits up or down)) + let result: number; + switch (newExp - oldExp) { + // New less than old, target lowered, difficulty increased + case -1: + result = ((oldNum << 8) * 100) / newNum - 100; + break; + // Same exponent, compare numbers as is. + case 0: + result = (oldNum * 100) / newNum - 100; + break; + // Old less than new, target raised, difficulty decreased + case 1: + result = (oldNum * 100) / (newNum << 8) - 100; + break; + default: + throw new Error('Impossible exponent difference'); + } + + // Min/Max values + return result > 300 ? 300 : result < -75 ? -75 : result; +} + export function calcDifficultyAdjustment( DATime: number, nowSeconds: number, From 5accbeb2d3d10f78c1f9fe14c3d689480767b325 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 1 Aug 2023 17:33:03 +0900 Subject: [PATCH 31/73] fix null coinbase fees in block summary --- backend/src/api/blocks.ts | 4 ++-- backend/src/api/common.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 20c015f44..0372d30a3 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -701,7 +701,7 @@ class Blocks { // fill in missing transaction fee data from verboseBlock for (let i = 0; i < transactions.length; i++) { if (!transactions[i].fee && transactions[i].txid === verboseBlock.tx[i].txid) { - transactions[i].fee = verboseBlock.tx[i].fee * 100_000_000; + transactions[i].fee = (verboseBlock.tx[i].fee * 100_000_000) || 0; } } @@ -941,7 +941,7 @@ class Blocks { transactions: cpfpSummary.transactions.map(tx => { return { txid: tx.txid, - fee: tx.fee, + fee: tx.fee || 0, vsize: tx.vsize, value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0)), rate: tx.effectiveFeePerVsize diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index cd9da3d2a..0e03fe32c 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -108,7 +108,7 @@ export class Common { static stripTransaction(tx: TransactionExtended): TransactionStripped { return { txid: tx.txid, - fee: tx.fee, + fee: tx.fee || 0, vsize: tx.weight / 4, value: tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0), rate: tx.effectiveFeePerVsize, From 946e2e700d69f420f1b737616e7e316d17fff59b Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Tue, 1 Aug 2023 17:24:26 +0900 Subject: [PATCH 32/73] Add space below tagline --- .../components/global-footer/global-footer.component.scss | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.scss b/frontend/src/app/shared/components/global-footer/global-footer.component.scss index cec41f25b..387a27ffc 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.scss +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.scss @@ -21,6 +21,10 @@ footer .row.main .branding { text-align: center; } +footer .row.main .branding > p { + margin-bottom: 45px; +} + footer .row.main .branding .btn { display: inline-block; color: #fff !important; From 14b386d2b2d4e02cb14d974be6c09f3926a243c2 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Tue, 1 Aug 2023 17:26:13 +0900 Subject: [PATCH 33/73] Change more networks to networks --- .../components/global-footer/global-footer.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index 58b7c5b1e..e1d853848 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -52,7 +52,7 @@
-

More Networks

+

Networks

Mainnet Explorer

Testnet Explorer

Signet Explorer

From bae3682e051629e17e737670413930f7ab294a31 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Tue, 1 Aug 2023 17:35:18 +0900 Subject: [PATCH 34/73] Improve show conditions for explore links --- .../components/global-footer/global-footer.component.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index e1d853848..7206cc34f 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -26,10 +26,10 @@

Explore

Mining Dashboard

-

Lightning Dashboard

+

Lightning Dashboard

Recent Blocks

Broadcast Transaction

-

Connect to our Nodes

+

Connect to our Nodes

API Documentation

From 7d4c9267c89b4d8ef47c10ae6381d116d1cfc236 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Tue, 1 Aug 2023 17:36:12 +0900 Subject: [PATCH 35/73] Remove commented/inactive social links --- .../global-footer/global-footer.component.html | 9 --------- 1 file changed, 9 deletions(-) diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index 7206cc34f..a5792bb17 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -40,15 +40,6 @@

Why isn't my transaction confirming?

More FAQs ›

-
From 04430fadd192f6c6f22d462aae559e111d9efa85 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Tue, 1 Aug 2023 18:08:49 +0900 Subject: [PATCH 36/73] Improve conditions for showing network links --- .../global-footer/global-footer.component.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index a5792bb17..903ba70bd 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -42,13 +42,13 @@
-
+

Networks

-

Mainnet Explorer

-

Testnet Explorer

-

Signet Explorer

-

Liquid Explorer

-

Bisq Explorer

+

Mainnet Explorer

+

Testnet Explorer

+

Signet Explorer

+

Liquid Explorer

+

Bisq Explorer

Legal

From c0724e20515a5ba71bde77c7984f20764516826c Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 1 Aug 2023 18:41:48 +0900 Subject: [PATCH 37/73] fix partially indexed difficulty bug --- backend/src/api/mining/mining.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index 3eeb3e9e5..7376e7cf4 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -438,6 +438,7 @@ class Mining { const oldestConsecutiveBlock = await BlocksRepository.$getOldestConsecutiveBlock(); if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== -1) { currentBits = oldestConsecutiveBlock.bits; + currentDifficulty = oldestConsecutiveBlock.difficulty; } let totalBlockChecked = 0; From 35edfef0ddb7634b5d42bc1a1692edcb92fe3d56 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Tue, 1 Aug 2023 18:35:51 +0900 Subject: [PATCH 38/73] Show tool list when no other networks available --- .../global-footer/global-footer.component.html | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index 903ba70bd..a556b0783 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -42,13 +42,20 @@
-
+

Networks

Mainnet Explorer

Testnet Explorer

Signet Explorer

-

Liquid Explorer

-

Bisq Explorer

+

Liquid Testnet Explorer

+

Liquid Explorer

+

Bisq Explorer

+
+
+

Tools

+

Clock (Mempool)

+

Clock (Mined)

+

BTC/Fiat Converter

Legal

From 47674033743059a91e6aafcd84485b3d3bb8336d Mon Sep 17 00:00:00 2001 From: softsimon Date: Wed, 2 Aug 2023 14:34:59 +0900 Subject: [PATCH 39/73] Minor ngIf else template optimization --- .../components/global-footer/global-footer.component.html | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index a556b0783..2b30714d1 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -42,7 +42,7 @@
-
+

Networks

Mainnet Explorer

Testnet Explorer

@@ -51,12 +51,14 @@

Liquid Explorer

Bisq Explorer

-
+ +

Tools

Clock (Mempool)

Clock (Mined)

BTC/Fiat Converter

+

Legal

Terms of Service

From 6e1d39eb455d2891184359d3b464bb9d270b3fdc Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 2 Aug 2023 13:24:56 +0900 Subject: [PATCH 40/73] use bulk mempool/txs post api to batch mempool update requests --- .../bitcoin/bitcoin-api-abstract-factory.ts | 3 +- backend/src/api/bitcoin/bitcoin-api.ts | 9 +++- backend/src/api/bitcoin/esplora-api.ts | 25 ++++++++- backend/src/api/mempool.ts | 54 ++++++++++--------- backend/src/api/transaction-utils.ts | 19 +++++++ 5 files changed, 82 insertions(+), 28 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts index 7f4a5e53a..f14c5525d 100644 --- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -3,7 +3,8 @@ import { IEsploraApi } from './esplora-api.interface'; export interface AbstractBitcoinApi { $getRawMempool(): Promise; $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, lazyPrevouts?: boolean): Promise; - $getMempoolTransactions(lastTxid: string); + $getMempoolTransactions(txids: string[]): Promise; + $getAllMempoolTransactions(lastTxid: string); $getTransactionHex(txId: string): Promise; $getBlockHeightTip(): Promise; $getBlockHashTip(): Promise; diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 132cda91a..b315ed0f7 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -60,8 +60,13 @@ class BitcoinApi implements AbstractBitcoinApi { }); } - $getMempoolTransactions(lastTxid: string): Promise { - return Promise.resolve([]); + $getMempoolTransactions(txids: string[]): Promise { + throw new Error('Method getMempoolTransactions not supported by the Bitcoin RPC API.'); + } + + $getAllMempoolTransactions(lastTxid: string): Promise { + throw new Error('Method getAllMempoolTransactions not supported by the Bitcoin RPC API.'); + } async $getTransactionHex(txId: string): Promise { diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index ff10751e0..77c6d80fc 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -61,6 +61,25 @@ class ElectrsApi implements AbstractBitcoinApi { }); } + $postWrapper(url, body, responseType = 'json', params: any = undefined): Promise { + return axiosConnection.post(url, body, { ...this.activeAxiosConfig, responseType: responseType, params }) + .then((response) => response.data) + .catch((e) => { + if (e?.code === 'ECONNREFUSED') { + this.fallbackToTcpSocket(); + // Retry immediately + return axiosConnection.post(url, body, this.activeAxiosConfig) + .then((response) => response.data) + .catch((e) => { + logger.warn(`Cannot query esplora through the unix socket nor the tcp socket. Exception ${e}`); + throw e; + }); + } else { + throw e; + } + }); + } + $getRawMempool(): Promise { return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/mempool/txids'); } @@ -69,7 +88,11 @@ class ElectrsApi implements AbstractBitcoinApi { return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/tx/' + txId); } - async $getMempoolTransactions(lastSeenTxid?: string): Promise { + async $getMempoolTransactions(txids: string[]): Promise { + return this.$postWrapper(config.ESPLORA.REST_API_URL + '/mempool/txs', txids, 'json'); + } + + async $getAllMempoolTransactions(lastSeenTxid?: string): Promise { return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/mempool/txs' + (lastSeenTxid ? '/' + lastSeenTxid : '')); } diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 85b6e6101..ccdd307f8 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -123,7 +123,7 @@ class Mempool { loadingIndicators.setProgress('mempool', count / expectedCount * 100); while (!done) { try { - const result = await bitcoinApi.$getMempoolTransactions(last_txid); + const result = await bitcoinApi.$getAllMempoolTransactions(last_txid); if (result) { for (const tx of result) { const extendedTransaction = transactionUtils.extendMempoolTransaction(tx); @@ -231,31 +231,37 @@ class Mempool { } if (!loaded) { - for (const txid of transactions) { - if (!this.mempoolCache[txid]) { - try { - const transaction = await transactionUtils.$getMempoolTransactionExtended(txid, false, false, false); - this.updateTimerProgress(timer, 'fetched new transaction'); - this.mempoolCache[txid] = transaction; - if (this.inSync) { - this.txPerSecondArray.push(new Date().getTime()); - this.vBytesPerSecondArray.push({ - unixTime: new Date().getTime(), - vSize: transaction.vsize, - }); - } - hasChange = true; - newTransactions.push(transaction); + const remainingTxids = transactions.filter(txid => !this.mempoolCache[txid]); + const sliceLength = 10000; + for (let i = 0; i < Math.ceil(remainingTxids.length / sliceLength); i++) { + const slice = remainingTxids.slice(i * sliceLength, (i + 1) * sliceLength); + const txs = await transactionUtils.$getMempoolTransactionsExtended(slice, false, false, false); + logger.debug(`fetched ${txs.length} transactions`); + this.updateTimerProgress(timer, 'fetched new transactions'); - if (config.REDIS.ENABLED) { - await redisCache.$addTransaction(transaction); - } - } catch (e: any) { - if (config.MEMPOOL.BACKEND === 'esplora' && e.response?.status === 404) { - this.missingTxCount++; - } - logger.debug(`Error finding transaction '${txid}' in the mempool: ` + (e instanceof Error ? e.message : e)); + for (const transaction of txs) { + this.mempoolCache[transaction.txid] = transaction; + if (this.inSync) { + this.txPerSecondArray.push(new Date().getTime()); + this.vBytesPerSecondArray.push({ + unixTime: new Date().getTime(), + vSize: transaction.vsize, + }); } + hasChange = true; + newTransactions.push(transaction); + + if (config.REDIS.ENABLED) { + await redisCache.$addTransaction(transaction); + } + } + + if (txs.length < slice.length) { + const missing = slice.length - txs.length; + if (config.MEMPOOL.BACKEND === 'esplora') { + this.missingTxCount += missing; + } + logger.debug(`Error finding ${missing} transactions in the mempool: `); } if (Date.now() - intervalTimer > Math.max(pollRate * 2, 5_000)) { diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index e141a6076..02ee7c055 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -4,6 +4,7 @@ import { Common } from './common'; import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; import * as bitcoinjs from 'bitcoinjs-lib'; import logger from '../logger'; +import config from '../config'; class TransactionUtils { constructor() { } @@ -71,6 +72,24 @@ class TransactionUtils { return (await this.$getTransactionExtended(txId, addPrevouts, lazyPrevouts, forceCore, true)) as MempoolTransactionExtended; } + public async $getMempoolTransactionsExtended(txids: string[], addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise { + if (forceCore || config.MEMPOOL.BACKEND !== 'esplora') { + const results = await Promise.allSettled(txids.map(txid => this.$getTransactionExtended(txid, addPrevouts, lazyPrevouts, forceCore, true))); + return (results.filter(r => r.status === 'fulfilled') as PromiseFulfilledResult[]).map(r => r.value); + } else { + const transactions = await bitcoinApi.$getMempoolTransactions(txids); + return transactions.map(transaction => { + if (Common.isLiquid()) { + if (!isFinite(Number(transaction.fee))) { + transaction.fee = Object.values(transaction.fee || {}).reduce((total, output) => total + output, 0); + } + } + + return this.extendMempoolTransaction(transaction); + }); + } + } + public extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended { // @ts-ignore if (transaction.vsize) { From fa9523ad6d109c2058222eed967e83ee67cac212 Mon Sep 17 00:00:00 2001 From: andrewtoth Date: Thu, 3 Aug 2023 03:46:15 +0000 Subject: [PATCH 41/73] Accept CLA --- contributors/andrewtoth.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 contributors/andrewtoth.txt diff --git a/contributors/andrewtoth.txt b/contributors/andrewtoth.txt new file mode 100644 index 000000000..bd2ed3d06 --- /dev/null +++ b/contributors/andrewtoth.txt @@ -0,0 +1,3 @@ +I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of August 2, 2023. + +Signed: andrewtoth From 67b8305c90b2d238a73b286218fb174d55475780 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Mon, 31 Jul 2023 11:06:52 +0900 Subject: [PATCH 42/73] [mining] send pool unique id in /pools API --- backend/src/api/mining/mining.ts | 1 + backend/src/mempool.interfaces.ts | 1 + backend/src/repositories/PoolsRepository.ts | 3 ++- frontend/src/app/interfaces/node-api.interface.ts | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index e190492b8..79b255e7f 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -107,6 +107,7 @@ class Mining { slug: poolInfo.slug, avgMatchRate: poolInfo.avgMatchRate !== null ? Math.round(100 * poolInfo.avgMatchRate) / 100 : null, avgFeeDelta: poolInfo.avgFeeDelta, + poolUniqueId: poolInfo.poolUniqueId }; poolsStats.push(poolStat); }); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 4715440e4..c08846191 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -20,6 +20,7 @@ export interface PoolInfo { slug: string; avgMatchRate: number | null; avgFeeDelta: number | null; + poolUniqueId: number; } export interface PoolStats extends PoolInfo { diff --git a/backend/src/repositories/PoolsRepository.ts b/backend/src/repositories/PoolsRepository.ts index 899712266..eda792bb3 100644 --- a/backend/src/repositories/PoolsRepository.ts +++ b/backend/src/repositories/PoolsRepository.ts @@ -40,7 +40,8 @@ class PoolsRepository { pools.link AS link, slug, AVG(blocks_audits.match_rate) AS avgMatchRate, - AVG((CAST(blocks.fees as SIGNED) - CAST(blocks_audits.expected_fees as SIGNED)) / NULLIF(CAST(blocks_audits.expected_fees as SIGNED), 0)) AS avgFeeDelta + AVG((CAST(blocks.fees as SIGNED) - CAST(blocks_audits.expected_fees as SIGNED)) / NULLIF(CAST(blocks_audits.expected_fees as SIGNED), 0)) AS avgFeeDelta, + unique_id as poolUniqueId FROM blocks JOIN pools on pools.id = pool_id LEFT JOIN blocks_audits ON blocks_audits.height = blocks.height diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index cb42eaed3..5316b1cdc 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -111,6 +111,7 @@ export interface PoolInfo { regexes: string; // JSON array addresses: string; // JSON array emptyBlocks: number; + poolUniqueId: number; } export interface PoolStat { pool: PoolInfo; From adcf9946dd0183ceb6e5c0f2a2b35cb084347d7e Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 17 Feb 2023 18:54:48 +0900 Subject: [PATCH 43/73] [sponsors] show profile by tiers on about page --- .../app/components/about/about.component.html | 49 +++++++++++++++---- .../app/components/about/about.component.scss | 3 ++ .../app/components/about/about.component.ts | 24 ++++++--- frontend/src/app/services/api.service.ts | 14 ++---- frontend/src/resources/profile/unknown.svg | 1 + 5 files changed, 62 insertions(+), 29 deletions(-) create mode 100644 frontend/src/resources/profile/unknown.svg diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index 381353948..55f406d00 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -31,6 +31,12 @@ +

Support the Project

+ +
+ Become a Sponsor! +
+

Enterprise Sponsors 🚀

@@ -191,16 +197,39 @@
-
-

Community Sponsors ❤️

+
+
+

Whale Sponsors

+
+ + + + + + + +
+
-
- - - - +
+

Chad Sponsors

+
+ + + +
+
+
+ +
+

OG Sponsors ❤️

+
+ + + +
@@ -340,7 +369,7 @@
- +
@@ -354,7 +383,7 @@
- + {{ contributor.name }} @@ -366,7 +395,7 @@
- + {{ contributor.name }} diff --git a/frontend/src/app/components/about/about.component.scss b/frontend/src/app/components/about/about.component.scss index e1d6c829a..2a5710ca1 100644 --- a/frontend/src/app/components/about/about.component.scss +++ b/frontend/src/app/components/about/about.component.scss @@ -10,6 +10,9 @@ margin: 25px; line-height: 32px; } + .unknown { + border: 1px solid #b4b4b4; + } .image.not-rounded { border-radius: 0; diff --git a/frontend/src/app/components/about/about.component.ts b/frontend/src/app/components/about/about.component.ts index 176490add..4bf7869de 100644 --- a/frontend/src/app/components/about/about.component.ts +++ b/frontend/src/app/components/about/about.component.ts @@ -6,7 +6,7 @@ import { Observable } from 'rxjs'; import { ApiService } from '../../services/api.service'; import { IBackendInfo } from '../../interfaces/websocket.interface'; import { Router, ActivatedRoute } from '@angular/router'; -import { map, tap } from 'rxjs/operators'; +import { map, share, tap } from 'rxjs/operators'; import { ITranslators } from '../../interfaces/node-api.interface'; import { DOCUMENT } from '@angular/common'; @@ -19,14 +19,16 @@ import { DOCUMENT } from '@angular/common'; export class AboutComponent implements OnInit { @ViewChild('promoVideo') promoVideo: ElementRef; backendInfo$: Observable; - sponsors$: Observable; - translators$: Observable; - allContributors$: Observable; frontendGitCommitHash = this.stateService.env.GIT_COMMIT_HASH; packetJsonVersion = this.stateService.env.PACKAGE_JSON_VERSION; officialMempoolSpace = this.stateService.env.OFFICIAL_MEMPOOL_SPACE; showNavigateToSponsor = false; + profiles$: Observable; + translators$: Observable; + allContributors$: Observable; + ogs$: Observable; + constructor( private websocketService: WebsocketService, private seoService: SeoService, @@ -43,10 +45,13 @@ export class AboutComponent implements OnInit { this.seoService.setTitle($localize`:@@004b222ff9ef9dd4771b777950ca1d0e4cd4348a:About`); this.websocketService.want(['blocks']); - this.sponsors$ = this.apiService.getDonation$() - .pipe( - tap(() => this.goToAnchor()) - ); + this.profiles$ = this.apiService.getAboutPageProfiles$().pipe( + tap(() => { + this.goToAnchor() + }), + share(), + ) + this.translators$ = this.apiService.getTranslators$() .pipe( map((translators) => { @@ -59,6 +64,9 @@ export class AboutComponent implements OnInit { }), tap(() => this.goToAnchor()) ); + + this.ogs$ = this.apiService.getOgs$(); + this.allContributors$ = this.apiService.getContributor$().pipe( map((contributors) => { return { diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index e2d3be9be..24dbe734a 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -92,15 +92,11 @@ export class ApiService { return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/outspends', { params }); } - requestDonation$(amount: number, orderId: string): Observable { - const params = { - amount: amount, - orderId: orderId, - }; - return this.httpClient.post(this.apiBaseUrl + '/api/v1/donations', params); + getAboutPageProfiles$(): Observable { + return this.httpClient.get(this.apiBaseUrl + '/api/v1/about-page'); } - getDonation$(): Observable { + getOgs$(): Observable { return this.httpClient.get(this.apiBaseUrl + '/api/v1/donations'); } @@ -112,10 +108,6 @@ export class ApiService { return this.httpClient.get(this.apiBaseUrl + '/api/v1/contributors'); } - checkDonation$(orderId: string): Observable { - return this.httpClient.get(this.apiBaseUrl + '/api/v1/donations/check?order_id=' + orderId); - } - getInitData$(): Observable { return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/init-data'); } diff --git a/frontend/src/resources/profile/unknown.svg b/frontend/src/resources/profile/unknown.svg new file mode 100644 index 000000000..50a548c3c --- /dev/null +++ b/frontend/src/resources/profile/unknown.svg @@ -0,0 +1 @@ + \ No newline at end of file From 9c21f4ea1bfea3d032512c9278bcc69d8ab2bcdf Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 1 Apr 2023 11:40:04 +0900 Subject: [PATCH 44/73] [lightning] claim your node button --- .../src/app/lightning/lightning.module.ts | 3 +++ .../node-owner/node-owner.component.html | 17 ++++++++++++ .../node-owner/node-owner.component.scss | 4 +++ .../node-owner/node-owner.component.ts | 20 ++++++++++++++ .../app/lightning/node/node.component.html | 8 ++++-- .../app/lightning/node/node.component.scss | 14 ++++++++++ .../src/app/lightning/node/node.component.ts | 26 ++++++++++++++++--- frontend/src/app/services/api.service.ts | 11 ++++++++ 8 files changed, 98 insertions(+), 5 deletions(-) create mode 100644 frontend/src/app/lightning/node-owner/node-owner.component.html create mode 100644 frontend/src/app/lightning/node-owner/node-owner.component.scss create mode 100644 frontend/src/app/lightning/node-owner/node-owner.component.ts diff --git a/frontend/src/app/lightning/lightning.module.ts b/frontend/src/app/lightning/lightning.module.ts index 0b824ad78..f0154a15f 100644 --- a/frontend/src/app/lightning/lightning.module.ts +++ b/frontend/src/app/lightning/lightning.module.ts @@ -34,6 +34,7 @@ import { OldestNodes } from '../lightning/nodes-ranking/oldest-nodes/oldest-node import { NodesRankingsDashboard } from '../lightning/nodes-rankings-dashboard/nodes-rankings-dashboard.component'; import { NodeChannels } from '../lightning/nodes-channels/node-channels.component'; import { GroupComponent } from './group/group.component'; +import { NodeOwnerComponent } from './node-owner/node-owner.component'; @NgModule({ declarations: [ @@ -66,6 +67,7 @@ import { GroupComponent } from './group/group.component'; NodesRankingsDashboard, NodeChannels, GroupComponent, + NodeOwnerComponent, ], imports: [ CommonModule, @@ -103,6 +105,7 @@ import { GroupComponent } from './group/group.component'; OldestNodes, NodesRankingsDashboard, NodeChannels, + NodeOwnerComponent, ], providers: [ LightningApiService, diff --git a/frontend/src/app/lightning/node-owner/node-owner.component.html b/frontend/src/app/lightning/node-owner/node-owner.component.html new file mode 100644 index 000000000..e37b1e027 --- /dev/null +++ b/frontend/src/app/lightning/node-owner/node-owner.component.html @@ -0,0 +1,17 @@ +
+ +
+ +
+ + + +
+ +
+ Claim +
+ +
+ +
\ No newline at end of file diff --git a/frontend/src/app/lightning/node-owner/node-owner.component.scss b/frontend/src/app/lightning/node-owner/node-owner.component.scss new file mode 100644 index 000000000..6734168cf --- /dev/null +++ b/frontend/src/app/lightning/node-owner/node-owner.component.scss @@ -0,0 +1,4 @@ +.profile-photo { + width: 31px; + height: 31px; +} \ No newline at end of file diff --git a/frontend/src/app/lightning/node-owner/node-owner.component.ts b/frontend/src/app/lightning/node-owner/node-owner.component.ts new file mode 100644 index 000000000..a03c04901 --- /dev/null +++ b/frontend/src/app/lightning/node-owner/node-owner.component.ts @@ -0,0 +1,20 @@ +import { ChangeDetectionStrategy, Component, Input } from '@angular/core'; +import { Observable } from 'rxjs'; +import { StateService } from '../../services/state.service'; + +@Component({ + selector: 'app-node-owner', + templateUrl: './node-owner.component.html', + styleUrls: ['./node-owner.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class NodeOwnerComponent{ + @Input() publicKey: string = ''; + @Input() alias: string = ''; + @Input() nodeOwner$: Observable; + + constructor( + public stateService: StateService + ) { + } +} diff --git a/frontend/src/app/lightning/node/node.component.html b/frontend/src/app/lightning/node/node.component.html index c6c693a3a..11ddbc0eb 100644 --- a/frontend/src/app/lightning/node/node.component.html +++ b/frontend/src/app/lightning/node/node.component.html @@ -3,13 +3,17 @@
Lightning node
-

{{ node.alias }}

- +
+

{{ node.alias }}

+ +
+ +
diff --git a/frontend/src/app/lightning/node/node.component.scss b/frontend/src/app/lightning/node/node.component.scss index 272de4b09..117fc8a2c 100644 --- a/frontend/src/app/lightning/node/node.component.scss +++ b/frontend/src/app/lightning/node/node.component.scss @@ -111,3 +111,17 @@ app-fiat { margin: 0 0.25em; color: slategrey; } + +.claim-btn { + max-height: 32px; + @media (min-width: 850px) { + display: none; + } +} + +.claim-btn-mobile { + max-height: 32px; + @media (max-width: 850px) { + display: none; + } +} \ No newline at end of file diff --git a/frontend/src/app/lightning/node/node.component.ts b/frontend/src/app/lightning/node/node.component.ts index 719136d79..6447eb6bd 100644 --- a/frontend/src/app/lightning/node/node.component.ts +++ b/frontend/src/app/lightning/node/node.component.ts @@ -1,7 +1,7 @@ -import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; +import { ChangeDetectionStrategy, Component, OnInit, ChangeDetectorRef } from '@angular/core'; import { ActivatedRoute, ParamMap } from '@angular/router'; -import { Observable } from 'rxjs'; -import { catchError, map, switchMap, tap } from 'rxjs/operators'; +import { Observable, of, EMPTY } from 'rxjs'; +import { catchError, map, switchMap, tap, share } from 'rxjs/operators'; import { SeoService } from '../../services/seo.service'; import { ApiService } from '../../services/api.service'; import { LightningApiService } from '../lightning-api.service'; @@ -38,6 +38,7 @@ export class NodeComponent implements OnInit { tlvRecords: CustomRecord[]; avgChannelDistance$: Observable; showFeatures = false; + nodeOwner$: Observable; kmToMiles = kmToMiles; constructor( @@ -45,6 +46,7 @@ export class NodeComponent implements OnInit { private lightningApiService: LightningApiService, private activatedRoute: ActivatedRoute, private seoService: SeoService, + private cd: ChangeDetectorRef, ) { } ngOnInit(): void { @@ -147,6 +149,24 @@ export class NodeComponent implements OnInit { return null; }) ) as Observable; + + this.nodeOwner$ = this.activatedRoute.paramMap + .pipe( + switchMap((params: ParamMap) => { + return this.apiService.getNodeOwner$(params.get('public_key')).pipe( + switchMap((response) => { + if (response.status === 204) { + return of(false); + } + return of(response.body); + }), + catchError(() => { + return of(false); + }) + ) + }), + share(), + ); } toggleShowDetails(): void { diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index 24dbe734a..1ed9d2f5c 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -8,6 +8,8 @@ import { WebsocketResponse } from '../interfaces/websocket.interface'; import { Outspend, Transaction } from '../interfaces/electrs.interface'; import { Conversion } from './price.service'; +const SERVICES_API_PREFIX = `/api/v1/services`; + @Injectable({ providedIn: 'root' }) @@ -315,4 +317,13 @@ export class ApiService { (timestamp ? `?timestamp=${timestamp}` : '') ); } + + /** + * Services + */ + getNodeOwner$(publicKey: string) { + let params = new HttpParams() + .set('node_public_key', publicKey); + return this.httpClient.get(`${SERVICES_API_PREFIX}/lightning/claim/current`, { params, observe: 'response' }); + } } From a6b9c5ddd9d754cd1a5f66d79fd9768b8f31b986 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Wed, 3 May 2023 16:32:00 +0200 Subject: [PATCH 45/73] [tx] add accel shortcut into transaction component --- .../transaction/transaction.component.html | 10 ++++++++-- .../transaction/transaction.component.scss | 19 +++++++++++++++++++ .../transaction/transaction.component.ts | 2 +- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index 7422c6894..59404bb77 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -99,14 +99,20 @@ - In several hours (or more) + + In several hours (or more) + Accelerate + - + + + Accelerate + diff --git a/frontend/src/app/components/transaction/transaction.component.scss b/frontend/src/app/components/transaction/transaction.component.scss index bea8e82bc..5bef401d7 100644 --- a/frontend/src/app/components/transaction/transaction.component.scss +++ b/frontend/src/app/components/transaction/transaction.component.scss @@ -216,4 +216,23 @@ .alert-link { display: block; } +} + +.eta { + display: flex; + justify-content: end; + flex-wrap: wrap; + align-content: center; + @media (min-width: 850px) { + justify-content: space-between; + } +} + +.accelerate { + align-self: auto; + margin-top: 3px; + @media (min-width: 850px) { + justify-self: start; + margin-left: 0px; + } } \ No newline at end of file diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index f1d218a79..26e39515e 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -97,7 +97,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { private router: Router, private relativeUrlPipe: RelativeUrlPipe, private electrsApiService: ElectrsApiService, - private stateService: StateService, + public stateService: StateService, private cacheService: CacheService, private websocketService: WebsocketService, private audioService: AudioService, From 350cb66c390c7a42426044351b6f7df8a5072188 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 11 May 2023 18:00:52 +0200 Subject: [PATCH 46/73] [footer] dynamic CTA button based on login status --- .../app/components/about/about.component.html | 2 +- .../global-footer.component.html | 8 ++++-- .../global-footer/global-footer.component.ts | 26 +++++++++++++++++-- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index 55f406d00..b9309c916 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -34,7 +34,7 @@

Support the Project

- Become a Sponsor! + Become a Sponsor!
diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index 8e5279a94..30727e662 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -14,8 +14,12 @@
- -

Sign In

+
+
Logged in as {{ username}}
+ + +
+

Sign In

Broadcast Transaction

Connect to our Nodes

diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.ts b/frontend/src/app/shared/components/global-footer/global-footer.component.ts index d4f1946ea..cdb0f1159 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.ts +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.ts @@ -1,10 +1,13 @@ -import { ChangeDetectionStrategy, Component, OnInit, Inject, LOCALE_ID } from '@angular/core'; -import { Observable, merge, of, Subject } from 'rxjs'; +import { ChangeDetectionStrategy, ChangeDetectorRef, Component, OnInit, Inject, LOCALE_ID } from '@angular/core'; +import { ActivatedRoute } from '@angular/router'; +import { Observable, merge, of, Subject, Subscription } from 'rxjs'; import { tap, takeUntil } from 'rxjs/operators'; import { Env, StateService } from '../../../services/state.service'; import { IBackendInfo } from '../../../interfaces/websocket.interface'; import { LanguageService } from '../../../services/language.service'; import { NavigationService } from '../../../services/navigation.service'; +import { StorageService } from '../../../services/storage.service'; +import { WebsocketService } from '../../../services/websocket.service'; @Component({ selector: 'app-global-footer', @@ -23,12 +26,19 @@ export class GlobalFooterComponent implements OnInit { network$: Observable; networkPaths: { [network: string]: string }; currentNetwork = ''; + loggedIn = false; + username = null; + urlSubscription: Subscription; constructor( public stateService: StateService, private languageService: LanguageService, private navigationService: NavigationService, @Inject(LOCALE_ID) public locale: string, + private storageService: StorageService, + private route: ActivatedRoute, + private cd: ChangeDetectorRef, + private websocketService: WebsocketService ) {} ngOnInit(): void { @@ -46,11 +56,23 @@ export class GlobalFooterComponent implements OnInit { this.network$.pipe(takeUntil(this.destroy$)).subscribe((network) => { this.currentNetwork = network; }); + + this.urlSubscription = this.route.url.subscribe((url) => { + this.loggedIn = JSON.parse(this.storageService.getValue('auth')) !== null; + const auth = JSON.parse(this.storageService.getValue('auth')); + if (auth?.user?.username) { + this.username = auth.user.username; + } else { + this.username = null; + } + this.cd.markForCheck(); + }) } ngOnDestroy(): void { this.destroy$.next(true); this.destroy$.complete(); + this.urlSubscription.unsubscribe(); } networkLink(network) { From 72e83615d4c5a99b7ba2b2671e63ed66481fe0a5 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Wed, 24 May 2023 21:49:08 -0700 Subject: [PATCH 47/73] [footer] main cta points to /accelerator --- .../src/app/components/about/about.component.html | 5 ++--- .../global-footer/global-footer.component.html | 14 ++++++-------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index b9309c916..93644a201 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -31,10 +31,9 @@ -

Support the Project

- +

Sponsor the project

- Become a Sponsor! + See plans
diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index 30727e662..658e0ac72 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -13,14 +13,12 @@
- -
-
Logged in as {{ username}}
- - -
-

Sign In

- +
+
Logged in as {{ username}}
+ + +
+

Sign In

Broadcast Transaction

Connect to our Nodes

About The Mempool Open Source Project™

From 7a2883d4b37bae9932e4b331a822e9fb5262703d Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 3 Aug 2023 14:29:10 +0900 Subject: [PATCH 48/73] toggle header visibility in master page component --- .../master-page/master-page.component.html | 2 +- .../master-page/master-page.component.ts | 15 +++++++++++---- frontend/src/app/shared/shared.module.ts | 1 + 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/frontend/src/app/components/master-page/master-page.component.html b/frontend/src/app/components/master-page/master-page.component.html index 79daf7de6..f12fbc960 100644 --- a/frontend/src/app/components/master-page/master-page.component.html +++ b/frontend/src/app/components/master-page/master-page.component.html @@ -1,5 +1,5 @@ -
+
-
-
-

Whale Sponsors

-
- - + +
+
+

Whale Sponsors

+
+ + + + + + + +
+
+ +
+

Chad Sponsors

+
+ - +
- -
-

Chad Sponsors

-
- - - - - -
-
-
+

OG Sponsors ❤️

diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index 59404bb77..9584cecfd 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -101,7 +101,7 @@ In several hours (or more) - Accelerate + Accelerate @@ -111,7 +111,7 @@ - Accelerate + Accelerate diff --git a/frontend/src/app/services/state.service.ts b/frontend/src/app/services/state.service.ts index 9ab8a7e93..675cf88d1 100644 --- a/frontend/src/app/services/state.service.ts +++ b/frontend/src/app/services/state.service.ts @@ -47,6 +47,7 @@ export interface Env { TESTNET_BLOCK_AUDIT_START_HEIGHT: number; SIGNET_BLOCK_AUDIT_START_HEIGHT: number; HISTORICAL_PRICE: boolean; + ACCELERATOR: boolean; } const defaultEnv: Env = { @@ -77,6 +78,7 @@ const defaultEnv: Env = { 'TESTNET_BLOCK_AUDIT_START_HEIGHT': 0, 'SIGNET_BLOCK_AUDIT_START_HEIGHT': 0, 'HISTORICAL_PRICE': true, + 'ACCELERATOR': false, }; @Injectable({ diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index 658e0ac72..c4436c4b6 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -13,7 +13,7 @@
-
+
Logged in as {{ username}}
From 361730e33b30c164e057555a057d81650056b9f7 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 30 Jul 2023 17:59:35 +0900 Subject: [PATCH 50/73] Improve difficulty tooltip display on mobile --- .../difficulty-tooltip.component.html | 54 ++++++++++++------- .../difficulty-tooltip.component.ts | 12 ++++- .../difficulty/difficulty.component.ts | 1 + 3 files changed, 47 insertions(+), 20 deletions(-) diff --git a/frontend/src/app/components/difficulty/difficulty-tooltip.component.html b/frontend/src/app/components/difficulty/difficulty-tooltip.component.html index d06bb5e91..7e4b421e1 100644 --- a/frontend/src/app/components/difficulty/difficulty-tooltip.component.html +++ b/frontend/src/app/components/difficulty/difficulty-tooltip.component.html @@ -4,38 +4,56 @@ class="difficulty-tooltip" [style.visibility]="status ? 'visible' : 'hidden'" [style.left]="tooltipPosition.x + 'px'" - [style.top]="tooltipPosition.y + 'px'" + [style.top]="tooltipPosition.y + (isMobile ? -60 : 0) + 'px'" > - + - - {{ i }} blocks expected - {{ i }} block expected + - - {{ i }} blocks mined - {{ i }} block mined + - - {{ i }} blocks remaining - {{ i }} block remaining + - - {{ i }} blocks ahead - {{ i }} block ahead + - - {{ i }} blocks behind - {{ i }} block behind + Next Block -
\ No newline at end of file + + + + + + + +
+ +
+ + + + + + + +
+ +{{ i }} blocks expected +{{ i }} block expected +{{ i }} blocks mined +{{ i }} block mined +{{ i }} blocks remaining +{{ i }} block remaining +{{ i }} blocks ahead +{{ i }} block ahead +{{ i }} blocks behind +{{ i }} block behind \ No newline at end of file diff --git a/frontend/src/app/components/difficulty/difficulty-tooltip.component.ts b/frontend/src/app/components/difficulty/difficulty-tooltip.component.ts index c7d26f61a..42d2a61b4 100644 --- a/frontend/src/app/components/difficulty/difficulty-tooltip.component.ts +++ b/frontend/src/app/components/difficulty/difficulty-tooltip.component.ts @@ -1,4 +1,4 @@ -import { Component, ElementRef, ViewChild, Input, OnChanges } from '@angular/core'; +import { Component, ElementRef, ViewChild, Input, OnChanges, HostListener } from '@angular/core'; interface EpochProgress { base: string; @@ -35,12 +35,15 @@ export class DifficultyTooltipComponent implements OnChanges { remaining: number; isAhead: boolean; isBehind: boolean; + isMobile: boolean; tooltipPosition = { x: 0, y: 0 }; @ViewChild('tooltip') tooltipElement: ElementRef; - constructor() {} + constructor() { + this.onResize(); + } ngOnChanges(changes): void { if (changes.cursorPosition && changes.cursorPosition.currentValue) { @@ -63,4 +66,9 @@ export class DifficultyTooltipComponent implements OnChanges { this.isBehind = this.behind > 0; } } + + @HostListener('window:resize', ['$event']) + onResize(): void { + this.isMobile = window.innerWidth <= 767.98; + } } diff --git a/frontend/src/app/components/difficulty/difficulty.component.ts b/frontend/src/app/components/difficulty/difficulty.component.ts index a2c03dc56..093dde79a 100644 --- a/frontend/src/app/components/difficulty/difficulty.component.ts +++ b/frontend/src/app/components/difficulty/difficulty.component.ts @@ -193,6 +193,7 @@ export class DifficultyComponent implements OnInit { @HostListener('pointerdown', ['$event']) onPointerDown(event) { this.onPointerMove(event); + event.preventDefault(); } @HostListener('pointermove', ['$event']) From 3cae43c7e92698bb778aae60d451b0c08bffd5f3 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Thu, 3 Aug 2023 15:49:15 +0900 Subject: [PATCH 51/73] limit mouse events to difficulty bar --- .../difficulty/difficulty.component.html | 4 +-- .../difficulty/difficulty.component.ts | 28 +++++++++++-------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/frontend/src/app/components/difficulty/difficulty.component.html b/frontend/src/app/components/difficulty/difficulty.component.html index 27cddc043..f08ea06f5 100644 --- a/frontend/src/app/components/difficulty/difficulty.component.html +++ b/frontend/src/app/components/difficulty/difficulty.component.html @@ -4,7 +4,7 @@
- + @@ -22,7 +22,7 @@ class="rect {{rect.status}}" [class.hover]="hoverSection && rect.status === hoverSection.status" (pointerover)="onHover($event, rect);" - (pointerout)="onBlur($event);" + (pointerout)="onBlur();" > ; isLoadingWebSocket$: Observable; difficultyEpoch$: Observable; @@ -191,22 +193,26 @@ export class DifficultyComponent implements OnInit { } @HostListener('pointerdown', ['$event']) - onPointerDown(event) { - this.onPointerMove(event); - event.preventDefault(); + onPointerDown(event): void { + if (this.epochSvgElement.nativeElement?.contains(event.target)) { + this.onPointerMove(event); + event.preventDefault(); + } } @HostListener('pointermove', ['$event']) - onPointerMove(event) { - this.tooltipPosition = { x: event.clientX, y: event.clientY }; - this.cd.markForCheck(); + onPointerMove(event): void { + if (this.epochSvgElement.nativeElement?.contains(event.target)) { + this.tooltipPosition = { x: event.clientX, y: event.clientY }; + this.cd.markForCheck(); + } } - onHover(event, rect): void { + onHover(_, rect): void { this.hoverSection = rect; } - onBlur(event): void { + onBlur(): void { this.hoverSection = null; } } From 646cab1fd701c5b39c415416296441360e752e60 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 3 Aug 2023 16:12:59 +0900 Subject: [PATCH 52/73] [footer] polish cta using update footer with logo --- .../global-footer/global-footer.component.html | 13 +++++-------- .../global-footer/global-footer.component.scss | 8 ++------ 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index 7dc98ca27..d4f303221 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -1,7 +1,7 @@
-
+
-
Logged in as {{ username}}
- - +
-

Sign In

-

Broadcast Transaction

-

Connect to our Nodes

-

About The Mempool Open Source Project™

diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.scss b/frontend/src/app/shared/components/global-footer/global-footer.component.scss index 387a27ffc..6b67711b6 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.scss +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.scss @@ -22,7 +22,7 @@ footer .row.main .branding { } footer .row.main .branding > p { - margin-bottom: 45px; + margin-bottom: 25px; } footer .row.main .branding .btn { @@ -35,11 +35,7 @@ footer .row.main .branding button.account { } footer .row.main .branding .cta { - margin: 20px auto 25px auto; -} - -footer .row.main .branding .cta-secondary { - + margin: 25px auto 25px auto; } footer .row.main .links.outer { From 7b37dc9289d096233721ee3c50a1ea719885e611 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Thu, 3 Aug 2023 16:17:17 +0900 Subject: [PATCH 53/73] resume tracking subscriptions after websocket reconnect --- frontend/src/app/services/websocket.service.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/services/websocket.service.ts b/frontend/src/app/services/websocket.service.ts index 4bd20e987..af2a15e8c 100644 --- a/frontend/src/app/services/websocket.service.ts +++ b/frontend/src/app/services/websocket.service.ts @@ -28,8 +28,9 @@ export class WebsocketService { private isTrackingTx = false; private trackingTxId: string; private isTrackingMempoolBlock = false; - private isTrackingRbf = false; + private isTrackingRbf: 'all' | 'fullRbf' | false = false; private isTrackingRbfSummary = false; + private isTrackingAddress: string | false = false; private trackingMempoolBlock: number; private latestGitCommit = ''; private onlineCheckTimeout: number; @@ -110,6 +111,15 @@ export class WebsocketService { if (this.isTrackingMempoolBlock) { this.startTrackMempoolBlock(this.trackingMempoolBlock); } + if (this.isTrackingRbf) { + this.startTrackRbf(this.isTrackingRbf); + } + if (this.isTrackingRbfSummary) { + this.startTrackRbfSummary(); + } + if (this.isTrackingAddress) { + this.startTrackAddress(this.isTrackingAddress); + } this.stateService.connectionState$.next(2); } @@ -151,10 +161,12 @@ export class WebsocketService { startTrackAddress(address: string) { this.websocketSubject.next({ 'track-address': address }); + this.isTrackingAddress = address; } stopTrackingAddress() { this.websocketSubject.next({ 'track-address': 'stop' }); + this.isTrackingAddress = false; } startTrackAsset(asset: string) { @@ -178,7 +190,7 @@ export class WebsocketService { startTrackRbf(mode: 'all' | 'fullRbf') { this.websocketSubject.next({ 'track-rbf': mode }); - this.isTrackingRbf = true; + this.isTrackingRbf = mode; } stopTrackRbf() { From da2cc8d1625665729629bde6eeb34b3ca784b97e Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 29 Jul 2023 14:23:06 +0900 Subject: [PATCH 54/73] add /prices api endpoint --- backend/src/api/prices/prices.routes.ts | 19 +++++++++++++++++++ backend/src/index.ts | 2 ++ 2 files changed, 21 insertions(+) create mode 100644 backend/src/api/prices/prices.routes.ts diff --git a/backend/src/api/prices/prices.routes.ts b/backend/src/api/prices/prices.routes.ts new file mode 100644 index 000000000..03951ec05 --- /dev/null +++ b/backend/src/api/prices/prices.routes.ts @@ -0,0 +1,19 @@ +import { Application, Request, Response } from 'express'; +import config from '../../config'; +import pricesUpdater from '../../tasks/price-updater'; + +class PricesRoutes { + public initRoutes(app: Application): void { + app.get(config.MEMPOOL.API_URL_PREFIX + 'prices', this.$getCurrentPrices.bind(this)); + } + + private async $getCurrentPrices(req: Request, res: Response): Promise { + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); + + res.json(pricesUpdater.getLatestPrices()); + } +} + +export default new PricesRoutes(); diff --git a/backend/src/index.ts b/backend/src/index.ts index 185a47067..b00909bc6 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -30,6 +30,7 @@ import generalLightningRoutes from './api/explorer/general.routes'; import lightningStatsUpdater from './tasks/lightning/stats-updater.service'; import networkSyncService from './tasks/lightning/network-sync.service'; import statisticsRoutes from './api/statistics/statistics.routes'; +import pricesRoutes from './api/prices/prices.routes'; import miningRoutes from './api/mining/mining-routes'; import bisqRoutes from './api/bisq/bisq.routes'; import liquidRoutes from './api/liquid/liquid.routes'; @@ -261,6 +262,7 @@ class Server { setUpHttpApiRoutes(): void { bitcoinRoutes.initRoutes(this.app); + pricesRoutes.initRoutes(this.app); if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && config.MEMPOOL.ENABLED) { statisticsRoutes.initRoutes(this.app); } From 59937c2ce91fa6598332e794fe802d763290be9b Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 29 Jul 2023 14:45:00 +0900 Subject: [PATCH 55/73] configurable price update frequency --- backend/mempool-config.sample.json | 1 + backend/src/__fixtures__/mempool-config.template.json | 1 + backend/src/__tests__/config.test.ts | 1 + backend/src/config.ts | 2 ++ backend/src/tasks/price-updater.ts | 4 ++-- docker/backend/mempool-config.json | 1 + docker/backend/start.sh | 2 ++ production/mempool-config.mainnet.json | 3 +++ 8 files changed, 13 insertions(+), 2 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 7948049fc..fddd44781 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -116,6 +116,7 @@ "PASSWORD": "" }, "PRICE_DATA_SERVER": { + "UPDATE_FREQUENCY": 300, "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices", "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices" }, diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index ab700c466..3a2b151a5 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -93,6 +93,7 @@ "PASSWORD": "__SOCKS5PROXY_PASSWORD__" }, "PRICE_DATA_SERVER": { + "UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY", "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" }, diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index edfcc7f47..cb564a334 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -102,6 +102,7 @@ describe('Mempool Backend Config', () => { }); expect(config.PRICE_DATA_SERVER).toStrictEqual({ + UPDATE_FREQUENCY: 3600, TOR_URL: 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', CLEARNET_URL: 'https://price.bisq.wiz.biz/getAllMarketPrices' }); diff --git a/backend/src/config.ts b/backend/src/config.ts index 3a028d0cd..5117983fb 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -116,6 +116,7 @@ interface IConfig { PASSWORD: string; }; PRICE_DATA_SERVER: { + UPDATE_FREQUENCY: number; TOR_URL: string; CLEARNET_URL: string; }; @@ -259,6 +260,7 @@ const defaults: IConfig = { 'PASSWORD': '' }, 'PRICE_DATA_SERVER': { + 'UPDATE_FREQUENCY': 3600, 'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', 'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices' }, diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index fafe2b913..335a61065 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -108,8 +108,8 @@ class PriceUpdater { this.lastRun = await PricesRepository.$getLatestPriceTime(); } - if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < 3600) { - // Refresh only once every hour + if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < config.PRICE_DATA_SERVER.UPDATE_FREQUENCY) { + // Refresh every UPDATE_FREQUENCY seconds at most return; } diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 8b47d53b8..8b8a79969 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -112,6 +112,7 @@ "PASSWORD": "__SOCKS5PROXY_PASSWORD__" }, "PRICE_DATA_SERVER": { + "UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY__", "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" }, diff --git a/docker/backend/start.sh b/docker/backend/start.sh index e05c73710..add483568 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -95,6 +95,7 @@ __SOCKS5PROXY_USERNAME__=${SOCKS5PROXY_USERNAME:=""} __SOCKS5PROXY_PASSWORD__=${SOCKS5PROXY_PASSWORD:=""} # PRICE_DATA_SERVER +__PRICE_DATA_SERVER_UPDATE_FREQUENCY__=${PRICE_DATA_SERVER_UPDATE_FREQUENCY:=300} __PRICE_DATA_SERVER_TOR_URL__=${PRICE_DATA_SERVER_TOR_URL:=http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices} __PRICE_DATA_SERVER_CLEARNET_URL__=${PRICE_DATA_SERVER_CLEARNET_URL:=https://price.bisq.wiz.biz/getAllMarketPrices} @@ -226,6 +227,7 @@ sed -i "s!__SOCKS5PROXY_PORT__!${__SOCKS5PROXY_PORT__}!g" mempool-config.json sed -i "s!__SOCKS5PROXY_USERNAME__!${__SOCKS5PROXY_USERNAME__}!g" mempool-config.json sed -i "s!__SOCKS5PROXY_PASSWORD__!${__SOCKS5PROXY_PASSWORD__}!g" mempool-config.json +sed -i "s!__PRICE_DATA_SERVER_UPDATE_FREQUENCY__!${__PRICE_DATA_SERVER_UPDATE_FREQUENCY__}!g" mempool-config.json sed -i "s!__PRICE_DATA_SERVER_TOR_URL__!${__PRICE_DATA_SERVER_TOR_URL__}!g" mempool-config.json sed -i "s!__PRICE_DATA_SERVER_CLEARNET_URL__!${__PRICE_DATA_SERVER_CLEARNET_URL__}!g" mempool-config.json diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index 5e25bcb76..769dd8019 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -49,6 +49,9 @@ "ENABLED": true, "TX_PER_SECOND_SAMPLE_PERIOD": 150 }, + "PRICE_DATA_SERVER": { + "UPDATE_FREQUENCY": 300 + }, "REPLICATION": { "ENABLED": true, "AUDIT": true, From 2e4306d795c5afd8486ca59b698b481361016931 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 29 Jul 2023 16:29:36 +0900 Subject: [PATCH 56/73] Update backend/src/__fixtures__/mempool-config.template.json --- backend/src/__fixtures__/mempool-config.template.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 3a2b151a5..5ed888962 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -93,7 +93,7 @@ "PASSWORD": "__SOCKS5PROXY_PASSWORD__" }, "PRICE_DATA_SERVER": { - "UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY", + "UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY__", "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" }, From bb7f73637bea426245280fc3b527f4d79cc63b68 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 29 Jul 2023 16:34:47 +0900 Subject: [PATCH 57/73] Update backend/src/api/prices/prices.routes.ts --- backend/src/api/prices/prices.routes.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/prices/prices.routes.ts b/backend/src/api/prices/prices.routes.ts index 03951ec05..c6a643dd8 100644 --- a/backend/src/api/prices/prices.routes.ts +++ b/backend/src/api/prices/prices.routes.ts @@ -7,7 +7,7 @@ class PricesRoutes { app.get(config.MEMPOOL.API_URL_PREFIX + 'prices', this.$getCurrentPrices.bind(this)); } - private async $getCurrentPrices(req: Request, res: Response): Promise { + private $getCurrentPrices(req: Request, res: Response): void { res.header('Pragma', 'public'); res.header('Cache-control', 'public'); res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); From 8c442fe2078af00f10cdfe8d919c46951d8c780e Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 29 Jul 2023 19:27:19 +0900 Subject: [PATCH 58/73] Refactoring price update config. Fixing last price time. --- backend/mempool-config.sample.json | 8 ++------ .../src/__fixtures__/mempool-config.template.json | 12 ++++-------- backend/src/__tests__/config.test.ts | 9 +-------- backend/src/config.ts | 14 ++------------ backend/src/indexer.ts | 8 ++++++-- backend/src/repositories/PricesRepository.ts | 2 +- backend/src/tasks/price-updater.ts | 10 +++++----- docker/backend/mempool-config.json | 8 ++------ production/mempool-config.mainnet.json | 6 ++---- 9 files changed, 25 insertions(+), 52 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index fddd44781..7f8bb4e22 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -32,7 +32,8 @@ "CPFP_INDEXING": false, "DISK_CACHE_BLOCK_INTERVAL": 6, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, - "ALLOW_UNREACHABLE": true + "ALLOW_UNREACHABLE": true, + "PRICE_UPDATE_FREQUENCY": 3600 }, "CORE_RPC": { "HOST": "127.0.0.1", @@ -115,11 +116,6 @@ "USERNAME": "", "PASSWORD": "" }, - "PRICE_DATA_SERVER": { - "UPDATE_FREQUENCY": 300, - "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices", - "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices" - }, "EXTERNAL_DATA_SERVER": { "MEMPOOL_API": "https://mempool.space/api/v1", "MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1", diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 5ed888962..583f8fb2d 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -23,8 +23,8 @@ "USER_AGENT": "__MEMPOOL_USER_AGENT__", "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__", "INDEXING_BLOCKS_AMOUNT": 14, - "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__", - "POOLS_JSON_URL": "__POOLS_JSON_URL__", + "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", + "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", "AUDIT": true, "ADVANCED_GBT_AUDIT": true, "ADVANCED_GBT_MEMPOOL": true, @@ -33,7 +33,8 @@ "MAX_BLOCKS_BULK_QUERY": 999, "DISK_CACHE_BLOCK_INTERVAL": 999, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, - "ALLOW_UNREACHABLE": true + "ALLOW_UNREACHABLE": true, + "PRICE_UPDATE_FREQUENCY": "__MEMPOOL_PRICE_UPDATE_FREQUENCY__" }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", @@ -92,11 +93,6 @@ "USERNAME": "__SOCKS5PROXY_USERNAME__", "PASSWORD": "__SOCKS5PROXY_PASSWORD__" }, - "PRICE_DATA_SERVER": { - "UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY__", - "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", - "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" - }, "EXTERNAL_DATA_SERVER": { "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__", "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__", diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index cb564a334..c42ccd223 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -47,6 +47,7 @@ describe('Mempool Backend Config', () => { DISK_CACHE_BLOCK_INTERVAL: 6, MAX_PUSH_TX_SIZE_WEIGHT: 400000, ALLOW_UNREACHABLE: true, + PRICE_UPDATE_FREQUENCY: 3600, }); expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true }); @@ -101,12 +102,6 @@ describe('Mempool Backend Config', () => { PASSWORD: '' }); - expect(config.PRICE_DATA_SERVER).toStrictEqual({ - UPDATE_FREQUENCY: 3600, - TOR_URL: 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', - CLEARNET_URL: 'https://price.bisq.wiz.biz/getAllMarketPrices' - }); - expect(config.EXTERNAL_DATA_SERVER).toStrictEqual({ MEMPOOL_API: 'https://mempool.space/api/v1', MEMPOOL_ONION: 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1', @@ -164,8 +159,6 @@ describe('Mempool Backend Config', () => { expect(config.SOCKS5PROXY).toStrictEqual(fixture.SOCKS5PROXY); - expect(config.PRICE_DATA_SERVER).toStrictEqual(fixture.PRICE_DATA_SERVER); - expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER); expect(config.REDIS).toStrictEqual(fixture.REDIS); diff --git a/backend/src/config.ts b/backend/src/config.ts index 5117983fb..1c59bcc46 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -38,6 +38,7 @@ interface IConfig { DISK_CACHE_BLOCK_INTERVAL: number; MAX_PUSH_TX_SIZE_WEIGHT: number; ALLOW_UNREACHABLE: boolean; + PRICE_UPDATE_FREQUENCY: number; }; ESPLORA: { REST_API_URL: string; @@ -115,11 +116,6 @@ interface IConfig { USERNAME: string; PASSWORD: string; }; - PRICE_DATA_SERVER: { - UPDATE_FREQUENCY: number; - TOR_URL: string; - CLEARNET_URL: string; - }; EXTERNAL_DATA_SERVER: { MEMPOOL_API: string; MEMPOOL_ONION: string; @@ -182,6 +178,7 @@ const defaults: IConfig = { 'DISK_CACHE_BLOCK_INTERVAL': 6, 'MAX_PUSH_TX_SIZE_WEIGHT': 400000, 'ALLOW_UNREACHABLE': true, + 'PRICE_UPDATE_FREQUENCY': 3600, }, 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', @@ -259,11 +256,6 @@ const defaults: IConfig = { 'USERNAME': '', 'PASSWORD': '' }, - 'PRICE_DATA_SERVER': { - 'UPDATE_FREQUENCY': 3600, - 'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', - 'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices' - }, 'EXTERNAL_DATA_SERVER': { 'MEMPOOL_API': 'https://mempool.space/api/v1', 'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1', @@ -304,7 +296,6 @@ class Config implements IConfig { LND: IConfig['LND']; CLIGHTNING: IConfig['CLIGHTNING']; SOCKS5PROXY: IConfig['SOCKS5PROXY']; - PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER']; EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER']; MAXMIND: IConfig['MAXMIND']; REPLICATION: IConfig['REPLICATION']; @@ -325,7 +316,6 @@ class Config implements IConfig { this.LND = configs.LND; this.CLIGHTNING = configs.CLIGHTNING; this.SOCKS5PROXY = configs.SOCKS5PROXY; - this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER; this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER; this.MAXMIND = configs.MAXMIND; this.REPLICATION = configs.REPLICATION; diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index d89a2647f..7ec65d9c9 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -105,6 +105,12 @@ class Indexer { return; } + try { + await priceUpdater.$run(); + } catch (e) { + logger.err(`Running priceUpdater failed. Reason: ` + (e instanceof Error ? e.message : e)); + } + // Do not attempt to index anything unless Bitcoin Core is fully synced const blockchainInfo = await bitcoinClient.getBlockchainInfo(); if (blockchainInfo.blocks !== blockchainInfo.headers) { @@ -119,8 +125,6 @@ class Indexer { await this.checkAvailableCoreIndexes(); try { - await priceUpdater.$run(); - const chainValid = await blocks.$generateBlockDatabase(); if (chainValid === false) { // Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index ed9d1fd72..7d77834f5 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -65,7 +65,7 @@ class PricesRepository { try { await DB.query(` - INSERT INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY) + INSERT IGNORE INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY) VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ? )`, [time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY] ); diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index 335a61065..91a36d04b 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -108,8 +108,8 @@ class PriceUpdater { this.lastRun = await PricesRepository.$getLatestPriceTime(); } - if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < config.PRICE_DATA_SERVER.UPDATE_FREQUENCY) { - // Refresh every UPDATE_FREQUENCY seconds at most + if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < config.MEMPOOL.PRICE_UPDATE_FREQUENCY) { + // Refresh every PRICE_UPDATE_FREQUENCY seconds at most return; } @@ -146,14 +146,11 @@ class PriceUpdater { } } - logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`); - if (config.DATABASE.ENABLED === true) { // Save everything in db try { const p = 60 * 60 * 1000; // milliseconds in an hour const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042 - this.latestPrices.time = nowRounded.getTime() / 1000; await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices); } catch (e) { this.lastRun = previousRun + 5 * 60; @@ -161,6 +158,9 @@ class PriceUpdater { } } + this.latestPrices.time = Math.round(new Date().getTime() / 1000); + logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`); + if (this.ratesChangedCallback) { this.ratesChangedCallback(this.latestPrices); } diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 8b8a79969..19d7965fd 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -33,7 +33,8 @@ "MAX_PUSH_TX_SIZE_WEIGHT": __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__, "ALLOW_UNREACHABLE": __MEMPOOL_ALLOW_UNREACHABLE__, "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", - "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__" + "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", + "PRICE_UPDATE_FREQUENCY": __MEMPOOL_PRICE_UPDATE_FREQUENCY__ }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", @@ -111,11 +112,6 @@ "USERNAME": "__SOCKS5PROXY_USERNAME__", "PASSWORD": "__SOCKS5PROXY_PASSWORD__" }, - "PRICE_DATA_SERVER": { - "UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY__", - "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", - "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" - }, "EXTERNAL_DATA_SERVER": { "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__", "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__", diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index 769dd8019..02078cf78 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -18,7 +18,8 @@ "USE_SECOND_NODE_FOR_MINFEE": true, "DISK_CACHE_BLOCK_INTERVAL": 1, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, - "ALLOW_UNREACHABLE": true + "ALLOW_UNREACHABLE": true, + "PRICE_UPDATE_FREQUENCY": 300 }, "SYSLOG" : { "MIN_PRIORITY": "debug" @@ -49,9 +50,6 @@ "ENABLED": true, "TX_PER_SECOND_SAMPLE_PERIOD": 150 }, - "PRICE_DATA_SERVER": { - "UPDATE_FREQUENCY": 300 - }, "REPLICATION": { "ENABLED": true, "AUDIT": true, From 9b24c0700242b235142cbfd11c717293f5f70768 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 30 Jul 2023 12:09:05 +0900 Subject: [PATCH 59/73] Updating start.sh and README --- docker/README.md | 4 +++- docker/backend/start.sh | 12 ++---------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/docker/README.md b/docker/README.md index d95bc7aee..f70b600e1 100644 --- a/docker/README.md +++ b/docker/README.md @@ -113,7 +113,8 @@ Below we list all settings from `mempool-config.json` and the corresponding over "ADVANCED_GBT_MEMPOOL": false, "CPFP_INDEXING": false, "MAX_BLOCKS_BULK_QUERY": 0, - "DISK_CACHE_BLOCK_INTERVAL": 6 + "DISK_CACHE_BLOCK_INTERVAL": 6, + "PRICE_UPDATE_FREQUENCY": 3600 }, ``` @@ -146,6 +147,7 @@ Corresponding `docker-compose.yml` overrides: MEMPOOL_CPFP_INDEXING: "" MEMPOOL_MAX_BLOCKS_BULK_QUERY: "" MEMPOOL_DISK_CACHE_BLOCK_INTERVAL: "" + MEMPOOL_PRICE_UPDATE_FREQUENCY: "" ... ``` diff --git a/docker/backend/start.sh b/docker/backend/start.sh index add483568..8f8c79b41 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -35,7 +35,7 @@ __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0} __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6} __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__=${MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT:=4000000} __MEMPOOL_ALLOW_UNREACHABLE__=${MEMPOOL_ALLOW_UNREACHABLE:=true} - +__MEMPOOL_PRICE_UPDATE_FREQUENCY__=${MEMPOOL_PRICE_UPDATE_FREQUENCY:=3600} # CORE_RPC __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1} @@ -94,11 +94,6 @@ __SOCKS5PROXY_PORT__=${SOCKS5PROXY_PORT:=9050} __SOCKS5PROXY_USERNAME__=${SOCKS5PROXY_USERNAME:=""} __SOCKS5PROXY_PASSWORD__=${SOCKS5PROXY_PASSWORD:=""} -# PRICE_DATA_SERVER -__PRICE_DATA_SERVER_UPDATE_FREQUENCY__=${PRICE_DATA_SERVER_UPDATE_FREQUENCY:=300} -__PRICE_DATA_SERVER_TOR_URL__=${PRICE_DATA_SERVER_TOR_URL:=http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices} -__PRICE_DATA_SERVER_CLEARNET_URL__=${PRICE_DATA_SERVER_CLEARNET_URL:=https://price.bisq.wiz.biz/getAllMarketPrices} - # EXTERNAL_DATA_SERVER __EXTERNAL_DATA_SERVER_MEMPOOL_API__=${EXTERNAL_DATA_SERVER_MEMPOOL_API:=https://mempool.space/api/v1} __EXTERNAL_DATA_SERVER_MEMPOOL_ONION__=${EXTERNAL_DATA_SERVER_MEMPOOL_ONION:=http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1} @@ -178,6 +173,7 @@ sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__} sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json sed -i "s!__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__!${__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__}!g" mempool-config.json sed -i "s!__MEMPOOL_ALLOW_UNREACHABLE__!${__MEMPOOL_ALLOW_UNREACHABLE__}!g" mempool-config.json +sed -i "s!__MEMPOOL_PRICE_UPDATE_FREQUENCY__!${__MEMPOOL_PRICE_UPDATE_FREQUENCY__}!g" mempool-config.json sed -i "s!__CORE_RPC_HOST__!${__CORE_RPC_HOST__}!g" mempool-config.json sed -i "s!__CORE_RPC_PORT__!${__CORE_RPC_PORT__}!g" mempool-config.json @@ -227,10 +223,6 @@ sed -i "s!__SOCKS5PROXY_PORT__!${__SOCKS5PROXY_PORT__}!g" mempool-config.json sed -i "s!__SOCKS5PROXY_USERNAME__!${__SOCKS5PROXY_USERNAME__}!g" mempool-config.json sed -i "s!__SOCKS5PROXY_PASSWORD__!${__SOCKS5PROXY_PASSWORD__}!g" mempool-config.json -sed -i "s!__PRICE_DATA_SERVER_UPDATE_FREQUENCY__!${__PRICE_DATA_SERVER_UPDATE_FREQUENCY__}!g" mempool-config.json -sed -i "s!__PRICE_DATA_SERVER_TOR_URL__!${__PRICE_DATA_SERVER_TOR_URL__}!g" mempool-config.json -sed -i "s!__PRICE_DATA_SERVER_CLEARNET_URL__!${__PRICE_DATA_SERVER_CLEARNET_URL__}!g" mempool-config.json - sed -i "s!__EXTERNAL_DATA_SERVER_MEMPOOL_API__!${__EXTERNAL_DATA_SERVER_MEMPOOL_API__}!g" mempool-config.json sed -i "s!__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__!${__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__}!g" mempool-config.json sed -i "s!__EXTERNAL_DATA_SERVER_LIQUID_API__!${__EXTERNAL_DATA_SERVER_LIQUID_API__}!g" mempool-config.json From 833153556214776ab6f2c505cd1db51608cd973b Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 30 Jul 2023 13:52:47 +0900 Subject: [PATCH 60/73] Setting fixture to fixed number --- backend/src/__fixtures__/mempool-config.template.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 583f8fb2d..e3e8f5b14 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -34,7 +34,7 @@ "DISK_CACHE_BLOCK_INTERVAL": 999, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, - "PRICE_UPDATE_FREQUENCY": "__MEMPOOL_PRICE_UPDATE_FREQUENCY__" + "PRICE_UPDATE_FREQUENCY": 3600 }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", From 4ffc3d0ce51be9ce5eb1a9f7c847468fbb37eb62 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 30 Jul 2023 15:30:23 +0900 Subject: [PATCH 61/73] Allow priceUpdater to run without storing to database --- backend/src/index.ts | 1 + backend/src/indexer.ts | 2 +- backend/src/tasks/price-updater.ts | 8 ++++---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index b00909bc6..adb3f2e02 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -194,6 +194,7 @@ class Server { await memPool.$updateMempool(newMempool, pollRate); } indexer.$run(); + priceUpdater.$run(); // rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS const elapsed = Date.now() - start; diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 7ec65d9c9..392174ecf 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -106,7 +106,7 @@ class Indexer { } try { - await priceUpdater.$run(); + await priceUpdater.$run(true); } catch (e) { logger.err(`Running priceUpdater failed. Reason: ` + (e instanceof Error ? e.message : e)); } diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index 91a36d04b..f96616c12 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -72,7 +72,7 @@ class PriceUpdater { this.latestPrices = await PricesRepository.$getLatestConversionRates(); } - public async $run(): Promise { + public async $run(storeInDb: boolean = false): Promise { if (config.MEMPOOL.NETWORK === 'signet' || config.MEMPOOL.NETWORK === 'testnet') { // Coins have no value on testnet/signet, so we want to always show 0 return; @@ -89,7 +89,7 @@ class PriceUpdater { } try { - await this.$updatePrice(); + await this.$updatePrice(storeInDb); if (this.historyInserted === false && config.DATABASE.ENABLED === true) { await this.$insertHistoricalPrices(); } @@ -103,7 +103,7 @@ class PriceUpdater { /** * Fetch last BTC price from exchanges, average them, and save it in the database once every hour */ - private async $updatePrice(): Promise { + private async $updatePrice(storeInDb: boolean): Promise { if (this.lastRun === 0 && config.DATABASE.ENABLED === true) { this.lastRun = await PricesRepository.$getLatestPriceTime(); } @@ -146,7 +146,7 @@ class PriceUpdater { } } - if (config.DATABASE.ENABLED === true) { + if (config.DATABASE.ENABLED === true && storeInDb) { // Save everything in db try { const p = 60 * 60 * 1000; // milliseconds in an hour From 16b75c6ab344af420b2d35e8a09a8dc26c018482 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 30 Jul 2023 16:35:37 +0900 Subject: [PATCH 62/73] Update price at least every hour --- backend/src/tasks/price-updater.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index f96616c12..05d6f0bec 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -108,8 +108,8 @@ class PriceUpdater { this.lastRun = await PricesRepository.$getLatestPriceTime(); } - if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < config.MEMPOOL.PRICE_UPDATE_FREQUENCY) { - // Refresh every PRICE_UPDATE_FREQUENCY seconds at most + if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < Math.min(config.MEMPOOL.PRICE_UPDATE_FREQUENCY, 3600)) { + // Refresh at least every hour or PRICE_UPDATE_FREQUENCY seconds return; } From f7d1e2818036e49c34c074c3b586cb4da93eed6d Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 30 Jul 2023 16:37:24 +0900 Subject: [PATCH 63/73] Erasing unused PRICE_DATA_SERVER from readme --- docker/README.md | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/docker/README.md b/docker/README.md index f70b600e1..4270d11c8 100644 --- a/docker/README.md +++ b/docker/README.md @@ -365,25 +365,6 @@ Corresponding `docker-compose.yml` overrides:
-`mempool-config.json`: -```json - "PRICE_DATA_SERVER": { - "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices", - "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices" - } -``` - -Corresponding `docker-compose.yml` overrides: -```yaml - api: - environment: - PRICE_DATA_SERVER_TOR_URL: "" - PRICE_DATA_SERVER_CLEARNET_URL: "" - ... -``` - -
- `mempool-config.json`: ```json "LIGHTNING": { From a59144ec843640da5dfd2171b458990e13ec381c Mon Sep 17 00:00:00 2001 From: softsimon Date: Thu, 3 Aug 2023 17:42:41 +0900 Subject: [PATCH 64/73] Changing setting to per hour --- backend/mempool-config.sample.json | 2 +- .../__fixtures__/mempool-config.template.json | 2 +- backend/src/__tests__/config.test.ts | 2 +- backend/src/config.ts | 4 +- backend/src/indexer.ts | 2 +- backend/src/repositories/PricesRepository.ts | 2 +- backend/src/tasks/price-updater.ts | 56 ++++++++++++++----- docker/README.md | 4 +- docker/backend/mempool-config.json | 2 +- docker/backend/start.sh | 4 +- production/mempool-config.mainnet.json | 2 +- 11 files changed, 55 insertions(+), 27 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 7f8bb4e22..8fd59a430 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -33,7 +33,7 @@ "DISK_CACHE_BLOCK_INTERVAL": 6, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, - "PRICE_UPDATE_FREQUENCY": 3600 + "PRICE_UPDATES_PER_HOUR": 3600 }, "CORE_RPC": { "HOST": "127.0.0.1", diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index e3e8f5b14..471b7bb47 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -34,7 +34,7 @@ "DISK_CACHE_BLOCK_INTERVAL": 999, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, - "PRICE_UPDATE_FREQUENCY": 3600 + "PRICE_UPDATES_PER_HOUR": 1 }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index c42ccd223..9cd547b90 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -47,7 +47,7 @@ describe('Mempool Backend Config', () => { DISK_CACHE_BLOCK_INTERVAL: 6, MAX_PUSH_TX_SIZE_WEIGHT: 400000, ALLOW_UNREACHABLE: true, - PRICE_UPDATE_FREQUENCY: 3600, + PRICE_UPDATES_PER_HOUR: 1, }); expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true }); diff --git a/backend/src/config.ts b/backend/src/config.ts index 1c59bcc46..2e89bd525 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -38,7 +38,7 @@ interface IConfig { DISK_CACHE_BLOCK_INTERVAL: number; MAX_PUSH_TX_SIZE_WEIGHT: number; ALLOW_UNREACHABLE: boolean; - PRICE_UPDATE_FREQUENCY: number; + PRICE_UPDATES_PER_HOUR: number; }; ESPLORA: { REST_API_URL: string; @@ -178,7 +178,7 @@ const defaults: IConfig = { 'DISK_CACHE_BLOCK_INTERVAL': 6, 'MAX_PUSH_TX_SIZE_WEIGHT': 400000, 'ALLOW_UNREACHABLE': true, - 'PRICE_UPDATE_FREQUENCY': 3600, + 'PRICE_UPDATES_PER_HOUR': 1, }, 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 392174ecf..7ec65d9c9 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -106,7 +106,7 @@ class Indexer { } try { - await priceUpdater.$run(true); + await priceUpdater.$run(); } catch (e) { logger.err(`Running priceUpdater failed. Reason: ` + (e instanceof Error ? e.message : e)); } diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index 7d77834f5..ed9d1fd72 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -65,7 +65,7 @@ class PricesRepository { try { await DB.query(` - INSERT IGNORE INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY) + INSERT INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY) VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ? )`, [time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY] ); diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index 05d6f0bec..bbfb7d167 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -25,7 +25,9 @@ export interface PriceHistory { class PriceUpdater { public historyInserted = false; - private lastRun = 0; + private timeBetweenUpdatesMs = 3600000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR; + private cyclePosition = -1; + private firstRun = true; private lastHistoricalRun = 0; private running = false; private feeds: PriceFeed[] = []; @@ -41,6 +43,8 @@ class PriceUpdater { this.feeds.push(new CoinbaseApi()); this.feeds.push(new BitfinexApi()); this.feeds.push(new GeminiApi()); + + this.setCyclePosition(); } public getLatestPrices(): ApiPrice { @@ -72,7 +76,7 @@ class PriceUpdater { this.latestPrices = await PricesRepository.$getLatestConversionRates(); } - public async $run(storeInDb: boolean = false): Promise { + public async $run(): Promise { if (config.MEMPOOL.NETWORK === 'signet' || config.MEMPOOL.NETWORK === 'testnet') { // Coins have no value on testnet/signet, so we want to always show 0 return; @@ -89,7 +93,7 @@ class PriceUpdater { } try { - await this.$updatePrice(storeInDb); + await this.$updatePrice(); if (this.historyInserted === false && config.DATABASE.ENABLED === true) { await this.$insertHistoricalPrices(); } @@ -100,22 +104,42 @@ class PriceUpdater { this.running = false; } + private getMillisecondsSinceBeginningOfHour(): number { + const now = new Date(); + const beginningOfHour = new Date(now); + beginningOfHour.setMinutes(0, 0, 0); + return now.getTime() - beginningOfHour.getTime(); + } + + private setCyclePosition(): void { + const millisecondsSinceBeginningOfHour = this.getMillisecondsSinceBeginningOfHour(); + for (let i = 0; i < config.MEMPOOL.PRICE_UPDATES_PER_HOUR; i++) { + if (this.timeBetweenUpdatesMs * i > millisecondsSinceBeginningOfHour) { + this.cyclePosition = i; + return; + } + } + this.cyclePosition = config.MEMPOOL.PRICE_UPDATES_PER_HOUR; + } + /** * Fetch last BTC price from exchanges, average them, and save it in the database once every hour */ - private async $updatePrice(storeInDb: boolean): Promise { - if (this.lastRun === 0 && config.DATABASE.ENABLED === true) { - this.lastRun = await PricesRepository.$getLatestPriceTime(); + private async $updatePrice(): Promise { + let forceUpdate = false; + if (this.firstRun === true && config.DATABASE.ENABLED === true) { + const lastUpdate = await PricesRepository.$getLatestPriceTime(); + if (new Date().getTime() / 1000 - lastUpdate > this.timeBetweenUpdatesMs / 1000) { + forceUpdate = true; + } + this.firstRun = false; } - if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < Math.min(config.MEMPOOL.PRICE_UPDATE_FREQUENCY, 3600)) { - // Refresh at least every hour or PRICE_UPDATE_FREQUENCY seconds + const millisecondsSinceBeginningOfHour = this.getMillisecondsSinceBeginningOfHour(); + if (millisecondsSinceBeginningOfHour < this.timeBetweenUpdatesMs * this.cyclePosition && !forceUpdate) { return; } - const previousRun = this.lastRun; - this.lastRun = new Date().getTime() / 1000; - for (const currency of this.currencies) { let prices: number[] = []; @@ -146,14 +170,13 @@ class PriceUpdater { } } - if (config.DATABASE.ENABLED === true && storeInDb) { + if (config.DATABASE.ENABLED === true && this.cyclePosition === 0) { // Save everything in db try { const p = 60 * 60 * 1000; // milliseconds in an hour const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042 await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices); } catch (e) { - this.lastRun = previousRun + 5 * 60; logger.err(`Cannot save latest prices into db. Trying again in 5 minutes. Reason: ${(e instanceof Error ? e.message : e)}`); } } @@ -165,7 +188,12 @@ class PriceUpdater { this.ratesChangedCallback(this.latestPrices); } - this.lastRun = new Date().getTime() / 1000; + if (!forceUpdate) { + this.cyclePosition++; + if (this.cyclePosition > config.MEMPOOL.PRICE_UPDATES_PER_HOUR) { + this.cyclePosition = 0; + } + } if (this.latestPrices.USD === -1) { this.latestPrices = await PricesRepository.$getLatestConversionRates(); diff --git a/docker/README.md b/docker/README.md index 4270d11c8..13bda7ec6 100644 --- a/docker/README.md +++ b/docker/README.md @@ -114,7 +114,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over "CPFP_INDEXING": false, "MAX_BLOCKS_BULK_QUERY": 0, "DISK_CACHE_BLOCK_INTERVAL": 6, - "PRICE_UPDATE_FREQUENCY": 3600 + "PRICE_UPDATES_PER_HOUR": 1 }, ``` @@ -147,7 +147,7 @@ Corresponding `docker-compose.yml` overrides: MEMPOOL_CPFP_INDEXING: "" MEMPOOL_MAX_BLOCKS_BULK_QUERY: "" MEMPOOL_DISK_CACHE_BLOCK_INTERVAL: "" - MEMPOOL_PRICE_UPDATE_FREQUENCY: "" + MEMPOOL_PRICE_UPDATES_PER_HOUR: "" ... ``` diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 19d7965fd..216ad95c2 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -34,7 +34,7 @@ "ALLOW_UNREACHABLE": __MEMPOOL_ALLOW_UNREACHABLE__, "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", - "PRICE_UPDATE_FREQUENCY": __MEMPOOL_PRICE_UPDATE_FREQUENCY__ + "PRICE_UPDATES_PER_HOUR": __MEMPOOL_PRICE_UPDATES_PER_HOUR__ }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 8f8c79b41..30d2ac332 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -35,7 +35,7 @@ __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0} __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6} __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__=${MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT:=4000000} __MEMPOOL_ALLOW_UNREACHABLE__=${MEMPOOL_ALLOW_UNREACHABLE:=true} -__MEMPOOL_PRICE_UPDATE_FREQUENCY__=${MEMPOOL_PRICE_UPDATE_FREQUENCY:=3600} +__MEMPOOL_PRICE_UPDATES_PER_HOUR__=${MEMPOOL_PRICE_UPDATES_PER_HOUR:=3600} # CORE_RPC __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1} @@ -173,7 +173,7 @@ sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__} sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json sed -i "s!__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__!${__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__}!g" mempool-config.json sed -i "s!__MEMPOOL_ALLOW_UNREACHABLE__!${__MEMPOOL_ALLOW_UNREACHABLE__}!g" mempool-config.json -sed -i "s!__MEMPOOL_PRICE_UPDATE_FREQUENCY__!${__MEMPOOL_PRICE_UPDATE_FREQUENCY__}!g" mempool-config.json +sed -i "s!__MEMPOOL_PRICE_UPDATES_PER_HOUR__!${__MEMPOOL_PRICE_UPDATES_PER_HOUR__}!g" mempool-config.json sed -i "s!__CORE_RPC_HOST__!${__CORE_RPC_HOST__}!g" mempool-config.json sed -i "s!__CORE_RPC_PORT__!${__CORE_RPC_PORT__}!g" mempool-config.json diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index 02078cf78..f54635415 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -19,7 +19,7 @@ "DISK_CACHE_BLOCK_INTERVAL": 1, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, - "PRICE_UPDATE_FREQUENCY": 300 + "PRICE_UPDATES_PER_HOUR": 12 }, "SYSLOG" : { "MIN_PRIORITY": "debug" From 8aeab0fb3d525b2b8171578945c39934898a1a1c Mon Sep 17 00:00:00 2001 From: softsimon Date: Thu, 3 Aug 2023 18:26:24 +0900 Subject: [PATCH 65/73] Updating config sample --- backend/mempool-config.sample.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 8fd59a430..c53139b0b 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -33,7 +33,7 @@ "DISK_CACHE_BLOCK_INTERVAL": 6, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, - "PRICE_UPDATES_PER_HOUR": 3600 + "PRICE_UPDATES_PER_HOUR": 1 }, "CORE_RPC": { "HOST": "127.0.0.1", From 7ddb097e02bc9c1c633739127fe995a3aaf34310 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 4 Aug 2023 10:23:15 +0900 Subject: [PATCH 66/73] [about] re-enable two sponsor buttons --- frontend/src/app/components/about/about.component.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index c93353e4f..551489b45 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -34,7 +34,8 @@

Sponsor the project

- See plans + Community + Enterprise
From dab95ef55dc3102698aed9192a25ded0f58382f4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Aug 2023 02:40:58 +0000 Subject: [PATCH 67/73] Bump mysql2 from 3.5.2 to 3.6.0 in /backend Bumps [mysql2](https://github.com/sidorares/node-mysql2) from 3.5.2 to 3.6.0. - [Release notes](https://github.com/sidorares/node-mysql2/releases) - [Changelog](https://github.com/sidorares/node-mysql2/blob/master/Changelog.md) - [Commits](https://github.com/sidorares/node-mysql2/compare/v3.5.2...v3.6.0) --- updated-dependencies: - dependency-name: mysql2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- backend/package-lock.json | 16 ++++++++-------- backend/package.json | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/backend/package-lock.json b/backend/package-lock.json index 1a92552cb..f5452e908 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -17,9 +17,9 @@ "crypto-js": "~4.1.1", "express": "~4.18.2", "maxmind": "~4.3.11", - "mysql2": "~3.5.2", - "rust-gbt": "file:./rust-gbt", + "mysql2": "~3.6.0", "redis": "^4.6.6", + "rust-gbt": "file:./rust-gbt", "socks-proxy-agent": "~7.0.0", "typescript": "~4.9.3", "ws": "~8.13.0" @@ -6102,9 +6102,9 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node_modules/mysql2": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.5.2.tgz", - "integrity": "sha512-cptobmhYkYeTBIFp2c0piw2+gElpioga1rUw5UidHvo8yaHijMZoo8A3zyBVoo/K71f7ZFvrShA9iMIy9dCzCA==", + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.6.0.tgz", + "integrity": "sha512-EWUGAhv6SphezurlfI2Fpt0uJEWLmirrtQR7SkbTHFC+4/mJBrPiSzHESHKAWKG7ALVD6xaG/NBjjd1DGJGQQQ==", "dependencies": { "denque": "^2.1.0", "generate-function": "^2.3.1", @@ -12212,9 +12212,9 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "mysql2": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.5.2.tgz", - "integrity": "sha512-cptobmhYkYeTBIFp2c0piw2+gElpioga1rUw5UidHvo8yaHijMZoo8A3zyBVoo/K71f7ZFvrShA9iMIy9dCzCA==", + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.6.0.tgz", + "integrity": "sha512-EWUGAhv6SphezurlfI2Fpt0uJEWLmirrtQR7SkbTHFC+4/mJBrPiSzHESHKAWKG7ALVD6xaG/NBjjd1DGJGQQQ==", "requires": { "denque": "^2.1.0", "generate-function": "^2.3.1", diff --git a/backend/package.json b/backend/package.json index 24da55e17..36268227c 100644 --- a/backend/package.json +++ b/backend/package.json @@ -45,7 +45,7 @@ "crypto-js": "~4.1.1", "express": "~4.18.2", "maxmind": "~4.3.11", - "mysql2": "~3.5.2", + "mysql2": "~3.6.0", "rust-gbt": "file:./rust-gbt", "redis": "^4.6.6", "socks-proxy-agent": "~7.0.0", From 6bcea3abf9630e0d9dd4254af07d6af5e0b560f7 Mon Sep 17 00:00:00 2001 From: softsimon Date: Fri, 4 Aug 2023 13:23:09 +0900 Subject: [PATCH 68/73] Fixing cycle reset at top of the hour --- backend/src/tasks/price-updater.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index bbfb7d167..79173ad37 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -28,6 +28,7 @@ class PriceUpdater { private timeBetweenUpdatesMs = 3600000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR; private cyclePosition = -1; private firstRun = true; + private lastTime = -1; private lastHistoricalRun = 0; private running = false; private feeds: PriceFeed[] = []; @@ -136,7 +137,13 @@ class PriceUpdater { } const millisecondsSinceBeginningOfHour = this.getMillisecondsSinceBeginningOfHour(); - if (millisecondsSinceBeginningOfHour < this.timeBetweenUpdatesMs * this.cyclePosition && !forceUpdate) { + + // Reset the cycle on new hour + if (this.lastTime > millisecondsSinceBeginningOfHour) { + this.cyclePosition = 0; + } + this.lastTime = millisecondsSinceBeginningOfHour; + if (millisecondsSinceBeginningOfHour < this.timeBetweenUpdatesMs * this.cyclePosition && !forceUpdate && this.cyclePosition !== 0) { return; } @@ -190,9 +197,6 @@ class PriceUpdater { if (!forceUpdate) { this.cyclePosition++; - if (this.cyclePosition > config.MEMPOOL.PRICE_UPDATES_PER_HOUR) { - this.cyclePosition = 0; - } } if (this.latestPrices.USD === -1) { From c779a0f55bf36bf12c4f4bce5736ec689493a17a Mon Sep 17 00:00:00 2001 From: softsimon Date: Fri, 4 Aug 2023 13:26:19 +0900 Subject: [PATCH 69/73] Base expiry on update frequency --- backend/src/api/prices/prices.routes.ts | 2 +- backend/src/tasks/price-updater.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/api/prices/prices.routes.ts b/backend/src/api/prices/prices.routes.ts index c6a643dd8..b46331b73 100644 --- a/backend/src/api/prices/prices.routes.ts +++ b/backend/src/api/prices/prices.routes.ts @@ -10,7 +10,7 @@ class PricesRoutes { private $getCurrentPrices(req: Request, res: Response): void { res.header('Pragma', 'public'); res.header('Cache-control', 'public'); - res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); + res.setHeader('Expires', new Date(Date.now() + 360_0000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR).toUTCString()); res.json(pricesUpdater.getLatestPrices()); } diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index 79173ad37..fd799fb87 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -25,7 +25,7 @@ export interface PriceHistory { class PriceUpdater { public historyInserted = false; - private timeBetweenUpdatesMs = 3600000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR; + private timeBetweenUpdatesMs = 360_0000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR; private cyclePosition = -1; private firstRun = true; private lastTime = -1; From 35518d093760a8e806ec1485afe568c562ec4663 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 4 Aug 2023 14:30:40 +0900 Subject: [PATCH 70/73] hide sponsor buttons --- frontend/src/app/components/about/about.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index 551489b45..e9d5ec3b2 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -31,7 +31,7 @@ - +

Sponsor the project

Community From 2b59d517351e2e21ea5bb2ce4d4808b29151d6b2 Mon Sep 17 00:00:00 2001 From: mononaut <83316221+mononaut@users.noreply.github.com> Date: Fri, 4 Aug 2023 15:23:55 +0900 Subject: [PATCH 71/73] Fix PRICE_UPDATES_PER_HOUR docker default Co-authored-by: nymkappa <9780671+nymkappa@users.noreply.github.com> --- docker/backend/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 30d2ac332..008c0aaf5 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -35,7 +35,7 @@ __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0} __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6} __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__=${MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT:=4000000} __MEMPOOL_ALLOW_UNREACHABLE__=${MEMPOOL_ALLOW_UNREACHABLE:=true} -__MEMPOOL_PRICE_UPDATES_PER_HOUR__=${MEMPOOL_PRICE_UPDATES_PER_HOUR:=3600} +__MEMPOOL_PRICE_UPDATES_PER_HOUR__=${MEMPOOL_PRICE_UPDATES_PER_HOUR:=1} # CORE_RPC __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1} From bffb7b144ba65c559d9a5917b0c34e16591d7d17 Mon Sep 17 00:00:00 2001 From: junderw Date: Thu, 3 Aug 2023 18:46:18 -0700 Subject: [PATCH 72/73] Fix: Rust-GBT packaging needs fixing --- backend/npm_package.sh | 14 ++++++++++++++ backend/npm_package_rm_build_deps.sh | 12 ++++++++++++ backend/package.json | 9 ++++----- backend/src/__tests__/gbt/gbt-tests.ts | 2 +- backend/src/api/mempool-blocks.ts | 2 +- 5 files changed, 32 insertions(+), 7 deletions(-) create mode 100755 backend/npm_package.sh create mode 100755 backend/npm_package_rm_build_deps.sh diff --git a/backend/npm_package.sh b/backend/npm_package.sh new file mode 100755 index 000000000..627a77360 --- /dev/null +++ b/backend/npm_package.sh @@ -0,0 +1,14 @@ +#/bin/sh +set -e + +npm run build +# Remove previous package folder +rm -rf package +# Move JS and deps +mv dist package +mv node_modules package +# Remove symlink for rust-gbt and insert real folder +rm package/node_modules/rust-gbt +mv rust-gbt package/node_modules +# Clean up deps +npm run package-rm-build-deps diff --git a/backend/npm_package_rm_build_deps.sh b/backend/npm_package_rm_build_deps.sh new file mode 100755 index 000000000..6b260d84d --- /dev/null +++ b/backend/npm_package_rm_build_deps.sh @@ -0,0 +1,12 @@ +#/bin/sh +set -e + +# Cleaning up inside the node_modules folder +cd package/node_modules +rm -r \ + typescript \ + @typescript-eslint \ + @napi-rs \ + ./rust-gbt/src \ + ./rust-gbt/Cargo.toml \ + ./rust-gbt/build.rs diff --git a/backend/package.json b/backend/package.json index 36268227c..e21924207 100644 --- a/backend/package.json +++ b/backend/package.json @@ -22,10 +22,10 @@ "main": "index.ts", "scripts": { "tsc": "./node_modules/typescript/bin/tsc -p tsconfig.build.json", - "build": "npm run build-rust && npm run tsc && npm run create-resources", + "build": "npm run tsc && npm run create-resources", "create-resources": "cp ./src/tasks/price-feeds/mtgox-weekly.json ./dist/tasks && node dist/api/fetch-version.js", - "package": "npm run build && rm -rf package && mv dist package && mv node_modules package && mv rust-gbt package && npm run package-rm-build-deps", - "package-rm-build-deps": "(cd package/node_modules; rm -r typescript @typescript-eslint @napi-rs ../rust-gbt/target ../rust-gbt/node_modules ../rust-gbt/src)", + "package": "./npm_package.sh", + "package-rm-build-deps": "./npm_package_rm_build_deps.sh", "start": "node --max-old-space-size=2048 dist/index.js", "start-production": "node --max-old-space-size=16384 dist/index.js", "reindex-updated-pools": "npm run start-production --update-pools", @@ -33,8 +33,7 @@ "test": "./node_modules/.bin/jest --coverage", "lint": "./node_modules/.bin/eslint . --ext .ts", "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix", - "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\"", - "build-rust": "cd rust-gbt && npm install" + "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\"" }, "dependencies": { "@babel/core": "^7.21.3", diff --git a/backend/src/__tests__/gbt/gbt-tests.ts b/backend/src/__tests__/gbt/gbt-tests.ts index 0c2eb0176..8a3995f71 100644 --- a/backend/src/__tests__/gbt/gbt-tests.ts +++ b/backend/src/__tests__/gbt/gbt-tests.ts @@ -1,5 +1,5 @@ import fs from 'fs'; -import { GbtGenerator, ThreadTransaction } from '../../../rust-gbt'; +import { GbtGenerator, ThreadTransaction } from 'rust-gbt'; import path from 'path'; const baseline = require('./test-data/target-template.json'); diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 1ea4d2af8..16cf60a2f 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,4 +1,4 @@ -import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from '../../rust-gbt'; +import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from 'rust-gbt'; import logger from '../logger'; import { MempoolBlock, MempoolTransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, PoolTag } from '../mempool.interfaces'; import { Common, OnlineFeeStatsCalculator } from './common'; From 8d6e5a8ad95e877b2bde151687ac226efba4a2cf Mon Sep 17 00:00:00 2001 From: softsimon Date: Fri, 4 Aug 2023 15:55:22 +0900 Subject: [PATCH 73/73] Removing console log --- backend/src/api/mempool-blocks.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 16cf60a2f..1de4bbee7 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -341,7 +341,6 @@ class MempoolBlocks { } public async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { - console.log('$rustMakeBlockTemplates'); const start = Date.now(); // reset mempool short ids