From ce950d63cb4fcb9f2a5c530753f2599b4e7f0588 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Mon, 12 Jun 2023 07:07:15 -0400 Subject: [PATCH 01/73] Replace disclaimer text regarding tx acceleration --- frontend/src/app/docs/api-docs/api-docs.component.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/docs/api-docs/api-docs.component.html b/frontend/src/app/docs/api-docs/api-docs.component.html index 392cda19e..b788d2739 100644 --- a/frontend/src/app/docs/api-docs/api-docs.component.html +++ b/frontend/src/app/docs/api-docs/api-docs.component.html @@ -10,8 +10,8 @@
-

mempool.space merely provides data about the Bitcoin network. It cannot help you with retrieving funds, confirming your transaction quicker, etc.

For any such requests, you need to get in touch with the entity that helped make the transaction (wallet software, exchange company, etc).

-

mempool.space merely provides data about the Bitcoin network. It cannot help you with retrieving funds, confirming your transaction quicker, etc.

For any such requests, you need to get in touch with the entity that helped make the transaction (wallet software, exchange company, etc).

+

mempool.space merely provides data about the Bitcoin network. It cannot help you with retrieving funds, wallet issues, etc.

For any such requests, you need to get in touch with the entity that helped make the transaction (wallet software, exchange company, etc).

+

mempool.space merely provides data about the Bitcoin network. It cannot help you with retrieving funds, wallet issues, etc.

For any such requests, you need to get in touch with the entity that helped make the transaction (wallet software, exchange company, etc).

From 2d4bc9dbd6d5082097f4dfdac3d03e46543df629 Mon Sep 17 00:00:00 2001 From: wiz Date: Sun, 16 Jul 2023 17:50:36 +0900 Subject: [PATCH 02/73] ops: Move electrs scripts to mempool/electrs repo --- production/electrs-start-liquid | 24 ------------------ production/electrs-start-liquidtestnet | 24 ------------------ production/electrs-start-mainnet | 22 ---------------- production/electrs-start-signet | 23 ----------------- production/electrs-start-testnet | 23 ----------------- production/install | 35 +++----------------------- 6 files changed, 3 insertions(+), 148 deletions(-) delete mode 100755 production/electrs-start-liquid delete mode 100755 production/electrs-start-liquidtestnet delete mode 100755 production/electrs-start-mainnet delete mode 100755 production/electrs-start-signet delete mode 100755 production/electrs-start-testnet diff --git a/production/electrs-start-liquid b/production/electrs-start-liquid deleted file mode 100755 index a28135836..000000000 --- a/production/electrs-start-liquid +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env zsh -cd "${HOME}/electrs" -#source "${HOME}/.cargo/env" -#export PATH="${HOME}/.cargo/bin:${PATH}" - -until false -do - cargo run \ - --release \ - --features liquid \ - --bin electrs \ - -- \ - -vvv \ - --asset-db-path "${HOME}/asset_registry_db" \ - --address-search \ - --cors '*' \ - --db-dir __ELECTRS_DATA_ROOT__ \ - --network liquid \ - --daemon-dir "${HOME}" \ - --http-socket-file '/elements/socket/esplora-liquid-mainnet' \ - --cookie '__ELEMENTS_RPC_USER__:__ELEMENTS_RPC_PASS__' \ - --precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt" - sleep 1 -done diff --git a/production/electrs-start-liquidtestnet b/production/electrs-start-liquidtestnet deleted file mode 100755 index 828e96533..000000000 --- a/production/electrs-start-liquidtestnet +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/local/bin/zsh -cd "${HOME}/electrs" -#source "${HOME}/.cargo/env" -#export PATH="${HOME}/.cargo/bin:${PATH}" - -until false -do - cargo run \ - --release \ - --features liquid \ - --bin electrs \ - -- \ - -vv \ - --asset-db-path "${HOME}/asset_registry_testnet_db" \ - --address-search \ - --cors '*' \ - --db-dir __ELECTRS_DATA_ROOT__ \ - --network liquidtestnet \ - --daemon-dir "${HOME}" \ - --http-socket-file '/elements/socket/esplora-liquid-testnet' \ - --cookie '__ELEMENTS_RPC_USER__:__ELEMENTS_RPC_PASS__' \ - --precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt" - sleep 1 -done diff --git a/production/electrs-start-mainnet b/production/electrs-start-mainnet deleted file mode 100755 index c6a8c4d54..000000000 --- a/production/electrs-start-mainnet +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env zsh -cd "${HOME}/electrs" -#source "${HOME}/.cargo/env" -#export PATH="${HOME}/.cargo/bin:${PATH}" - -until false -do - cargo run \ - --release \ - --bin electrs \ - -- \ - -vvvv \ - --address-search \ - --cors '*' \ - --db-dir __ELECTRS_DATA_ROOT__ \ - --daemon-dir "${HOME}" \ - --http-socket-file '/bitcoin/socket/esplora-bitcoin-mainnet' \ - --cookie '__BITCOIN_RPC_USER__:__BITCOIN_RPC_PASS__' \ - --precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt" - - sleep 3 -done diff --git a/production/electrs-start-signet b/production/electrs-start-signet deleted file mode 100755 index 40e1d1115..000000000 --- a/production/electrs-start-signet +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env zsh -cd "${HOME}/electrs" -#source "${HOME}/.cargo/env" -#export PATH="${HOME}/.cargo/bin:${PATH}" - -until false -do - cargo run \ - --release \ - --bin electrs \ - -- \ - -vv \ - --network signet \ - --address-search \ - --cors '*' \ - --db-dir __ELECTRS_DATA_ROOT__ \ - --daemon-rpc-addr '127.0.0.1:38332' \ - --daemon-dir "${HOME}" \ - --http-socket-file '/bitcoin/socket/esplora-bitcoin-signet' \ - --cookie '__BITCOIN_RPC_USER__:__BITCOIN_RPC_PASS__' \ - --precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt" - sleep 1 -done diff --git a/production/electrs-start-testnet b/production/electrs-start-testnet deleted file mode 100755 index ce05de2de..000000000 --- a/production/electrs-start-testnet +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env zsh -cd "${HOME}/electrs" -#source $HOME/.cargo/env -#export PATH=$HOME/.cargo/bin:$PATH - -until false -do - cargo run \ - --release \ - --bin electrs \ - -- \ - -vvvv \ - --network testnet \ - --address-search \ - --cors '*' \ - --db-dir __ELECTRS_DATA_ROOT__ \ - --daemon-dir "${HOME}" \ - --http-socket-file '/bitcoin/socket/esplora-bitcoin-testnet' \ - --cookie '__BITCOIN_RPC_USER__:__BITCOIN_RPC_PASS__' \ - --precache-scripts "${HOME}/electrs/contrib/popular-scripts.txt" - - sleep 3 -done diff --git a/production/install b/production/install index 1121f5b4f..05b868ba0 100755 --- a/production/install +++ b/production/install @@ -1600,15 +1600,8 @@ fi ######################################## # Electrs instance for Bitcoin Mainnet # ######################################## - if [ "${BITCOIN_MAINNET_ENABLE}" = ON ];then - echo "[*] Installing Bitcoin Mainnet electrs start script" - osSudo "${ROOT_USER}" install -c -o "${BITCOIN_USER}" -g "${BITCOIN_GROUP}" -m 755 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/electrs-start-mainnet" "${BITCOIN_ELECTRS_HOME}" - - echo "[*] Configuring Bitcoin Mainnet RPC credentials in electrs start script" - osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_USER__/${BITCOIN_RPC_USER}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-mainnet" - osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_PASS__/${BITCOIN_RPC_PASS}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-mainnet" - osSudo "${ROOT_USER}" sed -i.orig "s!__ELECTRS_DATA_ROOT__!${ELECTRS_DATA_ROOT}!" "${BITCOIN_ELECTRS_HOME}/electrs-start-mainnet" + echo "[*] FIXME: must only crontab enabled daemons" fi ######################################## @@ -1616,13 +1609,7 @@ fi ######################################## if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then - echo "[*] Installing Bitcoin Testnet electrs start script" - osSudo "${ROOT_USER}" install -c -o "${BITCOIN_USER}" -g "${BITCOIN_GROUP}" -m 755 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/electrs-start-testnet" "${BITCOIN_ELECTRS_HOME}" - - echo "[*] Configuring Bitcoin Testnet RPC credentials in electrs start script" - osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_USER__/${BITCOIN_RPC_USER}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-testnet" - osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_PASS__/${BITCOIN_RPC_PASS}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-testnet" - osSudo "${ROOT_USER}" sed -i.orig "s!__ELECTRS_DATA_ROOT__!${ELECTRS_DATA_ROOT}!" "${BITCOIN_ELECTRS_HOME}/electrs-start-testnet" + echo "[*] FIXME: must only crontab enabled daemons" fi ####################################### @@ -1630,13 +1617,7 @@ fi ####################################### if [ "${BITCOIN_SIGNET_ENABLE}" = ON ];then - echo "[*] Installing Bitcoin Signet electrs start script" - osSudo "${ROOT_USER}" install -c -o "${BITCOIN_USER}" -g "${BITCOIN_GROUP}" -m 755 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/electrs-start-signet" "${BITCOIN_ELECTRS_HOME}" - - echo "[*] Configuring Bitcoin Signet RPC credentials in electrs start script" - osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_USER__/${BITCOIN_RPC_USER}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-signet" - osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_PASS__/${BITCOIN_RPC_PASS}/" "${BITCOIN_ELECTRS_HOME}/electrs-start-signet" - osSudo "${ROOT_USER}" sed -i.orig "s!__ELECTRS_DATA_ROOT__!${ELECTRS_DATA_ROOT}!" "${BITCOIN_ELECTRS_HOME}/electrs-start-signet" + echo "[*] FIXME: must only crontab enabled daemons" fi ######################################## @@ -1644,21 +1625,12 @@ fi ######################################## if [ "${ELEMENTS_LIQUID_ENABLE}" = ON ];then - echo "[*] Installing Elements Liquid electrs start script" - osSudo "${ROOT_USER}" install -c -o "${ELEMENTS_USER}" -g "${ELEMENTS_GROUP}" -m 755 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/electrs-start-liquid" "${ELEMENTS_ELECTRS_HOME}" - echo "[*] Installing Elements crontab" case $OS in FreeBSD) - echo "[*] FIXME: must only crontab enabled daemons" osSudo "${ROOT_USER}" crontab -u "${ELEMENTS_USER}" "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/elements.crontab" ;; esac - - echo "[*] Configuring Elements Liquid RPC credentials in electrs start script" - osSudo "${ROOT_USER}" sed -i.orig "s/__ELEMENTS_RPC_USER__/${ELEMENTS_RPC_USER}/" "${ELEMENTS_ELECTRS_HOME}/electrs-start-liquid" - osSudo "${ROOT_USER}" sed -i.orig "s/__ELEMENTS_RPC_PASS__/${ELEMENTS_RPC_PASS}/" "${ELEMENTS_ELECTRS_HOME}/electrs-start-liquid" - osSudo "${ROOT_USER}" sed -i.orig "s!__ELECTRS_DATA_ROOT__!${ELECTRS_DATA_ROOT}!" "${ELEMENTS_ELECTRS_HOME}/electrs-start-liquid" fi ################################################ @@ -1687,7 +1659,6 @@ fi echo "[*] Installing crontabs" case $OS in FreeBSD) - echo "[*] FIXME: must only crontab enabled daemons" osSudo "${ROOT_USER}" crontab -u "${BITCOIN_USER}" "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/bitcoin.crontab" osSudo "${ROOT_USER}" crontab -u "${MINFEE_USER}" "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/minfee.crontab" ;; From e489f713eb46f8ece0d4190f1b49d72920d8ffdd Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 22 May 2023 18:16:58 -0400 Subject: [PATCH 03/73] include accelerated tx data in block audits --- backend/src/api/audit.ts | 11 ++++++++--- backend/src/api/database-migration.ts | 7 ++++++- backend/src/api/mempool-blocks.ts | 2 +- backend/src/api/websocket-handler.ts | 3 ++- backend/src/mempool.interfaces.ts | 1 + backend/src/repositories/BlocksAuditsRepository.ts | 9 ++++++--- .../app/components/block-overview-graph/tx-view.ts | 5 ++++- .../block-overview-tooltip.component.html | 1 + frontend/src/app/components/block/block.component.ts | 10 ++++++++++ frontend/src/app/interfaces/node-api.interface.ts | 3 ++- frontend/src/app/interfaces/websocket.interface.ts | 2 +- 11 files changed, 42 insertions(+), 12 deletions(-) diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index a909fc2b6..63da288a1 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -6,16 +6,17 @@ import rbfCache from './rbf-cache'; const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners class Audit { - auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }) - : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], score: number, similarity: number } { + auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }, accelerations: { [txid: string]: number }) + : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } { if (!projectedBlocks?.[0]?.transactionIds || !mempool) { - return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], score: 0, similarity: 1 }; + return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 0, similarity: 1 }; } const matches: string[] = []; // present in both mined block and template const added: string[] = []; // present in mined block, not in template const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN const rbf: string[] = []; // either missing or present, and either part of a full-rbf replacement, or a conflict with the mined block + const accelerated: string[] = []; // prioritized by the mempool accelerator const isCensored = {}; // missing, without excuse const isDisplaced = {}; let displacedWeight = 0; @@ -28,6 +29,9 @@ class Audit { const now = Math.round((Date.now() / 1000)); for (const tx of transactions) { inBlock[tx.txid] = tx; + if (accelerations[tx.txid]) { + accelerated.push(tx.txid); + } } // coinbase is always expected if (transactions[0]) { @@ -149,6 +153,7 @@ class Audit { fresh, sigop: [], fullrbf: rbf, + accelerated, score, similarity, }; diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 7c7608aff..b7dc39493 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 64; + private static currentVersion = 65; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -548,6 +548,11 @@ class DatabaseMigration { await this.$executeQuery('ALTER TABLE `nodes` ADD features text NULL'); await this.updateToSchemaVersion(64); } + + if (databaseSchemaVersion < 65 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `blocks_audits` ADD accelerated_txs JSON DEFAULT "[]"'); + await this.updateToSchemaVersion(65); + } } /** diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 08508310d..5ca5cff09 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -206,7 +206,7 @@ class MempoolBlocks { return mempoolBlockDeltas; } - public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): Promise { + public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, accelerations: { [txid: string]: number } = {}): Promise { const start = Date.now(); // reset mempool short ids diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 56c8513cd..91004e292 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -666,7 +666,7 @@ class WebsocketHandler { } if (Common.indexingEnabled()) { - const { censored, added, fresh, sigop, fullrbf, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool); + const { censored, added, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool, accelerations); const matchRate = Math.round(score * 100 * 100) / 100; const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : []; @@ -695,6 +695,7 @@ class WebsocketHandler { freshTxs: fresh, sigopTxs: sigop, fullrbfTxs: fullrbf, + acceleratedTxs: accelerated, matchRate: matchRate, expectedFees: totalFees, expectedWeight: totalWeight, diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 25e7f0387..d0549cd97 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -36,6 +36,7 @@ export interface BlockAudit { sigopTxs: string[], fullrbfTxs: string[], addedTxs: string[], + acceleratedTxs: string[], matchRate: number, expectedFees?: number, expectedWeight?: number, diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index f7a2a59b5..9c7568567 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -6,9 +6,9 @@ import { BlockAudit, AuditScore } from '../mempool.interfaces'; class BlocksAuditRepositories { public async $saveAudit(audit: BlockAudit): Promise { try { - await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, fullrbf_txs, match_rate, expected_fees, expected_weight) - VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs), - JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]); + await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, fullrbf_txs, accelerated_txs, match_rate, expected_fees, expected_weight) + VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs), + JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), JSON.stringify(audit.acceleratedTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]); } catch (e: any) { if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`); @@ -69,6 +69,7 @@ class BlocksAuditRepositories { fresh_txs as freshTxs, sigop_txs as sigopTxs, fullrbf_txs as fullrbfTxs, + accelerated_txs as acceleratedTxs, match_rate as matchRate, expected_fees as expectedFees, expected_weight as expectedWeight @@ -83,6 +84,8 @@ class BlocksAuditRepositories { rows[0].freshTxs = JSON.parse(rows[0].freshTxs); rows[0].sigopTxs = JSON.parse(rows[0].sigopTxs); rows[0].fullrbfTxs = JSON.parse(rows[0].fullrbfTxs); + rows[0].acceleratedTxs = JSON.parse(rows[0].acceleratedTxs); + rows[0].transactions = JSON.parse(rows[0].transactions); rows[0].template = JSON.parse(rows[0].template); return rows[0]; diff --git a/frontend/src/app/components/block-overview-graph/tx-view.ts b/frontend/src/app/components/block-overview-graph/tx-view.ts index 1b8c88704..ac93eee0c 100644 --- a/frontend/src/app/components/block-overview-graph/tx-view.ts +++ b/frontend/src/app/components/block-overview-graph/tx-view.ts @@ -17,6 +17,7 @@ const auditColors = { missing: darken(desaturate(hexToColor('f344df'), 0.3), 0.7), added: hexToColor('0099ff'), selected: darken(desaturate(hexToColor('0099ff'), 0.3), 0.7), + accelerated: hexToColor('8F5FF6'), }; // convert from this class's update format to TxSprite's update format @@ -38,7 +39,7 @@ export default class TxView implements TransactionStripped { value: number; feerate: number; rate?: number; - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; scene?: BlockScene; @@ -216,6 +217,8 @@ export default class TxView implements TransactionStripped { return auditColors.added; case 'selected': return marginalFeeColors[feeLevelIndex] || marginalFeeColors[mempoolFeeColors.length - 1]; + case 'accelerated': + return auditColors.accelerated; case 'found': if (this.context === 'projected') { return auditFeeColors[feeLevelIndex] || auditFeeColors[mempoolFeeColors.length - 1]; diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html index c62779b69..8a410f3df 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html @@ -54,6 +54,7 @@ Added Marginal fee rate Conflicting + Accelerated diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index ec9a49504..1e94b1a9c 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -340,6 +340,7 @@ export class BlockComponent implements OnInit, OnDestroy { const isFresh = {}; const isSigop = {}; const isRbf = {}; + const isAccelerated = {}; this.numMissing = 0; this.numUnexpected = 0; @@ -365,6 +366,9 @@ export class BlockComponent implements OnInit, OnDestroy { for (const txid of blockAudit.fullrbfTxs || []) { isRbf[txid] = true; } + for (const txid of blockAudit.acceleratedTxs || []) { + isAccelerated[txid] = true; + } // set transaction statuses for (const tx of blockAudit.template) { tx.context = 'projected'; @@ -389,6 +393,9 @@ export class BlockComponent implements OnInit, OnDestroy { isMissing[tx.txid] = true; this.numMissing++; } + if (isAccelerated[tx.txid]) { + tx.status = 'accelerated'; + } } for (const [index, tx] of transactions.entries()) { tx.context = 'actual'; @@ -405,6 +412,9 @@ export class BlockComponent implements OnInit, OnDestroy { isSelected[tx.txid] = true; this.numUnexpected++; } + if (isAccelerated[tx.txid]) { + tx.status = 'accelerated'; + } } for (const tx of transactions) { inBlock[tx.txid] = true; diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 4249fd9db..7e7acfcf3 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -158,6 +158,7 @@ export interface BlockAudit extends BlockExtended { freshTxs: string[], sigopTxs: string[], fullrbfTxs: string[], + acceleratedTxs: string[], matchRate: number, expectedFees: number, expectedWeight: number, @@ -174,7 +175,7 @@ export interface TransactionStripped { vsize: number; value: number; rate?: number; // effective fee rate - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; } diff --git a/frontend/src/app/interfaces/websocket.interface.ts b/frontend/src/app/interfaces/websocket.interface.ts index e0ecdfeda..fb3c6d0c2 100644 --- a/frontend/src/app/interfaces/websocket.interface.ts +++ b/frontend/src/app/interfaces/websocket.interface.ts @@ -89,7 +89,7 @@ export interface TransactionStripped { vsize: number; value: number; rate?: number; // effective fee rate - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; } From aa24f6a84dbb5af04d2886462f2c752b056f7da6 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 26 May 2023 21:10:32 -0400 Subject: [PATCH 04/73] use accelerated rates for block templates & show in viz --- backend/src/api/audit.ts | 4 +-- backend/src/api/bitcoin/bitcoin.routes.ts | 1 + backend/src/api/common.ts | 1 + backend/src/api/mempool-blocks.ts | 15 +++++----- backend/src/api/mempool.ts | 29 +++++++++++++++++++ backend/src/api/websocket-handler.ts | 5 +++- backend/src/mempool.interfaces.ts | 2 ++ .../block-overview-graph.component.ts | 2 +- .../block-overview-graph/block-scene.ts | 3 +- .../block-overview-graph/tx-view.ts | 13 ++++++++- .../block-overview-tooltip.component.html | 3 +- .../block-overview-tooltip.component.ts | 2 ++ .../mempool-block-overview.component.ts | 1 - .../transaction/transaction.component.html | 3 +- .../transaction/transaction.component.ts | 3 ++ .../src/app/interfaces/electrs.interface.ts | 1 + .../src/app/interfaces/node-api.interface.ts | 1 + .../src/app/interfaces/websocket.interface.ts | 3 +- 18 files changed, 75 insertions(+), 17 deletions(-) diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index 63da288a1..9710d0362 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -6,7 +6,7 @@ import rbfCache from './rbf-cache'; const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners class Audit { - auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }, accelerations: { [txid: string]: number }) + auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }) : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } { if (!projectedBlocks?.[0]?.transactionIds || !mempool) { return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 0, similarity: 1 }; @@ -29,7 +29,7 @@ class Audit { const now = Math.round((Date.now() / 1000)); for (const tx of transactions) { inBlock[tx.txid] = tx; - if (accelerations[tx.txid]) { + if (tx.acceleration) { accelerated.push(tx.txid); } } diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index ffdb2e629..e2887b706 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -214,6 +214,7 @@ class BitcoinRoutes { effectiveFeePerVsize: tx.effectiveFeePerVsize || null, sigops: tx.sigops, adjustedVsize: tx.adjustedVsize, + acceleration: tx.acceleration }); return; } diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index cd9da3d2a..775da2643 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -111,6 +111,7 @@ export class Common { fee: tx.fee, vsize: tx.weight / 4, value: tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0), + acc: tx.acceleration || undefined, rate: tx.effectiveFeePerVsize, }; } diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 5ca5cff09..81f2092a0 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -170,7 +170,7 @@ class MempoolBlocks { for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) { let added: TransactionStripped[] = []; let removed: string[] = []; - const changed: { txid: string, rate: number | undefined }[] = []; + const changed: { txid: string, rate: number | undefined, acc: number | undefined }[] = []; if (mempoolBlocks[i] && !prevBlocks[i]) { added = mempoolBlocks[i].transactions; } else if (!mempoolBlocks[i] && prevBlocks[i]) { @@ -192,8 +192,8 @@ class MempoolBlocks { mempoolBlocks[i].transactions.forEach(tx => { if (!prevIds[tx.txid]) { added.push(tx); - } else if (tx.rate !== prevIds[tx.txid].rate) { - changed.push({ txid: tx.txid, rate: tx.rate }); + } else if (tx.rate !== prevIds[tx.txid].rate || tx.acc !== prevIds[tx.txid].acc) { + changed.push({ txid: tx.txid, rate: tx.rate, acc: tx.acc }); } }); } @@ -206,7 +206,7 @@ class MempoolBlocks { return mempoolBlockDeltas; } - public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, accelerations: { [txid: string]: number } = {}): Promise { + public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): Promise { const start = Date.now(); // reset mempool short ids @@ -222,7 +222,7 @@ class MempoolBlocks { if (entry.uid !== null && entry.uid !== undefined) { const stripped = { uid: entry.uid, - fee: entry.fee, + fee: entry.fee + (entry.acceleration || 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, @@ -285,13 +285,14 @@ class MempoolBlocks { for (const tx of Object.values(added)) { this.setUid(tx, true); } - const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => (uid !== null && uid !== undefined)) as number[]; + + const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => uid != null) as number[]; // prepare a stripped down version of the mempool with only the minimum necessary data // to reduce the overhead of passing this data to the worker thread const addedStripped: CompactThreadTransaction[] = added.filter(entry => (entry.uid !== null && entry.uid !== undefined)).map(entry => { return { uid: entry.uid || 0, - fee: entry.fee, + fee: entry.fee + (entry.acceleration || 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 945b78738..8e350c4fe 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -23,6 +23,8 @@ class Mempool { private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]) => Promise) | undefined; + private accelerations: { [txId: string]: number } = {}; + private txPerSecondArray: number[] = []; private txPerSecond: number = 0; @@ -301,6 +303,17 @@ class Mempool { const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx)); this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6); + const newAccelerations: { txid: string, delta: number }[] = []; + newTransactions.forEach(tx => { + if (tx.txid.startsWith('00')) { + const delta = Math.floor(Math.random() * 100000) + 100000; + newAccelerations.push({ txid: tx.txid, delta }); + tx.acceleration = delta; + } + }); + this.addAccelerations(newAccelerations); + this.removeAccelerations(deletedTransactions.map(tx => tx.txid)); + this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize); if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) { @@ -325,6 +338,22 @@ class Mempool { this.clearTimer(timer); } + public getAccelerations(): { [txid: string]: number } { + return this.accelerations; + } + + public addAccelerations(newAccelerations: { txid: string, delta: number }[]): void { + for (const acceleration of newAccelerations) { + this.accelerations[acceleration.txid] = acceleration.delta; + } + } + + public removeAccelerations(txids: string[]): void { + for (const txid of txids) { + delete this.accelerations[txid]; + } + } + private startTimer() { const state: any = { start: Date.now(), diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 91004e292..a33a0f0fa 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -21,6 +21,7 @@ import Audit from './audit'; import { deepClone } from '../utils/clone'; import priceUpdater from '../tasks/price-updater'; import { ApiPrice } from '../repositories/PricesRepository'; +import mempool from './mempool'; // valid 'want' subscriptions const wantable = [ @@ -666,7 +667,7 @@ class WebsocketHandler { } if (Common.indexingEnabled()) { - const { censored, added, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool, accelerations); + const { censored, added, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool); const matchRate = Math.round(score * 100 * 100) / 100; const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : []; @@ -737,6 +738,8 @@ class WebsocketHandler { const fees = feeApi.getRecommendedFee(); const mempoolInfo = memPool.getMempoolInfo(); + memPool.removeAccelerations(txIds); + // update init data this.updateSocketDataFields({ 'mempoolInfo': mempoolInfo, diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index d0549cd97..185256619 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -92,6 +92,7 @@ export interface TransactionExtended extends IEsploraApi.Transaction { block: number, vsize: number, }; + acceleration?: number; uid?: number; } @@ -183,6 +184,7 @@ export interface TransactionStripped { fee: number; vsize: number; value: number; + acc?: number; rate?: number; // effective fee rate } diff --git a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts index 49da16d55..fe847103f 100644 --- a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts +++ b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts @@ -147,7 +147,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On } } - update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { + update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { if (this.scene) { this.scene.update(add, remove, change, direction, resetLayout); this.start(); diff --git a/frontend/src/app/components/block-overview-graph/block-scene.ts b/frontend/src/app/components/block-overview-graph/block-scene.ts index 510803f03..94984bae2 100644 --- a/frontend/src/app/components/block-overview-graph/block-scene.ts +++ b/frontend/src/app/components/block-overview-graph/block-scene.ts @@ -150,7 +150,7 @@ export default class BlockScene { this.updateAll(startTime, 200, direction); } - update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { + update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { const startTime = performance.now(); const removed = this.removeBatch(remove, startTime, direction); @@ -175,6 +175,7 @@ export default class BlockScene { // update effective rates change.forEach(tx => { if (this.txs[tx.txid]) { + this.txs[tx.txid].acc = tx.acc; this.txs[tx.txid].feerate = tx.rate || (this.txs[tx.txid].fee / this.txs[tx.txid].vsize); this.txs[tx.txid].rate = tx.rate; this.txs[tx.txid].dirty = true; diff --git a/frontend/src/app/components/block-overview-graph/tx-view.ts b/frontend/src/app/components/block-overview-graph/tx-view.ts index ac93eee0c..690b974e3 100644 --- a/frontend/src/app/components/block-overview-graph/tx-view.ts +++ b/frontend/src/app/components/block-overview-graph/tx-view.ts @@ -38,6 +38,7 @@ export default class TxView implements TransactionStripped { vsize: number; value: number; feerate: number; + acc?: number; rate?: number; status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; @@ -64,6 +65,7 @@ export default class TxView implements TransactionStripped { this.vsize = tx.vsize; this.value = tx.value; this.feerate = tx.rate || (tx.fee / tx.vsize); // sort by effective fee rate where available + this.acc = tx.acc; this.rate = tx.rate; this.status = tx.status; this.initialised = false; @@ -200,6 +202,11 @@ export default class TxView implements TransactionStripped { const feeLevelColor = feeColors[feeLevelIndex] || feeColors[mempoolFeeColors.length - 1]; // Normal mode if (!this.scene?.highlightingEnabled) { + if (this.acc) { + return auditColors.accelerated; + } else { + return feeLevelColor; + } return feeLevelColor; } // Block audit @@ -226,7 +233,11 @@ export default class TxView implements TransactionStripped { return feeLevelColor; } default: - return feeLevelColor; + if (this.acc) { + return auditColors.accelerated; + } else { + return feeLevelColor; + } } } } diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html index 8a410f3df..a53cfdc9c 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html @@ -29,7 +29,8 @@ - Effective fee rate + Effective fee rate + Accelerated fee rate diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts index 61c294263..65d0f984c 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts @@ -21,6 +21,7 @@ export class BlockOverviewTooltipComponent implements OnChanges { vsize = 1; feeRate = 0; effectiveRate; + acceleration; tooltipPosition: Position = { x: 0, y: 0 }; @@ -53,6 +54,7 @@ export class BlockOverviewTooltipComponent implements OnChanges { this.vsize = tx.vsize || 1; this.feeRate = this.fee / this.vsize; this.effectiveRate = tx.rate; + this.acceleration = tx.acc; } } } diff --git a/frontend/src/app/components/mempool-block-overview/mempool-block-overview.component.ts b/frontend/src/app/components/mempool-block-overview/mempool-block-overview.component.ts index 30632a862..226be5210 100644 --- a/frontend/src/app/components/mempool-block-overview/mempool-block-overview.component.ts +++ b/frontend/src/app/components/mempool-block-overview/mempool-block-overview.component.ts @@ -94,7 +94,6 @@ export class MempoolBlockOverviewComponent implements OnInit, OnDestroy, OnChang updateBlock(delta: MempoolBlockDelta): void { const blockMined = (this.stateService.latestBlockHeight > this.lastBlockHeight); - if (this.blockIndex !== this.index) { const direction = (this.blockIndex == null || this.index < this.blockIndex) ? this.poolDirection : this.chainDirection; this.blockGraph.replace(delta.added, direction); diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index d4cd6913d..81a6106db 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -488,7 +488,8 @@ - Effective fee rate + Accelerated fee rate + Effective fee rate
diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index e856f34eb..f1d218a79 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -183,6 +183,9 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { } else { this.tx.effectiveFeePerVsize = cpfpInfo.effectiveFeePerVsize; } + if (cpfpInfo.acceleration) { + this.tx.acceleration = cpfpInfo.acceleration; + } this.cpfpInfo = cpfpInfo; this.hasEffectiveFeeRate = hasRelatives || (this.tx.effectiveFeePerVsize && (Math.abs(this.tx.effectiveFeePerVsize - this.tx.feePerVsize) > 0.01)); diff --git a/frontend/src/app/interfaces/electrs.interface.ts b/frontend/src/app/interfaces/electrs.interface.ts index df19f7491..5c15b0ae4 100644 --- a/frontend/src/app/interfaces/electrs.interface.ts +++ b/frontend/src/app/interfaces/electrs.interface.ts @@ -19,6 +19,7 @@ export interface Transaction { ancestors?: Ancestor[]; bestDescendant?: BestDescendant | null; cpfpChecked?: boolean; + acceleration?: number; deleteAfter?: number; _unblinded?: any; _deduced?: boolean; diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 7e7acfcf3..fe6233866 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -27,6 +27,7 @@ export interface CpfpInfo { effectiveFeePerVsize?: number; sigops?: number; adjustedVsize?: number; + acceleration?: number; } export interface RbfInfo { diff --git a/frontend/src/app/interfaces/websocket.interface.ts b/frontend/src/app/interfaces/websocket.interface.ts index fb3c6d0c2..f8686042b 100644 --- a/frontend/src/app/interfaces/websocket.interface.ts +++ b/frontend/src/app/interfaces/websocket.interface.ts @@ -70,7 +70,7 @@ export interface MempoolBlockWithTransactions extends MempoolBlock { export interface MempoolBlockDelta { added: TransactionStripped[], removed: string[], - changed?: { txid: string, rate: number | undefined }[]; + changed?: { txid: string, rate: number | undefined, acc: number | undefined }[]; } export interface MempoolInfo { @@ -88,6 +88,7 @@ export interface TransactionStripped { fee: number; vsize: number; value: number; + acc?: number; // acceleration delta rate?: number; // effective fee rate status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; From c246db1cf9d116a3796d9a5e41c245e6880f9460 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 30 May 2023 19:35:39 -0400 Subject: [PATCH 05/73] Refactor acceleration tracking --- backend/src/api/mempool-blocks.ts | 20 ++++--- backend/src/api/mempool.ts | 72 +++++++++++++++--------- backend/src/api/services/acceleration.ts | 20 +++++++ backend/src/api/websocket-handler.ts | 6 +- 4 files changed, 79 insertions(+), 39 deletions(-) create mode 100644 backend/src/api/services/acceleration.ts diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 81f2092a0..bc0def7a8 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -5,6 +5,7 @@ import { Common, OnlineFeeStatsCalculator } from './common'; import config from '../config'; import { Worker } from 'worker_threads'; import path from 'path'; +import mempool from './mempool'; const MAX_UINT32 = Math.pow(2, 32) - 1; @@ -212,9 +213,11 @@ class MempoolBlocks { // reset mempool short ids this.resetUids(); for (const tx of Object.values(newMempool)) { - this.setUid(tx); + this.setUid(tx, true); } + const accelerations = mempool.getAccelerations(); + // prepare a stripped down version of the mempool with only the minimum necessary data // to reduce the overhead of passing this data to the worker thread const strippedMempool: Map = new Map(); @@ -222,7 +225,7 @@ class MempoolBlocks { if (entry.uid !== null && entry.uid !== undefined) { const stripped = { uid: entry.uid, - fee: entry.fee + (entry.acceleration || 0), + fee: entry.fee + (accelerations[entry.txid] || 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, @@ -273,7 +276,7 @@ class MempoolBlocks { return this.mempoolBlocks; } - public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], saveResults: boolean = false): Promise { + public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], accelerationDelta: string[] = [], saveResults: boolean = false): Promise { if (!this.txSelectionWorker) { // need to reset the worker await this.$makeBlockTemplates(newMempool, saveResults); @@ -282,17 +285,20 @@ class MempoolBlocks { const start = Date.now(); - for (const tx of Object.values(added)) { + const accelerations = mempool.getAccelerations(); + const addedAndChanged: MempoolTransactionExtended[] = accelerationDelta.map(txid => newMempool[txid]).filter(tx => tx != null).concat(added); + + for (const tx of addedAndChanged) { this.setUid(tx, true); } - const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => uid != null) as number[]; + // prepare a stripped down version of the mempool with only the minimum necessary data // to reduce the overhead of passing this data to the worker thread - const addedStripped: CompactThreadTransaction[] = added.filter(entry => (entry.uid !== null && entry.uid !== undefined)).map(entry => { + const addedStripped: CompactThreadTransaction[] = addedAndChanged.filter(entry => entry.uid != null).map(entry => { return { uid: entry.uid || 0, - fee: entry.fee + (entry.acceleration || 0), + fee: entry.fee + (accelerations[entry.txid] || 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 8e350c4fe..7bde83df9 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -1,5 +1,5 @@ import config from '../config'; -import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; +import bitcoinApi from './bitcoin/bitcoin-api-factory'; import { MempoolTransactionExtended, TransactionExtended, VbytesPerSecond } from '../mempool.interfaces'; import logger from '../logger'; import { Common } from './common'; @@ -9,7 +9,7 @@ import loadingIndicators from './loading-indicators'; import bitcoinClient from './bitcoin/bitcoin-client'; import bitcoinSecondClient from './bitcoin/bitcoin-second-client'; import rbfCache from './rbf-cache'; -import { IEsploraApi } from './bitcoin/esplora-api.interface'; +import accelerationApi from './services/acceleration'; class Mempool { private inSync: boolean = false; @@ -19,9 +19,9 @@ class Mempool { private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0, maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 }; private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[], - deletedTransactions: MempoolTransactionExtended[]) => void) | undefined; + deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => void) | undefined; private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[], - deletedTransactions: MempoolTransactionExtended[]) => Promise) | undefined; + deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise) | undefined; private accelerations: { [txId: string]: number } = {}; @@ -68,12 +68,12 @@ class Mempool { } public setMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; }, - newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]) => void): void { + newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => void): void { this.mempoolChangedCallback = fn; } public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; }, mempoolSize: number, - newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]) => Promise): void { + newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise): void { this.$asyncMempoolChangedCallback = fn; } @@ -98,10 +98,10 @@ class Mempool { count++; } if (this.mempoolChangedCallback) { - this.mempoolChangedCallback(this.mempoolCache, [], []); + this.mempoolChangedCallback(this.mempoolCache, [], [], []); } if (this.$asyncMempoolChangedCallback) { - await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], []); + await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], [], []); } this.addToSpendMap(Object.values(this.mempoolCache)); } @@ -303,25 +303,19 @@ class Mempool { const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx)); this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6); - const newAccelerations: { txid: string, delta: number }[] = []; - newTransactions.forEach(tx => { - if (tx.txid.startsWith('00')) { - const delta = Math.floor(Math.random() * 100000) + 100000; - newAccelerations.push({ txid: tx.txid, delta }); - tx.acceleration = delta; - } - }); - this.addAccelerations(newAccelerations); - this.removeAccelerations(deletedTransactions.map(tx => tx.txid)); + const accelerationDelta = await this.$updateAccelerations(); + if (accelerationDelta.length) { + hasChange = true; + } this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize); if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) { - this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions); + this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions, accelerationDelta); } if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) { this.updateTimerProgress(timer, 'running async mempool callback'); - await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions); + await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions, accelerationDelta); this.updateTimerProgress(timer, 'completed async mempool callback'); } @@ -342,15 +336,37 @@ class Mempool { return this.accelerations; } - public addAccelerations(newAccelerations: { txid: string, delta: number }[]): void { - for (const acceleration of newAccelerations) { - this.accelerations[acceleration.txid] = acceleration.delta; - } - } + public async $updateAccelerations(): Promise { + try { + const newAccelerations = await accelerationApi.fetchAccelerations$(); - public removeAccelerations(txids: string[]): void { - for (const txid of txids) { - delete this.accelerations[txid]; + const changed: string[] = []; + + const newAccelerationMap: { [txid: string]: number } = {}; + for (const acceleration of newAccelerations) { + newAccelerationMap[acceleration.txid] = acceleration.feeDelta; + if (this.accelerations[acceleration.txid] == null) { + // new acceleration + changed.push(acceleration.txid); + } else if (this.accelerations[acceleration.txid] !== acceleration.feeDelta) { + // feeDelta changed + changed.push(acceleration.txid); + } + } + + for (const oldTxid of Object.keys(this.accelerations)) { + if (!newAccelerationMap[oldTxid]) { + // removed + changed.push(oldTxid); + } + } + + this.accelerations = newAccelerationMap; + + return changed; + } catch (e: any) { + logger.debug(`Failed to update accelerations: ` + (e instanceof Error ? e.message : e)); + return []; } } diff --git a/backend/src/api/services/acceleration.ts b/backend/src/api/services/acceleration.ts new file mode 100644 index 000000000..efbf32f5d --- /dev/null +++ b/backend/src/api/services/acceleration.ts @@ -0,0 +1,20 @@ +import { query } from '../../utils/axios-query'; +import config from '../../config'; + +export interface Acceleration { + txid: string, + feeDelta: number, +} + +class AccelerationApi { + public async fetchAccelerations$(): Promise { + if (config.MEMPOOL_SERVICES.ACCELERATIONS) { + const response = await query(`${config.MEMPOOL_SERVICES.API}/accelerations`); + return (response as Acceleration[]) || []; + } else { + return []; + } + } +} + +export default new AccelerationApi(); \ No newline at end of file diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index a33a0f0fa..3a90b29f4 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -381,7 +381,7 @@ class WebsocketHandler { } async $handleMempoolChange(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, - newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]): Promise { + newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]): Promise { if (!this.wss) { throw new Error('WebSocket.Server is not set'); } @@ -392,7 +392,7 @@ class WebsocketHandler { if (config.MEMPOOL.RUST_GBT) { await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions); } else { - await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, true); + await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true); } } else { mempoolBlocks.updateMempoolBlocks(newMempool, true); @@ -738,8 +738,6 @@ class WebsocketHandler { const fees = feeApi.getRecommendedFee(); const mempoolInfo = memPool.getMempoolInfo(); - memPool.removeAccelerations(txIds); - // update init data this.updateSocketDataFields({ 'mempoolInfo': mempoolInfo, From 20b3ceab1e318e088c8e2d9e266d367143c64bbc Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 30 May 2023 19:51:11 -0400 Subject: [PATCH 06/73] Implement accelerations API & config setting --- backend/mempool-config.sample.json | 4 ++++ .../src/__fixtures__/mempool-config.template.json | 4 ++++ backend/src/__tests__/config.test.ts | 7 +++++++ backend/src/api/mempool.ts | 4 ++++ backend/src/config.ts | 12 +++++++++++- docker/backend/mempool-config.json | 4 ++++ docker/backend/start.sh | 9 +++++++++ 7 files changed, 43 insertions(+), 1 deletion(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index e3df7d2fe..8f8b82475 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -136,5 +136,9 @@ "trusted", "servers" ] + }, + "MEMPOOL_SERVICES": { + "API": "https://mempool.space/api", + "ACCELERATIONS": false } } diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 4213f0ffb..d6754f966 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -127,5 +127,9 @@ "AUDIT": false, "AUDIT_START_HEIGHT": 774000, "SERVERS": [] + }, + "MEMPOOl_SERVICES": { + "API": "__MEMPOOL_SERVICES_API__", + "ACCELERATIONS": "__MEMPOOL_SERVICES_ACCELERATIONS__" } } diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index dc1beaa46..2c83d1af5 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -127,6 +127,11 @@ describe('Mempool Backend Config', () => { AUDIT_START_HEIGHT: 774000, SERVERS: [] }); + + expect(config.MEMPOOL_SERVICES).toStrictEqual({ + API: "", + ACCELERATIONS: false, + }); }); }); @@ -160,6 +165,8 @@ describe('Mempool Backend Config', () => { expect(config.PRICE_DATA_SERVER).toStrictEqual(fixture.PRICE_DATA_SERVER); expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER); + + expect(config.MEMPOOL_SERVICES).toStrictEqual(fixture.MEMPOOL_SERVICES); }); }); diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 7bde83df9..b114c89fc 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -337,6 +337,10 @@ class Mempool { } public async $updateAccelerations(): Promise { + if (!config.MEMPOOL_SERVICES.ACCELERATIONS) { + return []; + } + try { const newAccelerations = await accelerationApi.fetchAccelerations$(); diff --git a/backend/src/config.ts b/backend/src/config.ts index 09d279537..ceb569e06 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -137,7 +137,11 @@ interface IConfig { AUDIT: boolean; AUDIT_START_HEIGHT: number; SERVERS: string[]; - } + }, + MEMPOOL_SERVICES: { + API: string; + ACCELERATIONS: boolean; + }, } const defaults: IConfig = { @@ -275,6 +279,10 @@ const defaults: IConfig = { 'AUDIT': false, 'AUDIT_START_HEIGHT': 774000, 'SERVERS': [], + }, + 'MEMPOOL_SERVICES': { + 'API': '', + 'ACCELERATIONS': false, } }; @@ -296,6 +304,7 @@ class Config implements IConfig { EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER']; MAXMIND: IConfig['MAXMIND']; REPLICATION: IConfig['REPLICATION']; + MEMPOOL_SERVICES: IConfig['MEMPOOL_SERVICES']; constructor() { const configs = this.merge(configFromFile, defaults); @@ -316,6 +325,7 @@ class Config implements IConfig { this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER; this.MAXMIND = configs.MAXMIND; this.REPLICATION = configs.REPLICATION; + this.MEMPOOL_SERVICES = configs.MEMPOOL_SERVICES; } merge = (...objects: object[]): IConfig => { diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 2ff76d5dd..71fe3bd65 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -133,5 +133,9 @@ "AUDIT": __REPLICATION_AUDIT__, "AUDIT_START_HEIGHT": __REPLICATION_AUDIT_START_HEIGHT__, "SERVERS": __REPLICATION_SERVERS__ + }, + "MEMPOOL_SERVICES": { + "API": "__MEMPOOL_SERVICES_API__", + "ACCELERATIONS": __MEMPOOL_SERVICES_ACCELERATIONS__ } } diff --git a/docker/backend/start.sh b/docker/backend/start.sh index c34d804b4..66f873605 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -136,6 +136,10 @@ __REPLICATION_AUDIT__=${REPLICATION_AUDIT:=true} __REPLICATION_AUDIT_START_HEIGHT__=${REPLICATION_AUDIT_START_HEIGHT:=774000} __REPLICATION_SERVERS__=${REPLICATION_SERVERS:=[]} +# MEMPOOL_SERVICES +__MEMPOOL_SERVICES_API__==${MEMPOOL_SERVICES_API:=""} +__MEMPOOL_SERVICES_ACCELERATIONS__==${MEMPOOL_SERVICES_ACCELERATIONS:=false} + mkdir -p "${__MEMPOOL_CACHE_DIR__}" @@ -262,4 +266,9 @@ sed -i "s!__REPLICATION_AUDIT__!${__REPLICATION_AUDIT__}!g" mempool-config.json sed -i "s!__REPLICATION_AUDIT_START_HEIGHT__!${__REPLICATION_AUDIT_START_HEIGHT__}!g" mempool-config.json sed -i "s!__REPLICATION_SERVERS__!${__REPLICATION_SERVERS__}!g" mempool-config.json +# MEMPOOL_SERVICES +sed -i "s!__MEMPOOL_SERVICES_API__!${__MEMPOOL_SERVICES_API__}!g" mempool-config.json +sed -i "s!__MEMPOOL_SERVICES_ACCELERATIONS__!${__MEMPOOL_SERVICES_ACCELERATIONS__}!g" mempool-config.json + + node /backend/package/index.js From 083bfdba06875fd8d254b16c6db9a6070b6b2d0d Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 3 Jun 2023 16:54:12 -0400 Subject: [PATCH 07/73] Refactor accelerated audits --- backend/src/api/audit.ts | 4 ++-- backend/src/api/mempool-blocks.ts | 28 +++++++++++++----------- backend/src/api/mempool.ts | 2 +- backend/src/api/services/acceleration.ts | 20 ++++++++++++++++- backend/src/api/websocket-handler.ts | 17 ++++++++++---- 5 files changed, 50 insertions(+), 21 deletions(-) diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index 9710d0362..e78b2796f 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -6,7 +6,7 @@ import rbfCache from './rbf-cache'; const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners class Audit { - auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }) + auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }, useAccelerations: boolean = false) : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } { if (!projectedBlocks?.[0]?.transactionIds || !mempool) { return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 0, similarity: 1 }; @@ -29,7 +29,7 @@ class Audit { const now = Math.round((Date.now() / 1000)); for (const tx of transactions) { inBlock[tx.txid] = tx; - if (tx.acceleration) { + if (mempool[tx.txid] && mempool[tx.txid].acceleration) { accelerated.push(tx.txid); } } diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index bc0def7a8..0ce0ea522 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -207,7 +207,7 @@ class MempoolBlocks { return mempoolBlockDeltas; } - public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): Promise { + public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false): Promise { const start = Date.now(); // reset mempool short ids @@ -216,7 +216,7 @@ class MempoolBlocks { this.setUid(tx, true); } - const accelerations = mempool.getAccelerations(); + const accelerations = useAccelerations ? mempool.getAccelerations() : {}; // prepare a stripped down version of the mempool with only the minimum necessary data // to reduce the overhead of passing this data to the worker thread @@ -225,7 +225,7 @@ class MempoolBlocks { if (entry.uid !== null && entry.uid !== undefined) { const stripped = { uid: entry.uid, - fee: entry.fee + (accelerations[entry.txid] || 0), + fee: entry.fee + (useAccelerations ? (accelerations[entry.txid] || 0) : 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, @@ -265,7 +265,7 @@ class MempoolBlocks { // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); - const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), saveResults); + const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, saveResults); logger.debug(`makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); @@ -276,17 +276,17 @@ class MempoolBlocks { return this.mempoolBlocks; } - public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], accelerationDelta: string[] = [], saveResults: boolean = false): Promise { + public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], accelerationDelta: string[] = [], saveResults: boolean = false, useAccelerations: boolean = false): Promise { if (!this.txSelectionWorker) { // need to reset the worker - await this.$makeBlockTemplates(newMempool, saveResults); + await this.$makeBlockTemplates(newMempool, saveResults, useAccelerations); return; } const start = Date.now(); - const accelerations = mempool.getAccelerations(); - const addedAndChanged: MempoolTransactionExtended[] = accelerationDelta.map(txid => newMempool[txid]).filter(tx => tx != null).concat(added); + const accelerations = useAccelerations ? mempool.getAccelerations() : {}; + const addedAndChanged: MempoolTransactionExtended[] = useAccelerations ? accelerationDelta.map(txid => newMempool[txid]).filter(tx => tx != null).concat(added) : added; for (const tx of addedAndChanged) { this.setUid(tx, true); @@ -298,7 +298,7 @@ class MempoolBlocks { const addedStripped: CompactThreadTransaction[] = addedAndChanged.filter(entry => entry.uid != null).map(entry => { return { uid: entry.uid || 0, - fee: entry.fee + (accelerations[entry.txid] || 0), + fee: entry.fee + (useAccelerations ? (accelerations[entry.txid] || 0) : 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, @@ -325,7 +325,7 @@ class MempoolBlocks { // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); - this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), saveResults); + this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, saveResults); logger.debug(`updateBlockTemplates completed in ${(Date.now() - start) / 1000} seconds`); } catch (e) { logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e)); @@ -362,7 +362,7 @@ class MempoolBlocks { if (saveResults) { this.rustInitialized = true; } - const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, saveResults); + const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, {}, saveResults); logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); return processed; } catch (e) { @@ -414,7 +414,7 @@ class MempoolBlocks { if (mempoolSize !== resultMempoolSize) { throw new Error('GBT returned wrong number of transactions, cache is probably out of sync'); } else { - this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, true); + this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, {}, true); } this.removeUids(removedUids); logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); @@ -424,7 +424,7 @@ class MempoolBlocks { } } - private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], saveResults): MempoolBlockWithTransactions[] { + private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], accelerations, saveResults): MempoolBlockWithTransactions[] { for (const [txid, rate] of rates) { if (txid in mempool) { mempool[txid].effectiveFeePerVsize = rate; @@ -503,6 +503,8 @@ class MempoolBlocks { mempoolTx.cpfpChecked = true; } + mempoolTx.acceleration = accelerations[txid]; + // online calculation of stack-of-blocks fee stats if (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) { feeStatsCalculator.processNext(mempoolTx); diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index b114c89fc..4efe17731 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -342,7 +342,7 @@ class Mempool { } try { - const newAccelerations = await accelerationApi.fetchAccelerations$(); + const newAccelerations = await accelerationApi.$fetchAccelerations(); const changed: string[] = []; diff --git a/backend/src/api/services/acceleration.ts b/backend/src/api/services/acceleration.ts index efbf32f5d..8c849cdd8 100644 --- a/backend/src/api/services/acceleration.ts +++ b/backend/src/api/services/acceleration.ts @@ -1,5 +1,6 @@ import { query } from '../../utils/axios-query'; import config from '../../config'; +import { BlockExtended, PoolTag } from '../../mempool.interfaces'; export interface Acceleration { txid: string, @@ -7,7 +8,7 @@ export interface Acceleration { } class AccelerationApi { - public async fetchAccelerations$(): Promise { + public async $fetchAccelerations(): Promise { if (config.MEMPOOL_SERVICES.ACCELERATIONS) { const response = await query(`${config.MEMPOOL_SERVICES.API}/accelerations`); return (response as Acceleration[]) || []; @@ -15,6 +16,23 @@ class AccelerationApi { return []; } } + + public async $fetchPools(): Promise { + if (config.MEMPOOL_SERVICES.ACCELERATIONS) { + const response = await query(`${config.MEMPOOL_SERVICES.API}/partners`); + return (response as PoolTag[]) || []; + } else { + return []; + } + } + + public async $isAcceleratedBlock(block: BlockExtended): Promise { + const pools = await this.$fetchPools(); + if (block?.extras?.pool?.id == null) { + return false; + } + return pools.reduce((match, tag) => match || tag.uniqueId === block.extras.pool.id, false); + } } export default new AccelerationApi(); \ No newline at end of file diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 3a90b29f4..51fccbcbb 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -21,7 +21,7 @@ import Audit from './audit'; import { deepClone } from '../utils/clone'; import priceUpdater from '../tasks/price-updater'; import { ApiPrice } from '../repositories/PricesRepository'; -import mempool from './mempool'; +import accelerationApi from './services/acceleration'; // valid 'want' subscriptions const wantable = [ @@ -392,7 +392,7 @@ class WebsocketHandler { if (config.MEMPOOL.RUST_GBT) { await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions); } else { - await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true); + await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS); } } else { mempoolBlocks.updateMempoolBlocks(newMempool, true); @@ -648,6 +648,7 @@ class WebsocketHandler { if (config.MEMPOOL.AUDIT && memPool.isInSync()) { let projectedBlocks; let auditMempool = _memPool; + const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && await accelerationApi.$isAcceleratedBlock(block); // template calculation functions have mempool side effects, so calculate audits using // a cloned copy of the mempool if we're running a different algorithm for mempool updates const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL; @@ -657,13 +658,17 @@ class WebsocketHandler { if (config.MEMPOOL.RUST_GBT) { projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool); } else { - projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false); + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated); } } else { projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false); } } else { - projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); + if ((config.MEMPOOL_SERVICES.ACCELERATIONS && !isAccelerated)) { + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated); + } else { + projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); + } } if (Common.indexingEnabled()) { @@ -723,11 +728,15 @@ class WebsocketHandler { } if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { +<<<<<<< HEAD if (config.MEMPOOL.RUST_GBT) { await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions); } else { await mempoolBlocks.$makeBlockTemplates(_memPool, true); } +======= + await mempoolBlocks.$makeBlockTemplates(_memPool, true, config.MEMPOOL_SERVICES.ACCELERATIONS); +>>>>>>> 77b0a8ecc (Refactor accelerated audits) } else { mempoolBlocks.updateMempoolBlocks(_memPool, true); } From ba54bc9d1593076a9ce380216580731df22518bb Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 13 Jun 2023 13:35:25 -0400 Subject: [PATCH 08/73] support for acceleration mempool blocks animation --- backend/src/api/websocket-handler.ts | 24 ++++++++++----- .../mempool-blocks.component.html | 2 +- .../mempool-blocks.component.scss | 30 +++++++++++++++++++ .../mempool-blocks.component.ts | 1 + .../transaction/transaction.component.html | 4 +-- .../src/app/interfaces/node-api.interface.ts | 1 + 6 files changed, 51 insertions(+), 11 deletions(-) diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 51fccbcbb..7716ed7b7 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -173,9 +173,15 @@ class WebsocketHandler { } const tx = memPool.getMempool()[trackTxid]; if (tx && tx.position) { + const position: { block: number, vsize: number, accelerated?: number } = { + ...tx.position + }; + if (tx.acceleration) { + position.accelerated = tx.acceleration; + } response['txPosition'] = JSON.stringify({ txid: trackTxid, - position: tx.position, + position }); } } else { @@ -600,7 +606,10 @@ class WebsocketHandler { if (mempoolTx && mempoolTx.position) { response['txPosition'] = JSON.stringify({ txid: trackTxid, - position: mempoolTx.position, + position: { + ...mempoolTx.position, + accelerated: mempoolTx.acceleration || undefined, + } }); } } @@ -728,15 +737,11 @@ class WebsocketHandler { } if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { -<<<<<<< HEAD if (config.MEMPOOL.RUST_GBT) { await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions); } else { - await mempoolBlocks.$makeBlockTemplates(_memPool, true); + await mempoolBlocks.$makeBlockTemplates(_memPool, true, config.MEMPOOL_SERVICES.ACCELERATIONS); } -======= - await mempoolBlocks.$makeBlockTemplates(_memPool, true, config.MEMPOOL_SERVICES.ACCELERATIONS); ->>>>>>> 77b0a8ecc (Refactor accelerated audits) } else { mempoolBlocks.updateMempoolBlocks(_memPool, true); } @@ -799,7 +804,10 @@ class WebsocketHandler { if (mempoolTx && mempoolTx.position) { response['txPosition'] = JSON.stringify({ txid: trackTxid, - position: mempoolTx.position, + position: { + ...mempoolTx.position, + accelerated: mempoolTx.acceleration || undefined, + } }); } } diff --git a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.html b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.html index 9c5c338c0..59d35c91e 100644 --- a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.html +++ b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.html @@ -49,7 +49,7 @@
-
+
diff --git a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.scss b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.scss index 40f43a015..606699d93 100644 --- a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.scss +++ b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.scss @@ -169,4 +169,34 @@ transform: translate(calc(-0.2 * var(--block-size)), calc(1.1 * var(--block-size))); border-radius: 2px; z-index: -1; +} + +.blink{ + width:400px; + height:400px; + border-bottom: 35px solid #FFF; + animation: blink 0.2s infinite; +} +@keyframes blink{ + 0% { + border-bottom: 35px solid green; + } + 50% { + border-bottom: 35px solid yellow; + } + 100% { + border-bottom: 35px solid orange; + } +} + +@-webkit-keyframes blink{ + 0% { + border-bottom: 35px solid green; + } + 50% { + border-bottom: 35px solid yellow; + } + 100% { + border-bottom: 35px solid orange; + } } \ No newline at end of file diff --git a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts index 71075b261..33db897a5 100644 --- a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts +++ b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts @@ -26,6 +26,7 @@ import { animate, style, transition, trigger } from '@angular/animations'; export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy { @Input() minimal: boolean = false; @Input() blockWidth: number = 125; + @Input() containerWidth: number = null; @Input() count: number = null; @Input() spotlight: number = 0; @Input() getHref?: (index) => string = (index) => `/mempool-block/${index}`; diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index 81a6106db..7422c6894 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -488,8 +488,8 @@ - Accelerated fee rate - Effective fee rate + Accelerated fee rate + Effective fee rate
diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index fe6233866..a0a3da5e6 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -188,6 +188,7 @@ export interface RbfTransaction extends TransactionStripped { export interface MempoolPosition { block: number, vsize: number, + accelerated?: boolean } export interface RewardStats { From 6494f890fe587655e5ce3c6ef56633402f3823d4 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 13 Jun 2023 17:03:36 -0400 Subject: [PATCH 09/73] include per-tx pools in /accelerations endpoint --- backend/src/api/mempool-blocks.ts | 8 +++--- backend/src/api/mempool.ts | 36 +++++++++++++++++++----- backend/src/api/services/acceleration.ts | 20 ++++--------- backend/src/api/websocket-handler.ts | 5 ++-- 4 files changed, 42 insertions(+), 27 deletions(-) diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 0ce0ea522..606c98c83 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,6 +1,6 @@ import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction } from '../../rust-gbt'; import logger from '../logger'; -import { MempoolBlock, MempoolTransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats } from '../mempool.interfaces'; +import { MempoolBlock, MempoolTransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, PoolTag } from '../mempool.interfaces'; import { Common, OnlineFeeStatsCalculator } from './common'; import config from '../config'; import { Worker } from 'worker_threads'; @@ -207,7 +207,7 @@ class MempoolBlocks { return mempoolBlockDeltas; } - public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false): Promise { + public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { const start = Date.now(); // reset mempool short ids @@ -225,7 +225,7 @@ class MempoolBlocks { if (entry.uid !== null && entry.uid !== undefined) { const stripped = { uid: entry.uid, - fee: entry.fee + (useAccelerations ? (accelerations[entry.txid] || 0) : 0), + fee: entry.fee + (useAccelerations && (!accelerationPool || accelerations[entry.txid]?.pools?.includes(accelerationPool)) ? (accelerations[entry.txid]?.feeDelta || 0) : 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, @@ -298,7 +298,7 @@ class MempoolBlocks { const addedStripped: CompactThreadTransaction[] = addedAndChanged.filter(entry => entry.uid != null).map(entry => { return { uid: entry.uid || 0, - fee: entry.fee + (useAccelerations ? (accelerations[entry.txid] || 0) : 0), + fee: entry.fee + (useAccelerations ? (accelerations[entry.txid]?.feeDelta || 0) : 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 4efe17731..107aa41cb 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -23,7 +23,7 @@ class Mempool { private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise) | undefined; - private accelerations: { [txId: string]: number } = {}; + private accelerations: { [txId: string]: Acceleration } = {}; private txPerSecondArray: number[] = []; private txPerSecond: number = 0; @@ -332,7 +332,7 @@ class Mempool { this.clearTimer(timer); } - public getAccelerations(): { [txid: string]: number } { + public getAccelerations(): { [txid: string]: Acceleration } { return this.accelerations; } @@ -346,15 +346,37 @@ class Mempool { const changed: string[] = []; - const newAccelerationMap: { [txid: string]: number } = {}; + const newAccelerationMap: { [txid: string]: Acceleration } = {}; for (const acceleration of newAccelerations) { - newAccelerationMap[acceleration.txid] = acceleration.feeDelta; + newAccelerationMap[acceleration.txid] = acceleration; if (this.accelerations[acceleration.txid] == null) { // new acceleration changed.push(acceleration.txid); - } else if (this.accelerations[acceleration.txid] !== acceleration.feeDelta) { - // feeDelta changed - changed.push(acceleration.txid); + } else { + if (this.accelerations[acceleration.txid].feeDelta !== acceleration.feeDelta) { + // feeDelta changed + changed.push(acceleration.txid); + } else if (this.accelerations[acceleration.txid].pools?.length) { + let poolsChanged = false; + const pools = new Set(); + this.accelerations[acceleration.txid].pools.forEach(pool => { + pools.add(pool); + }); + acceleration.pools.forEach(pool => { + if (!pools.has(pool)) { + poolsChanged = true; + } else { + pools.delete(pool); + } + }); + if (pools.size > 0) { + poolsChanged = true; + } + if (poolsChanged) { + // pools changed + changed.push(acceleration.txid); + } + } } } diff --git a/backend/src/api/services/acceleration.ts b/backend/src/api/services/acceleration.ts index 8c849cdd8..1c2fff7c6 100644 --- a/backend/src/api/services/acceleration.ts +++ b/backend/src/api/services/acceleration.ts @@ -5,6 +5,7 @@ import { BlockExtended, PoolTag } from '../../mempool.interfaces'; export interface Acceleration { txid: string, feeDelta: number, + pools: number[], } class AccelerationApi { @@ -17,21 +18,12 @@ class AccelerationApi { } } - public async $fetchPools(): Promise { - if (config.MEMPOOL_SERVICES.ACCELERATIONS) { - const response = await query(`${config.MEMPOOL_SERVICES.API}/partners`); - return (response as PoolTag[]) || []; - } else { - return []; + public isAcceleratedBlock(block: BlockExtended, accelerations: Acceleration[]): boolean { + let anyAccelerated = false; + for (let i = 0; i < accelerations.length && !anyAccelerated; i++) { + anyAccelerated = anyAccelerated || accelerations[i].pools?.includes(block.extras.pool.id); } - } - - public async $isAcceleratedBlock(block: BlockExtended): Promise { - const pools = await this.$fetchPools(); - if (block?.extras?.pool?.id == null) { - return false; - } - return pools.reduce((match, tag) => match || tag.uniqueId === block.extras.pool.id, false); + return anyAccelerated; } } diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 7716ed7b7..ced6642d8 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -22,6 +22,7 @@ import { deepClone } from '../utils/clone'; import priceUpdater from '../tasks/price-updater'; import { ApiPrice } from '../repositories/PricesRepository'; import accelerationApi from './services/acceleration'; +import mempool from './mempool'; // valid 'want' subscriptions const wantable = [ @@ -657,7 +658,7 @@ class WebsocketHandler { if (config.MEMPOOL.AUDIT && memPool.isInSync()) { let projectedBlocks; let auditMempool = _memPool; - const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && await accelerationApi.$isAcceleratedBlock(block); + const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations())); // template calculation functions have mempool side effects, so calculate audits using // a cloned copy of the mempool if we're running a different algorithm for mempool updates const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL; @@ -667,7 +668,7 @@ class WebsocketHandler { if (config.MEMPOOL.RUST_GBT) { projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool); } else { - projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated); + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id); } } else { projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false); From ffc2b6c53cecbd9f5c1b2e40732469ce05bf27f3 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 18 Jul 2023 15:05:44 +0900 Subject: [PATCH 10/73] Add acceleration support to rust gbt --- backend/rust-gbt/index.d.ts | 8 ++- backend/rust-gbt/src/audit_transaction.rs | 27 +++++---- backend/rust-gbt/src/gbt.rs | 13 ++++- backend/rust-gbt/src/lib.rs | 10 +++- backend/src/api/mempool-blocks.ts | 56 ++++++++++++++----- backend/src/api/mempool.ts | 2 +- backend/src/api/websocket-handler.ts | 8 +-- backend/src/mempool.interfaces.ts | 4 +- backend/src/replication/AuditReplication.ts | 1 + .../block-overview-graph.component.ts | 2 +- .../block-overview-graph/block-scene.ts | 2 +- .../block-overview-graph/tx-view.ts | 2 +- .../fee-distribution-graph.component.ts | 2 +- .../src/app/interfaces/websocket.interface.ts | 4 +- 14 files changed, 96 insertions(+), 45 deletions(-) diff --git a/backend/rust-gbt/index.d.ts b/backend/rust-gbt/index.d.ts index 33ae32bdf..2bd8a620a 100644 --- a/backend/rust-gbt/index.d.ts +++ b/backend/rust-gbt/index.d.ts @@ -12,6 +12,10 @@ export interface ThreadTransaction { effectiveFeePerVsize: number inputs: Array } +export interface ThreadAcceleration { + uid: number + delta: number +} export class GbtGenerator { constructor() /** @@ -19,13 +23,13 @@ export class GbtGenerator { * * Rejects if the thread panics or if the Mutex is poisoned. */ - make(mempool: Array, maxUid: number): Promise + make(mempool: Array, accelerations: Array, maxUid: number): Promise /** * # Errors * * Rejects if the thread panics or if the Mutex is poisoned. */ - update(newTxs: Array, removeTxs: Array, maxUid: number): Promise + update(newTxs: Array, removeTxs: Array, accelerations: Array, maxUid: number): Promise } /** * The result from calling the gbt function. diff --git a/backend/rust-gbt/src/audit_transaction.rs b/backend/rust-gbt/src/audit_transaction.rs index 3e25a18a0..9b7472c30 100644 --- a/backend/rust-gbt/src/audit_transaction.rs +++ b/backend/rust-gbt/src/audit_transaction.rs @@ -1,6 +1,6 @@ use crate::{ u32_hasher_types::{u32hashset_new, U32HasherState}, - ThreadTransaction, + ThreadTransaction, thread_acceleration::ThreadAcceleration, }; use std::{ cmp::Ordering, @@ -88,37 +88,42 @@ impl Ord for AuditTransaction { } #[inline] -fn calc_fee_rate(fee: f64, vsize: f64) -> f64 { - fee / (if vsize == 0.0 { 1.0 } else { vsize }) +fn calc_fee_rate(fee: u64, vsize: f64) -> f64 { + (fee as f64) / (if vsize == 0.0 { 1.0 } else { vsize }) } impl AuditTransaction { - pub fn from_thread_transaction(tx: &ThreadTransaction) -> Self { + pub fn from_thread_transaction(tx: &ThreadTransaction, maybe_acceleration: Option>) -> Self { + let fee_delta = match maybe_acceleration { + Some(Some(acceleration)) => acceleration.delta, + _ => 0.0 + }; + let fee = (tx.fee as u64) + (fee_delta as u64); // rounded up to the nearest integer let is_adjusted = tx.weight < (tx.sigops * 20); let sigop_adjusted_vsize = ((tx.weight + 3) / 4).max(tx.sigops * 5); let sigop_adjusted_weight = tx.weight.max(tx.sigops * 20); - let effective_fee_per_vsize = if is_adjusted { - calc_fee_rate(tx.fee, f64::from(sigop_adjusted_weight) / 4.0) + let effective_fee_per_vsize = if is_adjusted || fee_delta > 0.0 { + calc_fee_rate(fee, f64::from(sigop_adjusted_weight) / 4.0) } else { tx.effective_fee_per_vsize }; Self { uid: tx.uid, order: tx.order, - fee: tx.fee as u64, + fee, weight: tx.weight, sigop_adjusted_weight, sigop_adjusted_vsize, sigops: tx.sigops, - adjusted_fee_per_vsize: calc_fee_rate(tx.fee, f64::from(sigop_adjusted_vsize)), + adjusted_fee_per_vsize: calc_fee_rate(fee, f64::from(sigop_adjusted_vsize)), effective_fee_per_vsize, dependency_rate: f64::INFINITY, inputs: tx.inputs.clone(), relatives_set_flag: false, ancestors: u32hashset_new(), children: u32hashset_new(), - ancestor_fee: tx.fee as u64, + ancestor_fee: fee, ancestor_sigop_adjusted_weight: sigop_adjusted_weight, ancestor_sigop_adjusted_vsize: sigop_adjusted_vsize, ancestor_sigops: tx.sigops, @@ -156,7 +161,7 @@ impl AuditTransaction { // grows, so if we think of 0 as "grew infinitely" then dependency_rate would be // the smaller of the two. If either side is NaN, the other side is returned. self.dependency_rate.min(calc_fee_rate( - self.ancestor_fee as f64, + self.ancestor_fee, f64::from(self.ancestor_sigop_adjusted_weight) / 4.0, )) } @@ -172,7 +177,7 @@ impl AuditTransaction { #[inline] fn calc_new_score(&mut self) { self.score = self.adjusted_fee_per_vsize.min(calc_fee_rate( - self.ancestor_fee as f64, + self.ancestor_fee, f64::from(self.ancestor_sigop_adjusted_vsize), )); } diff --git a/backend/rust-gbt/src/gbt.rs b/backend/rust-gbt/src/gbt.rs index 09b6377e6..0bf7f9999 100644 --- a/backend/rust-gbt/src/gbt.rs +++ b/backend/rust-gbt/src/gbt.rs @@ -5,7 +5,7 @@ use tracing::{info, trace}; use crate::{ audit_transaction::{partial_cmp_uid_score, AuditTransaction}, u32_hasher_types::{u32hashset_new, u32priority_queue_with_capacity, U32HasherState}, - GbtResult, ThreadTransactionsMap, + GbtResult, ThreadTransactionsMap, thread_acceleration::ThreadAcceleration, }; const MAX_BLOCK_WEIGHT_UNITS: u32 = 4_000_000 - 4_000; @@ -53,7 +53,13 @@ impl Ord for TxPriority { // TODO: Make gbt smaller to fix these lints. #[allow(clippy::too_many_lines)] #[allow(clippy::cognitive_complexity)] -pub fn gbt(mempool: &mut ThreadTransactionsMap, max_uid: usize) -> GbtResult { +pub fn gbt(mempool: &mut ThreadTransactionsMap, accelerations: &[ThreadAcceleration], max_uid: usize) -> GbtResult { + let mut indexed_accelerations = Vec::with_capacity(max_uid + 1); + indexed_accelerations.resize(max_uid + 1, None); + for acceleration in accelerations { + indexed_accelerations[acceleration.uid as usize] = Some(acceleration); + } + let mempool_len = mempool.len(); let mut audit_pool: AuditPool = Vec::with_capacity(max_uid + 1); audit_pool.resize(max_uid + 1, None); @@ -63,7 +69,8 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap, max_uid: usize) -> GbtResult { info!("Initializing working structs"); for (uid, tx) in &mut *mempool { - let audit_tx = AuditTransaction::from_thread_transaction(tx); + let acceleration = indexed_accelerations.get(*uid as usize); + let audit_tx = AuditTransaction::from_thread_transaction(tx, acceleration.copied()); // Safety: audit_pool and mempool_stack must always contain the same transactions audit_pool[*uid as usize] = Some(ManuallyDrop::new(audit_tx)); mempool_stack.push(*uid); diff --git a/backend/rust-gbt/src/lib.rs b/backend/rust-gbt/src/lib.rs index 516a26402..53db0ba21 100644 --- a/backend/rust-gbt/src/lib.rs +++ b/backend/rust-gbt/src/lib.rs @@ -9,6 +9,7 @@ use napi::bindgen_prelude::Result; use napi_derive::napi; use thread_transaction::ThreadTransaction; +use thread_acceleration::ThreadAcceleration; use tracing::{debug, info, trace}; use tracing_log::LogTracer; use tracing_subscriber::{EnvFilter, FmtSubscriber}; @@ -19,6 +20,7 @@ use std::sync::{Arc, Mutex}; mod audit_transaction; mod gbt; mod thread_transaction; +mod thread_acceleration; mod u32_hasher_types; use u32_hasher_types::{u32hashmap_with_capacity, U32HasherState}; @@ -74,10 +76,11 @@ impl GbtGenerator { /// /// Rejects if the thread panics or if the Mutex is poisoned. #[napi] - pub async fn make(&self, mempool: Vec, max_uid: u32) -> Result { + pub async fn make(&self, mempool: Vec, accelerations: Vec, max_uid: u32) -> Result { trace!("make: Current State {:#?}", self.thread_transactions); run_task( Arc::clone(&self.thread_transactions), + accelerations, max_uid as usize, move |map| { for tx in mempool { @@ -96,11 +99,13 @@ impl GbtGenerator { &self, new_txs: Vec, remove_txs: Vec, + accelerations: Vec, max_uid: u32, ) -> Result { trace!("update: Current State {:#?}", self.thread_transactions); run_task( Arc::clone(&self.thread_transactions), + accelerations, max_uid as usize, move |map| { for tx in new_txs { @@ -141,6 +146,7 @@ pub struct GbtResult { /// to the `HashMap` as the only argument. (A move closure is recommended to meet the bounds) async fn run_task( thread_transactions: Arc>, + accelerations: Vec, max_uid: usize, callback: F, ) -> Result @@ -159,7 +165,7 @@ where callback(&mut map); info!("Starting gbt algorithm for {} elements...", map.len()); - let result = gbt::gbt(&mut map, max_uid); + let result = gbt::gbt(&mut map, &accelerations, max_uid); info!("Finished gbt algorithm for {} elements...", map.len()); debug!( diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 606c98c83..16772b7cd 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,4 +1,4 @@ -import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction } from '../../rust-gbt'; +import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from '../../rust-gbt'; import logger from '../logger'; import { MempoolBlock, MempoolTransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, PoolTag } from '../mempool.interfaces'; import { Common, OnlineFeeStatsCalculator } from './common'; @@ -171,7 +171,7 @@ class MempoolBlocks { for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) { let added: TransactionStripped[] = []; let removed: string[] = []; - const changed: { txid: string, rate: number | undefined, acc: number | undefined }[] = []; + const changed: { txid: string, rate: number | undefined, acc: boolean | undefined }[] = []; if (mempoolBlocks[i] && !prevBlocks[i]) { added = mempoolBlocks[i].transactions; } else if (!mempoolBlocks[i] && prevBlocks[i]) { @@ -265,7 +265,7 @@ class MempoolBlocks { // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); - const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, saveResults); + const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, accelerationPool, saveResults); logger.debug(`makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); @@ -325,7 +325,7 @@ class MempoolBlocks { // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); - this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, saveResults); + this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, null, saveResults); logger.debug(`updateBlockTemplates completed in ${(Date.now() - start) / 1000} seconds`); } catch (e) { logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e)); @@ -337,7 +337,7 @@ class MempoolBlocks { this.rustGbtGenerator = new GbtGenerator(); } - private async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): Promise { + private async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { const start = Date.now(); // reset mempool short ids @@ -353,16 +353,25 @@ class MempoolBlocks { tx.inputs = tx.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[]; } + const accelerations = useAccelerations ? mempool.getAccelerations() : {}; + const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]); + const convertedAccelerations = acceleratedList.map(acc => { + return { + uid: this.getUid(newMempool[acc.txid]), + delta: acc.feeDelta, + }; + }); + // run the block construction algorithm in a separate thread, and wait for a result const rustGbt = saveResults ? this.rustGbtGenerator : new GbtGenerator(); try { const { blocks, blockWeights, rates, clusters } = this.convertNapiResultTxids( - await rustGbt.make(Object.values(newMempool) as RustThreadTransaction[], this.nextUid), + await rustGbt.make(Object.values(newMempool) as RustThreadTransaction[], convertedAccelerations as RustThreadAcceleration[], this.nextUid), ); if (saveResults) { this.rustInitialized = true; } - const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, {}, saveResults); + const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, saveResults); logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); return processed; } catch (e) { @@ -374,19 +383,20 @@ class MempoolBlocks { return this.mempoolBlocks; } - public async $oneOffRustBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }): Promise { - return this.$rustMakeBlockTemplates(newMempool, false); + public async $oneOffRustBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, useAccelerations: boolean, accelerationPool?: number): Promise { + return this.$rustMakeBlockTemplates(newMempool, false, useAccelerations, accelerationPool); } - public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[]): Promise { + public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], useAccelerations: boolean, accelerationPool?: number): Promise { // GBT optimization requires that uids never get too sparse // as a sanity check, we should also explicitly prevent uint32 uid overflow if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * mempoolSize), MAX_UINT32)) { this.resetRustGbt(); } + if (!this.rustInitialized) { // need to reset the worker - await this.$rustMakeBlockTemplates(newMempool, true); + await this.$rustMakeBlockTemplates(newMempool, true, useAccelerations, accelerationPool); return; } @@ -401,12 +411,22 @@ class MempoolBlocks { } const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => (uid !== null && uid !== undefined)) as number[]; + const accelerations = useAccelerations ? mempool.getAccelerations() : {}; + const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]); + const convertedAccelerations = acceleratedList.map(acc => { + return { + uid: this.getUid(newMempool[acc.txid]), + delta: acc.feeDelta, + }; + }); + // run the block construction algorithm in a separate thread, and wait for a result try { const { blocks, blockWeights, rates, clusters } = this.convertNapiResultTxids( await this.rustGbtGenerator.update( added as RustThreadTransaction[], removedUids, + convertedAccelerations as RustThreadAcceleration[], this.nextUid, ), ); @@ -414,7 +434,7 @@ class MempoolBlocks { if (mempoolSize !== resultMempoolSize) { throw new Error('GBT returned wrong number of transactions, cache is probably out of sync'); } else { - this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, {}, true); + this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, true); } this.removeUids(removedUids); logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); @@ -424,7 +444,7 @@ class MempoolBlocks { } } - private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], accelerations, saveResults): MempoolBlockWithTransactions[] { + private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], accelerations, accelerationPool, saveResults): MempoolBlockWithTransactions[] { for (const [txid, rate] of rates) { if (txid in mempool) { mempool[txid].effectiveFeePerVsize = rate; @@ -503,7 +523,15 @@ class MempoolBlocks { mempoolTx.cpfpChecked = true; } - mempoolTx.acceleration = accelerations[txid]; + const acceleration = accelerations[txid]; + if (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool))) { + mempoolTx.acceleration = true; + for (const ancestor of mempoolTx.ancestors || []) { + mempool[ancestor.txid].acceleration = true; + } + } else { + delete mempoolTx.acceleration; + } // online calculation of stack-of-blocks fee stats if (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) { diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 107aa41cb..d1a2d70a9 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -9,7 +9,7 @@ import loadingIndicators from './loading-indicators'; import bitcoinClient from './bitcoin/bitcoin-client'; import bitcoinSecondClient from './bitcoin/bitcoin-second-client'; import rbfCache from './rbf-cache'; -import accelerationApi from './services/acceleration'; +import accelerationApi, { Acceleration } from './services/acceleration'; class Mempool { private inSync: boolean = false; diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index ced6642d8..e42e87acd 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -174,7 +174,7 @@ class WebsocketHandler { } const tx = memPool.getMempool()[trackTxid]; if (tx && tx.position) { - const position: { block: number, vsize: number, accelerated?: number } = { + const position: { block: number, vsize: number, accelerated?: boolean } = { ...tx.position }; if (tx.acceleration) { @@ -397,7 +397,7 @@ class WebsocketHandler { if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { if (config.MEMPOOL.RUST_GBT) { - await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions); + await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions, true,); } else { await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS); } @@ -666,7 +666,7 @@ class WebsocketHandler { auditMempool = deepClone(_memPool); if (config.MEMPOOL.ADVANCED_GBT_AUDIT) { if (config.MEMPOOL.RUST_GBT) { - projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool); + projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool, isAccelerated, block.extras.pool.id); } else { projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id); } @@ -739,7 +739,7 @@ class WebsocketHandler { if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { if (config.MEMPOOL.RUST_GBT) { - await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions); + await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions, true); } else { await mempoolBlocks.$makeBlockTemplates(_memPool, true, config.MEMPOOL_SERVICES.ACCELERATIONS); } diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 185256619..4715440e4 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -92,7 +92,7 @@ export interface TransactionExtended extends IEsploraApi.Transaction { block: number, vsize: number, }; - acceleration?: number; + acceleration?: boolean; uid?: number; } @@ -184,7 +184,7 @@ export interface TransactionStripped { fee: number; vsize: number; value: number; - acc?: number; + acc?: boolean; rate?: number; // effective fee rate } diff --git a/backend/src/replication/AuditReplication.ts b/backend/src/replication/AuditReplication.ts index 26bf6dad7..5de9de0da 100644 --- a/backend/src/replication/AuditReplication.ts +++ b/backend/src/replication/AuditReplication.ts @@ -116,6 +116,7 @@ class AuditReplication { freshTxs: auditSummary.freshTxs || [], sigopTxs: auditSummary.sigopTxs || [], fullrbfTxs: auditSummary.fullrbfTxs || [], + acceleratedTxs: auditSummary.acceleratedTxs || [], matchRate: auditSummary.matchRate, expectedFees: auditSummary.expectedFees, expectedWeight: auditSummary.expectedWeight, diff --git a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts index fe847103f..634d0f524 100644 --- a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts +++ b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts @@ -147,7 +147,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On } } - update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { + update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: boolean | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { if (this.scene) { this.scene.update(add, remove, change, direction, resetLayout); this.start(); diff --git a/frontend/src/app/components/block-overview-graph/block-scene.ts b/frontend/src/app/components/block-overview-graph/block-scene.ts index 94984bae2..cb0537e2a 100644 --- a/frontend/src/app/components/block-overview-graph/block-scene.ts +++ b/frontend/src/app/components/block-overview-graph/block-scene.ts @@ -150,7 +150,7 @@ export default class BlockScene { this.updateAll(startTime, 200, direction); } - update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { + update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: boolean | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { const startTime = performance.now(); const removed = this.removeBatch(remove, startTime, direction); diff --git a/frontend/src/app/components/block-overview-graph/tx-view.ts b/frontend/src/app/components/block-overview-graph/tx-view.ts index 690b974e3..db2c4f6ae 100644 --- a/frontend/src/app/components/block-overview-graph/tx-view.ts +++ b/frontend/src/app/components/block-overview-graph/tx-view.ts @@ -38,7 +38,7 @@ export default class TxView implements TransactionStripped { vsize: number; value: number; feerate: number; - acc?: number; + acc?: boolean; rate?: number; status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; diff --git a/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts b/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts index f275588a1..1d9c289d8 100644 --- a/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts +++ b/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts @@ -64,7 +64,7 @@ export class FeeDistributionGraphComponent implements OnInit, OnChanges, OnDestr return; } const samples = []; - const txs = this.transactions.map(tx => { return { vsize: tx.vsize, rate: tx.rate || (tx.fee / tx.vsize) }; }).sort((a, b) => { return b.rate - a.rate; }); + const txs = this.transactions.filter(tx => !tx.acc).map(tx => { return { vsize: tx.vsize, rate: tx.rate || (tx.fee / tx.vsize) }; }).sort((a, b) => { return b.rate - a.rate; }); const maxBlockVSize = this.stateService.env.BLOCK_WEIGHT_UNITS / 4; const sampleInterval = maxBlockVSize / this.numSamples; let cumVSize = 0; diff --git a/frontend/src/app/interfaces/websocket.interface.ts b/frontend/src/app/interfaces/websocket.interface.ts index f8686042b..43ab1e5f4 100644 --- a/frontend/src/app/interfaces/websocket.interface.ts +++ b/frontend/src/app/interfaces/websocket.interface.ts @@ -70,7 +70,7 @@ export interface MempoolBlockWithTransactions extends MempoolBlock { export interface MempoolBlockDelta { added: TransactionStripped[], removed: string[], - changed?: { txid: string, rate: number | undefined, acc: number | undefined }[]; + changed?: { txid: string, rate: number | undefined, acc: boolean | undefined }[]; } export interface MempoolInfo { @@ -88,7 +88,7 @@ export interface TransactionStripped { fee: number; vsize: number; value: number; - acc?: number; // acceleration delta + acc?: boolean; // is accelerated? rate?: number; // effective fee rate status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; From 3838d947b13f24e1abf7ef2e03147e811a22922a Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 18 Jul 2023 15:14:33 +0900 Subject: [PATCH 11/73] fix tests --- backend/src/__fixtures__/mempool-config.template.json | 6 +++--- backend/src/__tests__/gbt/gbt-tests.ts | 2 +- docker/backend/start.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index d6754f966..21c8e51c0 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -128,8 +128,8 @@ "AUDIT_START_HEIGHT": 774000, "SERVERS": [] }, - "MEMPOOl_SERVICES": { - "API": "__MEMPOOL_SERVICES_API__", - "ACCELERATIONS": "__MEMPOOL_SERVICES_ACCELERATIONS__" + "MEMPOOL_SERVICES": { + "API": "", + "ACCELERATIONS": false } } diff --git a/backend/src/__tests__/gbt/gbt-tests.ts b/backend/src/__tests__/gbt/gbt-tests.ts index 0651faac4..0c2eb0176 100644 --- a/backend/src/__tests__/gbt/gbt-tests.ts +++ b/backend/src/__tests__/gbt/gbt-tests.ts @@ -15,7 +15,7 @@ describe('Rust GBT', () => { test('should produce the same template as getBlockTemplate from Bitcoin Core', async () => { const rustGbt = new GbtGenerator(); const { mempool, maxUid } = mempoolFromArrayBuffer(vectorBuffer.buffer); - const result = await rustGbt.make(mempool, maxUid); + const result = await rustGbt.make(mempool, [], maxUid); const blocks: [string, number][][] = result.blocks.map(block => { return block.map(uid => [vectorUidMap.get(uid) || 'missing', uid]); diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 66f873605..c9088c0a6 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -137,8 +137,8 @@ __REPLICATION_AUDIT_START_HEIGHT__=${REPLICATION_AUDIT_START_HEIGHT:=774000} __REPLICATION_SERVERS__=${REPLICATION_SERVERS:=[]} # MEMPOOL_SERVICES -__MEMPOOL_SERVICES_API__==${MEMPOOL_SERVICES_API:=""} -__MEMPOOL_SERVICES_ACCELERATIONS__==${MEMPOOL_SERVICES_ACCELERATIONS:=false} +__MEMPOOL_SERVICES_API__=${MEMPOOL_SERVICES_API:=""} +__MEMPOOL_SERVICES_ACCELERATIONS__=${MEMPOOL_SERVICES_ACCELERATIONS:=false} mkdir -p "${__MEMPOOL_CACHE_DIR__}" From 2a2aee21fb466128225d2f3791780152c9e5372f Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 18 Jul 2023 16:08:25 +0900 Subject: [PATCH 12/73] fix audit highlightning and fee ranges --- backend/src/api/common.ts | 2 +- backend/src/api/mempool-blocks.ts | 5 +++-- backend/src/api/websocket-handler.ts | 6 +++--- backend/src/repositories/BlocksAuditsRepository.ts | 1 - frontend/src/app/components/block/block.component.ts | 3 +++ frontend/src/app/interfaces/node-api.interface.ts | 1 + 6 files changed, 11 insertions(+), 7 deletions(-) diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 775da2643..b2f476da3 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -461,7 +461,7 @@ export class Common { }; } - static calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string }[]): EffectiveFeeStats { + static calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string, acceleration?: boolean }[]): EffectiveFeeStats { const sortedTxs = transactions.map(tx => { return { txid: tx.txid, weight: tx.weight, rate: tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4)) }; }).sort((a, b) => a.rate - b.rate); let weightCount = 0; diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 16772b7cd..599a53c23 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -337,7 +337,8 @@ class MempoolBlocks { this.rustGbtGenerator = new GbtGenerator(); } - private async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { + public async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { + console.log('$rustMakeBlockTemplates'); const start = Date.now(); // reset mempool short ids @@ -569,7 +570,7 @@ class MempoolBlocks { private dataToMempoolBlocks(transactionIds: string[], transactions: MempoolTransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, feeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions { if (!feeStats) { - feeStats = Common.calcEffectiveFeeStatistics(transactions); + feeStats = Common.calcEffectiveFeeStatistics(transactions.filter(tx => !tx.acceleration)); } return { blockSize: totalSize, diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index e42e87acd..9eb624b2e 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -397,7 +397,7 @@ class WebsocketHandler { if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { if (config.MEMPOOL.RUST_GBT) { - await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions, true,); + await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions, config.MEMPOOL_SERVICES.ACCELERATIONS); } else { await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS); } @@ -661,7 +661,7 @@ class WebsocketHandler { const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations())); // template calculation functions have mempool side effects, so calculate audits using // a cloned copy of the mempool if we're running a different algorithm for mempool updates - const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL; + const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL || isAccelerated; if (separateAudit) { auditMempool = deepClone(_memPool); if (config.MEMPOOL.ADVANCED_GBT_AUDIT) { @@ -675,7 +675,7 @@ class WebsocketHandler { } } else { if ((config.MEMPOOL_SERVICES.ACCELERATIONS && !isAccelerated)) { - projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated); + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, false); } else { projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); } diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index 9c7568567..c17958d2b 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -85,7 +85,6 @@ class BlocksAuditRepositories { rows[0].sigopTxs = JSON.parse(rows[0].sigopTxs); rows[0].fullrbfTxs = JSON.parse(rows[0].fullrbfTxs); rows[0].acceleratedTxs = JSON.parse(rows[0].acceleratedTxs); - rows[0].transactions = JSON.parse(rows[0].transactions); rows[0].template = JSON.parse(rows[0].template); return rows[0]; diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 1e94b1a9c..1345717bd 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -347,6 +347,9 @@ export class BlockComponent implements OnInit, OnDestroy { if (blockAudit?.template) { for (const tx of blockAudit.template) { inTemplate[tx.txid] = true; + if (tx.acc) { + isAccelerated[tx.txid] = true; + } } for (const tx of transactions) { inBlock[tx.txid] = true; diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index a0a3da5e6..cb42eaed3 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -176,6 +176,7 @@ export interface TransactionStripped { vsize: number; value: number; rate?: number; // effective fee rate + acc?: boolean; status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; } From 7c641544b2fda1c48fe6d2df43bb343537ef98cc Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 18 Jul 2023 16:32:52 +0900 Subject: [PATCH 13/73] check in missing rust-gbt file --- backend/rust-gbt/src/thread_acceleration.rs | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 backend/rust-gbt/src/thread_acceleration.rs diff --git a/backend/rust-gbt/src/thread_acceleration.rs b/backend/rust-gbt/src/thread_acceleration.rs new file mode 100644 index 000000000..618cac3db --- /dev/null +++ b/backend/rust-gbt/src/thread_acceleration.rs @@ -0,0 +1,8 @@ +use napi_derive::napi; + +#[derive(Debug)] +#[napi(object)] +pub struct ThreadAcceleration { + pub uid: u32, + pub delta: f64, // fee delta +} From 928a8be846779820c4290aabd02dceae65983dd7 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 18 Jul 2023 17:30:51 +0900 Subject: [PATCH 14/73] fix pool-dependent accelerated audit handling --- backend/rust-gbt/src/audit_transaction.rs | 2 +- backend/src/api/mempool-blocks.ts | 13 +++++++------ backend/src/api/websocket-handler.ts | 6 +++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/backend/rust-gbt/src/audit_transaction.rs b/backend/rust-gbt/src/audit_transaction.rs index 9b7472c30..fe20e5a14 100644 --- a/backend/rust-gbt/src/audit_transaction.rs +++ b/backend/rust-gbt/src/audit_transaction.rs @@ -130,7 +130,7 @@ impl AuditTransaction { score: 0.0, used: false, modified: false, - dirty: effective_fee_per_vsize != tx.effective_fee_per_vsize, + dirty: effective_fee_per_vsize != tx.effective_fee_per_vsize || fee_delta > 0.0, } } diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 599a53c23..6d3fa0036 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -388,7 +388,7 @@ class MempoolBlocks { return this.$rustMakeBlockTemplates(newMempool, false, useAccelerations, accelerationPool); } - public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], useAccelerations: boolean, accelerationPool?: number): Promise { + public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], useAccelerations: boolean, accelerationPool?: number): Promise { // GBT optimization requires that uids never get too sparse // as a sanity check, we should also explicitly prevent uint32 uid overflow if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * mempoolSize), MAX_UINT32)) { @@ -397,8 +397,7 @@ class MempoolBlocks { if (!this.rustInitialized) { // need to reset the worker - await this.$rustMakeBlockTemplates(newMempool, true, useAccelerations, accelerationPool); - return; + return this.$rustMakeBlockTemplates(newMempool, true, useAccelerations, accelerationPool); } const start = Date.now(); @@ -435,13 +434,15 @@ class MempoolBlocks { if (mempoolSize !== resultMempoolSize) { throw new Error('GBT returned wrong number of transactions, cache is probably out of sync'); } else { - this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, true); + const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, true); + this.removeUids(removedUids); + logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); + return processed; } - this.removeUids(removedUids); - logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); } catch (e) { logger.err('RUST updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e)); this.resetRustGbt(); + return this.mempoolBlocks; } } diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 9eb624b2e..bed059ad2 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -661,7 +661,7 @@ class WebsocketHandler { const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations())); // template calculation functions have mempool side effects, so calculate audits using // a cloned copy of the mempool if we're running a different algorithm for mempool updates - const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL || isAccelerated; + const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL; if (separateAudit) { auditMempool = deepClone(_memPool); if (config.MEMPOOL.ADVANCED_GBT_AUDIT) { @@ -674,8 +674,8 @@ class WebsocketHandler { projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false); } } else { - if ((config.MEMPOOL_SERVICES.ACCELERATIONS && !isAccelerated)) { - projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, false); + if ((config.MEMPOOL_SERVICES.ACCELERATIONS)) { + projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(auditMempool, Object.keys(auditMempool).length, [], [], isAccelerated, block.extras.pool.id); } else { projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); } From cde4af59305e9feb99ad061b8645c676720bb156 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 19 Jul 2023 11:18:04 +0900 Subject: [PATCH 15/73] fix mismatched use of gbt implementations --- backend/src/api/mempool-blocks.ts | 12 +++++++++--- backend/src/api/websocket-handler.ts | 6 +++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 6d3fa0036..1ea4d2af8 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -211,9 +211,12 @@ class MempoolBlocks { const start = Date.now(); // reset mempool short ids - this.resetUids(); + if (saveResults) { + this.resetUids(); + } + // set missing short ids for (const tx of Object.values(newMempool)) { - this.setUid(tx, true); + this.setUid(tx, !saveResults); } const accelerations = useAccelerations ? mempool.getAccelerations() : {}; @@ -497,6 +500,8 @@ class MempoolBlocks { } } + const isAccelerated : { [txid: string]: boolean } = {}; + const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2; // update this thread's mempool with the results let mempoolTx: MempoolTransactionExtended; @@ -526,10 +531,11 @@ class MempoolBlocks { } const acceleration = accelerations[txid]; - if (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool))) { + if (isAccelerated[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) { mempoolTx.acceleration = true; for (const ancestor of mempoolTx.ancestors || []) { mempool[ancestor.txid].acceleration = true; + isAccelerated[ancestor.txid] = true; } } else { delete mempoolTx.acceleration; diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index bed059ad2..f5c940218 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -675,7 +675,11 @@ class WebsocketHandler { } } else { if ((config.MEMPOOL_SERVICES.ACCELERATIONS)) { - projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(auditMempool, Object.keys(auditMempool).length, [], [], isAccelerated, block.extras.pool.id); + if (config.MEMPOOL.RUST_GBT) { + projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(auditMempool, Object.keys(auditMempool).length, [], [], isAccelerated, block.extras.pool.id); + } else { + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id); + } } else { projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); } From 67cff804a6979ed6bf5220bdf829be8e394b82b6 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 22 Jul 2023 09:46:35 +0900 Subject: [PATCH 16/73] /accelerations -> /accelerator/accelerations --- backend/src/api/services/acceleration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/services/acceleration.ts b/backend/src/api/services/acceleration.ts index 1c2fff7c6..635dc8300 100644 --- a/backend/src/api/services/acceleration.ts +++ b/backend/src/api/services/acceleration.ts @@ -11,7 +11,7 @@ export interface Acceleration { class AccelerationApi { public async $fetchAccelerations(): Promise { if (config.MEMPOOL_SERVICES.ACCELERATIONS) { - const response = await query(`${config.MEMPOOL_SERVICES.API}/accelerations`); + const response = await query(`${config.MEMPOOL_SERVICES.API}/accelerator/accelerations`); return (response as Acceleration[]) || []; } else { return []; From 945a8ce92e7bc99d3ae3b1dcde746cf1799536c1 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 30 Jul 2023 18:56:57 +0900 Subject: [PATCH 17/73] Use log10 scale for projected block fee graph --- .../fee-distribution-graph.component.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts b/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts index f275588a1..212510e71 100644 --- a/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts +++ b/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts @@ -74,14 +74,14 @@ export class FeeDistributionGraphComponent implements OnInit, OnChanges, OnDestr this.labelInterval = this.numSamples / this.numLabels; while (nextSample <= maxBlockVSize) { if (txIndex >= txs.length) { - samples.push([(1 - (sampleIndex / this.numSamples)) * 100, 0]); + samples.push([(1 - (sampleIndex / this.numSamples)) * 100, 0.000001]); nextSample += sampleInterval; sampleIndex++; continue; } while (txs[txIndex] && nextSample < cumVSize + txs[txIndex].vsize) { - samples.push([(1 - (sampleIndex / this.numSamples)) * 100, txs[txIndex].rate]); + samples.push([(1 - (sampleIndex / this.numSamples)) * 100, txs[txIndex].rate || 0.000001]); nextSample += sampleInterval; sampleIndex++; } @@ -118,7 +118,9 @@ export class FeeDistributionGraphComponent implements OnInit, OnChanges, OnDestr }, }, yAxis: { - type: 'value', + type: 'log', + min: 1, + max: this.data.reduce((min, val) => Math.max(min, val[1]), 1), // name: 'Effective Fee Rate s/vb', // nameLocation: 'middle', splitLine: { @@ -129,12 +131,16 @@ export class FeeDistributionGraphComponent implements OnInit, OnChanges, OnDestr } }, axisLabel: { + show: true, formatter: (value: number): string => { const unitValue = this.weightMode ? value / 4 : value; const selectedPowerOfTen = selectPowerOfTen(unitValue); const newVal = Math.round(unitValue / selectedPowerOfTen.divider); return `${newVal}${selectedPowerOfTen.unit}`; }, + }, + axisTick: { + show: true, } }, series: [{ From 363fc1b00bd336618008bfc9fa62cc5a6594f3a2 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 31 Jul 2023 15:03:54 +0900 Subject: [PATCH 18/73] Get blocks from electrs again --- backend/src/api/bitcoin/bitcoin.routes.ts | 4 ++-- backend/src/api/blocks.ts | 12 ++++++------ backend/src/api/mempool.ts | 2 +- backend/src/api/mining/mining.ts | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index ffdb2e629..b24c83932 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -6,7 +6,7 @@ import websocketHandler from '../websocket-handler'; import mempool from '../mempool'; import feeApi from '../fee-api'; import mempoolBlocks from '../mempool-blocks'; -import bitcoinApi, { bitcoinCoreApi } from './bitcoin-api-factory'; +import bitcoinApi from './bitcoin-api-factory'; import { Common } from '../common'; import backendInfo from '../backend-info'; import transactionUtils from '../transaction-utils'; @@ -483,7 +483,7 @@ class BitcoinRoutes { returnBlocks.push(localBlock); nextHash = localBlock.previousblockhash; } else { - const block = await bitcoinCoreApi.$getBlock(nextHash); + const block = await bitcoinApi.$getBlock(nextHash); returnBlocks.push(block); nextHash = block.previousblockhash; } diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index f86bc53e9..88f85ba6e 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -599,7 +599,7 @@ class Blocks { loadingIndicators.setProgress('block-indexing', progress, false); } const blockHash = await bitcoinApi.$getBlockHash(blockHeight); - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, null, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -656,7 +656,7 @@ class Blocks { const heightDiff = blockHeightTip % 2016; const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff); this.updateTimerProgress(timer, 'got block hash for initial difficulty adjustment'); - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); this.updateTimerProgress(timer, 'got block for initial difficulty adjustment'); this.lastDifficultyAdjustmentTime = block.timestamp; this.currentDifficulty = block.difficulty; @@ -664,7 +664,7 @@ class Blocks { if (blockHeightTip >= 2016) { const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016); this.updateTimerProgress(timer, 'got previous block hash for initial difficulty adjustment'); - const previousPeriodBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(previousPeriodBlockHash); + const previousPeriodBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(previousPeriodBlockHash); this.updateTimerProgress(timer, 'got previous block for initial difficulty adjustment'); this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100; logger.debug(`Initial difficulty adjustment data set.`); @@ -862,7 +862,7 @@ class Blocks { } const blockHash = await bitcoinApi.$getBlockHash(height); - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -874,7 +874,7 @@ class Blocks { } public async $indexStaleBlock(hash: string): Promise { - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash); const transactions = await this.$getTransactionsExtended(hash, block.height, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -899,7 +899,7 @@ class Blocks { } // Bitcoin network, add our custom data on top - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash); if (block.stale) { return await this.$indexStaleBlock(hash); } else { diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index d5214de5d..b9b648012 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -1,5 +1,5 @@ import config from '../config'; -import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; +import bitcoinApi from './bitcoin/bitcoin-api-factory'; import { MempoolTransactionExtended, TransactionExtended, VbytesPerSecond } from '../mempool.interfaces'; import logger from '../logger'; import { Common } from './common'; diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index e190492b8..beda0bc48 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -11,7 +11,7 @@ import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjust import config from '../../config'; import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository'; import PricesRepository from '../../repositories/PricesRepository'; -import { bitcoinCoreApi } from '../bitcoin/bitcoin-api-factory'; +import bitcoinApi from '../bitcoin/bitcoin-api-factory'; import { IEsploraApi } from '../bitcoin/esplora-api.interface'; import database from '../../database'; @@ -201,7 +201,7 @@ class Mining { try { const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp; - const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); const genesisTimestamp = genesisBlock.timestamp * 1000; const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps(); @@ -312,7 +312,7 @@ class Mining { const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp; try { - const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); const genesisTimestamp = genesisBlock.timestamp * 1000; const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp); const lastMidnight = this.getDateMidnight(new Date()); @@ -421,7 +421,7 @@ class Mining { } const blocks: any = await BlocksRepository.$getBlocksDifficulty(); - const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); let currentDifficulty = genesisBlock.difficulty; let totalIndexed = 0; From bafc0bd9cfb579ae0e2d428164dc4c8ebb408039 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 31 Jul 2023 16:03:18 +0900 Subject: [PATCH 19/73] fix indexing log prints --- backend/src/api/blocks.ts | 42 +++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 88f85ba6e..2e792bdba 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -419,8 +419,8 @@ class Blocks { let newlyIndexed = 0; let totalIndexed = indexedBlockSummariesHashesArray.length; let indexedThisRun = 0; - let timer = new Date().getTime() / 1000; - const startedAt = new Date().getTime() / 1000; + let timer = Date.now() / 1000; + const startedAt = Date.now() / 1000; for (const block of indexedBlocks) { if (indexedBlockSummariesHashes[block.hash] === true) { @@ -428,13 +428,13 @@ class Blocks { } // Logging - const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer)); + const elapsedSeconds = (Date.now() / 1000) - timer; if (elapsedSeconds > 5) { - const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds); + const runningFor = (Date.now() / 1000) - startedAt; + const blockPerSeconds = indexedThisRun / elapsedSeconds; const progress = Math.round(totalIndexed / indexedBlocks.length * 10000) / 100; - logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining); - timer = new Date().getTime() / 1000; + logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor.toFixed(2)} seconds`, logger.tags.mining); + timer = Date.now() / 1000; indexedThisRun = 0; } @@ -477,18 +477,18 @@ class Blocks { // Logging let count = 0; let countThisRun = 0; - let timer = new Date().getTime() / 1000; - const startedAt = new Date().getTime() / 1000; + let timer = Date.now() / 1000; + const startedAt = Date.now() / 1000; for (const height of unindexedBlockHeights) { // Logging const hash = await bitcoinApi.$getBlockHash(height); - const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer); + const elapsedSeconds = (Date.now() / 1000) - timer; if (elapsedSeconds > 5) { - const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - const blockPerSeconds = (countThisRun / elapsedSeconds); + const runningFor = (Date.now() / 1000) - startedAt; + const blockPerSeconds = countThisRun / elapsedSeconds; const progress = Math.round(count / unindexedBlockHeights.length * 10000) / 100; - logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor} seconds`); - timer = new Date().getTime() / 1000; + logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor.toFixed(2)} seconds`); + timer = Date.now() / 1000; countThisRun = 0; } @@ -567,8 +567,8 @@ class Blocks { let totalIndexed = await blocksRepository.$blockCountBetweenHeight(currentBlockHeight, lastBlockToIndex); let indexedThisRun = 0; let newlyIndexed = 0; - const startedAt = new Date().getTime() / 1000; - let timer = new Date().getTime() / 1000; + const startedAt = Date.now() / 1000; + let timer = Date.now() / 1000; while (currentBlockHeight >= lastBlockToIndex) { const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1); @@ -588,13 +588,13 @@ class Blocks { } ++indexedThisRun; ++totalIndexed; - const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer); + const elapsedSeconds = (Date.now() / 1000) - timer; if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) { - const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds); + const runningFor = (Date.now() / 1000) - startedAt; + const blockPerSeconds = indexedThisRun / elapsedSeconds; const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100; - logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining); - timer = new Date().getTime() / 1000; + logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress.toFixed(2)}%) | elapsed: ${runningFor.toFixed(2)} seconds`, logger.tags.mining); + timer = Date.now() / 1000; indexedThisRun = 0; loadingIndicators.setProgress('block-indexing', progress, false); } From 6b7d8d95f7beab28d8f5b6457c431bfbcc791517 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 31 Jul 2023 16:41:36 +0900 Subject: [PATCH 20/73] reduce mempool poll rate while indexing --- backend/src/index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index 51d407f6f..222537d0b 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -195,7 +195,8 @@ class Server { // rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS const elapsed = Date.now() - start; - const remainingTime = Math.max(0, config.MEMPOOL.POLL_RATE_MS - elapsed) + const pollRate = config.MEMPOOL.POLL_RATE_MS * (indexer.indexerRunning ? 10 : 1); + const remainingTime = Math.max(0, pollRate - elapsed); setTimeout(this.runMainUpdateLoop.bind(this), numHandledBlocks > 0 ? 0 : remainingTime); this.backendRetryCount = 0; } catch (e: any) { From 0d25ef0b5b8a05a83fde361addaae6c2ad41110f Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 31 Jul 2023 18:02:18 +0900 Subject: [PATCH 21/73] Get block txs from esplora, index CPFP together with summaries --- backend/src/api/blocks.ts | 59 +++++++++++++++----- backend/src/repositories/BlocksRepository.ts | 15 ++++- 2 files changed, 58 insertions(+), 16 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 2e792bdba..20c015f44 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -438,7 +438,14 @@ class Blocks { indexedThisRun = 0; } - await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary + + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(block.hash)).map(tx => transactionUtils.extendTransaction(tx)); + const cpfpSummary = await this.$indexCPFP(block.hash, block.height, txs); + await this.$getStrippedBlockTransactions(block.hash, true, true, cpfpSummary, block.height); // This will index the block summary + } else { + await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary + } // Logging indexedThisRun++; @@ -942,10 +949,15 @@ class Blocks { }), }; } else { - // Call Core RPC - const block = await bitcoinClient.getBlock(hash, 2); - summary = this.summarizeBlock(block); - height = block.height; + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx)); + summary = this.summarizeBlockTransactions(hash, txs); + } else { + // Call Core RPC + const block = await bitcoinClient.getBlock(hash, 2); + summary = this.summarizeBlock(block); + height = block.height; + } } if (height == null) { const block = await bitcoinApi.$getBlock(hash); @@ -1068,8 +1080,17 @@ class Blocks { if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) { cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash); if (cleanBlock.fee_amt_percentiles === null) { - const block = await bitcoinClient.getBlock(cleanBlock.hash, 2); - const summary = this.summarizeBlock(block); + + let summary; + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(cleanBlock.hash)).map(tx => transactionUtils.extendTransaction(tx)); + summary = this.summarizeBlockTransactions(cleanBlock.hash, txs); + } else { + // Call Core RPC + const block = await bitcoinClient.getBlock(cleanBlock.hash, 2); + summary = this.summarizeBlock(block); + } + await BlocksSummariesRepository.$saveTransactions(cleanBlock.height, cleanBlock.hash, summary.transactions); cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash); } @@ -1129,19 +1150,29 @@ class Blocks { return this.currentBlockHeight; } - public async $indexCPFP(hash: string, height: number): Promise { - const block = await bitcoinClient.getBlock(hash, 2); - const transactions = block.tx.map(tx => { - tx.fee *= 100_000_000; - return tx; - }); + public async $indexCPFP(hash: string, height: number, txs?: TransactionExtended[]): Promise { + let transactions = txs; + if (!transactions) { + if (config.MEMPOOL.BACKEND === 'esplora') { + transactions = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx)); + } + if (!transactions) { + const block = await bitcoinClient.getBlock(hash, 2); + transactions = block.tx.map(tx => { + tx.fee *= 100_000_000; + return tx; + }); + } + } - const summary = Common.calculateCpfp(height, transactions); + const summary = Common.calculateCpfp(height, transactions as TransactionExtended[]); await this.$saveCpfp(hash, height, summary); const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions); await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats); + + return summary; } public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise { diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 078b85a03..5f20f5abd 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -1,3 +1,4 @@ +import bitcoinApi from '../api/bitcoin/bitcoin-api-factory'; import { BlockExtended, BlockExtension, BlockPrice, EffectiveFeeStats } from '../mempool.interfaces'; import DB from '../database'; import logger from '../logger'; @@ -12,6 +13,7 @@ import config from '../config'; import chainTips from '../api/chain-tips'; import blocks from '../api/blocks'; import BlocksAuditsRepository from './BlocksAuditsRepository'; +import transactionUtils from '../api/transaction-utils'; interface DatabaseBlock { id: string; @@ -1036,8 +1038,17 @@ class BlocksRepository { { extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id); if (extras.feePercentiles === null) { - const block = await bitcoinClient.getBlock(dbBlk.id, 2); - const summary = blocks.summarizeBlock(block); + + let summary; + if (config.MEMPOOL.BACKEND === 'esplora') { + const txs = (await bitcoinApi.$getTxsForBlock(dbBlk.id)).map(tx => transactionUtils.extendTransaction(tx)); + summary = blocks.summarizeBlockTransactions(dbBlk.id, txs); + } else { + // Call Core RPC + const block = await bitcoinClient.getBlock(dbBlk.id, 2); + summary = blocks.summarizeBlock(block); + } + await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.id, summary.transactions); extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id); } From 17f1cb8648db406a3887647b4d8be73e9951b1f8 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Mon, 31 Jul 2023 14:10:49 -0700 Subject: [PATCH 22/73] Fix config unit test that was returning early --- backend/src/__tests__/config.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 0c06b03e1..edfcc7f47 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -181,12 +181,12 @@ describe('Mempool Backend Config', () => { // We have a few cases where we can't follow the pattern if (root === 'MEMPOOL' && key === 'HTTP_PORT') { console.log('skipping check for MEMPOOL_HTTP_PORT'); - return; + continue; } switch (typeof value) { case 'object': { if (Array.isArray(value)) { - return; + continue; } else { parseJson(value, key); } From b7b6548cce486bace954820b6baa273a92fa52b7 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Mon, 31 Jul 2023 14:11:31 -0700 Subject: [PATCH 23/73] Fix RUST GBT Docker override --- docker/backend/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 7071493fa..e05c73710 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -170,7 +170,7 @@ sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-co sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json -sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_GBT__}!g" mempool-config.json +sed -i "s!__MEMPOOL_RUST_GBT__!${__MEMPOOL_RUST_GBT__}!g" mempool-config.json sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json From aa17f8203c58ceefd0cd120b05463cb1c7823a6b Mon Sep 17 00:00:00 2001 From: Bastien Guillaumat Date: Tue, 1 Aug 2023 03:13:58 +0200 Subject: [PATCH 24/73] Add bguillaumat.txt to contributors --- contributors/bguillaumat.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 contributors/bguillaumat.txt diff --git a/contributors/bguillaumat.txt b/contributors/bguillaumat.txt new file mode 100644 index 000000000..ac14a07c7 --- /dev/null +++ b/contributors/bguillaumat.txt @@ -0,0 +1,3 @@ +I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of January 25, 2022. + +Signed: bguillaumat From 910e67ff36c404276cfba0f1f25edd8d366e2de9 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 31 Jul 2023 15:03:54 +0900 Subject: [PATCH 25/73] Get blocks from electrs again --- backend/src/api/bitcoin/bitcoin.routes.ts | 4 ++-- backend/src/api/blocks.ts | 12 ++++++------ backend/src/api/mempool.ts | 2 +- backend/src/api/mining/mining.ts | 8 ++++---- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index d2a77c995..f27bb7797 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -6,7 +6,7 @@ import websocketHandler from '../websocket-handler'; import mempool from '../mempool'; import feeApi from '../fee-api'; import mempoolBlocks from '../mempool-blocks'; -import bitcoinApi, { bitcoinCoreApi } from './bitcoin-api-factory'; +import bitcoinApi from './bitcoin-api-factory'; import { Common } from '../common'; import backendInfo from '../backend-info'; import transactionUtils from '../transaction-utils'; @@ -483,7 +483,7 @@ class BitcoinRoutes { returnBlocks.push(localBlock); nextHash = localBlock.previousblockhash; } else { - const block = await bitcoinCoreApi.$getBlock(nextHash); + const block = await bitcoinApi.$getBlock(nextHash); returnBlocks.push(block); nextHash = block.previousblockhash; } diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index f86bc53e9..88f85ba6e 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -599,7 +599,7 @@ class Blocks { loadingIndicators.setProgress('block-indexing', progress, false); } const blockHash = await bitcoinApi.$getBlockHash(blockHeight); - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, null, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -656,7 +656,7 @@ class Blocks { const heightDiff = blockHeightTip % 2016; const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff); this.updateTimerProgress(timer, 'got block hash for initial difficulty adjustment'); - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); this.updateTimerProgress(timer, 'got block for initial difficulty adjustment'); this.lastDifficultyAdjustmentTime = block.timestamp; this.currentDifficulty = block.difficulty; @@ -664,7 +664,7 @@ class Blocks { if (blockHeightTip >= 2016) { const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016); this.updateTimerProgress(timer, 'got previous block hash for initial difficulty adjustment'); - const previousPeriodBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(previousPeriodBlockHash); + const previousPeriodBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(previousPeriodBlockHash); this.updateTimerProgress(timer, 'got previous block for initial difficulty adjustment'); this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100; logger.debug(`Initial difficulty adjustment data set.`); @@ -862,7 +862,7 @@ class Blocks { } const blockHash = await bitcoinApi.$getBlockHash(height); - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); const transactions = await this.$getTransactionsExtended(blockHash, block.height, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -874,7 +874,7 @@ class Blocks { } public async $indexStaleBlock(hash: string): Promise { - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash); const transactions = await this.$getTransactionsExtended(hash, block.height, true); const blockExtended = await this.$getBlockExtended(block, transactions); @@ -899,7 +899,7 @@ class Blocks { } // Bitcoin network, add our custom data on top - const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash); + const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash); if (block.stale) { return await this.$indexStaleBlock(hash); } else { diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index d5214de5d..b9b648012 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -1,5 +1,5 @@ import config from '../config'; -import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; +import bitcoinApi from './bitcoin/bitcoin-api-factory'; import { MempoolTransactionExtended, TransactionExtended, VbytesPerSecond } from '../mempool.interfaces'; import logger from '../logger'; import { Common } from './common'; diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index e190492b8..beda0bc48 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -11,7 +11,7 @@ import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjust import config from '../../config'; import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository'; import PricesRepository from '../../repositories/PricesRepository'; -import { bitcoinCoreApi } from '../bitcoin/bitcoin-api-factory'; +import bitcoinApi from '../bitcoin/bitcoin-api-factory'; import { IEsploraApi } from '../bitcoin/esplora-api.interface'; import database from '../../database'; @@ -201,7 +201,7 @@ class Mining { try { const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp; - const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); const genesisTimestamp = genesisBlock.timestamp * 1000; const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps(); @@ -312,7 +312,7 @@ class Mining { const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp; try { - const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); const genesisTimestamp = genesisBlock.timestamp * 1000; const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp); const lastMidnight = this.getDateMidnight(new Date()); @@ -421,7 +421,7 @@ class Mining { } const blocks: any = await BlocksRepository.$getBlocksDifficulty(); - const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0)); + const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); let currentDifficulty = genesisBlock.difficulty; let totalIndexed = 0; From 0924bb6ac09c4543979539738dfacd02ee8a4613 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 31 Jul 2023 18:28:08 +0900 Subject: [PATCH 26/73] Use bits to detect difficulty adjustments, not difficulty --- backend/src/api/mining/mining.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index beda0bc48..3eeb3e9e5 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -423,6 +423,7 @@ class Mining { const blocks: any = await BlocksRepository.$getBlocksDifficulty(); const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0)); let currentDifficulty = genesisBlock.difficulty; + let currentBits = genesisBlock.bits; let totalIndexed = 0; if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && indexedHeights[0] !== true) { @@ -436,17 +437,18 @@ class Mining { const oldestConsecutiveBlock = await BlocksRepository.$getOldestConsecutiveBlock(); if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== -1) { - currentDifficulty = oldestConsecutiveBlock.difficulty; + currentBits = oldestConsecutiveBlock.bits; } let totalBlockChecked = 0; let timer = new Date().getTime() / 1000; for (const block of blocks) { - if (block.difficulty !== currentDifficulty) { + if (block.bits !== currentBits) { if (indexedHeights[block.height] === true) { // Already indexed if (block.height >= oldestConsecutiveBlock.height) { currentDifficulty = block.difficulty; + currentBits = block.bits; } continue; } @@ -464,6 +466,7 @@ class Mining { totalIndexed++; if (block.height >= oldestConsecutiveBlock.height) { currentDifficulty = block.difficulty; + currentBits = block.bits; } } From a863c17408bade6bd3021816037992021d585e61 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 1 Aug 2023 11:16:35 +0900 Subject: [PATCH 27/73] Fix difficulty indexing db queries to return bits --- backend/src/repositories/BlocksRepository.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 078b85a03..ca54fc1b7 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -539,7 +539,7 @@ class BlocksRepository { */ public async $getBlocksDifficulty(): Promise { try { - const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty FROM blocks`); + const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty, bits FROM blocks`); return rows; } catch (e) { logger.err('Cannot get blocks difficulty list from the db. Reason: ' + (e instanceof Error ? e.message : e)); @@ -848,7 +848,7 @@ class BlocksRepository { */ public async $getOldestConsecutiveBlock(): Promise { try { - const [rows]: any = await DB.query(`SELECT height, UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty FROM blocks ORDER BY height DESC`); + const [rows]: any = await DB.query(`SELECT height, UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty, bits FROM blocks ORDER BY height DESC`); for (let i = 0; i < rows.length - 1; ++i) { if (rows[i].height - rows[i + 1].height > 1) { return rows[i]; From b0080a5859c7a2c50fa9384fb8f29ed7d9ab5654 Mon Sep 17 00:00:00 2001 From: softsimon Date: Tue, 1 Aug 2023 15:55:03 +0900 Subject: [PATCH 28/73] Base mempool break limit of current poll rate --- backend/src/api/mempool.ts | 6 +++--- backend/src/index.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index b9b648012..85b6e6101 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -182,7 +182,7 @@ class Mempool { return txTimes; } - public async $updateMempool(transactions: string[]): Promise { + public async $updateMempool(transactions: string[], pollRate: number): Promise { logger.debug(`Updating mempool...`); // warn if this run stalls the main loop for more than 2 minutes @@ -258,7 +258,7 @@ class Mempool { } } - if (Date.now() - intervalTimer > 5_000) { + if (Date.now() - intervalTimer > Math.max(pollRate * 2, 5_000)) { if (this.inSync) { // Break and restart mempool loop if we spend too much time processing // new transactions that may lead to falling behind on block height @@ -270,7 +270,7 @@ class Mempool { if (Math.floor(progress) < 100) { loadingIndicators.setProgress('mempool', progress); } - intervalTimer = Date.now() + intervalTimer = Date.now(); } } } diff --git a/backend/src/index.ts b/backend/src/index.ts index 222537d0b..185a47067 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -188,14 +188,14 @@ class Server { } const newMempool = await bitcoinApi.$getRawMempool(); const numHandledBlocks = await blocks.$updateBlocks(); + const pollRate = config.MEMPOOL.POLL_RATE_MS * (indexer.indexerRunning ? 10 : 1); if (numHandledBlocks === 0) { - await memPool.$updateMempool(newMempool); + await memPool.$updateMempool(newMempool, pollRate); } indexer.$run(); // rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS const elapsed = Date.now() - start; - const pollRate = config.MEMPOOL.POLL_RATE_MS * (indexer.indexerRunning ? 10 : 1); const remainingTime = Math.max(0, pollRate - elapsed); setTimeout(this.runMainUpdateLoop.bind(this), numHandledBlocks > 0 ? 0 : remainingTime); this.backendRetryCount = 0; From a1af41804a6a8f2a0d61b5ad6fe814c40d2fd837 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Tue, 1 Aug 2023 16:00:16 +0900 Subject: [PATCH 29/73] Implement wiz footer suggestions --- .../global-footer/global-footer.component.html | 10 ++++++---- .../global-footer/global-footer.component.scss | 5 +++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index b1ba84109..58b7c5b1e 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -2,7 +2,10 @@
-
The Mempool Open Source Project ®
+

Explore the full Bitcoin ecosystem ™

@@ -17,9 +20,6 @@

Sign In

-

Broadcast Transaction

-

Connect to our Nodes

-

About The Mempool Open Source Project®

@@ -28,6 +28,8 @@

Mining Dashboard

Lightning Dashboard

Recent Blocks

+

Broadcast Transaction

+

Connect to our Nodes

API Documentation

diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.scss b/frontend/src/app/shared/components/global-footer/global-footer.component.scss index defbe937e..cec41f25b 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.scss +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.scss @@ -89,6 +89,11 @@ footer .row.version p a { color: #09a3ba; } +.main-logo { + max-width: 220px; + margin: 0 auto 20px auto; +} + @media (max-width: 992px) { footer .row.main .links.outer { From 9bf334a22d05b4e7d0bc3f3fbe224d5d28129665 Mon Sep 17 00:00:00 2001 From: junderw Date: Mon, 31 Jul 2023 18:22:13 -0700 Subject: [PATCH 30/73] Fix: Use bits to calculate difficulty instead of floating points --- .../api/difficulty-adjustment.test.ts | 48 +++++++++++++- backend/src/api/blocks.ts | 17 +++-- backend/src/api/difficulty-adjustment.ts | 62 +++++++++++++++++++ 3 files changed, 120 insertions(+), 7 deletions(-) diff --git a/backend/src/__tests__/api/difficulty-adjustment.test.ts b/backend/src/__tests__/api/difficulty-adjustment.test.ts index da1ac0d2c..c3e8e1a88 100644 --- a/backend/src/__tests__/api/difficulty-adjustment.test.ts +++ b/backend/src/__tests__/api/difficulty-adjustment.test.ts @@ -1,4 +1,8 @@ -import { calcDifficultyAdjustment, DifficultyAdjustment } from '../../api/difficulty-adjustment'; +import { + calcBitsDifference, + calcDifficultyAdjustment, + DifficultyAdjustment, +} from '../../api/difficulty-adjustment'; describe('Mempool Difficulty Adjustment', () => { test('should calculate Difficulty Adjustments properly', () => { @@ -86,4 +90,46 @@ describe('Mempool Difficulty Adjustment', () => { expect(result).toStrictEqual(vector[1]); } }); + + test('should calculate Difficulty change from bits fields of two blocks', () => { + // Check same exponent + check min max for output + expect(calcBitsDifference(0x1d000200, 0x1d000100)).toEqual(100); + expect(calcBitsDifference(0x1d000400, 0x1d000100)).toEqual(300); + expect(calcBitsDifference(0x1d000800, 0x1d000100)).toEqual(300); // Actually 700 + expect(calcBitsDifference(0x1d000100, 0x1d000200)).toEqual(-50); + expect(calcBitsDifference(0x1d000100, 0x1d000400)).toEqual(-75); + expect(calcBitsDifference(0x1d000100, 0x1d000800)).toEqual(-75); // Actually -87.5 + // Check new higher exponent + expect(calcBitsDifference(0x1c000200, 0x1d000001)).toEqual(100); + expect(calcBitsDifference(0x1c000400, 0x1d000001)).toEqual(300); + expect(calcBitsDifference(0x1c000800, 0x1d000001)).toEqual(300); + expect(calcBitsDifference(0x1c000100, 0x1d000002)).toEqual(-50); + expect(calcBitsDifference(0x1c000100, 0x1d000004)).toEqual(-75); + expect(calcBitsDifference(0x1c000100, 0x1d000008)).toEqual(-75); + // Check new lower exponent + expect(calcBitsDifference(0x1d000002, 0x1c000100)).toEqual(100); + expect(calcBitsDifference(0x1d000004, 0x1c000100)).toEqual(300); + expect(calcBitsDifference(0x1d000008, 0x1c000100)).toEqual(300); + expect(calcBitsDifference(0x1d000001, 0x1c000200)).toEqual(-50); + expect(calcBitsDifference(0x1d000001, 0x1c000400)).toEqual(-75); + expect(calcBitsDifference(0x1d000001, 0x1c000800)).toEqual(-75); + // Check error when exponents are too far apart + expect(() => calcBitsDifference(0x1d000001, 0x1a000800)).toThrow( + /Impossible exponent difference/ + ); + // Check invalid inputs + expect(() => calcBitsDifference(0x7f000001, 0x1a000800)).toThrow( + /Invalid bits/ + ); + expect(() => calcBitsDifference(0, 0x1a000800)).toThrow(/Invalid bits/); + expect(() => calcBitsDifference(100.2783, 0x1a000800)).toThrow( + /Invalid bits/ + ); + expect(() => calcBitsDifference(0x00800000, 0x1a000800)).toThrow( + /Invalid bits/ + ); + expect(() => calcBitsDifference(0x1c000000, 0x1a000800)).toThrow( + /Invalid bits/ + ); + }); }); diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 20c015f44..4919e71a6 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -28,12 +28,13 @@ import chainTips from './chain-tips'; import websocketHandler from './websocket-handler'; import redisCache from './redis-cache'; import rbfCache from './rbf-cache'; +import { calcBitsDifference } from './difficulty-adjustment'; class Blocks { private blocks: BlockExtended[] = []; private blockSummaries: BlockSummary[] = []; private currentBlockHeight = 0; - private currentDifficulty = 0; + private currentBits = 0; private lastDifficultyAdjustmentTime = 0; private previousDifficultyRetarget = 0; private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = []; @@ -666,14 +667,14 @@ class Blocks { const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash); this.updateTimerProgress(timer, 'got block for initial difficulty adjustment'); this.lastDifficultyAdjustmentTime = block.timestamp; - this.currentDifficulty = block.difficulty; + this.currentBits = block.bits; if (blockHeightTip >= 2016) { const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016); this.updateTimerProgress(timer, 'got previous block hash for initial difficulty adjustment'); const previousPeriodBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(previousPeriodBlockHash); this.updateTimerProgress(timer, 'got previous block for initial difficulty adjustment'); - this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100; + this.previousDifficultyRetarget = calcBitsDifference(previousPeriodBlock.bits, block.bits); logger.debug(`Initial difficulty adjustment data set.`); } } else { @@ -786,14 +787,18 @@ class Blocks { time: block.timestamp, height: block.height, difficulty: block.difficulty, - adjustment: Math.round((block.difficulty / this.currentDifficulty) * 1000000) / 1000000, // Remove float point noise + adjustment: Math.round( + // calcBitsDifference returns +- percentage, +100 returns to positive, /100 returns to ratio. + // Instead of actually doing /100, just reduce the multiplier. + (calcBitsDifference(this.currentBits, block.bits) + 100) * 10000 + ) / 1000000, // Remove float point noise }); this.updateTimerProgress(timer, `saved difficulty adjustment for ${this.currentBlockHeight}`); } - this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100; + this.previousDifficultyRetarget = calcBitsDifference(this.currentBits, block.bits); this.lastDifficultyAdjustmentTime = block.timestamp; - this.currentDifficulty = block.difficulty; + this.currentBits = block.bits; } // wait for pending async callbacks to finish diff --git a/backend/src/api/difficulty-adjustment.ts b/backend/src/api/difficulty-adjustment.ts index 1f37d8be9..23d0c33de 100644 --- a/backend/src/api/difficulty-adjustment.ts +++ b/backend/src/api/difficulty-adjustment.ts @@ -16,6 +16,68 @@ export interface DifficultyAdjustment { expectedBlocks: number; // Block count } +/** + * Calculate the difficulty increase/decrease by using the `bits` integer contained in two + * block headers. + * + * Warning: Only compare `bits` from blocks in two adjacent difficulty periods. This code + * assumes the maximum difference is x4 or /4 (as per the protocol) and will throw an + * error if an exponent difference of 2 or more is seen. + * + * @param {number} oldBits The 32 bit `bits` integer from a block header. + * @param {number} newBits The 32 bit `bits` integer from a block header in the next difficulty period. + * @returns {number} A floating point decimal of the difficulty change from old to new. + * (ie. 21.3 means 21.3% increase in difficulty, -21.3 is a 21.3% decrease in difficulty) + */ +export function calcBitsDifference(oldBits: number, newBits: number): number { + // Must be + // - integer + // - highest exponent is 0x1f, so max value (as integer) is 0x1f0000ff + // - min value is 1 (exponent = 0) + // - highest bit of the number-part is +- sign, it must not be 1 + const verifyBits = (bits: number): void => { + if ( + Math.floor(bits) !== bits || + bits > 0x1f0000ff || + bits < 1 || + (bits & 0x00800000) !== 0 || + (bits & 0x007fffff) === 0 + ) { + throw new Error('Invalid bits'); + } + }; + verifyBits(oldBits); + verifyBits(newBits); + + // No need to mask exponents because we checked the bounds above + const oldExp = oldBits >> 24; + const newExp = newBits >> 24; + const oldNum = oldBits & 0x007fffff; + const newNum = newBits & 0x007fffff; + // The diff can only possibly be 1, 0, -1 + // (because maximum difficulty change is x4 or /4 (2 bits up or down)) + let result: number; + switch (newExp - oldExp) { + // New less than old, target lowered, difficulty increased + case -1: + result = ((oldNum << 8) * 100) / newNum - 100; + break; + // Same exponent, compare numbers as is. + case 0: + result = (oldNum * 100) / newNum - 100; + break; + // Old less than new, target raised, difficulty decreased + case 1: + result = (oldNum * 100) / (newNum << 8) - 100; + break; + default: + throw new Error('Impossible exponent difference'); + } + + // Min/Max values + return result > 300 ? 300 : result < -75 ? -75 : result; +} + export function calcDifficultyAdjustment( DATime: number, nowSeconds: number, From 543357f1db64b2b0e15f821766d3efad7eadcd99 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 1 Aug 2023 17:33:03 +0900 Subject: [PATCH 31/73] fix null coinbase fees in block summary --- backend/src/api/blocks.ts | 4 ++-- backend/src/api/common.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 20c015f44..0372d30a3 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -701,7 +701,7 @@ class Blocks { // fill in missing transaction fee data from verboseBlock for (let i = 0; i < transactions.length; i++) { if (!transactions[i].fee && transactions[i].txid === verboseBlock.tx[i].txid) { - transactions[i].fee = verboseBlock.tx[i].fee * 100_000_000; + transactions[i].fee = (verboseBlock.tx[i].fee * 100_000_000) || 0; } } @@ -941,7 +941,7 @@ class Blocks { transactions: cpfpSummary.transactions.map(tx => { return { txid: tx.txid, - fee: tx.fee, + fee: tx.fee || 0, vsize: tx.vsize, value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0)), rate: tx.effectiveFeePerVsize diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index cd9da3d2a..0e03fe32c 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -108,7 +108,7 @@ export class Common { static stripTransaction(tx: TransactionExtended): TransactionStripped { return { txid: tx.txid, - fee: tx.fee, + fee: tx.fee || 0, vsize: tx.weight / 4, value: tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0), rate: tx.effectiveFeePerVsize, From 316028fe6633da73d9e816bfb660d6dad41a9771 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Tue, 1 Aug 2023 17:24:26 +0900 Subject: [PATCH 32/73] Add space below tagline --- .../components/global-footer/global-footer.component.scss | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.scss b/frontend/src/app/shared/components/global-footer/global-footer.component.scss index cec41f25b..387a27ffc 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.scss +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.scss @@ -21,6 +21,10 @@ footer .row.main .branding { text-align: center; } +footer .row.main .branding > p { + margin-bottom: 45px; +} + footer .row.main .branding .btn { display: inline-block; color: #fff !important; From 4cb379bb0ffc9e3c11578e644d567b057bad7989 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Tue, 1 Aug 2023 17:26:13 +0900 Subject: [PATCH 33/73] Change more networks to networks --- .../components/global-footer/global-footer.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index 58b7c5b1e..e1d853848 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -52,7 +52,7 @@
-

More Networks

+

Networks

Mainnet Explorer

Testnet Explorer

Signet Explorer

From d3532eb734182d359ff296a56dc2a3be176269ed Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Tue, 1 Aug 2023 17:35:18 +0900 Subject: [PATCH 34/73] Improve show conditions for explore links --- .../components/global-footer/global-footer.component.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index e1d853848..7206cc34f 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -26,10 +26,10 @@

Explore

Mining Dashboard

-

Lightning Dashboard

+

Lightning Dashboard

Recent Blocks

Broadcast Transaction

-

Connect to our Nodes

+

Connect to our Nodes

API Documentation

From f166cb79741fb54881054f6803f6889e110438a2 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Tue, 1 Aug 2023 17:36:12 +0900 Subject: [PATCH 35/73] Remove commented/inactive social links --- .../global-footer/global-footer.component.html | 9 --------- 1 file changed, 9 deletions(-) diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index 7206cc34f..a5792bb17 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -40,15 +40,6 @@

Why isn't my transaction confirming?

More FAQs ›

-
From 8fe78fa12b995997cc473db3997d5e284c26a661 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Tue, 1 Aug 2023 18:08:49 +0900 Subject: [PATCH 36/73] Improve conditions for showing network links --- .../global-footer/global-footer.component.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index a5792bb17..903ba70bd 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -42,13 +42,13 @@
-
+

Networks

-

Mainnet Explorer

-

Testnet Explorer

-

Signet Explorer

-

Liquid Explorer

-

Bisq Explorer

+

Mainnet Explorer

+

Testnet Explorer

+

Signet Explorer

+

Liquid Explorer

+

Bisq Explorer

Legal

From ab784cede23f9f9fbc8a4a3ac07cefbd4a5c29b6 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 1 Aug 2023 18:41:48 +0900 Subject: [PATCH 37/73] fix partially indexed difficulty bug --- backend/src/api/mining/mining.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index 3eeb3e9e5..7376e7cf4 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -438,6 +438,7 @@ class Mining { const oldestConsecutiveBlock = await BlocksRepository.$getOldestConsecutiveBlock(); if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== -1) { currentBits = oldestConsecutiveBlock.bits; + currentDifficulty = oldestConsecutiveBlock.difficulty; } let totalBlockChecked = 0; From ddd5baf44e4d2f06f9f90cda7fff45a15174f0dc Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Tue, 1 Aug 2023 18:35:51 +0900 Subject: [PATCH 38/73] Show tool list when no other networks available --- .../global-footer/global-footer.component.html | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index 903ba70bd..a556b0783 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -42,13 +42,20 @@
-
+

Networks

Mainnet Explorer

Testnet Explorer

Signet Explorer

-

Liquid Explorer

-

Bisq Explorer

+

Liquid Testnet Explorer

+

Liquid Explorer

+

Bisq Explorer

+
+
+

Tools

+

Clock (Mempool)

+

Clock (Mined)

+

BTC/Fiat Converter

Legal

From 9373fb3dd11ceaedf1fafe7ddef6156a74e66fc5 Mon Sep 17 00:00:00 2001 From: softsimon Date: Wed, 2 Aug 2023 14:34:59 +0900 Subject: [PATCH 39/73] Minor ngIf else template optimization --- .../components/global-footer/global-footer.component.html | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index a556b0783..2b30714d1 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -42,7 +42,7 @@
-
+

Networks

Mainnet Explorer

Testnet Explorer

@@ -51,12 +51,14 @@

Liquid Explorer

Bisq Explorer

-
+ +

Tools

Clock (Mempool)

Clock (Mined)

BTC/Fiat Converter

+

Legal

Terms of Service

From ca0c6b5e6e431ebfd27f1c31de61ffcaa0923c42 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 2 Aug 2023 13:24:56 +0900 Subject: [PATCH 40/73] use bulk mempool/txs post api to batch mempool update requests --- .../bitcoin/bitcoin-api-abstract-factory.ts | 3 +- backend/src/api/bitcoin/bitcoin-api.ts | 9 +++- backend/src/api/bitcoin/esplora-api.ts | 25 ++++++++- backend/src/api/mempool.ts | 54 ++++++++++--------- backend/src/api/transaction-utils.ts | 19 +++++++ 5 files changed, 82 insertions(+), 28 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts index 7f4a5e53a..f14c5525d 100644 --- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -3,7 +3,8 @@ import { IEsploraApi } from './esplora-api.interface'; export interface AbstractBitcoinApi { $getRawMempool(): Promise; $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, lazyPrevouts?: boolean): Promise; - $getMempoolTransactions(lastTxid: string); + $getMempoolTransactions(txids: string[]): Promise; + $getAllMempoolTransactions(lastTxid: string); $getTransactionHex(txId: string): Promise; $getBlockHeightTip(): Promise; $getBlockHashTip(): Promise; diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 132cda91a..b315ed0f7 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -60,8 +60,13 @@ class BitcoinApi implements AbstractBitcoinApi { }); } - $getMempoolTransactions(lastTxid: string): Promise { - return Promise.resolve([]); + $getMempoolTransactions(txids: string[]): Promise { + throw new Error('Method getMempoolTransactions not supported by the Bitcoin RPC API.'); + } + + $getAllMempoolTransactions(lastTxid: string): Promise { + throw new Error('Method getAllMempoolTransactions not supported by the Bitcoin RPC API.'); + } async $getTransactionHex(txId: string): Promise { diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index ff10751e0..77c6d80fc 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -61,6 +61,25 @@ class ElectrsApi implements AbstractBitcoinApi { }); } + $postWrapper(url, body, responseType = 'json', params: any = undefined): Promise { + return axiosConnection.post(url, body, { ...this.activeAxiosConfig, responseType: responseType, params }) + .then((response) => response.data) + .catch((e) => { + if (e?.code === 'ECONNREFUSED') { + this.fallbackToTcpSocket(); + // Retry immediately + return axiosConnection.post(url, body, this.activeAxiosConfig) + .then((response) => response.data) + .catch((e) => { + logger.warn(`Cannot query esplora through the unix socket nor the tcp socket. Exception ${e}`); + throw e; + }); + } else { + throw e; + } + }); + } + $getRawMempool(): Promise { return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/mempool/txids'); } @@ -69,7 +88,11 @@ class ElectrsApi implements AbstractBitcoinApi { return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/tx/' + txId); } - async $getMempoolTransactions(lastSeenTxid?: string): Promise { + async $getMempoolTransactions(txids: string[]): Promise { + return this.$postWrapper(config.ESPLORA.REST_API_URL + '/mempool/txs', txids, 'json'); + } + + async $getAllMempoolTransactions(lastSeenTxid?: string): Promise { return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/mempool/txs' + (lastSeenTxid ? '/' + lastSeenTxid : '')); } diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 85b6e6101..ccdd307f8 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -123,7 +123,7 @@ class Mempool { loadingIndicators.setProgress('mempool', count / expectedCount * 100); while (!done) { try { - const result = await bitcoinApi.$getMempoolTransactions(last_txid); + const result = await bitcoinApi.$getAllMempoolTransactions(last_txid); if (result) { for (const tx of result) { const extendedTransaction = transactionUtils.extendMempoolTransaction(tx); @@ -231,31 +231,37 @@ class Mempool { } if (!loaded) { - for (const txid of transactions) { - if (!this.mempoolCache[txid]) { - try { - const transaction = await transactionUtils.$getMempoolTransactionExtended(txid, false, false, false); - this.updateTimerProgress(timer, 'fetched new transaction'); - this.mempoolCache[txid] = transaction; - if (this.inSync) { - this.txPerSecondArray.push(new Date().getTime()); - this.vBytesPerSecondArray.push({ - unixTime: new Date().getTime(), - vSize: transaction.vsize, - }); - } - hasChange = true; - newTransactions.push(transaction); + const remainingTxids = transactions.filter(txid => !this.mempoolCache[txid]); + const sliceLength = 10000; + for (let i = 0; i < Math.ceil(remainingTxids.length / sliceLength); i++) { + const slice = remainingTxids.slice(i * sliceLength, (i + 1) * sliceLength); + const txs = await transactionUtils.$getMempoolTransactionsExtended(slice, false, false, false); + logger.debug(`fetched ${txs.length} transactions`); + this.updateTimerProgress(timer, 'fetched new transactions'); - if (config.REDIS.ENABLED) { - await redisCache.$addTransaction(transaction); - } - } catch (e: any) { - if (config.MEMPOOL.BACKEND === 'esplora' && e.response?.status === 404) { - this.missingTxCount++; - } - logger.debug(`Error finding transaction '${txid}' in the mempool: ` + (e instanceof Error ? e.message : e)); + for (const transaction of txs) { + this.mempoolCache[transaction.txid] = transaction; + if (this.inSync) { + this.txPerSecondArray.push(new Date().getTime()); + this.vBytesPerSecondArray.push({ + unixTime: new Date().getTime(), + vSize: transaction.vsize, + }); } + hasChange = true; + newTransactions.push(transaction); + + if (config.REDIS.ENABLED) { + await redisCache.$addTransaction(transaction); + } + } + + if (txs.length < slice.length) { + const missing = slice.length - txs.length; + if (config.MEMPOOL.BACKEND === 'esplora') { + this.missingTxCount += missing; + } + logger.debug(`Error finding ${missing} transactions in the mempool: `); } if (Date.now() - intervalTimer > Math.max(pollRate * 2, 5_000)) { diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index e141a6076..02ee7c055 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -4,6 +4,7 @@ import { Common } from './common'; import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; import * as bitcoinjs from 'bitcoinjs-lib'; import logger from '../logger'; +import config from '../config'; class TransactionUtils { constructor() { } @@ -71,6 +72,24 @@ class TransactionUtils { return (await this.$getTransactionExtended(txId, addPrevouts, lazyPrevouts, forceCore, true)) as MempoolTransactionExtended; } + public async $getMempoolTransactionsExtended(txids: string[], addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise { + if (forceCore || config.MEMPOOL.BACKEND !== 'esplora') { + const results = await Promise.allSettled(txids.map(txid => this.$getTransactionExtended(txid, addPrevouts, lazyPrevouts, forceCore, true))); + return (results.filter(r => r.status === 'fulfilled') as PromiseFulfilledResult[]).map(r => r.value); + } else { + const transactions = await bitcoinApi.$getMempoolTransactions(txids); + return transactions.map(transaction => { + if (Common.isLiquid()) { + if (!isFinite(Number(transaction.fee))) { + transaction.fee = Object.values(transaction.fee || {}).reduce((total, output) => total + output, 0); + } + } + + return this.extendMempoolTransaction(transaction); + }); + } + } + public extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended { // @ts-ignore if (transaction.vsize) { From e0b2ffa527b37f9541b32c9cff1c591636ee2170 Mon Sep 17 00:00:00 2001 From: andrewtoth Date: Thu, 3 Aug 2023 03:46:15 +0000 Subject: [PATCH 41/73] Accept CLA --- contributors/andrewtoth.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 contributors/andrewtoth.txt diff --git a/contributors/andrewtoth.txt b/contributors/andrewtoth.txt new file mode 100644 index 000000000..bd2ed3d06 --- /dev/null +++ b/contributors/andrewtoth.txt @@ -0,0 +1,3 @@ +I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of August 2, 2023. + +Signed: andrewtoth From d236d897178fcaf06270eff1de1a343d24229f30 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Mon, 31 Jul 2023 11:06:52 +0900 Subject: [PATCH 42/73] [mining] send pool unique id in /pools API --- backend/src/api/mining/mining.ts | 1 + backend/src/mempool.interfaces.ts | 1 + backend/src/repositories/PoolsRepository.ts | 3 ++- frontend/src/app/interfaces/node-api.interface.ts | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index e190492b8..79b255e7f 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -107,6 +107,7 @@ class Mining { slug: poolInfo.slug, avgMatchRate: poolInfo.avgMatchRate !== null ? Math.round(100 * poolInfo.avgMatchRate) / 100 : null, avgFeeDelta: poolInfo.avgFeeDelta, + poolUniqueId: poolInfo.poolUniqueId }; poolsStats.push(poolStat); }); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 4715440e4..c08846191 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -20,6 +20,7 @@ export interface PoolInfo { slug: string; avgMatchRate: number | null; avgFeeDelta: number | null; + poolUniqueId: number; } export interface PoolStats extends PoolInfo { diff --git a/backend/src/repositories/PoolsRepository.ts b/backend/src/repositories/PoolsRepository.ts index 899712266..eda792bb3 100644 --- a/backend/src/repositories/PoolsRepository.ts +++ b/backend/src/repositories/PoolsRepository.ts @@ -40,7 +40,8 @@ class PoolsRepository { pools.link AS link, slug, AVG(blocks_audits.match_rate) AS avgMatchRate, - AVG((CAST(blocks.fees as SIGNED) - CAST(blocks_audits.expected_fees as SIGNED)) / NULLIF(CAST(blocks_audits.expected_fees as SIGNED), 0)) AS avgFeeDelta + AVG((CAST(blocks.fees as SIGNED) - CAST(blocks_audits.expected_fees as SIGNED)) / NULLIF(CAST(blocks_audits.expected_fees as SIGNED), 0)) AS avgFeeDelta, + unique_id as poolUniqueId FROM blocks JOIN pools on pools.id = pool_id LEFT JOIN blocks_audits ON blocks_audits.height = blocks.height diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index cb42eaed3..5316b1cdc 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -111,6 +111,7 @@ export interface PoolInfo { regexes: string; // JSON array addresses: string; // JSON array emptyBlocks: number; + poolUniqueId: number; } export interface PoolStat { pool: PoolInfo; From f2ae8580971abb8d62054b1c690a55408ab4fbe0 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 17 Feb 2023 18:54:48 +0900 Subject: [PATCH 43/73] [sponsors] show profile by tiers on about page --- .../app/components/about/about.component.html | 49 +++++++++++++++---- .../app/components/about/about.component.scss | 3 ++ .../app/components/about/about.component.ts | 24 ++++++--- frontend/src/app/services/api.service.ts | 14 ++---- frontend/src/resources/profile/unknown.svg | 1 + 5 files changed, 62 insertions(+), 29 deletions(-) create mode 100644 frontend/src/resources/profile/unknown.svg diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index 381353948..55f406d00 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -31,6 +31,12 @@ +

Support the Project

+ +
+ Become a Sponsor! +
+

Enterprise Sponsors 🚀

@@ -191,16 +197,39 @@
-
-

Community Sponsors ❤️

+
+
+

Whale Sponsors

+
+ + + + + + + +
+
-
- - - - +
+

Chad Sponsors

+
+ + + +
+
+
+ +
+

OG Sponsors ❤️

+
+ + + +
@@ -340,7 +369,7 @@
- +
@@ -354,7 +383,7 @@
- + {{ contributor.name }} @@ -366,7 +395,7 @@
- + {{ contributor.name }} diff --git a/frontend/src/app/components/about/about.component.scss b/frontend/src/app/components/about/about.component.scss index e1d6c829a..2a5710ca1 100644 --- a/frontend/src/app/components/about/about.component.scss +++ b/frontend/src/app/components/about/about.component.scss @@ -10,6 +10,9 @@ margin: 25px; line-height: 32px; } + .unknown { + border: 1px solid #b4b4b4; + } .image.not-rounded { border-radius: 0; diff --git a/frontend/src/app/components/about/about.component.ts b/frontend/src/app/components/about/about.component.ts index 176490add..4bf7869de 100644 --- a/frontend/src/app/components/about/about.component.ts +++ b/frontend/src/app/components/about/about.component.ts @@ -6,7 +6,7 @@ import { Observable } from 'rxjs'; import { ApiService } from '../../services/api.service'; import { IBackendInfo } from '../../interfaces/websocket.interface'; import { Router, ActivatedRoute } from '@angular/router'; -import { map, tap } from 'rxjs/operators'; +import { map, share, tap } from 'rxjs/operators'; import { ITranslators } from '../../interfaces/node-api.interface'; import { DOCUMENT } from '@angular/common'; @@ -19,14 +19,16 @@ import { DOCUMENT } from '@angular/common'; export class AboutComponent implements OnInit { @ViewChild('promoVideo') promoVideo: ElementRef; backendInfo$: Observable; - sponsors$: Observable; - translators$: Observable; - allContributors$: Observable; frontendGitCommitHash = this.stateService.env.GIT_COMMIT_HASH; packetJsonVersion = this.stateService.env.PACKAGE_JSON_VERSION; officialMempoolSpace = this.stateService.env.OFFICIAL_MEMPOOL_SPACE; showNavigateToSponsor = false; + profiles$: Observable; + translators$: Observable; + allContributors$: Observable; + ogs$: Observable; + constructor( private websocketService: WebsocketService, private seoService: SeoService, @@ -43,10 +45,13 @@ export class AboutComponent implements OnInit { this.seoService.setTitle($localize`:@@004b222ff9ef9dd4771b777950ca1d0e4cd4348a:About`); this.websocketService.want(['blocks']); - this.sponsors$ = this.apiService.getDonation$() - .pipe( - tap(() => this.goToAnchor()) - ); + this.profiles$ = this.apiService.getAboutPageProfiles$().pipe( + tap(() => { + this.goToAnchor() + }), + share(), + ) + this.translators$ = this.apiService.getTranslators$() .pipe( map((translators) => { @@ -59,6 +64,9 @@ export class AboutComponent implements OnInit { }), tap(() => this.goToAnchor()) ); + + this.ogs$ = this.apiService.getOgs$(); + this.allContributors$ = this.apiService.getContributor$().pipe( map((contributors) => { return { diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index e2d3be9be..24dbe734a 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -92,15 +92,11 @@ export class ApiService { return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/outspends', { params }); } - requestDonation$(amount: number, orderId: string): Observable { - const params = { - amount: amount, - orderId: orderId, - }; - return this.httpClient.post(this.apiBaseUrl + '/api/v1/donations', params); + getAboutPageProfiles$(): Observable { + return this.httpClient.get(this.apiBaseUrl + '/api/v1/about-page'); } - getDonation$(): Observable { + getOgs$(): Observable { return this.httpClient.get(this.apiBaseUrl + '/api/v1/donations'); } @@ -112,10 +108,6 @@ export class ApiService { return this.httpClient.get(this.apiBaseUrl + '/api/v1/contributors'); } - checkDonation$(orderId: string): Observable { - return this.httpClient.get(this.apiBaseUrl + '/api/v1/donations/check?order_id=' + orderId); - } - getInitData$(): Observable { return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/init-data'); } diff --git a/frontend/src/resources/profile/unknown.svg b/frontend/src/resources/profile/unknown.svg new file mode 100644 index 000000000..50a548c3c --- /dev/null +++ b/frontend/src/resources/profile/unknown.svg @@ -0,0 +1 @@ + \ No newline at end of file From 6c2c62ba2eb7d50f499cf71e457104adfccb0672 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 1 Apr 2023 11:40:04 +0900 Subject: [PATCH 44/73] [lightning] claim your node button --- .../src/app/lightning/lightning.module.ts | 3 +++ .../node-owner/node-owner.component.html | 17 ++++++++++++ .../node-owner/node-owner.component.scss | 4 +++ .../node-owner/node-owner.component.ts | 20 ++++++++++++++ .../app/lightning/node/node.component.html | 8 ++++-- .../app/lightning/node/node.component.scss | 14 ++++++++++ .../src/app/lightning/node/node.component.ts | 26 ++++++++++++++++--- frontend/src/app/services/api.service.ts | 11 ++++++++ 8 files changed, 98 insertions(+), 5 deletions(-) create mode 100644 frontend/src/app/lightning/node-owner/node-owner.component.html create mode 100644 frontend/src/app/lightning/node-owner/node-owner.component.scss create mode 100644 frontend/src/app/lightning/node-owner/node-owner.component.ts diff --git a/frontend/src/app/lightning/lightning.module.ts b/frontend/src/app/lightning/lightning.module.ts index 0b824ad78..f0154a15f 100644 --- a/frontend/src/app/lightning/lightning.module.ts +++ b/frontend/src/app/lightning/lightning.module.ts @@ -34,6 +34,7 @@ import { OldestNodes } from '../lightning/nodes-ranking/oldest-nodes/oldest-node import { NodesRankingsDashboard } from '../lightning/nodes-rankings-dashboard/nodes-rankings-dashboard.component'; import { NodeChannels } from '../lightning/nodes-channels/node-channels.component'; import { GroupComponent } from './group/group.component'; +import { NodeOwnerComponent } from './node-owner/node-owner.component'; @NgModule({ declarations: [ @@ -66,6 +67,7 @@ import { GroupComponent } from './group/group.component'; NodesRankingsDashboard, NodeChannels, GroupComponent, + NodeOwnerComponent, ], imports: [ CommonModule, @@ -103,6 +105,7 @@ import { GroupComponent } from './group/group.component'; OldestNodes, NodesRankingsDashboard, NodeChannels, + NodeOwnerComponent, ], providers: [ LightningApiService, diff --git a/frontend/src/app/lightning/node-owner/node-owner.component.html b/frontend/src/app/lightning/node-owner/node-owner.component.html new file mode 100644 index 000000000..e37b1e027 --- /dev/null +++ b/frontend/src/app/lightning/node-owner/node-owner.component.html @@ -0,0 +1,17 @@ +
+ +
+ +
+ + + +
+ +
+ Claim +
+ +
+ +
\ No newline at end of file diff --git a/frontend/src/app/lightning/node-owner/node-owner.component.scss b/frontend/src/app/lightning/node-owner/node-owner.component.scss new file mode 100644 index 000000000..6734168cf --- /dev/null +++ b/frontend/src/app/lightning/node-owner/node-owner.component.scss @@ -0,0 +1,4 @@ +.profile-photo { + width: 31px; + height: 31px; +} \ No newline at end of file diff --git a/frontend/src/app/lightning/node-owner/node-owner.component.ts b/frontend/src/app/lightning/node-owner/node-owner.component.ts new file mode 100644 index 000000000..a03c04901 --- /dev/null +++ b/frontend/src/app/lightning/node-owner/node-owner.component.ts @@ -0,0 +1,20 @@ +import { ChangeDetectionStrategy, Component, Input } from '@angular/core'; +import { Observable } from 'rxjs'; +import { StateService } from '../../services/state.service'; + +@Component({ + selector: 'app-node-owner', + templateUrl: './node-owner.component.html', + styleUrls: ['./node-owner.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class NodeOwnerComponent{ + @Input() publicKey: string = ''; + @Input() alias: string = ''; + @Input() nodeOwner$: Observable; + + constructor( + public stateService: StateService + ) { + } +} diff --git a/frontend/src/app/lightning/node/node.component.html b/frontend/src/app/lightning/node/node.component.html index c6c693a3a..11ddbc0eb 100644 --- a/frontend/src/app/lightning/node/node.component.html +++ b/frontend/src/app/lightning/node/node.component.html @@ -3,13 +3,17 @@
Lightning node
-

{{ node.alias }}

- +
+

{{ node.alias }}

+ +
+ +
diff --git a/frontend/src/app/lightning/node/node.component.scss b/frontend/src/app/lightning/node/node.component.scss index 272de4b09..117fc8a2c 100644 --- a/frontend/src/app/lightning/node/node.component.scss +++ b/frontend/src/app/lightning/node/node.component.scss @@ -111,3 +111,17 @@ app-fiat { margin: 0 0.25em; color: slategrey; } + +.claim-btn { + max-height: 32px; + @media (min-width: 850px) { + display: none; + } +} + +.claim-btn-mobile { + max-height: 32px; + @media (max-width: 850px) { + display: none; + } +} \ No newline at end of file diff --git a/frontend/src/app/lightning/node/node.component.ts b/frontend/src/app/lightning/node/node.component.ts index 719136d79..6447eb6bd 100644 --- a/frontend/src/app/lightning/node/node.component.ts +++ b/frontend/src/app/lightning/node/node.component.ts @@ -1,7 +1,7 @@ -import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; +import { ChangeDetectionStrategy, Component, OnInit, ChangeDetectorRef } from '@angular/core'; import { ActivatedRoute, ParamMap } from '@angular/router'; -import { Observable } from 'rxjs'; -import { catchError, map, switchMap, tap } from 'rxjs/operators'; +import { Observable, of, EMPTY } from 'rxjs'; +import { catchError, map, switchMap, tap, share } from 'rxjs/operators'; import { SeoService } from '../../services/seo.service'; import { ApiService } from '../../services/api.service'; import { LightningApiService } from '../lightning-api.service'; @@ -38,6 +38,7 @@ export class NodeComponent implements OnInit { tlvRecords: CustomRecord[]; avgChannelDistance$: Observable; showFeatures = false; + nodeOwner$: Observable; kmToMiles = kmToMiles; constructor( @@ -45,6 +46,7 @@ export class NodeComponent implements OnInit { private lightningApiService: LightningApiService, private activatedRoute: ActivatedRoute, private seoService: SeoService, + private cd: ChangeDetectorRef, ) { } ngOnInit(): void { @@ -147,6 +149,24 @@ export class NodeComponent implements OnInit { return null; }) ) as Observable; + + this.nodeOwner$ = this.activatedRoute.paramMap + .pipe( + switchMap((params: ParamMap) => { + return this.apiService.getNodeOwner$(params.get('public_key')).pipe( + switchMap((response) => { + if (response.status === 204) { + return of(false); + } + return of(response.body); + }), + catchError(() => { + return of(false); + }) + ) + }), + share(), + ); } toggleShowDetails(): void { diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index 24dbe734a..1ed9d2f5c 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -8,6 +8,8 @@ import { WebsocketResponse } from '../interfaces/websocket.interface'; import { Outspend, Transaction } from '../interfaces/electrs.interface'; import { Conversion } from './price.service'; +const SERVICES_API_PREFIX = `/api/v1/services`; + @Injectable({ providedIn: 'root' }) @@ -315,4 +317,13 @@ export class ApiService { (timestamp ? `?timestamp=${timestamp}` : '') ); } + + /** + * Services + */ + getNodeOwner$(publicKey: string) { + let params = new HttpParams() + .set('node_public_key', publicKey); + return this.httpClient.get(`${SERVICES_API_PREFIX}/lightning/claim/current`, { params, observe: 'response' }); + } } From 727d170c9c8df42919800e35ead381e428bf6b66 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Wed, 3 May 2023 16:32:00 +0200 Subject: [PATCH 45/73] [tx] add accel shortcut into transaction component --- .../transaction/transaction.component.html | 10 ++++++++-- .../transaction/transaction.component.scss | 19 +++++++++++++++++++ .../transaction/transaction.component.ts | 2 +- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index 7422c6894..59404bb77 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -99,14 +99,20 @@ - In several hours (or more) + + In several hours (or more) + Accelerate + - + + + Accelerate + diff --git a/frontend/src/app/components/transaction/transaction.component.scss b/frontend/src/app/components/transaction/transaction.component.scss index bea8e82bc..5bef401d7 100644 --- a/frontend/src/app/components/transaction/transaction.component.scss +++ b/frontend/src/app/components/transaction/transaction.component.scss @@ -216,4 +216,23 @@ .alert-link { display: block; } +} + +.eta { + display: flex; + justify-content: end; + flex-wrap: wrap; + align-content: center; + @media (min-width: 850px) { + justify-content: space-between; + } +} + +.accelerate { + align-self: auto; + margin-top: 3px; + @media (min-width: 850px) { + justify-self: start; + margin-left: 0px; + } } \ No newline at end of file diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index f1d218a79..26e39515e 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -97,7 +97,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { private router: Router, private relativeUrlPipe: RelativeUrlPipe, private electrsApiService: ElectrsApiService, - private stateService: StateService, + public stateService: StateService, private cacheService: CacheService, private websocketService: WebsocketService, private audioService: AudioService, From 5846862d550ec0a9d07c254ec13b98db6fb4f4b5 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 11 May 2023 18:00:52 +0200 Subject: [PATCH 46/73] [footer] dynamic CTA button based on login status --- .../app/components/about/about.component.html | 2 +- .../global-footer.component.html | 8 ++++-- .../global-footer/global-footer.component.ts | 26 +++++++++++++++++-- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index 55f406d00..b9309c916 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -34,7 +34,7 @@

Support the Project

- Become a Sponsor! + Become a Sponsor!
diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index 8e5279a94..30727e662 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -14,8 +14,12 @@
- -

Sign In

+
+
Logged in as {{ username}}
+ + +
+

Sign In

Broadcast Transaction

Connect to our Nodes

diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.ts b/frontend/src/app/shared/components/global-footer/global-footer.component.ts index d4f1946ea..cdb0f1159 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.ts +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.ts @@ -1,10 +1,13 @@ -import { ChangeDetectionStrategy, Component, OnInit, Inject, LOCALE_ID } from '@angular/core'; -import { Observable, merge, of, Subject } from 'rxjs'; +import { ChangeDetectionStrategy, ChangeDetectorRef, Component, OnInit, Inject, LOCALE_ID } from '@angular/core'; +import { ActivatedRoute } from '@angular/router'; +import { Observable, merge, of, Subject, Subscription } from 'rxjs'; import { tap, takeUntil } from 'rxjs/operators'; import { Env, StateService } from '../../../services/state.service'; import { IBackendInfo } from '../../../interfaces/websocket.interface'; import { LanguageService } from '../../../services/language.service'; import { NavigationService } from '../../../services/navigation.service'; +import { StorageService } from '../../../services/storage.service'; +import { WebsocketService } from '../../../services/websocket.service'; @Component({ selector: 'app-global-footer', @@ -23,12 +26,19 @@ export class GlobalFooterComponent implements OnInit { network$: Observable; networkPaths: { [network: string]: string }; currentNetwork = ''; + loggedIn = false; + username = null; + urlSubscription: Subscription; constructor( public stateService: StateService, private languageService: LanguageService, private navigationService: NavigationService, @Inject(LOCALE_ID) public locale: string, + private storageService: StorageService, + private route: ActivatedRoute, + private cd: ChangeDetectorRef, + private websocketService: WebsocketService ) {} ngOnInit(): void { @@ -46,11 +56,23 @@ export class GlobalFooterComponent implements OnInit { this.network$.pipe(takeUntil(this.destroy$)).subscribe((network) => { this.currentNetwork = network; }); + + this.urlSubscription = this.route.url.subscribe((url) => { + this.loggedIn = JSON.parse(this.storageService.getValue('auth')) !== null; + const auth = JSON.parse(this.storageService.getValue('auth')); + if (auth?.user?.username) { + this.username = auth.user.username; + } else { + this.username = null; + } + this.cd.markForCheck(); + }) } ngOnDestroy(): void { this.destroy$.next(true); this.destroy$.complete(); + this.urlSubscription.unsubscribe(); } networkLink(network) { From 1458e89f3a67c3d9885f6accd69a96eff3a447c0 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Wed, 24 May 2023 21:49:08 -0700 Subject: [PATCH 47/73] [footer] main cta points to /accelerator --- .../src/app/components/about/about.component.html | 5 ++--- .../global-footer/global-footer.component.html | 14 ++++++-------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index b9309c916..93644a201 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -31,10 +31,9 @@ -

Support the Project

- +

Sponsor the project

- Become a Sponsor! + See plans
diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index 30727e662..658e0ac72 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -13,14 +13,12 @@
- -
-
Logged in as {{ username}}
- - -
-

Sign In

- +
+
Logged in as {{ username}}
+ + +
+

Sign In

Broadcast Transaction

Connect to our Nodes

About The Mempool Open Source Project™

From 7ec5d8265fcea3c0ae5f3e3a01bdafc9ab31e2a8 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 3 Aug 2023 14:29:10 +0900 Subject: [PATCH 48/73] toggle header visibility in master page component --- .../master-page/master-page.component.html | 2 +- .../master-page/master-page.component.ts | 15 +++++++++++---- frontend/src/app/shared/shared.module.ts | 1 + 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/frontend/src/app/components/master-page/master-page.component.html b/frontend/src/app/components/master-page/master-page.component.html index 79daf7de6..f12fbc960 100644 --- a/frontend/src/app/components/master-page/master-page.component.html +++ b/frontend/src/app/components/master-page/master-page.component.html @@ -1,5 +1,5 @@ -
+
-
-
-

Whale Sponsors

-
- - + +
+
+

Whale Sponsors

+
+ + + + + + + +
+
+ +
+

Chad Sponsors

+
+ - +
- -
-

Chad Sponsors

-
- - - - - -
-
-
+

OG Sponsors ❤️

diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index 59404bb77..9584cecfd 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -101,7 +101,7 @@ In several hours (or more) - Accelerate + Accelerate @@ -111,7 +111,7 @@ - Accelerate + Accelerate diff --git a/frontend/src/app/services/state.service.ts b/frontend/src/app/services/state.service.ts index 9ab8a7e93..675cf88d1 100644 --- a/frontend/src/app/services/state.service.ts +++ b/frontend/src/app/services/state.service.ts @@ -47,6 +47,7 @@ export interface Env { TESTNET_BLOCK_AUDIT_START_HEIGHT: number; SIGNET_BLOCK_AUDIT_START_HEIGHT: number; HISTORICAL_PRICE: boolean; + ACCELERATOR: boolean; } const defaultEnv: Env = { @@ -77,6 +78,7 @@ const defaultEnv: Env = { 'TESTNET_BLOCK_AUDIT_START_HEIGHT': 0, 'SIGNET_BLOCK_AUDIT_START_HEIGHT': 0, 'HISTORICAL_PRICE': true, + 'ACCELERATOR': false, }; @Injectable({ diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index 658e0ac72..c4436c4b6 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -13,7 +13,7 @@
-
+
Logged in as {{ username}}
From d06fe83bd996a77fc5d476b11bb6025ab47c423f Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 30 Jul 2023 17:59:35 +0900 Subject: [PATCH 50/73] Improve difficulty tooltip display on mobile --- .../difficulty-tooltip.component.html | 54 ++++++++++++------- .../difficulty-tooltip.component.ts | 12 ++++- .../difficulty/difficulty.component.ts | 1 + 3 files changed, 47 insertions(+), 20 deletions(-) diff --git a/frontend/src/app/components/difficulty/difficulty-tooltip.component.html b/frontend/src/app/components/difficulty/difficulty-tooltip.component.html index d06bb5e91..7e4b421e1 100644 --- a/frontend/src/app/components/difficulty/difficulty-tooltip.component.html +++ b/frontend/src/app/components/difficulty/difficulty-tooltip.component.html @@ -4,38 +4,56 @@ class="difficulty-tooltip" [style.visibility]="status ? 'visible' : 'hidden'" [style.left]="tooltipPosition.x + 'px'" - [style.top]="tooltipPosition.y + 'px'" + [style.top]="tooltipPosition.y + (isMobile ? -60 : 0) + 'px'" > - + - - {{ i }} blocks expected - {{ i }} block expected + - - {{ i }} blocks mined - {{ i }} block mined + - - {{ i }} blocks remaining - {{ i }} block remaining + - - {{ i }} blocks ahead - {{ i }} block ahead + - - {{ i }} blocks behind - {{ i }} block behind + Next Block -
\ No newline at end of file + + + + + + + +
+ +
+ + + + + + + +
+ +{{ i }} blocks expected +{{ i }} block expected +{{ i }} blocks mined +{{ i }} block mined +{{ i }} blocks remaining +{{ i }} block remaining +{{ i }} blocks ahead +{{ i }} block ahead +{{ i }} blocks behind +{{ i }} block behind \ No newline at end of file diff --git a/frontend/src/app/components/difficulty/difficulty-tooltip.component.ts b/frontend/src/app/components/difficulty/difficulty-tooltip.component.ts index c7d26f61a..42d2a61b4 100644 --- a/frontend/src/app/components/difficulty/difficulty-tooltip.component.ts +++ b/frontend/src/app/components/difficulty/difficulty-tooltip.component.ts @@ -1,4 +1,4 @@ -import { Component, ElementRef, ViewChild, Input, OnChanges } from '@angular/core'; +import { Component, ElementRef, ViewChild, Input, OnChanges, HostListener } from '@angular/core'; interface EpochProgress { base: string; @@ -35,12 +35,15 @@ export class DifficultyTooltipComponent implements OnChanges { remaining: number; isAhead: boolean; isBehind: boolean; + isMobile: boolean; tooltipPosition = { x: 0, y: 0 }; @ViewChild('tooltip') tooltipElement: ElementRef; - constructor() {} + constructor() { + this.onResize(); + } ngOnChanges(changes): void { if (changes.cursorPosition && changes.cursorPosition.currentValue) { @@ -63,4 +66,9 @@ export class DifficultyTooltipComponent implements OnChanges { this.isBehind = this.behind > 0; } } + + @HostListener('window:resize', ['$event']) + onResize(): void { + this.isMobile = window.innerWidth <= 767.98; + } } diff --git a/frontend/src/app/components/difficulty/difficulty.component.ts b/frontend/src/app/components/difficulty/difficulty.component.ts index a2c03dc56..093dde79a 100644 --- a/frontend/src/app/components/difficulty/difficulty.component.ts +++ b/frontend/src/app/components/difficulty/difficulty.component.ts @@ -193,6 +193,7 @@ export class DifficultyComponent implements OnInit { @HostListener('pointerdown', ['$event']) onPointerDown(event) { this.onPointerMove(event); + event.preventDefault(); } @HostListener('pointermove', ['$event']) From 3d6a8a501da6b192634b00786eb1ed3ed229cc49 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Thu, 3 Aug 2023 15:49:15 +0900 Subject: [PATCH 51/73] limit mouse events to difficulty bar --- .../difficulty/difficulty.component.html | 4 +-- .../difficulty/difficulty.component.ts | 28 +++++++++++-------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/frontend/src/app/components/difficulty/difficulty.component.html b/frontend/src/app/components/difficulty/difficulty.component.html index 27cddc043..f08ea06f5 100644 --- a/frontend/src/app/components/difficulty/difficulty.component.html +++ b/frontend/src/app/components/difficulty/difficulty.component.html @@ -4,7 +4,7 @@
- + @@ -22,7 +22,7 @@ class="rect {{rect.status}}" [class.hover]="hoverSection && rect.status === hoverSection.status" (pointerover)="onHover($event, rect);" - (pointerout)="onBlur($event);" + (pointerout)="onBlur();" > ; isLoadingWebSocket$: Observable; difficultyEpoch$: Observable; @@ -191,22 +193,26 @@ export class DifficultyComponent implements OnInit { } @HostListener('pointerdown', ['$event']) - onPointerDown(event) { - this.onPointerMove(event); - event.preventDefault(); + onPointerDown(event): void { + if (this.epochSvgElement.nativeElement?.contains(event.target)) { + this.onPointerMove(event); + event.preventDefault(); + } } @HostListener('pointermove', ['$event']) - onPointerMove(event) { - this.tooltipPosition = { x: event.clientX, y: event.clientY }; - this.cd.markForCheck(); + onPointerMove(event): void { + if (this.epochSvgElement.nativeElement?.contains(event.target)) { + this.tooltipPosition = { x: event.clientX, y: event.clientY }; + this.cd.markForCheck(); + } } - onHover(event, rect): void { + onHover(_, rect): void { this.hoverSection = rect; } - onBlur(event): void { + onBlur(): void { this.hoverSection = null; } } From 4ee703325af4c9cc54f14514b5071d428dd7ea21 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 3 Aug 2023 16:12:59 +0900 Subject: [PATCH 52/73] [footer] polish cta using update footer with logo --- .../global-footer/global-footer.component.html | 13 +++++-------- .../global-footer/global-footer.component.scss | 8 ++------ 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index 7dc98ca27..d4f303221 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -1,7 +1,7 @@
-
+
-
Logged in as {{ username}}
- - +
-

Sign In

-

Broadcast Transaction

-

Connect to our Nodes

-

About The Mempool Open Source Project™

diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.scss b/frontend/src/app/shared/components/global-footer/global-footer.component.scss index 387a27ffc..6b67711b6 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.scss +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.scss @@ -22,7 +22,7 @@ footer .row.main .branding { } footer .row.main .branding > p { - margin-bottom: 45px; + margin-bottom: 25px; } footer .row.main .branding .btn { @@ -35,11 +35,7 @@ footer .row.main .branding button.account { } footer .row.main .branding .cta { - margin: 20px auto 25px auto; -} - -footer .row.main .branding .cta-secondary { - + margin: 25px auto 25px auto; } footer .row.main .links.outer { From cb5b96485cea4a1e9ed34926cd9a1d588881645c Mon Sep 17 00:00:00 2001 From: Mononaut Date: Thu, 3 Aug 2023 16:17:17 +0900 Subject: [PATCH 53/73] resume tracking subscriptions after websocket reconnect --- frontend/src/app/services/websocket.service.ts | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/services/websocket.service.ts b/frontend/src/app/services/websocket.service.ts index 4bd20e987..af2a15e8c 100644 --- a/frontend/src/app/services/websocket.service.ts +++ b/frontend/src/app/services/websocket.service.ts @@ -28,8 +28,9 @@ export class WebsocketService { private isTrackingTx = false; private trackingTxId: string; private isTrackingMempoolBlock = false; - private isTrackingRbf = false; + private isTrackingRbf: 'all' | 'fullRbf' | false = false; private isTrackingRbfSummary = false; + private isTrackingAddress: string | false = false; private trackingMempoolBlock: number; private latestGitCommit = ''; private onlineCheckTimeout: number; @@ -110,6 +111,15 @@ export class WebsocketService { if (this.isTrackingMempoolBlock) { this.startTrackMempoolBlock(this.trackingMempoolBlock); } + if (this.isTrackingRbf) { + this.startTrackRbf(this.isTrackingRbf); + } + if (this.isTrackingRbfSummary) { + this.startTrackRbfSummary(); + } + if (this.isTrackingAddress) { + this.startTrackAddress(this.isTrackingAddress); + } this.stateService.connectionState$.next(2); } @@ -151,10 +161,12 @@ export class WebsocketService { startTrackAddress(address: string) { this.websocketSubject.next({ 'track-address': address }); + this.isTrackingAddress = address; } stopTrackingAddress() { this.websocketSubject.next({ 'track-address': 'stop' }); + this.isTrackingAddress = false; } startTrackAsset(asset: string) { @@ -178,7 +190,7 @@ export class WebsocketService { startTrackRbf(mode: 'all' | 'fullRbf') { this.websocketSubject.next({ 'track-rbf': mode }); - this.isTrackingRbf = true; + this.isTrackingRbf = mode; } stopTrackRbf() { From 22665f149b9e172ea0bc85a3eeacc44be914b3e5 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 29 Jul 2023 14:23:06 +0900 Subject: [PATCH 54/73] add /prices api endpoint --- backend/src/api/prices/prices.routes.ts | 19 +++++++++++++++++++ backend/src/index.ts | 2 ++ 2 files changed, 21 insertions(+) create mode 100644 backend/src/api/prices/prices.routes.ts diff --git a/backend/src/api/prices/prices.routes.ts b/backend/src/api/prices/prices.routes.ts new file mode 100644 index 000000000..03951ec05 --- /dev/null +++ b/backend/src/api/prices/prices.routes.ts @@ -0,0 +1,19 @@ +import { Application, Request, Response } from 'express'; +import config from '../../config'; +import pricesUpdater from '../../tasks/price-updater'; + +class PricesRoutes { + public initRoutes(app: Application): void { + app.get(config.MEMPOOL.API_URL_PREFIX + 'prices', this.$getCurrentPrices.bind(this)); + } + + private async $getCurrentPrices(req: Request, res: Response): Promise { + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); + + res.json(pricesUpdater.getLatestPrices()); + } +} + +export default new PricesRoutes(); diff --git a/backend/src/index.ts b/backend/src/index.ts index 185a47067..b00909bc6 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -30,6 +30,7 @@ import generalLightningRoutes from './api/explorer/general.routes'; import lightningStatsUpdater from './tasks/lightning/stats-updater.service'; import networkSyncService from './tasks/lightning/network-sync.service'; import statisticsRoutes from './api/statistics/statistics.routes'; +import pricesRoutes from './api/prices/prices.routes'; import miningRoutes from './api/mining/mining-routes'; import bisqRoutes from './api/bisq/bisq.routes'; import liquidRoutes from './api/liquid/liquid.routes'; @@ -261,6 +262,7 @@ class Server { setUpHttpApiRoutes(): void { bitcoinRoutes.initRoutes(this.app); + pricesRoutes.initRoutes(this.app); if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && config.MEMPOOL.ENABLED) { statisticsRoutes.initRoutes(this.app); } From 09c23b1241332b06c80a87c35a03ff3224c490e9 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 29 Jul 2023 14:45:00 +0900 Subject: [PATCH 55/73] configurable price update frequency --- backend/mempool-config.sample.json | 1 + backend/src/__fixtures__/mempool-config.template.json | 1 + backend/src/__tests__/config.test.ts | 1 + backend/src/config.ts | 2 ++ backend/src/tasks/price-updater.ts | 4 ++-- docker/backend/mempool-config.json | 1 + docker/backend/start.sh | 2 ++ production/mempool-config.mainnet.json | 3 +++ 8 files changed, 13 insertions(+), 2 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 7948049fc..fddd44781 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -116,6 +116,7 @@ "PASSWORD": "" }, "PRICE_DATA_SERVER": { + "UPDATE_FREQUENCY": 300, "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices", "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices" }, diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index ab700c466..3a2b151a5 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -93,6 +93,7 @@ "PASSWORD": "__SOCKS5PROXY_PASSWORD__" }, "PRICE_DATA_SERVER": { + "UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY", "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" }, diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index edfcc7f47..cb564a334 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -102,6 +102,7 @@ describe('Mempool Backend Config', () => { }); expect(config.PRICE_DATA_SERVER).toStrictEqual({ + UPDATE_FREQUENCY: 3600, TOR_URL: 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', CLEARNET_URL: 'https://price.bisq.wiz.biz/getAllMarketPrices' }); diff --git a/backend/src/config.ts b/backend/src/config.ts index 3a028d0cd..5117983fb 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -116,6 +116,7 @@ interface IConfig { PASSWORD: string; }; PRICE_DATA_SERVER: { + UPDATE_FREQUENCY: number; TOR_URL: string; CLEARNET_URL: string; }; @@ -259,6 +260,7 @@ const defaults: IConfig = { 'PASSWORD': '' }, 'PRICE_DATA_SERVER': { + 'UPDATE_FREQUENCY': 3600, 'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', 'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices' }, diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index fafe2b913..335a61065 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -108,8 +108,8 @@ class PriceUpdater { this.lastRun = await PricesRepository.$getLatestPriceTime(); } - if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < 3600) { - // Refresh only once every hour + if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < config.PRICE_DATA_SERVER.UPDATE_FREQUENCY) { + // Refresh every UPDATE_FREQUENCY seconds at most return; } diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 8b47d53b8..8b8a79969 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -112,6 +112,7 @@ "PASSWORD": "__SOCKS5PROXY_PASSWORD__" }, "PRICE_DATA_SERVER": { + "UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY__", "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" }, diff --git a/docker/backend/start.sh b/docker/backend/start.sh index e05c73710..add483568 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -95,6 +95,7 @@ __SOCKS5PROXY_USERNAME__=${SOCKS5PROXY_USERNAME:=""} __SOCKS5PROXY_PASSWORD__=${SOCKS5PROXY_PASSWORD:=""} # PRICE_DATA_SERVER +__PRICE_DATA_SERVER_UPDATE_FREQUENCY__=${PRICE_DATA_SERVER_UPDATE_FREQUENCY:=300} __PRICE_DATA_SERVER_TOR_URL__=${PRICE_DATA_SERVER_TOR_URL:=http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices} __PRICE_DATA_SERVER_CLEARNET_URL__=${PRICE_DATA_SERVER_CLEARNET_URL:=https://price.bisq.wiz.biz/getAllMarketPrices} @@ -226,6 +227,7 @@ sed -i "s!__SOCKS5PROXY_PORT__!${__SOCKS5PROXY_PORT__}!g" mempool-config.json sed -i "s!__SOCKS5PROXY_USERNAME__!${__SOCKS5PROXY_USERNAME__}!g" mempool-config.json sed -i "s!__SOCKS5PROXY_PASSWORD__!${__SOCKS5PROXY_PASSWORD__}!g" mempool-config.json +sed -i "s!__PRICE_DATA_SERVER_UPDATE_FREQUENCY__!${__PRICE_DATA_SERVER_UPDATE_FREQUENCY__}!g" mempool-config.json sed -i "s!__PRICE_DATA_SERVER_TOR_URL__!${__PRICE_DATA_SERVER_TOR_URL__}!g" mempool-config.json sed -i "s!__PRICE_DATA_SERVER_CLEARNET_URL__!${__PRICE_DATA_SERVER_CLEARNET_URL__}!g" mempool-config.json diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index 5e25bcb76..769dd8019 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -49,6 +49,9 @@ "ENABLED": true, "TX_PER_SECOND_SAMPLE_PERIOD": 150 }, + "PRICE_DATA_SERVER": { + "UPDATE_FREQUENCY": 300 + }, "REPLICATION": { "ENABLED": true, "AUDIT": true, From 864d9239ceacea56139b5de78e2e3c9f39660b44 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 29 Jul 2023 16:29:36 +0900 Subject: [PATCH 56/73] Update backend/src/__fixtures__/mempool-config.template.json --- backend/src/__fixtures__/mempool-config.template.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 3a2b151a5..5ed888962 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -93,7 +93,7 @@ "PASSWORD": "__SOCKS5PROXY_PASSWORD__" }, "PRICE_DATA_SERVER": { - "UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY", + "UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY__", "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" }, From b2d4000b2d5b7326dbab9a337b0a469e7b4f478f Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 29 Jul 2023 16:34:47 +0900 Subject: [PATCH 57/73] Update backend/src/api/prices/prices.routes.ts --- backend/src/api/prices/prices.routes.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/prices/prices.routes.ts b/backend/src/api/prices/prices.routes.ts index 03951ec05..c6a643dd8 100644 --- a/backend/src/api/prices/prices.routes.ts +++ b/backend/src/api/prices/prices.routes.ts @@ -7,7 +7,7 @@ class PricesRoutes { app.get(config.MEMPOOL.API_URL_PREFIX + 'prices', this.$getCurrentPrices.bind(this)); } - private async $getCurrentPrices(req: Request, res: Response): Promise { + private $getCurrentPrices(req: Request, res: Response): void { res.header('Pragma', 'public'); res.header('Cache-control', 'public'); res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); From ae59f95ba9bdbfde43714c173a5af5ae3b846715 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 29 Jul 2023 19:27:19 +0900 Subject: [PATCH 58/73] Refactoring price update config. Fixing last price time. --- backend/mempool-config.sample.json | 8 ++------ .../src/__fixtures__/mempool-config.template.json | 12 ++++-------- backend/src/__tests__/config.test.ts | 9 +-------- backend/src/config.ts | 14 ++------------ backend/src/indexer.ts | 8 ++++++-- backend/src/repositories/PricesRepository.ts | 2 +- backend/src/tasks/price-updater.ts | 10 +++++----- docker/backend/mempool-config.json | 8 ++------ production/mempool-config.mainnet.json | 6 ++---- 9 files changed, 25 insertions(+), 52 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index fddd44781..7f8bb4e22 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -32,7 +32,8 @@ "CPFP_INDEXING": false, "DISK_CACHE_BLOCK_INTERVAL": 6, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, - "ALLOW_UNREACHABLE": true + "ALLOW_UNREACHABLE": true, + "PRICE_UPDATE_FREQUENCY": 3600 }, "CORE_RPC": { "HOST": "127.0.0.1", @@ -115,11 +116,6 @@ "USERNAME": "", "PASSWORD": "" }, - "PRICE_DATA_SERVER": { - "UPDATE_FREQUENCY": 300, - "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices", - "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices" - }, "EXTERNAL_DATA_SERVER": { "MEMPOOL_API": "https://mempool.space/api/v1", "MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1", diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 5ed888962..583f8fb2d 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -23,8 +23,8 @@ "USER_AGENT": "__MEMPOOL_USER_AGENT__", "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__", "INDEXING_BLOCKS_AMOUNT": 14, - "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__", - "POOLS_JSON_URL": "__POOLS_JSON_URL__", + "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", + "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", "AUDIT": true, "ADVANCED_GBT_AUDIT": true, "ADVANCED_GBT_MEMPOOL": true, @@ -33,7 +33,8 @@ "MAX_BLOCKS_BULK_QUERY": 999, "DISK_CACHE_BLOCK_INTERVAL": 999, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, - "ALLOW_UNREACHABLE": true + "ALLOW_UNREACHABLE": true, + "PRICE_UPDATE_FREQUENCY": "__MEMPOOL_PRICE_UPDATE_FREQUENCY__" }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", @@ -92,11 +93,6 @@ "USERNAME": "__SOCKS5PROXY_USERNAME__", "PASSWORD": "__SOCKS5PROXY_PASSWORD__" }, - "PRICE_DATA_SERVER": { - "UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY__", - "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", - "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" - }, "EXTERNAL_DATA_SERVER": { "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__", "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__", diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index cb564a334..c42ccd223 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -47,6 +47,7 @@ describe('Mempool Backend Config', () => { DISK_CACHE_BLOCK_INTERVAL: 6, MAX_PUSH_TX_SIZE_WEIGHT: 400000, ALLOW_UNREACHABLE: true, + PRICE_UPDATE_FREQUENCY: 3600, }); expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true }); @@ -101,12 +102,6 @@ describe('Mempool Backend Config', () => { PASSWORD: '' }); - expect(config.PRICE_DATA_SERVER).toStrictEqual({ - UPDATE_FREQUENCY: 3600, - TOR_URL: 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', - CLEARNET_URL: 'https://price.bisq.wiz.biz/getAllMarketPrices' - }); - expect(config.EXTERNAL_DATA_SERVER).toStrictEqual({ MEMPOOL_API: 'https://mempool.space/api/v1', MEMPOOL_ONION: 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1', @@ -164,8 +159,6 @@ describe('Mempool Backend Config', () => { expect(config.SOCKS5PROXY).toStrictEqual(fixture.SOCKS5PROXY); - expect(config.PRICE_DATA_SERVER).toStrictEqual(fixture.PRICE_DATA_SERVER); - expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER); expect(config.REDIS).toStrictEqual(fixture.REDIS); diff --git a/backend/src/config.ts b/backend/src/config.ts index 5117983fb..1c59bcc46 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -38,6 +38,7 @@ interface IConfig { DISK_CACHE_BLOCK_INTERVAL: number; MAX_PUSH_TX_SIZE_WEIGHT: number; ALLOW_UNREACHABLE: boolean; + PRICE_UPDATE_FREQUENCY: number; }; ESPLORA: { REST_API_URL: string; @@ -115,11 +116,6 @@ interface IConfig { USERNAME: string; PASSWORD: string; }; - PRICE_DATA_SERVER: { - UPDATE_FREQUENCY: number; - TOR_URL: string; - CLEARNET_URL: string; - }; EXTERNAL_DATA_SERVER: { MEMPOOL_API: string; MEMPOOL_ONION: string; @@ -182,6 +178,7 @@ const defaults: IConfig = { 'DISK_CACHE_BLOCK_INTERVAL': 6, 'MAX_PUSH_TX_SIZE_WEIGHT': 400000, 'ALLOW_UNREACHABLE': true, + 'PRICE_UPDATE_FREQUENCY': 3600, }, 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', @@ -259,11 +256,6 @@ const defaults: IConfig = { 'USERNAME': '', 'PASSWORD': '' }, - 'PRICE_DATA_SERVER': { - 'UPDATE_FREQUENCY': 3600, - 'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices', - 'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices' - }, 'EXTERNAL_DATA_SERVER': { 'MEMPOOL_API': 'https://mempool.space/api/v1', 'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1', @@ -304,7 +296,6 @@ class Config implements IConfig { LND: IConfig['LND']; CLIGHTNING: IConfig['CLIGHTNING']; SOCKS5PROXY: IConfig['SOCKS5PROXY']; - PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER']; EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER']; MAXMIND: IConfig['MAXMIND']; REPLICATION: IConfig['REPLICATION']; @@ -325,7 +316,6 @@ class Config implements IConfig { this.LND = configs.LND; this.CLIGHTNING = configs.CLIGHTNING; this.SOCKS5PROXY = configs.SOCKS5PROXY; - this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER; this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER; this.MAXMIND = configs.MAXMIND; this.REPLICATION = configs.REPLICATION; diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index d89a2647f..7ec65d9c9 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -105,6 +105,12 @@ class Indexer { return; } + try { + await priceUpdater.$run(); + } catch (e) { + logger.err(`Running priceUpdater failed. Reason: ` + (e instanceof Error ? e.message : e)); + } + // Do not attempt to index anything unless Bitcoin Core is fully synced const blockchainInfo = await bitcoinClient.getBlockchainInfo(); if (blockchainInfo.blocks !== blockchainInfo.headers) { @@ -119,8 +125,6 @@ class Indexer { await this.checkAvailableCoreIndexes(); try { - await priceUpdater.$run(); - const chainValid = await blocks.$generateBlockDatabase(); if (chainValid === false) { // Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index ed9d1fd72..7d77834f5 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -65,7 +65,7 @@ class PricesRepository { try { await DB.query(` - INSERT INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY) + INSERT IGNORE INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY) VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ? )`, [time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY] ); diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index 335a61065..91a36d04b 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -108,8 +108,8 @@ class PriceUpdater { this.lastRun = await PricesRepository.$getLatestPriceTime(); } - if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < config.PRICE_DATA_SERVER.UPDATE_FREQUENCY) { - // Refresh every UPDATE_FREQUENCY seconds at most + if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < config.MEMPOOL.PRICE_UPDATE_FREQUENCY) { + // Refresh every PRICE_UPDATE_FREQUENCY seconds at most return; } @@ -146,14 +146,11 @@ class PriceUpdater { } } - logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`); - if (config.DATABASE.ENABLED === true) { // Save everything in db try { const p = 60 * 60 * 1000; // milliseconds in an hour const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042 - this.latestPrices.time = nowRounded.getTime() / 1000; await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices); } catch (e) { this.lastRun = previousRun + 5 * 60; @@ -161,6 +158,9 @@ class PriceUpdater { } } + this.latestPrices.time = Math.round(new Date().getTime() / 1000); + logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`); + if (this.ratesChangedCallback) { this.ratesChangedCallback(this.latestPrices); } diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 8b8a79969..19d7965fd 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -33,7 +33,8 @@ "MAX_PUSH_TX_SIZE_WEIGHT": __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__, "ALLOW_UNREACHABLE": __MEMPOOL_ALLOW_UNREACHABLE__, "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", - "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__" + "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", + "PRICE_UPDATE_FREQUENCY": __MEMPOOL_PRICE_UPDATE_FREQUENCY__ }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", @@ -111,11 +112,6 @@ "USERNAME": "__SOCKS5PROXY_USERNAME__", "PASSWORD": "__SOCKS5PROXY_PASSWORD__" }, - "PRICE_DATA_SERVER": { - "UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY__", - "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__", - "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__" - }, "EXTERNAL_DATA_SERVER": { "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__", "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__", diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index 769dd8019..02078cf78 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -18,7 +18,8 @@ "USE_SECOND_NODE_FOR_MINFEE": true, "DISK_CACHE_BLOCK_INTERVAL": 1, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, - "ALLOW_UNREACHABLE": true + "ALLOW_UNREACHABLE": true, + "PRICE_UPDATE_FREQUENCY": 300 }, "SYSLOG" : { "MIN_PRIORITY": "debug" @@ -49,9 +50,6 @@ "ENABLED": true, "TX_PER_SECOND_SAMPLE_PERIOD": 150 }, - "PRICE_DATA_SERVER": { - "UPDATE_FREQUENCY": 300 - }, "REPLICATION": { "ENABLED": true, "AUDIT": true, From bcb3b39bd87e6c621db7a9a3f7e51e73689ddaed Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 30 Jul 2023 12:09:05 +0900 Subject: [PATCH 59/73] Updating start.sh and README --- docker/README.md | 4 +++- docker/backend/start.sh | 12 ++---------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/docker/README.md b/docker/README.md index d95bc7aee..f70b600e1 100644 --- a/docker/README.md +++ b/docker/README.md @@ -113,7 +113,8 @@ Below we list all settings from `mempool-config.json` and the corresponding over "ADVANCED_GBT_MEMPOOL": false, "CPFP_INDEXING": false, "MAX_BLOCKS_BULK_QUERY": 0, - "DISK_CACHE_BLOCK_INTERVAL": 6 + "DISK_CACHE_BLOCK_INTERVAL": 6, + "PRICE_UPDATE_FREQUENCY": 3600 }, ``` @@ -146,6 +147,7 @@ Corresponding `docker-compose.yml` overrides: MEMPOOL_CPFP_INDEXING: "" MEMPOOL_MAX_BLOCKS_BULK_QUERY: "" MEMPOOL_DISK_CACHE_BLOCK_INTERVAL: "" + MEMPOOL_PRICE_UPDATE_FREQUENCY: "" ... ``` diff --git a/docker/backend/start.sh b/docker/backend/start.sh index add483568..8f8c79b41 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -35,7 +35,7 @@ __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0} __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6} __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__=${MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT:=4000000} __MEMPOOL_ALLOW_UNREACHABLE__=${MEMPOOL_ALLOW_UNREACHABLE:=true} - +__MEMPOOL_PRICE_UPDATE_FREQUENCY__=${MEMPOOL_PRICE_UPDATE_FREQUENCY:=3600} # CORE_RPC __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1} @@ -94,11 +94,6 @@ __SOCKS5PROXY_PORT__=${SOCKS5PROXY_PORT:=9050} __SOCKS5PROXY_USERNAME__=${SOCKS5PROXY_USERNAME:=""} __SOCKS5PROXY_PASSWORD__=${SOCKS5PROXY_PASSWORD:=""} -# PRICE_DATA_SERVER -__PRICE_DATA_SERVER_UPDATE_FREQUENCY__=${PRICE_DATA_SERVER_UPDATE_FREQUENCY:=300} -__PRICE_DATA_SERVER_TOR_URL__=${PRICE_DATA_SERVER_TOR_URL:=http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices} -__PRICE_DATA_SERVER_CLEARNET_URL__=${PRICE_DATA_SERVER_CLEARNET_URL:=https://price.bisq.wiz.biz/getAllMarketPrices} - # EXTERNAL_DATA_SERVER __EXTERNAL_DATA_SERVER_MEMPOOL_API__=${EXTERNAL_DATA_SERVER_MEMPOOL_API:=https://mempool.space/api/v1} __EXTERNAL_DATA_SERVER_MEMPOOL_ONION__=${EXTERNAL_DATA_SERVER_MEMPOOL_ONION:=http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1} @@ -178,6 +173,7 @@ sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__} sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json sed -i "s!__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__!${__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__}!g" mempool-config.json sed -i "s!__MEMPOOL_ALLOW_UNREACHABLE__!${__MEMPOOL_ALLOW_UNREACHABLE__}!g" mempool-config.json +sed -i "s!__MEMPOOL_PRICE_UPDATE_FREQUENCY__!${__MEMPOOL_PRICE_UPDATE_FREQUENCY__}!g" mempool-config.json sed -i "s!__CORE_RPC_HOST__!${__CORE_RPC_HOST__}!g" mempool-config.json sed -i "s!__CORE_RPC_PORT__!${__CORE_RPC_PORT__}!g" mempool-config.json @@ -227,10 +223,6 @@ sed -i "s!__SOCKS5PROXY_PORT__!${__SOCKS5PROXY_PORT__}!g" mempool-config.json sed -i "s!__SOCKS5PROXY_USERNAME__!${__SOCKS5PROXY_USERNAME__}!g" mempool-config.json sed -i "s!__SOCKS5PROXY_PASSWORD__!${__SOCKS5PROXY_PASSWORD__}!g" mempool-config.json -sed -i "s!__PRICE_DATA_SERVER_UPDATE_FREQUENCY__!${__PRICE_DATA_SERVER_UPDATE_FREQUENCY__}!g" mempool-config.json -sed -i "s!__PRICE_DATA_SERVER_TOR_URL__!${__PRICE_DATA_SERVER_TOR_URL__}!g" mempool-config.json -sed -i "s!__PRICE_DATA_SERVER_CLEARNET_URL__!${__PRICE_DATA_SERVER_CLEARNET_URL__}!g" mempool-config.json - sed -i "s!__EXTERNAL_DATA_SERVER_MEMPOOL_API__!${__EXTERNAL_DATA_SERVER_MEMPOOL_API__}!g" mempool-config.json sed -i "s!__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__!${__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__}!g" mempool-config.json sed -i "s!__EXTERNAL_DATA_SERVER_LIQUID_API__!${__EXTERNAL_DATA_SERVER_LIQUID_API__}!g" mempool-config.json From 83a487ecae0193e871db3b491301e15037caac5f Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 30 Jul 2023 13:52:47 +0900 Subject: [PATCH 60/73] Setting fixture to fixed number --- backend/src/__fixtures__/mempool-config.template.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 583f8fb2d..e3e8f5b14 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -34,7 +34,7 @@ "DISK_CACHE_BLOCK_INTERVAL": 999, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, - "PRICE_UPDATE_FREQUENCY": "__MEMPOOL_PRICE_UPDATE_FREQUENCY__" + "PRICE_UPDATE_FREQUENCY": 3600 }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", From af4d0b4d3f6329c41c3f0742482440a6630bbed0 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 30 Jul 2023 15:30:23 +0900 Subject: [PATCH 61/73] Allow priceUpdater to run without storing to database --- backend/src/index.ts | 1 + backend/src/indexer.ts | 2 +- backend/src/tasks/price-updater.ts | 8 ++++---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index b00909bc6..adb3f2e02 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -194,6 +194,7 @@ class Server { await memPool.$updateMempool(newMempool, pollRate); } indexer.$run(); + priceUpdater.$run(); // rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS const elapsed = Date.now() - start; diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 7ec65d9c9..392174ecf 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -106,7 +106,7 @@ class Indexer { } try { - await priceUpdater.$run(); + await priceUpdater.$run(true); } catch (e) { logger.err(`Running priceUpdater failed. Reason: ` + (e instanceof Error ? e.message : e)); } diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index 91a36d04b..f96616c12 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -72,7 +72,7 @@ class PriceUpdater { this.latestPrices = await PricesRepository.$getLatestConversionRates(); } - public async $run(): Promise { + public async $run(storeInDb: boolean = false): Promise { if (config.MEMPOOL.NETWORK === 'signet' || config.MEMPOOL.NETWORK === 'testnet') { // Coins have no value on testnet/signet, so we want to always show 0 return; @@ -89,7 +89,7 @@ class PriceUpdater { } try { - await this.$updatePrice(); + await this.$updatePrice(storeInDb); if (this.historyInserted === false && config.DATABASE.ENABLED === true) { await this.$insertHistoricalPrices(); } @@ -103,7 +103,7 @@ class PriceUpdater { /** * Fetch last BTC price from exchanges, average them, and save it in the database once every hour */ - private async $updatePrice(): Promise { + private async $updatePrice(storeInDb: boolean): Promise { if (this.lastRun === 0 && config.DATABASE.ENABLED === true) { this.lastRun = await PricesRepository.$getLatestPriceTime(); } @@ -146,7 +146,7 @@ class PriceUpdater { } } - if (config.DATABASE.ENABLED === true) { + if (config.DATABASE.ENABLED === true && storeInDb) { // Save everything in db try { const p = 60 * 60 * 1000; // milliseconds in an hour From 284d39baa8046bf11280562e75b0c4bf9927bfb4 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 30 Jul 2023 16:35:37 +0900 Subject: [PATCH 62/73] Update price at least every hour --- backend/src/tasks/price-updater.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index f96616c12..05d6f0bec 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -108,8 +108,8 @@ class PriceUpdater { this.lastRun = await PricesRepository.$getLatestPriceTime(); } - if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < config.MEMPOOL.PRICE_UPDATE_FREQUENCY) { - // Refresh every PRICE_UPDATE_FREQUENCY seconds at most + if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < Math.min(config.MEMPOOL.PRICE_UPDATE_FREQUENCY, 3600)) { + // Refresh at least every hour or PRICE_UPDATE_FREQUENCY seconds return; } From b213f43a91c2dde554479935c6a299807a508452 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 30 Jul 2023 16:37:24 +0900 Subject: [PATCH 63/73] Erasing unused PRICE_DATA_SERVER from readme --- docker/README.md | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/docker/README.md b/docker/README.md index f70b600e1..4270d11c8 100644 --- a/docker/README.md +++ b/docker/README.md @@ -365,25 +365,6 @@ Corresponding `docker-compose.yml` overrides:
-`mempool-config.json`: -```json - "PRICE_DATA_SERVER": { - "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices", - "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices" - } -``` - -Corresponding `docker-compose.yml` overrides: -```yaml - api: - environment: - PRICE_DATA_SERVER_TOR_URL: "" - PRICE_DATA_SERVER_CLEARNET_URL: "" - ... -``` - -
- `mempool-config.json`: ```json "LIGHTNING": { From 172c77328be75d13389f5cb2251269f702355a60 Mon Sep 17 00:00:00 2001 From: softsimon Date: Thu, 3 Aug 2023 17:42:41 +0900 Subject: [PATCH 64/73] Changing setting to per hour --- backend/mempool-config.sample.json | 2 +- .../__fixtures__/mempool-config.template.json | 2 +- backend/src/__tests__/config.test.ts | 2 +- backend/src/config.ts | 4 +- backend/src/indexer.ts | 2 +- backend/src/repositories/PricesRepository.ts | 2 +- backend/src/tasks/price-updater.ts | 56 ++++++++++++++----- docker/README.md | 4 +- docker/backend/mempool-config.json | 2 +- docker/backend/start.sh | 4 +- production/mempool-config.mainnet.json | 2 +- 11 files changed, 55 insertions(+), 27 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 7f8bb4e22..8fd59a430 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -33,7 +33,7 @@ "DISK_CACHE_BLOCK_INTERVAL": 6, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, - "PRICE_UPDATE_FREQUENCY": 3600 + "PRICE_UPDATES_PER_HOUR": 3600 }, "CORE_RPC": { "HOST": "127.0.0.1", diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index e3e8f5b14..471b7bb47 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -34,7 +34,7 @@ "DISK_CACHE_BLOCK_INTERVAL": 999, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, - "PRICE_UPDATE_FREQUENCY": 3600 + "PRICE_UPDATES_PER_HOUR": 1 }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index c42ccd223..9cd547b90 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -47,7 +47,7 @@ describe('Mempool Backend Config', () => { DISK_CACHE_BLOCK_INTERVAL: 6, MAX_PUSH_TX_SIZE_WEIGHT: 400000, ALLOW_UNREACHABLE: true, - PRICE_UPDATE_FREQUENCY: 3600, + PRICE_UPDATES_PER_HOUR: 1, }); expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true }); diff --git a/backend/src/config.ts b/backend/src/config.ts index 1c59bcc46..2e89bd525 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -38,7 +38,7 @@ interface IConfig { DISK_CACHE_BLOCK_INTERVAL: number; MAX_PUSH_TX_SIZE_WEIGHT: number; ALLOW_UNREACHABLE: boolean; - PRICE_UPDATE_FREQUENCY: number; + PRICE_UPDATES_PER_HOUR: number; }; ESPLORA: { REST_API_URL: string; @@ -178,7 +178,7 @@ const defaults: IConfig = { 'DISK_CACHE_BLOCK_INTERVAL': 6, 'MAX_PUSH_TX_SIZE_WEIGHT': 400000, 'ALLOW_UNREACHABLE': true, - 'PRICE_UPDATE_FREQUENCY': 3600, + 'PRICE_UPDATES_PER_HOUR': 1, }, 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 392174ecf..7ec65d9c9 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -106,7 +106,7 @@ class Indexer { } try { - await priceUpdater.$run(true); + await priceUpdater.$run(); } catch (e) { logger.err(`Running priceUpdater failed. Reason: ` + (e instanceof Error ? e.message : e)); } diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index 7d77834f5..ed9d1fd72 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -65,7 +65,7 @@ class PricesRepository { try { await DB.query(` - INSERT IGNORE INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY) + INSERT INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY) VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ? )`, [time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY] ); diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index 05d6f0bec..bbfb7d167 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -25,7 +25,9 @@ export interface PriceHistory { class PriceUpdater { public historyInserted = false; - private lastRun = 0; + private timeBetweenUpdatesMs = 3600000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR; + private cyclePosition = -1; + private firstRun = true; private lastHistoricalRun = 0; private running = false; private feeds: PriceFeed[] = []; @@ -41,6 +43,8 @@ class PriceUpdater { this.feeds.push(new CoinbaseApi()); this.feeds.push(new BitfinexApi()); this.feeds.push(new GeminiApi()); + + this.setCyclePosition(); } public getLatestPrices(): ApiPrice { @@ -72,7 +76,7 @@ class PriceUpdater { this.latestPrices = await PricesRepository.$getLatestConversionRates(); } - public async $run(storeInDb: boolean = false): Promise { + public async $run(): Promise { if (config.MEMPOOL.NETWORK === 'signet' || config.MEMPOOL.NETWORK === 'testnet') { // Coins have no value on testnet/signet, so we want to always show 0 return; @@ -89,7 +93,7 @@ class PriceUpdater { } try { - await this.$updatePrice(storeInDb); + await this.$updatePrice(); if (this.historyInserted === false && config.DATABASE.ENABLED === true) { await this.$insertHistoricalPrices(); } @@ -100,22 +104,42 @@ class PriceUpdater { this.running = false; } + private getMillisecondsSinceBeginningOfHour(): number { + const now = new Date(); + const beginningOfHour = new Date(now); + beginningOfHour.setMinutes(0, 0, 0); + return now.getTime() - beginningOfHour.getTime(); + } + + private setCyclePosition(): void { + const millisecondsSinceBeginningOfHour = this.getMillisecondsSinceBeginningOfHour(); + for (let i = 0; i < config.MEMPOOL.PRICE_UPDATES_PER_HOUR; i++) { + if (this.timeBetweenUpdatesMs * i > millisecondsSinceBeginningOfHour) { + this.cyclePosition = i; + return; + } + } + this.cyclePosition = config.MEMPOOL.PRICE_UPDATES_PER_HOUR; + } + /** * Fetch last BTC price from exchanges, average them, and save it in the database once every hour */ - private async $updatePrice(storeInDb: boolean): Promise { - if (this.lastRun === 0 && config.DATABASE.ENABLED === true) { - this.lastRun = await PricesRepository.$getLatestPriceTime(); + private async $updatePrice(): Promise { + let forceUpdate = false; + if (this.firstRun === true && config.DATABASE.ENABLED === true) { + const lastUpdate = await PricesRepository.$getLatestPriceTime(); + if (new Date().getTime() / 1000 - lastUpdate > this.timeBetweenUpdatesMs / 1000) { + forceUpdate = true; + } + this.firstRun = false; } - if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < Math.min(config.MEMPOOL.PRICE_UPDATE_FREQUENCY, 3600)) { - // Refresh at least every hour or PRICE_UPDATE_FREQUENCY seconds + const millisecondsSinceBeginningOfHour = this.getMillisecondsSinceBeginningOfHour(); + if (millisecondsSinceBeginningOfHour < this.timeBetweenUpdatesMs * this.cyclePosition && !forceUpdate) { return; } - const previousRun = this.lastRun; - this.lastRun = new Date().getTime() / 1000; - for (const currency of this.currencies) { let prices: number[] = []; @@ -146,14 +170,13 @@ class PriceUpdater { } } - if (config.DATABASE.ENABLED === true && storeInDb) { + if (config.DATABASE.ENABLED === true && this.cyclePosition === 0) { // Save everything in db try { const p = 60 * 60 * 1000; // milliseconds in an hour const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042 await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices); } catch (e) { - this.lastRun = previousRun + 5 * 60; logger.err(`Cannot save latest prices into db. Trying again in 5 minutes. Reason: ${(e instanceof Error ? e.message : e)}`); } } @@ -165,7 +188,12 @@ class PriceUpdater { this.ratesChangedCallback(this.latestPrices); } - this.lastRun = new Date().getTime() / 1000; + if (!forceUpdate) { + this.cyclePosition++; + if (this.cyclePosition > config.MEMPOOL.PRICE_UPDATES_PER_HOUR) { + this.cyclePosition = 0; + } + } if (this.latestPrices.USD === -1) { this.latestPrices = await PricesRepository.$getLatestConversionRates(); diff --git a/docker/README.md b/docker/README.md index 4270d11c8..13bda7ec6 100644 --- a/docker/README.md +++ b/docker/README.md @@ -114,7 +114,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over "CPFP_INDEXING": false, "MAX_BLOCKS_BULK_QUERY": 0, "DISK_CACHE_BLOCK_INTERVAL": 6, - "PRICE_UPDATE_FREQUENCY": 3600 + "PRICE_UPDATES_PER_HOUR": 1 }, ``` @@ -147,7 +147,7 @@ Corresponding `docker-compose.yml` overrides: MEMPOOL_CPFP_INDEXING: "" MEMPOOL_MAX_BLOCKS_BULK_QUERY: "" MEMPOOL_DISK_CACHE_BLOCK_INTERVAL: "" - MEMPOOL_PRICE_UPDATE_FREQUENCY: "" + MEMPOOL_PRICE_UPDATES_PER_HOUR: "" ... ``` diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 19d7965fd..216ad95c2 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -34,7 +34,7 @@ "ALLOW_UNREACHABLE": __MEMPOOL_ALLOW_UNREACHABLE__, "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", - "PRICE_UPDATE_FREQUENCY": __MEMPOOL_PRICE_UPDATE_FREQUENCY__ + "PRICE_UPDATES_PER_HOUR": __MEMPOOL_PRICE_UPDATES_PER_HOUR__ }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 8f8c79b41..30d2ac332 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -35,7 +35,7 @@ __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0} __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6} __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__=${MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT:=4000000} __MEMPOOL_ALLOW_UNREACHABLE__=${MEMPOOL_ALLOW_UNREACHABLE:=true} -__MEMPOOL_PRICE_UPDATE_FREQUENCY__=${MEMPOOL_PRICE_UPDATE_FREQUENCY:=3600} +__MEMPOOL_PRICE_UPDATES_PER_HOUR__=${MEMPOOL_PRICE_UPDATES_PER_HOUR:=3600} # CORE_RPC __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1} @@ -173,7 +173,7 @@ sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__} sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json sed -i "s!__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__!${__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__}!g" mempool-config.json sed -i "s!__MEMPOOL_ALLOW_UNREACHABLE__!${__MEMPOOL_ALLOW_UNREACHABLE__}!g" mempool-config.json -sed -i "s!__MEMPOOL_PRICE_UPDATE_FREQUENCY__!${__MEMPOOL_PRICE_UPDATE_FREQUENCY__}!g" mempool-config.json +sed -i "s!__MEMPOOL_PRICE_UPDATES_PER_HOUR__!${__MEMPOOL_PRICE_UPDATES_PER_HOUR__}!g" mempool-config.json sed -i "s!__CORE_RPC_HOST__!${__CORE_RPC_HOST__}!g" mempool-config.json sed -i "s!__CORE_RPC_PORT__!${__CORE_RPC_PORT__}!g" mempool-config.json diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index 02078cf78..f54635415 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -19,7 +19,7 @@ "DISK_CACHE_BLOCK_INTERVAL": 1, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, - "PRICE_UPDATE_FREQUENCY": 300 + "PRICE_UPDATES_PER_HOUR": 12 }, "SYSLOG" : { "MIN_PRIORITY": "debug" From 813f3dc09deb7d4ada81c3776d253615e2fd0c55 Mon Sep 17 00:00:00 2001 From: softsimon Date: Thu, 3 Aug 2023 18:26:24 +0900 Subject: [PATCH 65/73] Updating config sample --- backend/mempool-config.sample.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 8fd59a430..c53139b0b 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -33,7 +33,7 @@ "DISK_CACHE_BLOCK_INTERVAL": 6, "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, - "PRICE_UPDATES_PER_HOUR": 3600 + "PRICE_UPDATES_PER_HOUR": 1 }, "CORE_RPC": { "HOST": "127.0.0.1", From 851d0308786736d0ed7734b0fa4e1d89dd01316e Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 4 Aug 2023 10:23:15 +0900 Subject: [PATCH 66/73] [about] re-enable two sponsor buttons --- frontend/src/app/components/about/about.component.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index c93353e4f..551489b45 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -34,7 +34,8 @@

Sponsor the project

- See plans + Community + Enterprise
From 356ab9c6ae7d646b6043493738b810e5bbb531dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Aug 2023 02:40:58 +0000 Subject: [PATCH 67/73] Bump mysql2 from 3.5.2 to 3.6.0 in /backend Bumps [mysql2](https://github.com/sidorares/node-mysql2) from 3.5.2 to 3.6.0. - [Release notes](https://github.com/sidorares/node-mysql2/releases) - [Changelog](https://github.com/sidorares/node-mysql2/blob/master/Changelog.md) - [Commits](https://github.com/sidorares/node-mysql2/compare/v3.5.2...v3.6.0) --- updated-dependencies: - dependency-name: mysql2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- backend/package-lock.json | 16 ++++++++-------- backend/package.json | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/backend/package-lock.json b/backend/package-lock.json index 1a92552cb..f5452e908 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -17,9 +17,9 @@ "crypto-js": "~4.1.1", "express": "~4.18.2", "maxmind": "~4.3.11", - "mysql2": "~3.5.2", - "rust-gbt": "file:./rust-gbt", + "mysql2": "~3.6.0", "redis": "^4.6.6", + "rust-gbt": "file:./rust-gbt", "socks-proxy-agent": "~7.0.0", "typescript": "~4.9.3", "ws": "~8.13.0" @@ -6102,9 +6102,9 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node_modules/mysql2": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.5.2.tgz", - "integrity": "sha512-cptobmhYkYeTBIFp2c0piw2+gElpioga1rUw5UidHvo8yaHijMZoo8A3zyBVoo/K71f7ZFvrShA9iMIy9dCzCA==", + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.6.0.tgz", + "integrity": "sha512-EWUGAhv6SphezurlfI2Fpt0uJEWLmirrtQR7SkbTHFC+4/mJBrPiSzHESHKAWKG7ALVD6xaG/NBjjd1DGJGQQQ==", "dependencies": { "denque": "^2.1.0", "generate-function": "^2.3.1", @@ -12212,9 +12212,9 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "mysql2": { - "version": "3.5.2", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.5.2.tgz", - "integrity": "sha512-cptobmhYkYeTBIFp2c0piw2+gElpioga1rUw5UidHvo8yaHijMZoo8A3zyBVoo/K71f7ZFvrShA9iMIy9dCzCA==", + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.6.0.tgz", + "integrity": "sha512-EWUGAhv6SphezurlfI2Fpt0uJEWLmirrtQR7SkbTHFC+4/mJBrPiSzHESHKAWKG7ALVD6xaG/NBjjd1DGJGQQQ==", "requires": { "denque": "^2.1.0", "generate-function": "^2.3.1", diff --git a/backend/package.json b/backend/package.json index 24da55e17..36268227c 100644 --- a/backend/package.json +++ b/backend/package.json @@ -45,7 +45,7 @@ "crypto-js": "~4.1.1", "express": "~4.18.2", "maxmind": "~4.3.11", - "mysql2": "~3.5.2", + "mysql2": "~3.6.0", "rust-gbt": "file:./rust-gbt", "redis": "^4.6.6", "socks-proxy-agent": "~7.0.0", From 2892bfa1d81299474ddb3de265a1ecf785b1736d Mon Sep 17 00:00:00 2001 From: softsimon Date: Fri, 4 Aug 2023 13:23:09 +0900 Subject: [PATCH 68/73] Fixing cycle reset at top of the hour --- backend/src/tasks/price-updater.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index bbfb7d167..79173ad37 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -28,6 +28,7 @@ class PriceUpdater { private timeBetweenUpdatesMs = 3600000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR; private cyclePosition = -1; private firstRun = true; + private lastTime = -1; private lastHistoricalRun = 0; private running = false; private feeds: PriceFeed[] = []; @@ -136,7 +137,13 @@ class PriceUpdater { } const millisecondsSinceBeginningOfHour = this.getMillisecondsSinceBeginningOfHour(); - if (millisecondsSinceBeginningOfHour < this.timeBetweenUpdatesMs * this.cyclePosition && !forceUpdate) { + + // Reset the cycle on new hour + if (this.lastTime > millisecondsSinceBeginningOfHour) { + this.cyclePosition = 0; + } + this.lastTime = millisecondsSinceBeginningOfHour; + if (millisecondsSinceBeginningOfHour < this.timeBetweenUpdatesMs * this.cyclePosition && !forceUpdate && this.cyclePosition !== 0) { return; } @@ -190,9 +197,6 @@ class PriceUpdater { if (!forceUpdate) { this.cyclePosition++; - if (this.cyclePosition > config.MEMPOOL.PRICE_UPDATES_PER_HOUR) { - this.cyclePosition = 0; - } } if (this.latestPrices.USD === -1) { From 07f95acc29440905b2c70d992f692a3157c605cb Mon Sep 17 00:00:00 2001 From: softsimon Date: Fri, 4 Aug 2023 13:26:19 +0900 Subject: [PATCH 69/73] Base expiry on update frequency --- backend/src/api/prices/prices.routes.ts | 2 +- backend/src/tasks/price-updater.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/api/prices/prices.routes.ts b/backend/src/api/prices/prices.routes.ts index c6a643dd8..b46331b73 100644 --- a/backend/src/api/prices/prices.routes.ts +++ b/backend/src/api/prices/prices.routes.ts @@ -10,7 +10,7 @@ class PricesRoutes { private $getCurrentPrices(req: Request, res: Response): void { res.header('Pragma', 'public'); res.header('Cache-control', 'public'); - res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); + res.setHeader('Expires', new Date(Date.now() + 360_0000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR).toUTCString()); res.json(pricesUpdater.getLatestPrices()); } diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index 79173ad37..fd799fb87 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -25,7 +25,7 @@ export interface PriceHistory { class PriceUpdater { public historyInserted = false; - private timeBetweenUpdatesMs = 3600000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR; + private timeBetweenUpdatesMs = 360_0000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR; private cyclePosition = -1; private firstRun = true; private lastTime = -1; From 11248821c50493fb6d04f875590cc79a60f89822 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 4 Aug 2023 14:30:40 +0900 Subject: [PATCH 70/73] hide sponsor buttons --- frontend/src/app/components/about/about.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index 551489b45..e9d5ec3b2 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -31,7 +31,7 @@ - +

Sponsor the project

Community From 101de3bac7cea2596b0113be20b943a5ddc6ecdb Mon Sep 17 00:00:00 2001 From: mononaut <83316221+mononaut@users.noreply.github.com> Date: Fri, 4 Aug 2023 15:23:55 +0900 Subject: [PATCH 71/73] Fix PRICE_UPDATES_PER_HOUR docker default Co-authored-by: nymkappa <9780671+nymkappa@users.noreply.github.com> --- docker/backend/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 30d2ac332..008c0aaf5 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -35,7 +35,7 @@ __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0} __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6} __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__=${MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT:=4000000} __MEMPOOL_ALLOW_UNREACHABLE__=${MEMPOOL_ALLOW_UNREACHABLE:=true} -__MEMPOOL_PRICE_UPDATES_PER_HOUR__=${MEMPOOL_PRICE_UPDATES_PER_HOUR:=3600} +__MEMPOOL_PRICE_UPDATES_PER_HOUR__=${MEMPOOL_PRICE_UPDATES_PER_HOUR:=1} # CORE_RPC __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1} From 0535e8c5f92f4c53c97d2a2623e80aa3b18312c0 Mon Sep 17 00:00:00 2001 From: junderw Date: Thu, 3 Aug 2023 18:46:18 -0700 Subject: [PATCH 72/73] Fix: Rust-GBT packaging needs fixing --- backend/npm_package.sh | 14 ++++++++++++++ backend/npm_package_rm_build_deps.sh | 12 ++++++++++++ backend/package.json | 9 ++++----- backend/src/__tests__/gbt/gbt-tests.ts | 2 +- backend/src/api/mempool-blocks.ts | 2 +- 5 files changed, 32 insertions(+), 7 deletions(-) create mode 100755 backend/npm_package.sh create mode 100755 backend/npm_package_rm_build_deps.sh diff --git a/backend/npm_package.sh b/backend/npm_package.sh new file mode 100755 index 000000000..627a77360 --- /dev/null +++ b/backend/npm_package.sh @@ -0,0 +1,14 @@ +#/bin/sh +set -e + +npm run build +# Remove previous package folder +rm -rf package +# Move JS and deps +mv dist package +mv node_modules package +# Remove symlink for rust-gbt and insert real folder +rm package/node_modules/rust-gbt +mv rust-gbt package/node_modules +# Clean up deps +npm run package-rm-build-deps diff --git a/backend/npm_package_rm_build_deps.sh b/backend/npm_package_rm_build_deps.sh new file mode 100755 index 000000000..6b260d84d --- /dev/null +++ b/backend/npm_package_rm_build_deps.sh @@ -0,0 +1,12 @@ +#/bin/sh +set -e + +# Cleaning up inside the node_modules folder +cd package/node_modules +rm -r \ + typescript \ + @typescript-eslint \ + @napi-rs \ + ./rust-gbt/src \ + ./rust-gbt/Cargo.toml \ + ./rust-gbt/build.rs diff --git a/backend/package.json b/backend/package.json index 36268227c..e21924207 100644 --- a/backend/package.json +++ b/backend/package.json @@ -22,10 +22,10 @@ "main": "index.ts", "scripts": { "tsc": "./node_modules/typescript/bin/tsc -p tsconfig.build.json", - "build": "npm run build-rust && npm run tsc && npm run create-resources", + "build": "npm run tsc && npm run create-resources", "create-resources": "cp ./src/tasks/price-feeds/mtgox-weekly.json ./dist/tasks && node dist/api/fetch-version.js", - "package": "npm run build && rm -rf package && mv dist package && mv node_modules package && mv rust-gbt package && npm run package-rm-build-deps", - "package-rm-build-deps": "(cd package/node_modules; rm -r typescript @typescript-eslint @napi-rs ../rust-gbt/target ../rust-gbt/node_modules ../rust-gbt/src)", + "package": "./npm_package.sh", + "package-rm-build-deps": "./npm_package_rm_build_deps.sh", "start": "node --max-old-space-size=2048 dist/index.js", "start-production": "node --max-old-space-size=16384 dist/index.js", "reindex-updated-pools": "npm run start-production --update-pools", @@ -33,8 +33,7 @@ "test": "./node_modules/.bin/jest --coverage", "lint": "./node_modules/.bin/eslint . --ext .ts", "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix", - "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\"", - "build-rust": "cd rust-gbt && npm install" + "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\"" }, "dependencies": { "@babel/core": "^7.21.3", diff --git a/backend/src/__tests__/gbt/gbt-tests.ts b/backend/src/__tests__/gbt/gbt-tests.ts index 0c2eb0176..8a3995f71 100644 --- a/backend/src/__tests__/gbt/gbt-tests.ts +++ b/backend/src/__tests__/gbt/gbt-tests.ts @@ -1,5 +1,5 @@ import fs from 'fs'; -import { GbtGenerator, ThreadTransaction } from '../../../rust-gbt'; +import { GbtGenerator, ThreadTransaction } from 'rust-gbt'; import path from 'path'; const baseline = require('./test-data/target-template.json'); diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 1ea4d2af8..16cf60a2f 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,4 +1,4 @@ -import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from '../../rust-gbt'; +import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from 'rust-gbt'; import logger from '../logger'; import { MempoolBlock, MempoolTransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, PoolTag } from '../mempool.interfaces'; import { Common, OnlineFeeStatsCalculator } from './common'; From b026f5a481558c99359998a23d60d2c5eabc0e5e Mon Sep 17 00:00:00 2001 From: softsimon Date: Fri, 4 Aug 2023 15:55:22 +0900 Subject: [PATCH 73/73] Removing console log --- backend/src/api/mempool-blocks.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 16cf60a2f..1de4bbee7 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -341,7 +341,6 @@ class MempoolBlocks { } public async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { - console.log('$rustMakeBlockTemplates'); const start = Date.now(); // reset mempool short ids