From a28544d0469a8b7e981c8c9cf156c4b2e27c6147 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sat, 22 Oct 2022 13:20:16 -0700 Subject: [PATCH 01/15] Update Cypress GHA to use newer checkout and setup-node actions --- .github/workflows/cypress.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml index 7cace626c..1bca63dd3 100644 --- a/.github/workflows/cypress.yml +++ b/.github/workflows/cypress.yml @@ -2,7 +2,7 @@ name: Cypress Tests on: pull_request: - types: [ opened, review_requested, synchronize ] + types: [opened, review_requested, synchronize] jobs: cypress: if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')" @@ -24,19 +24,19 @@ jobs: - module: "bisq" spec: | cypress/e2e/bisq/bisq.spec.ts - + name: E2E tests for ${{ matrix.module }} steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: path: ${{ matrix.module }} - + - name: Setup node - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: 16.15.0 - cache: 'npm' + cache: "npm" cache-dependency-path: ${{ matrix.module }}/frontend/package-lock.json - name: Chrome browser tests (${{ matrix.module }}) @@ -46,14 +46,14 @@ jobs: working-directory: ${{ matrix.module }}/frontend build: npm run config:defaults:${{ matrix.module }} start: npm run start:local-staging - wait-on: 'http://localhost:4200' + wait-on: "http://localhost:4200" wait-on-timeout: 120 record: true parallel: true spec: ${{ matrix.spec }} group: Tests on Chrome (${{ matrix.module }}) browser: "chrome" - ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}' + ci-build-id: "${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}" env: COMMIT_INFO_MESSAGE: ${{ github.event.pull_request.title }} CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} From 0d92779971cd41a72be1ba04ef7c3ba2c63a91ea Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Thu, 1 Dec 2022 08:14:10 -0800 Subject: [PATCH 02/15] Update ignore rules and frequency of dependabot updates --- .github/dependabot.yml | 44 +++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 001ea3cb3..1e2574971 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,20 +1,28 @@ version: 2 updates: -- package-ecosystem: npm - directory: "/backend" - schedule: - interval: daily - open-pull-requests-limit: 10 -- package-ecosystem: npm - directory: "/frontend" - schedule: - interval: daily - open-pull-requests-limit: 10 -- package-ecosystem: docker - directory: "/docker/backend" - schedule: - interval: weekly -- package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "weekly" + - package-ecosystem: npm + directory: "/backend" + schedule: + interval: daily + open-pull-requests-limit: 10 + ignore: + - update-types: ["version-update:semver-major"] + - package-ecosystem: npm + directory: "/frontend" + schedule: + interval: daily + open-pull-requests-limit: 10 + ignore: + - update-types: ["version-update:semver-major"] + - package-ecosystem: docker + directory: "/docker/backend" + schedule: + interval: daily + ignore: + - update-types: ["version-update:semver-major"] + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: daily + ignore: + - update-types: ["version-update:semver-major"] From 132e848fdc7bf2457f44f19f50c33933e3209d7a Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 3 Dec 2022 10:49:10 +0900 Subject: [PATCH 03/15] Fix block summaries repo upsert race condition --- backend/src/api/websocket-handler.ts | 2 +- .../repositories/BlocksSummariesRepository.ts | 41 +++++++++++++------ 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 31224fc0c..246b5b90d 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -439,7 +439,7 @@ class WebsocketHandler { }; }) : []; - BlocksSummariesRepository.$saveSummary({ + BlocksSummariesRepository.$saveTemplate({ height: block.height, template: { id: block.id, diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts index 28b3cc7eb..1406a1d07 100644 --- a/backend/src/repositories/BlocksSummariesRepository.ts +++ b/backend/src/repositories/BlocksSummariesRepository.ts @@ -17,19 +17,16 @@ class BlocksSummariesRepository { return undefined; } - public async $saveSummary(params: { height: number, mined?: BlockSummary, template?: BlockSummary}) { - const blockId = params.mined?.id ?? params.template?.id; + public async $saveSummary(params: { height: number, mined?: BlockSummary}) { + const blockId = params.mined?.id; try { - const [dbSummary]: any[] = await DB.query(`SELECT * FROM blocks_summaries WHERE id = "${blockId}"`); - if (dbSummary.length === 0) { // First insertion - await DB.query(`INSERT INTO blocks_summaries VALUE (?, ?, ?, ?)`, [ - params.height, blockId, JSON.stringify(params.mined?.transactions ?? []), JSON.stringify(params.template?.transactions ?? []) - ]); - } else if (params.mined !== undefined) { // Update mined block summary - await DB.query(`UPDATE blocks_summaries SET transactions = ? WHERE id = "${params.mined.id}"`, [JSON.stringify(params.mined.transactions)]); - } else if (params.template !== undefined) { // Update template block summary - await DB.query(`UPDATE blocks_summaries SET template = ? WHERE id = "${params.template.id}"`, [JSON.stringify(params.template?.transactions)]); - } + const transactions = JSON.stringify(params.mined?.transactions || []); + await DB.query(` + INSERT INTO blocks_summaries (height, id, transactions, template) + VALUE (?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + transactions = ? + `, [params.height, blockId, transactions, '[]', transactions]); } catch (e: any) { if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart logger.debug(`Cannot save block summary for ${blockId} because it has already been indexed, ignoring`); @@ -40,6 +37,26 @@ class BlocksSummariesRepository { } } + public async $saveTemplate(params: { height: number, template: BlockSummary}) { + const blockId = params.template?.id; + try { + const transactions = JSON.stringify(params.template?.transactions || []); + await DB.query(` + INSERT INTO blocks_summaries (height, id, transactions, template) + VALUE (?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + template = ? + `, [params.height, blockId, '[]', transactions, transactions]); + } catch (e: any) { + if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart + logger.debug(`Cannot save block template for ${blockId} because it has already been indexed, ignoring`); + } else { + logger.debug(`Cannot save block template for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`); + throw e; + } + } + } + public async $getIndexedSummariesId(): Promise { try { const [rows]: any[] = await DB.query(`SELECT id from blocks_summaries`); From 3126a559a03794cddd0d412b44293e8c25d8a6cb Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 3 Dec 2022 11:17:53 +0900 Subject: [PATCH 04/15] Make forensics backend call rate limiting configurable --- backend/mempool-config.sample.json | 3 ++- backend/src/__fixtures__/mempool-config.template.json | 3 ++- backend/src/config.ts | 2 ++ backend/src/tasks/lightning/forensics.service.ts | 7 +++---- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 6de690ad8..dbbc8412d 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -85,7 +85,8 @@ "STATS_REFRESH_INTERVAL": 600, "GRAPH_REFRESH_INTERVAL": 600, "LOGGER_UPDATE_INTERVAL": 30, - "FORENSICS_INTERVAL": 43200 + "FORENSICS_INTERVAL": 43200, + "FORENSICS_RATE_LIMIT": 20 }, "LND": { "TLS_CERT_PATH": "tls.cert", diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 7a988a70d..2e9221c7a 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -101,7 +101,8 @@ "STATS_REFRESH_INTERVAL": 600, "GRAPH_REFRESH_INTERVAL": 600, "LOGGER_UPDATE_INTERVAL": 30, - "FORENSICS_INTERVAL": 43200 + "FORENSICS_INTERVAL": 43200, + "FORENSICS_RATE_LIMIT": "__FORENSICS_RATE_LIMIT__" }, "LND": { "TLS_CERT_PATH": "", diff --git a/backend/src/config.ts b/backend/src/config.ts index 3a3d2b56d..e97deb5e5 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -44,6 +44,7 @@ interface IConfig { GRAPH_REFRESH_INTERVAL: number; LOGGER_UPDATE_INTERVAL: number; FORENSICS_INTERVAL: number; + FORENSICS_RATE_LIMIT: number; }; LND: { TLS_CERT_PATH: string; @@ -205,6 +206,7 @@ const defaults: IConfig = { 'GRAPH_REFRESH_INTERVAL': 600, 'LOGGER_UPDATE_INTERVAL': 30, 'FORENSICS_INTERVAL': 43200, + 'FORENSICS_RATE_LIMIT': 20, }, 'LND': { 'TLS_CERT_PATH': '', diff --git a/backend/src/tasks/lightning/forensics.service.ts b/backend/src/tasks/lightning/forensics.service.ts index 7acb36e89..c62639411 100644 --- a/backend/src/tasks/lightning/forensics.service.ts +++ b/backend/src/tasks/lightning/forensics.service.ts @@ -7,7 +7,6 @@ import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface'; import { Common } from '../../api/common'; import { ILightningApi } from '../../api/lightning/lightning-api.interface'; -const throttleDelay = 20; //ms const tempCacheSize = 10000; class ForensicsService { @@ -91,7 +90,7 @@ class ForensicsService { let outspends: IEsploraApi.Outspend[] | undefined; try { outspends = await bitcoinApi.$getOutspends(channel.closing_transaction_id); - await Common.sleep$(throttleDelay); + await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT); } catch (e) { logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`); continue; @@ -340,7 +339,7 @@ class ForensicsService { let outspends: IEsploraApi.Outspend[] | undefined; try { outspends = await bitcoinApi.$getOutspends(input.txid); - await Common.sleep$(throttleDelay); + await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT); } catch (e) { logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + input.txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`); } @@ -429,7 +428,7 @@ class ForensicsService { if (temp) { this.tempCached.push(txid); } - await Common.sleep$(throttleDelay); + await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT); } catch (e) { logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`); return null; From e54e896e569e09b6070e24792e5b1aa198fc8d43 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 3 Dec 2022 12:10:54 +0900 Subject: [PATCH 05/15] fix skipped descendant updates on tx inclusion --- backend/src/api/tx-selection-worker.ts | 54 +++++++++++++------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/backend/src/api/tx-selection-worker.ts b/backend/src/api/tx-selection-worker.ts index 206d26fe3..ca40af84f 100644 --- a/backend/src/api/tx-selection-worker.ts +++ b/backend/src/api/tx-selection-worker.ts @@ -106,44 +106,44 @@ function makeBlockTemplates({ mempool, blockLimit, weightLimit, condenseRest }: if (nextTx && !nextTx?.used) { // Check if the package fits into this block if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) { - blockWeight += nextTx.ancestorWeight; const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values()); const descendants: AuditTransaction[] = []; // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count) const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx]; const effectiveFeeRate = nextTx.ancestorFee / (nextTx.ancestorWeight / 4); - + const used: AuditTransaction[] = []; while (sortedTxSet.length) { const ancestor = sortedTxSet.pop(); const mempoolTx = mempool[ancestor.txid]; - if (ancestor && !ancestor?.used) { - ancestor.used = true; - // update original copy of this tx with effective fee rate & relatives data - mempoolTx.effectiveFeePerVsize = effectiveFeeRate; - mempoolTx.ancestors = sortedTxSet.map((a) => { - return { - txid: a.txid, - fee: a.fee, - weight: a.weight, - }; - }).reverse(); - mempoolTx.descendants = descendants.map((a) => { - return { - txid: a.txid, - fee: a.fee, - weight: a.weight, - }; - }); - descendants.push(ancestor); - mempoolTx.cpfpChecked = true; - transactions.push(ancestor); - blockSize += ancestor.size; - } + ancestor.used = true; + ancestor.usedBy = nextTx.txid; + // update original copy of this tx with effective fee rate & relatives data + mempoolTx.effectiveFeePerVsize = effectiveFeeRate; + mempoolTx.ancestors = sortedTxSet.map((a) => { + return { + txid: a.txid, + fee: a.fee, + weight: a.weight, + }; + }).reverse(); + mempoolTx.descendants = descendants.map((a) => { + return { + txid: a.txid, + fee: a.fee, + weight: a.weight, + }; + }); + descendants.push(ancestor); + mempoolTx.cpfpChecked = true; + transactions.push(ancestor); + blockSize += ancestor.size; + blockWeight += ancestor.weight; + used.push(ancestor); } // remove these as valid package ancestors for any descendants remaining in the mempool - if (sortedTxSet.length) { - sortedTxSet.forEach(tx => { + if (used.length) { + used.forEach(tx => { updateDescendants(tx, auditPool, modified); }); } From 79f6ae3b6ff2f2839704622316e05fb625169e6c Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 3 Dec 2022 12:11:21 +0900 Subject: [PATCH 06/15] fix post-block advanced gbt call --- backend/src/api/websocket-handler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 31224fc0c..c9b7ff233 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -468,7 +468,7 @@ class WebsocketHandler { } if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { - await mempoolBlocks.makeBlockTemplates(_memPool, 2); + await mempoolBlocks.makeBlockTemplates(_memPool, 8, null, true); } else { mempoolBlocks.updateMempoolBlocks(_memPool); } From 685433fe4cc59ef77ee4683e235b188f71ef2397 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 5 Dec 2022 08:11:46 +0100 Subject: [PATCH 07/15] Handle ISP with no nodes --- backend/src/api/explorer/nodes.api.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 28241ad20..cf1c77686 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -538,6 +538,10 @@ class NodesApi { const IPSIds = ISPId.split(','); const [rows]: any = await DB.query(query, [IPSIds, IPSIds]); + if (!rows || rows.length === 0) { + return []; + } + const nodes = {}; const intISPIds: number[] = []; From 5658e053d03485052cb3550aa65f54447b09efc2 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Mon, 5 Dec 2022 20:22:53 -0800 Subject: [PATCH 08/15] Update Cypress GHA to v5 --- .github/workflows/cypress.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml index 1bca63dd3..e8f6d1df1 100644 --- a/.github/workflows/cypress.yml +++ b/.github/workflows/cypress.yml @@ -40,7 +40,7 @@ jobs: cache-dependency-path: ${{ matrix.module }}/frontend/package-lock.json - name: Chrome browser tests (${{ matrix.module }}) - uses: cypress-io/github-action@v4 + uses: cypress-io/github-action@v5 with: tag: ${{ github.event_name }} working-directory: ${{ matrix.module }}/frontend From 7a7172bb64d4addb257528e52235e5b25ed61513 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Mon, 5 Dec 2022 21:56:42 -0800 Subject: [PATCH 09/15] Change Bisq staging host to fra so tests can pass --- frontend/proxy.conf.staging.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/proxy.conf.staging.js b/frontend/proxy.conf.staging.js index a99ead7a4..0281b66ce 100644 --- a/frontend/proxy.conf.staging.js +++ b/frontend/proxy.conf.staging.js @@ -5,7 +5,7 @@ let PROXY_CONFIG = require('./proxy.conf'); PROXY_CONFIG.forEach(entry => { entry.target = entry.target.replace("mempool.space", "mempool-staging.tk7.mempool.space"); entry.target = entry.target.replace("liquid.network", "liquid-staging.tk7.mempool.space"); - entry.target = entry.target.replace("bisq.markets", "bisq-staging.tk7.mempool.space"); + entry.target = entry.target.replace("bisq.markets", "bisq-staging.fra.mempool.space"); }); module.exports = PROXY_CONFIG; From 4d0637768d98c20dc8eb2a67d8be99d49b9062d6 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 6 Dec 2022 05:51:44 +0900 Subject: [PATCH 10/15] Refactor advanced gbt to minimize inter-thread comms --- backend/src/api/mempool-blocks.ts | 152 ++++++++++++++++++---- backend/src/api/mempool.ts | 2 +- backend/src/api/tx-selection-worker.ts | 170 +++++++++---------------- backend/src/api/websocket-handler.ts | 8 +- backend/src/mempool.interfaces.ts | 15 ++- 5 files changed, 211 insertions(+), 136 deletions(-) diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index e8ab48230..62bdc8f1b 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,17 +1,14 @@ import logger from '../logger'; -import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta } from '../mempool.interfaces'; +import { MempoolBlock, TransactionExtended, ThreadTransaction, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor } from '../mempool.interfaces'; import { Common } from './common'; import config from '../config'; -import { StaticPool } from 'node-worker-threads-pool'; +import { Worker } from 'worker_threads'; import path from 'path'; class MempoolBlocks { private mempoolBlocks: MempoolBlockWithTransactions[] = []; private mempoolBlockDeltas: MempoolBlockDelta[] = []; - private makeTemplatesPool = new StaticPool({ - size: 1, - task: path.resolve(__dirname, './tx-selection-worker.js'), - }); + private txSelectionWorker: Worker | null = null; constructor() {} @@ -146,27 +143,136 @@ class MempoolBlocks { return mempoolBlockDeltas; } - public async makeBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, blockLimit: number, weightLimit: number | null = null, condenseRest = false): Promise { - const { mempool, blocks } = await this.makeTemplatesPool.exec({ mempool: newMempool, blockLimit, weightLimit, condenseRest }); - const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks); - - // copy CPFP info across to main thread's mempool - Object.keys(newMempool).forEach((txid) => { - if (newMempool[txid] && mempool[txid]) { - newMempool[txid].effectiveFeePerVsize = mempool[txid].effectiveFeePerVsize; - newMempool[txid].ancestors = mempool[txid].ancestors; - newMempool[txid].descendants = mempool[txid].descendants; - newMempool[txid].bestDescendant = mempool[txid].bestDescendant; - newMempool[txid].cpfpChecked = mempool[txid].cpfpChecked; - } + public async makeBlockTemplates(newMempool: { [txid: string]: TransactionExtended }): Promise { + // prepare a stripped down version of the mempool with only the minimum necessary data + // to reduce the overhead of passing this data to the worker thread + const strippedMempool: { [txid: string]: ThreadTransaction } = {}; + Object.values(newMempool).forEach(entry => { + strippedMempool[entry.txid] = { + txid: entry.txid, + fee: entry.fee, + weight: entry.weight, + feePerVsize: entry.fee / (entry.weight / 4), + effectiveFeePerVsize: entry.fee / (entry.weight / 4), + vin: entry.vin.map(v => v.txid), + }; }); - this.mempoolBlocks = blocks; + if (!this.txSelectionWorker) { + this.txSelectionWorker = new Worker(path.resolve(__dirname, './tx-selection-worker.js')); + this.txSelectionWorker.once('error', () => { + this.txSelectionWorker = null; + }); + this.txSelectionWorker.once('exit', () => { + this.txSelectionWorker = null; + }); + } + + // run the block construction algorithm in a separate thread, and wait for a result + const workerResultPromise = new Promise<{ blocks: ThreadTransaction[][], clusters: { [root: string]: string[] } }>((resolve) => { + this.txSelectionWorker?.once('message', (result): void => { + resolve(result); + }); + }); + this.txSelectionWorker.postMessage({ type: 'set', mempool: strippedMempool }); + const { blocks, clusters } = await workerResultPromise; + + this.processBlockTemplates(newMempool, blocks, clusters); + } + + public async updateBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, added: TransactionExtended[], removed: string[]): Promise { + if (!this.txSelectionWorker) { + // need to reset the worker + return this.makeBlockTemplates(newMempool); + } + // prepare a stripped down version of the mempool with only the minimum necessary data + // to reduce the overhead of passing this data to the worker thread + const addedStripped: ThreadTransaction[] = added.map(entry => { + return { + txid: entry.txid, + fee: entry.fee, + weight: entry.weight, + feePerVsize: entry.fee / (entry.weight / 4), + effectiveFeePerVsize: entry.fee / (entry.weight / 4), + vin: entry.vin.map(v => v.txid), + }; + }); + + // run the block construction algorithm in a separate thread, and wait for a result + const workerResultPromise = new Promise<{ blocks: ThreadTransaction[][], clusters: { [root: string]: string[] } }>((resolve) => { + this.txSelectionWorker?.once('message', (result): void => { + resolve(result); + }); + }); + this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed }); + const { blocks, clusters } = await workerResultPromise; + + this.processBlockTemplates(newMempool, blocks, clusters); + } + + private processBlockTemplates(mempool, blocks, clusters): void { + // update this thread's mempool with the results + blocks.forEach(block => { + block.forEach(tx => { + if (tx.txid in mempool) { + if (tx.effectiveFeePerVsize != null) { + mempool[tx.txid].effectiveFeePerVsize = tx.effectiveFeePerVsize; + } + if (tx.cpfpRoot && tx.cpfpRoot in clusters) { + const ancestors: Ancestor[] = []; + const descendants: Ancestor[] = []; + const cluster = clusters[tx.cpfpRoot]; + let matched = false; + cluster.forEach(txid => { + if (txid === tx.txid) { + matched = true; + } else { + const relative = { + txid: txid, + fee: mempool[txid].fee, + weight: mempool[txid].weight, + }; + if (matched) { + descendants.push(relative); + } else { + ancestors.push(relative); + } + } + }); + mempool[tx.txid].ancestors = ancestors; + mempool[tx.txid].descendants = descendants; + mempool[tx.txid].bestDescendant = null; + } + mempool[tx.txid].cpfpChecked = tx.cpfpChecked; + } + }); + }); + + // unpack the condensed blocks into proper mempool blocks + const mempoolBlocks = blocks.map((transactions, blockIndex) => { + return this.dataToMempoolBlocks(transactions.map(tx => { + return mempool[tx.txid] || null; + }).filter(tx => !!tx), undefined, undefined, blockIndex); + }); + + const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, mempoolBlocks); + + this.mempoolBlocks = mempoolBlocks; this.mempoolBlockDeltas = deltas; } private dataToMempoolBlocks(transactions: TransactionExtended[], - blockSize: number, blockWeight: number, blocksIndex: number): MempoolBlockWithTransactions { + blockSize: number | undefined, blockWeight: number | undefined, blocksIndex: number): MempoolBlockWithTransactions { + let totalSize = blockSize || 0; + let totalWeight = blockWeight || 0; + if (blockSize === undefined && blockWeight === undefined) { + totalSize = 0; + totalWeight = 0; + transactions.forEach(tx => { + totalSize += tx.size; + totalWeight += tx.weight; + }); + } let rangeLength = 4; if (blocksIndex === 0) { rangeLength = 8; @@ -177,8 +283,8 @@ class MempoolBlocks { rangeLength = 8; } return { - blockSize: blockSize, - blockVSize: blockWeight / 4, + blockSize: totalSize, + blockVSize: totalWeight / 4, nTx: transactions.length, totalFees: transactions.reduce((acc, cur) => acc + cur.fee, 0), medianFee: Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE), diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 584ddf816..717f4eebb 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -21,7 +21,7 @@ class Mempool { private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) => void) | undefined; private asyncMempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[], - deletedTransactions: TransactionExtended[]) => void) | undefined; + deletedTransactions: TransactionExtended[]) => Promise) | undefined; private txPerSecondArray: number[] = []; private txPerSecond: number = 0; diff --git a/backend/src/api/tx-selection-worker.ts b/backend/src/api/tx-selection-worker.ts index ca40af84f..7297cbe88 100644 --- a/backend/src/api/tx-selection-worker.ts +++ b/backend/src/api/tx-selection-worker.ts @@ -1,17 +1,30 @@ import config from '../config'; import logger from '../logger'; -import { TransactionExtended, MempoolBlockWithTransactions, AuditTransaction } from '../mempool.interfaces'; +import { ThreadTransaction, MempoolBlockWithTransactions, AuditTransaction } from '../mempool.interfaces'; import { PairingHeap } from '../utils/pairing-heap'; import { Common } from './common'; import { parentPort } from 'worker_threads'; +let mempool: { [txid: string]: ThreadTransaction } = {}; + if (parentPort) { - parentPort.on('message', (params: { mempool: { [txid: string]: TransactionExtended }, blockLimit: number, weightLimit: number | null, condenseRest: boolean}) => { - const { mempool, blocks } = makeBlockTemplates(params); + parentPort.on('message', (params) => { + if (params.type === 'set') { + mempool = params.mempool; + } else if (params.type === 'update') { + params.added.forEach(tx => { + mempool[tx.txid] = tx; + }); + params.removed.forEach(txid => { + delete mempool[txid]; + }); + } + + const { blocks, clusters } = makeBlockTemplates(mempool); // return the result to main thread. if (parentPort) { - parentPort.postMessage({ mempool, blocks }); + parentPort.postMessage({ blocks, clusters }); } }); } @@ -19,35 +32,24 @@ if (parentPort) { /* * Build projected mempool blocks using an approximation of the transaction selection algorithm from Bitcoin Core * (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp) -* -* blockLimit: number of blocks to build in total. -* weightLimit: maximum weight of transactions to consider using the selection algorithm. -* if weightLimit is significantly lower than the mempool size, results may start to diverge from getBlockTemplate -* condenseRest: whether to ignore excess transactions or append them to the final block. */ -function makeBlockTemplates({ mempool, blockLimit, weightLimit, condenseRest }: { mempool: { [txid: string]: TransactionExtended }, blockLimit: number, weightLimit?: number | null, condenseRest?: boolean | null }) - : { mempool: { [txid: string]: TransactionExtended }, blocks: MempoolBlockWithTransactions[] } { +function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction }) + : { blocks: ThreadTransaction[][], clusters: { [root: string]: string[] } } { const start = Date.now(); const auditPool: { [txid: string]: AuditTransaction } = {}; const mempoolArray: AuditTransaction[] = []; - const restOfArray: TransactionExtended[] = []; + const restOfArray: ThreadTransaction[] = []; + const cpfpClusters: { [root: string]: string[] } = {}; - let weight = 0; - const maxWeight = weightLimit ? Math.max(4_000_000 * blockLimit, weightLimit) : Infinity; // grab the top feerate txs up to maxWeight Object.values(mempool).sort((a, b) => b.feePerVsize - a.feePerVsize).forEach(tx => { - weight += tx.weight; - if (weight >= maxWeight) { - restOfArray.push(tx); - return; - } // initializing everything up front helps V8 optimize property access later auditPool[tx.txid] = { txid: tx.txid, fee: tx.fee, - size: tx.size, weight: tx.weight, feePerVsize: tx.feePerVsize, + effectiveFeePerVsize: tx.feePerVsize, vin: tx.vin, relativesSet: false, ancestorMap: new Map(), @@ -74,7 +76,7 @@ function makeBlockTemplates({ mempool, blockLimit, weightLimit, condenseRest }: // Build blocks by greedily choosing the highest feerate package // (i.e. the package rooted in the transaction with the best ancestor score) - const blocks: MempoolBlockWithTransactions[] = []; + const blocks: ThreadTransaction[][] = []; let blockWeight = 4000; let blockSize = 0; let transactions: AuditTransaction[] = []; @@ -82,7 +84,7 @@ function makeBlockTemplates({ mempool, blockLimit, weightLimit, condenseRest }: let overflow: AuditTransaction[] = []; let failures = 0; let top = 0; - while ((top < mempoolArray.length || !modified.isEmpty()) && (condenseRest || blocks.length < blockLimit)) { + while ((top < mempoolArray.length || !modified.isEmpty())) { // skip invalid transactions while (top < mempoolArray.length && (mempoolArray[top].used || mempoolArray[top].modified)) { top++; @@ -107,9 +109,13 @@ function makeBlockTemplates({ mempool, blockLimit, weightLimit, condenseRest }: // Check if the package fits into this block if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) { const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values()); - const descendants: AuditTransaction[] = []; // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count) const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx]; + let isCluster = false; + if (sortedTxSet.length > 1) { + cpfpClusters[nextTx.txid] = sortedTxSet.map(tx => tx.txid); + isCluster = true; + } const effectiveFeeRate = nextTx.ancestorFee / (nextTx.ancestorWeight / 4); const used: AuditTransaction[] = []; while (sortedTxSet.length) { @@ -119,21 +125,9 @@ function makeBlockTemplates({ mempool, blockLimit, weightLimit, condenseRest }: ancestor.usedBy = nextTx.txid; // update original copy of this tx with effective fee rate & relatives data mempoolTx.effectiveFeePerVsize = effectiveFeeRate; - mempoolTx.ancestors = sortedTxSet.map((a) => { - return { - txid: a.txid, - fee: a.fee, - weight: a.weight, - }; - }).reverse(); - mempoolTx.descendants = descendants.map((a) => { - return { - txid: a.txid, - fee: a.fee, - weight: a.weight, - }; - }); - descendants.push(ancestor); + if (isCluster) { + mempoolTx.cpfpRoot = nextTx.txid; + } mempoolTx.cpfpChecked = true; transactions.push(ancestor); blockSize += ancestor.size; @@ -159,10 +153,10 @@ function makeBlockTemplates({ mempool, blockLimit, weightLimit, condenseRest }: // this block is full const exceededPackageTries = failures > 1000 && blockWeight > (config.MEMPOOL.BLOCK_WEIGHT_UNITS - 4000); const queueEmpty = top >= mempoolArray.length && modified.isEmpty(); - if ((exceededPackageTries || queueEmpty) && (!condenseRest || blocks.length < blockLimit - 1)) { + if ((exceededPackageTries || queueEmpty) && blocks.length < 7) { // construct this block if (transactions.length) { - blocks.push(dataToMempoolBlocks(transactions.map(t => mempool[t.txid]), blockSize, blockWeight, blocks.length)); + blocks.push(transactions.map(t => mempool[t.txid])); } // reset for the next block transactions = []; @@ -181,55 +175,40 @@ function makeBlockTemplates({ mempool, blockLimit, weightLimit, condenseRest }: overflow = []; } } - if (condenseRest) { - // pack any leftover transactions into the last block - for (const tx of overflow) { - if (!tx || tx?.used) { - continue; - } - blockWeight += tx.weight; - blockSize += tx.size; - const mempoolTx = mempool[tx.txid]; - // update original copy of this tx with effective fee rate & relatives data - mempoolTx.effectiveFeePerVsize = tx.score; - mempoolTx.ancestors = (Array.from(tx.ancestorMap?.values()) as AuditTransaction[]).map((a) => { - return { - txid: a.txid, - fee: a.fee, - weight: a.weight, - }; - }); - mempoolTx.bestDescendant = null; - mempoolTx.cpfpChecked = true; - transactions.push(tx); - tx.used = true; + // pack any leftover transactions into the last block + for (const tx of overflow) { + if (!tx || tx?.used) { + continue; } - const blockTransactions = transactions.map(t => mempool[t.txid]); - restOfArray.forEach(tx => { - blockWeight += tx.weight; - blockSize += tx.size; - tx.effectiveFeePerVsize = tx.feePerVsize; - tx.cpfpChecked = false; - tx.ancestors = []; - tx.bestDescendant = null; - blockTransactions.push(tx); - }); - if (blockTransactions.length) { - blocks.push(dataToMempoolBlocks(blockTransactions, blockSize, blockWeight, blocks.length)); + blockWeight += tx.weight; + const mempoolTx = mempool[tx.txid]; + // update original copy of this tx with effective fee rate & relatives data + mempoolTx.effectiveFeePerVsize = tx.score; + if (tx.ancestorMap.size > 0) { + cpfpClusters[tx.txid] = Array.from(tx.ancestorMap?.values()).map(a => a.txid); + mempoolTx.cpfpRoot = tx.txid; } - transactions = []; - } else if (transactions.length) { - blocks.push(dataToMempoolBlocks(transactions.map(t => mempool[t.txid]), blockSize, blockWeight, blocks.length)); + mempoolTx.cpfpChecked = true; + transactions.push(tx); + tx.used = true; } + const blockTransactions = transactions.map(t => mempool[t.txid]); + restOfArray.forEach(tx => { + blockWeight += tx.weight; + tx.effectiveFeePerVsize = tx.feePerVsize; + tx.cpfpChecked = false; + blockTransactions.push(tx); + }); + if (blockTransactions.length) { + blocks.push(blockTransactions); + } + transactions = []; const end = Date.now(); const time = end - start; logger.debug('Mempool templates calculated in ' + time / 1000 + ' seconds'); - return { - mempool, - blocks - }; + return { blocks, clusters: cpfpClusters }; } // traverse in-mempool ancestors @@ -239,9 +218,9 @@ function setRelatives( mempool: { [txid: string]: AuditTransaction }, ): void { for (const parent of tx.vin) { - const parentTx = mempool[parent.txid]; - if (parentTx && !tx.ancestorMap?.has(parent.txid)) { - tx.ancestorMap.set(parent.txid, parentTx); + const parentTx = mempool[parent]; + if (parentTx && !tx.ancestorMap?.has(parent)) { + tx.ancestorMap.set(parent, parentTx); parentTx.children.add(tx); // visit each node only once if (!parentTx.relativesSet) { @@ -312,27 +291,4 @@ function updateDescendants( }); } } -} - -function dataToMempoolBlocks(transactions: TransactionExtended[], - blockSize: number, blockWeight: number, blocksIndex: number): MempoolBlockWithTransactions { - let rangeLength = 4; - if (blocksIndex === 0) { - rangeLength = 8; - } - if (transactions.length > 4000) { - rangeLength = 6; - } else if (transactions.length > 10000) { - rangeLength = 8; - } - return { - blockSize: blockSize, - blockVSize: blockWeight / 4, - nTx: transactions.length, - totalFees: transactions.reduce((acc, cur) => acc + cur.fee, 0), - medianFee: Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE), - feeRange: Common.getFeesInRange(transactions, rangeLength), - transactionIds: transactions.map((tx) => tx.txid), - transactions: transactions.map((tx) => Common.stripTransaction(tx)), - }; } \ No newline at end of file diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index c78c93544..b6f32aa05 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -251,7 +251,7 @@ class WebsocketHandler { } if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { - await mempoolBlocks.makeBlockTemplates(newMempool, 8, null, true); + await mempoolBlocks.updateBlockTemplates(newMempool, newTransactions, deletedTransactions.map(tx => tx.txid)); } else { mempoolBlocks.updateMempoolBlocks(newMempool); } @@ -419,7 +419,7 @@ class WebsocketHandler { const _memPool = memPool.getMempool(); if (config.MEMPOOL.ADVANCED_GBT_AUDIT) { - await mempoolBlocks.makeBlockTemplates(_memPool, 2); + await mempoolBlocks.makeBlockTemplates(_memPool); } else { mempoolBlocks.updateMempoolBlocks(_memPool); } @@ -462,13 +462,15 @@ class WebsocketHandler { } } + const removed: string[] = []; // Update mempool to remove transactions included in the new block for (const txId of txIds) { delete _memPool[txId]; + removed.push(txId); } if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { - await mempoolBlocks.makeBlockTemplates(_memPool, 8, null, true); + await mempoolBlocks.updateBlockTemplates(_memPool, [], removed); } else { mempoolBlocks.updateMempoolBlocks(_memPool); } diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 11de304b8..046083322 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -81,10 +81,10 @@ export interface TransactionExtended extends IEsploraApi.Transaction { export interface AuditTransaction { txid: string; fee: number; - size: number; weight: number; feePerVsize: number; - vin: IEsploraApi.Vin[]; + effectiveFeePerVsize: number; + vin: string[]; relativesSet: boolean; ancestorMap: Map; children: Set; @@ -96,6 +96,17 @@ export interface AuditTransaction { modifiedNode: HeapNode; } +export interface ThreadTransaction { + txid: string; + fee: number; + weight: number; + feePerVsize: number; + effectiveFeePerVsize?: number; + vin: string[]; + cpfpRoot?: string; + cpfpChecked?: boolean; +} + export interface Ancestor { txid: string; weight: number; From 56b6f79f97c6227eec960b5d7983277eca2d3d7d Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 7 Dec 2022 14:51:26 -0600 Subject: [PATCH 11/15] improve thread error handling --- backend/src/api/mempool-blocks.ts | 51 ++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 62bdc8f1b..d94ed77bd 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -158,8 +158,11 @@ class MempoolBlocks { }; }); + // (re)initialize tx selection worker thread if (!this.txSelectionWorker) { this.txSelectionWorker = new Worker(path.resolve(__dirname, './tx-selection-worker.js')); + // if the thread throws an unexpected error, or exits for any other reason, + // reset worker state so that it will be re-initialized on the next run this.txSelectionWorker.once('error', () => { this.txSelectionWorker = null; }); @@ -169,15 +172,25 @@ class MempoolBlocks { } // run the block construction algorithm in a separate thread, and wait for a result - const workerResultPromise = new Promise<{ blocks: ThreadTransaction[][], clusters: { [root: string]: string[] } }>((resolve) => { - this.txSelectionWorker?.once('message', (result): void => { - resolve(result); + let threadErrorListener; + try { + const workerResultPromise = new Promise<{ blocks: ThreadTransaction[][], clusters: { [root: string]: string[] } }>((resolve, reject) => { + threadErrorListener = reject; + this.txSelectionWorker?.once('message', (result): void => { + resolve(result); + }); + this.txSelectionWorker?.once('error', reject); }); - }); - this.txSelectionWorker.postMessage({ type: 'set', mempool: strippedMempool }); - const { blocks, clusters } = await workerResultPromise; + this.txSelectionWorker.postMessage({ type: 'set', mempool: strippedMempool }); + const { blocks, clusters } = await workerResultPromise; - this.processBlockTemplates(newMempool, blocks, clusters); + this.processBlockTemplates(newMempool, blocks, clusters); + + // clean up thread error listener + this.txSelectionWorker?.removeListener('error', threadErrorListener); + } catch (e) { + logger.err('makeBlockTemplates failed. ' + (e instanceof Error ? e.message : e)); + } } public async updateBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, added: TransactionExtended[], removed: string[]): Promise { @@ -199,15 +212,25 @@ class MempoolBlocks { }); // run the block construction algorithm in a separate thread, and wait for a result - const workerResultPromise = new Promise<{ blocks: ThreadTransaction[][], clusters: { [root: string]: string[] } }>((resolve) => { - this.txSelectionWorker?.once('message', (result): void => { - resolve(result); + let threadErrorListener; + try { + const workerResultPromise = new Promise<{ blocks: ThreadTransaction[][], clusters: { [root: string]: string[] } }>((resolve, reject) => { + threadErrorListener = reject; + this.txSelectionWorker?.once('message', (result): void => { + resolve(result); + }); + this.txSelectionWorker?.once('error', reject); }); - }); - this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed }); - const { blocks, clusters } = await workerResultPromise; + this.txSelectionWorker.postMessage({ type: 'update', added: addedStripped, removed }); + const { blocks, clusters } = await workerResultPromise; - this.processBlockTemplates(newMempool, blocks, clusters); + this.processBlockTemplates(newMempool, blocks, clusters); + + // clean up thread error listener + this.txSelectionWorker?.removeListener('error', threadErrorListener); + } catch (e) { + logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e)); + } } private processBlockTemplates(mempool, blocks, clusters): void { From fb137e62470d2504cf7384c426d544f4b13d04ea Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 7 Dec 2022 18:58:03 -0600 Subject: [PATCH 12/15] Fix & reenable cpfp indexer optimized path --- backend/src/api/blocks.ts | 10 ++++++---- backend/src/repositories/TransactionRepository.ts | 4 +++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 8c8272262..98720207b 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -742,7 +742,7 @@ class Blocks { public async $indexCPFP(hash: string, height: number): Promise { let transactions; - if (false/*Common.blocksSummariesIndexingEnabled()*/) { + if (Common.blocksSummariesIndexingEnabled()) { transactions = await this.$getStrippedBlockTransactions(hash); const rawBlock = await bitcoinApi.$getRawBlock(hash); const block = Block.fromBuffer(rawBlock); @@ -751,10 +751,11 @@ class Blocks { txMap[tx.getId()] = tx; } for (const tx of transactions) { + // convert from bitcoinjs to esplora vin format if (txMap[tx.txid]?.ins) { tx.vin = txMap[tx.txid].ins.map(vin => { return { - txid: vin.hash + txid: vin.hash.slice().reverse().toString('hex') }; }); } @@ -763,6 +764,7 @@ class Blocks { const block = await bitcoinClient.getBlock(hash, 2); transactions = block.tx.map(tx => { tx.vsize = tx.weight / 4; + tx.fee *= 100_000_000; return tx; }); } @@ -778,9 +780,9 @@ class Blocks { totalFee += tx?.fee || 0; totalVSize += tx.vsize; }); - const effectiveFeePerVsize = (totalFee * 100_000_000) / totalVSize; + const effectiveFeePerVsize = totalFee / totalVSize; if (cluster.length > 1) { - await cpfpRepository.$saveCluster(height, cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: (tx.fee || 0) * 100_000_000 }; }), effectiveFeePerVsize); + await cpfpRepository.$saveCluster(height, cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: tx.fee || 0 }; }), effectiveFeePerVsize); for (const tx of cluster) { await transactionRepository.$setCluster(tx.txid, cluster[0].txid); } diff --git a/backend/src/repositories/TransactionRepository.ts b/backend/src/repositories/TransactionRepository.ts index 1c6e3719f..74debb833 100644 --- a/backend/src/repositories/TransactionRepository.ts +++ b/backend/src/repositories/TransactionRepository.ts @@ -44,7 +44,9 @@ class TransactionRepository { const [rows]: any = await DB.query(query, [txid]); if (rows.length) { rows[0].txs = JSON.parse(rows[0].txs) as Ancestor[]; - return this.convertCpfp(rows[0]); + if (rows[0]?.txs?.length) { + return this.convertCpfp(rows[0]); + } } } catch (e) { logger.err('Cannot get transaction cpfp info from db. Reason: ' + (e instanceof Error ? e.message : e)); From 79f79b0e3bf03d34a358ec62563454b8643670ee Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Fri, 9 Dec 2022 20:58:52 -0800 Subject: [PATCH 13/15] Fix the dependabot ignore settings - RTFM --- .github/dependabot.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 1e2574971..e0dfe74ef 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,23 +6,27 @@ updates: interval: daily open-pull-requests-limit: 10 ignore: - - update-types: ["version-update:semver-major"] + - dependency-name: "*" + update-types: ["version-update:semver-major"] - package-ecosystem: npm directory: "/frontend" schedule: interval: daily open-pull-requests-limit: 10 ignore: - - update-types: ["version-update:semver-major"] + - dependency-name: "*" + update-types: ["version-update:semver-major"] - package-ecosystem: docker directory: "/docker/backend" schedule: interval: daily ignore: - - update-types: ["version-update:semver-major"] + - dependency-name: "*" + update-types: ["version-update:semver-major"] - package-ecosystem: "github-actions" directory: "/" schedule: interval: daily ignore: - - update-types: ["version-update:semver-major"] + - dependency-name: "*" + update-types: ["version-update:semver-major"] From ea5ec7bc32bd79f93d15247abc9feb6e62d11b98 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Fri, 16 Dec 2022 16:32:58 -0800 Subject: [PATCH 14/15] Update Cypress dependencies --- frontend/package-lock.json | 31 ++++++++++++++++--------------- frontend/package.json | 4 ++-- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 2efccf07d..72f9e7afa 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -31,6 +31,7 @@ "bootstrap": "~4.6.1", "browserify": "^17.0.0", "clipboard": "^2.0.11", + "cypress": "^12.1.0", "domino": "^2.1.6", "echarts": "~5.4.0", "echarts-gl": "^2.0.9", @@ -57,8 +58,8 @@ "typescript": "~4.6.4" }, "optionalDependencies": { - "@cypress/schematic": "~2.3.0", - "cypress": "^11.2.0", + "@cypress/schematic": "^2.4.0", + "cypress": "^12.1.0", "cypress-fail-on-console-error": "~4.0.2", "cypress-wait-until": "^1.7.2", "mock-socket": "~9.1.5", @@ -3225,9 +3226,9 @@ } }, "node_modules/@cypress/schematic": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@cypress/schematic/-/schematic-2.3.0.tgz", - "integrity": "sha512-LBKX20MUUYF2Xu+1+KpVbLCoMvt2Osa80yQfonduVsLJ/p8JxtLHqufuf/ryJp9Gm9R5sDfk/YhHL+rB7a+gsg==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@cypress/schematic/-/schematic-2.4.0.tgz", + "integrity": "sha512-aor8hQ+gMXqx/ASdo7CUGo/sMEWwwfSRsLr99rM2GjvW+pZnCKKTnRG4UPf8Ro9SevLJj7KRZAZWxa5MAkJzZA==", "optional": true, "dependencies": { "@angular-devkit/architect": "^0.1402.1", @@ -7019,9 +7020,9 @@ "peer": true }, "node_modules/cypress": { - "version": "11.2.0", - "resolved": "https://registry.npmjs.org/cypress/-/cypress-11.2.0.tgz", - "integrity": "sha512-u61UGwtu7lpsNWLUma/FKNOsrjcI6wleNmda/TyKHe0dOBcVjbCPlp1N6uwFZ0doXev7f/91YDpU9bqDCFeBLA==", + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/cypress/-/cypress-12.1.0.tgz", + "integrity": "sha512-7fz8N84uhN1+ePNDsfQvoWEl4P3/VGKKmAg+bJQFY4onhA37Ys+6oBkGbNdwGeC7n2QqibNVPhk8x3YuQLwzfw==", "hasInstallScript": true, "optional": true, "dependencies": { @@ -7072,7 +7073,7 @@ "cypress": "bin/cypress" }, "engines": { - "node": ">=12.0.0" + "node": "^14.0.0 || ^16.0.0 || >=18.0.0" } }, "node_modules/cypress-fail-on-console-error": { @@ -19345,9 +19346,9 @@ } }, "@cypress/schematic": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/@cypress/schematic/-/schematic-2.3.0.tgz", - "integrity": "sha512-LBKX20MUUYF2Xu+1+KpVbLCoMvt2Osa80yQfonduVsLJ/p8JxtLHqufuf/ryJp9Gm9R5sDfk/YhHL+rB7a+gsg==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@cypress/schematic/-/schematic-2.4.0.tgz", + "integrity": "sha512-aor8hQ+gMXqx/ASdo7CUGo/sMEWwwfSRsLr99rM2GjvW+pZnCKKTnRG4UPf8Ro9SevLJj7KRZAZWxa5MAkJzZA==", "optional": true, "requires": { "@angular-devkit/architect": "^0.1402.1", @@ -22282,9 +22283,9 @@ "peer": true }, "cypress": { - "version": "11.2.0", - "resolved": "https://registry.npmjs.org/cypress/-/cypress-11.2.0.tgz", - "integrity": "sha512-u61UGwtu7lpsNWLUma/FKNOsrjcI6wleNmda/TyKHe0dOBcVjbCPlp1N6uwFZ0doXev7f/91YDpU9bqDCFeBLA==", + "version": "12.1.0", + "resolved": "https://registry.npmjs.org/cypress/-/cypress-12.1.0.tgz", + "integrity": "sha512-7fz8N84uhN1+ePNDsfQvoWEl4P3/VGKKmAg+bJQFY4onhA37Ys+6oBkGbNdwGeC7n2QqibNVPhk8x3YuQLwzfw==", "optional": true, "requires": { "@cypress/request": "^2.88.10", diff --git a/frontend/package.json b/frontend/package.json index 325bb6bc3..174392ef0 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -109,8 +109,8 @@ "typescript": "~4.6.4" }, "optionalDependencies": { - "@cypress/schematic": "~2.3.0", - "cypress": "^11.2.0", + "@cypress/schematic": "^2.4.0", + "cypress": "^12.1.0", "cypress-fail-on-console-error": "~4.0.2", "cypress-wait-until": "^1.7.2", "mock-socket": "~9.1.5", From 8ac514733a1f340843c12790b4a48fef4ec76812 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 25 Dec 2022 23:17:04 +0400 Subject: [PATCH 15/15] Fix for spelling error in indexing status --- .../nodes-networks-chart/nodes-networks-chart.component.ts | 2 +- .../statistics-chart/lightning-statistics-chart.component.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.ts b/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.ts index 46f8b12e8..abf104e2f 100644 --- a/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.ts +++ b/frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.ts @@ -118,7 +118,7 @@ export class NodesNetworksChartComponent implements OnInit { color: 'grey', fontSize: 15 }, - text: $localize`Indexing in progess`, + text: $localize`Indexing in progress`, left: 'center', top: 'center', }; diff --git a/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.ts b/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.ts index 4f209e7a0..916483781 100644 --- a/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.ts +++ b/frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.ts @@ -109,7 +109,7 @@ export class LightningStatisticsChartComponent implements OnInit { color: 'grey', fontSize: 15 }, - text: $localize`Indexing in progess`, + text: $localize`Indexing in progress`, left: 'center', top: 'center' };