From 73c55c450cfbc31777c653980c64656b6980c59d Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sun, 8 Jan 2023 11:43:18 +0100 Subject: [PATCH 01/15] Ignore pool logo download failure as it's not a critical error --- frontend/sync-assets.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/frontend/sync-assets.js b/frontend/sync-assets.js index 9c447bf7d..8b43f61e5 100644 --- a/frontend/sync-assets.js +++ b/frontend/sync-assets.js @@ -54,9 +54,13 @@ function downloadMiningPoolLogos() { response.on('end', () => { let response_body = Buffer.concat(chunks_of_data); - const poolLogos = JSON.parse(response_body.toString()); - for (const poolLogo of poolLogos) { - download(`${PATH}/mining-pools/${poolLogo.name}`, poolLogo.download_url); + try { + const poolLogos = JSON.parse(response_body.toString()); + for (const poolLogo of poolLogos) { + download(`${PATH}/mining-pools/${poolLogo.name}`, poolLogo.download_url); + } + } catch (e) { + console.error(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`); } }); From fcd34eb87616c704fd95a91f3d9b7d03720a1b9d Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Tue, 10 Jan 2023 16:13:16 +0100 Subject: [PATCH 02/15] Add new mempool lightning nodes and format the array so it's human readable --- backend/src/api/explorer/nodes.routes.ts | 63 ++++++++++++++++++++++-- 1 file changed, 60 insertions(+), 3 deletions(-) diff --git a/backend/src/api/explorer/nodes.routes.ts b/backend/src/api/explorer/nodes.routes.ts index c19bde236..e2dbcb0b6 100644 --- a/backend/src/api/explorer/nodes.routes.ts +++ b/backend/src/api/explorer/nodes.routes.ts @@ -41,13 +41,70 @@ class NodesRoutes { let nodes: any[] = []; switch (config.MEMPOOL.NETWORK) { case 'testnet': - nodesList = ['032c7c7819276c4f706a04df1a0f1e10a5495994a7be4c1d3d28ca766e5a2b957b', '025a7e38c2834dd843591a4d23d5f09cdeb77ddca85f673c2d944a14220ff14cf7', '0395e2731a1673ef21d7a16a727c4fc4d4c35a861c428ce2c819c53d2b81c8bd55', '032ab2028c0b614c6d87824e2373529652fd7e4221b4c70cc4da7c7005c49afcf0', '029001b22fe70b48bee12d014df91982eb85ff1bd404ec772d5c83c4ee3e88d2c3', '0212e2848d79f928411da5f2ff0a8c95ec6ccb5a09d2031b6f71e91309dcde63af', '03e871a2229523d34f76e6311ff197cfe7f26c2fbec13554b93a46f4e710c47dab', '032202ec98d976b0e928bd1d91924e8bd3eab07231fc39feb3737b010071073df8', '02fa7c5a948d03d563a9f36940c2205a814e594d17c0042ced242c71a857d72605', '039c14fdec2d958e3d14cebf657451bbd9e039196615785e82c917f274e3fb2205', '033589bbcb233ffc416cefd5437c7f37e9d7cb7942d405e39e72c4c846d9b37f18', '029293110441c6e2eacb57e1255bf6ef05c41a6a676fe474922d33c19f98a7d584']; + nodesList = [ + '032c7c7819276c4f706a04df1a0f1e10a5495994a7be4c1d3d28ca766e5a2b957b', + '025a7e38c2834dd843591a4d23d5f09cdeb77ddca85f673c2d944a14220ff14cf7', + '0395e2731a1673ef21d7a16a727c4fc4d4c35a861c428ce2c819c53d2b81c8bd55', + '032ab2028c0b614c6d87824e2373529652fd7e4221b4c70cc4da7c7005c49afcf0', + '029001b22fe70b48bee12d014df91982eb85ff1bd404ec772d5c83c4ee3e88d2c3', + '0212e2848d79f928411da5f2ff0a8c95ec6ccb5a09d2031b6f71e91309dcde63af', + '03e871a2229523d34f76e6311ff197cfe7f26c2fbec13554b93a46f4e710c47dab', + '032202ec98d976b0e928bd1d91924e8bd3eab07231fc39feb3737b010071073df8', + '02fa7c5a948d03d563a9f36940c2205a814e594d17c0042ced242c71a857d72605', + '039c14fdec2d958e3d14cebf657451bbd9e039196615785e82c917f274e3fb2205', + '033589bbcb233ffc416cefd5437c7f37e9d7cb7942d405e39e72c4c846d9b37f18', + '029293110441c6e2eacb57e1255bf6ef05c41a6a676fe474922d33c19f98a7d584', + '0235ad0b56ed8c42c4354444c24e971c05e769ec0b5fb0ccea42880095dc02ea2c', + '029700819a37afea630f80e6cc461f3fd3c4ace2598a21cfbbe64d1c78d0ee69a5', + '02c2d8b2dbf87c7894af2f1d321290e2fe6db5446cd35323987cee98f06e2e0075', + '030b0ca1ea7b1075716d2a555630e6fd47ef11bc7391fe68963ec06cf370a5e382', + '031adb9eb2d66693f85fa31a4adca0319ba68219f3ad5f9a2ef9b34a6b40755fa1', + '02ccd07faa47eda810ecf5591ccf5ca50f6c1034d0d175052898d32a00b9bae24f', + ]; break; case 'signet': - nodesList = ['03ddab321b760433cbf561b615ef62ac7d318630c5f51d523aaf5395b90b751956', '033d92c7bfd213ef1b34c90e985fb5dc77f9ec2409d391492484e57a44c4aca1de', '02ad010dda54253c1eb9efe38b0760657a3b43ecad62198c359c051c9d99d45781', '025196512905b8a3f1597428b867bec63ec9a95e5089eb7dc7e63e2d2691669029', '027c625aa1fbe3768db68ebcb05b53b6dc0ce68b7b54b8900d326d167363e684fe', '03f1629af3101fcc56b7aac2667016be84e3defbf3d0c8719f836c9b41c9a57a43', '02dfb81e2f7a3c4c9e8a51b70ef82b4a24549cc2fab1f5b2fd636501774a918991', '02d01ccf832944c68f10d39006093769c5b8bda886d561b128534e313d729fdb34', '02499ed23027d4698a6904ff4ec1b6085a61f10b9a6937f90438f9947e38e8ea86', '038310e3a786340f2bd7770704c7ccfe560fd163d9a1c99d67894597419d12cbf7', '03e5e9d879b72c7d67ecd483bae023bd33e695bb32b981a4021260f7b9d62bc761', '028d16e1a0ace4c0c0a421536d8d32ce484dfe6e2f726b7b0e7c30f12a195f8cc7']; + nodesList = [ + '03ddab321b760433cbf561b615ef62ac7d318630c5f51d523aaf5395b90b751956', + '033d92c7bfd213ef1b34c90e985fb5dc77f9ec2409d391492484e57a44c4aca1de', + '02ad010dda54253c1eb9efe38b0760657a3b43ecad62198c359c051c9d99d45781', + '025196512905b8a3f1597428b867bec63ec9a95e5089eb7dc7e63e2d2691669029', + '027c625aa1fbe3768db68ebcb05b53b6dc0ce68b7b54b8900d326d167363e684fe', + '03f1629af3101fcc56b7aac2667016be84e3defbf3d0c8719f836c9b41c9a57a43', + '02dfb81e2f7a3c4c9e8a51b70ef82b4a24549cc2fab1f5b2fd636501774a918991', + '02d01ccf832944c68f10d39006093769c5b8bda886d561b128534e313d729fdb34', + '02499ed23027d4698a6904ff4ec1b6085a61f10b9a6937f90438f9947e38e8ea86', + '038310e3a786340f2bd7770704c7ccfe560fd163d9a1c99d67894597419d12cbf7', + '03e5e9d879b72c7d67ecd483bae023bd33e695bb32b981a4021260f7b9d62bc761', + '028d16e1a0ace4c0c0a421536d8d32ce484dfe6e2f726b7b0e7c30f12a195f8cc7', + '02ff690d06c187ab994bf83c5a2114fe5bf50112c2c817af0f788f736be9fa2070', + '02a9f570c51a2526a5ee85802e88f9281bed771eb66a0c8a7d898430dd5d0eae45', + '038c3de773255d3bd7a50e31e58d423baac5c90826a74d75e64b74c95475de1097', + '0242c7f7d315095f37ad1421ae0a2fc967d4cbe65b61b079c5395a769436959853', + '02a909e70eb03742f12666ebb1f56ac42a5fbaab0c0e8b5b1df4aa9f10f8a09240', + '03a26efa12489803c07f3ac2f1dba63812e38f0f6e866ce3ebb34df7de1f458cd2', + ]; break; default: - nodesList = ['03fbc17549ec667bccf397ababbcb4cdc0e3394345e4773079ab2774612ec9be61', '03da9a8623241ccf95f19cd645c6cecd4019ac91570e976eb0a128bebbc4d8a437', '03ca5340cf85cb2e7cf076e489f785410838de174e40be62723e8a60972ad75144', '0238bd27f02d67d6c51e269692bc8c9a32357a00e7777cba7f4f1f18a2a700b108', '03f983dcabed6baa1eab5b56c8b2e8fdc846ab3fd931155377897335e85a9fa57c', '03e399589533581e48796e29a825839a010036a61b20744fda929d6709fcbffcc5', '021f5288b5f72c42cd0d8801086af7ce09a816d8ee9a4c47a4b436399b26cb601a', '032b01b7585f781420cd4148841a82831ba37fa952342052cec16750852d4f2dd9', '02848036488d4b8fb1f1c4064261ec36151f43b085f0b51bd239ade3ddfc940c34', '02b6b1640fe029e304c216951af9fbefdb23b0bdc9baaf327540d31b6107841fdf', '03694289827203a5b3156d753071ddd5bf92e371f5a462943f9555eef6d2d6606c', '0283d850db7c3e8ea7cc9c4abc7afaab12bbdf72b677dcba1d608350d2537d7d43']; + nodesList = [ + '03fbc17549ec667bccf397ababbcb4cdc0e3394345e4773079ab2774612ec9be61', + '03da9a8623241ccf95f19cd645c6cecd4019ac91570e976eb0a128bebbc4d8a437', + '03ca5340cf85cb2e7cf076e489f785410838de174e40be62723e8a60972ad75144', + '0238bd27f02d67d6c51e269692bc8c9a32357a00e7777cba7f4f1f18a2a700b108', + '03f983dcabed6baa1eab5b56c8b2e8fdc846ab3fd931155377897335e85a9fa57c', + '03e399589533581e48796e29a825839a010036a61b20744fda929d6709fcbffcc5', + '021f5288b5f72c42cd0d8801086af7ce09a816d8ee9a4c47a4b436399b26cb601a', + '032b01b7585f781420cd4148841a82831ba37fa952342052cec16750852d4f2dd9', + '02848036488d4b8fb1f1c4064261ec36151f43b085f0b51bd239ade3ddfc940c34', + '02b6b1640fe029e304c216951af9fbefdb23b0bdc9baaf327540d31b6107841fdf', + '03694289827203a5b3156d753071ddd5bf92e371f5a462943f9555eef6d2d6606c', + '0283d850db7c3e8ea7cc9c4abc7afaab12bbdf72b677dcba1d608350d2537d7d43', + '02521287789f851268a39c9eccc9d6180d2c614315b583c9e6ae0addbd6d79df06', + '0258c2a7b7f8af2585b4411b1ec945f70988f30412bb1df179de941f14d0b1bc3e', + '03c3389ff1a896f84d921ed01a19fc99c6724ce8dc4b960cd3b7b2362b62cd60d7', + '038d118996b3eaa15dcd317b32a539c9ecfdd7698f204acf8a087336af655a9192', + '02a928903d93d78877dacc3642b696128a3636e9566dd42d2d132325b2c8891c09', + '0328cd17f3a9d3d90b532ade0d1a67e05eb8a51835b3dce0a2e38eac04b5a62a57', + ]; } for (let pubKey of nodesList) { From 8ebe04baa7168deaa3348142dcbfd51e4dc26ac4 Mon Sep 17 00:00:00 2001 From: softsimon Date: Tue, 10 Jan 2023 21:27:26 +0400 Subject: [PATCH 03/15] Fallback alias name to pubkey fixes #2935 --- .../app/components/address-labels/address-labels.component.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/components/address-labels/address-labels.component.ts b/frontend/src/app/components/address-labels/address-labels.component.ts index be5bdeead..f4b3d0ca5 100644 --- a/frontend/src/app/components/address-labels/address-labels.component.ts +++ b/frontend/src/app/components/address-labels/address-labels.component.ts @@ -36,7 +36,9 @@ export class AddressLabelsComponent implements OnChanges { handleChannel() { const type = this.vout ? 'open' : 'close'; - this.label = `Channel ${type}: ${this.channel.node_left.alias} <> ${this.channel.node_right.alias}`; + const leftNodeName = this.channel.node_left.alias || this.channel.node_left.public_key.substring(0, 10); + const rightNodeName = this.channel.node_right.alias || this.channel.node_right.public_key.substring(0, 10); + this.label = `Channel ${type}: ${leftNodeName} <> ${rightNodeName}`; } handleVin() { From 5379ec0f308c31dc2d017b914101523d491b58d6 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Tue, 10 Jan 2023 21:54:34 -0800 Subject: [PATCH 04/15] Expose whether Lightning is enabled on the backend --- backend/src/api/backend-info.ts | 4 +++- backend/src/mempool.interfaces.ts | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/src/api/backend-info.ts b/backend/src/api/backend-info.ts index 57bb5fe13..fc3181524 100644 --- a/backend/src/api/backend-info.ts +++ b/backend/src/api/backend-info.ts @@ -2,6 +2,7 @@ import fs from 'fs'; import path from 'path'; import os from 'os'; import { IBackendInfo } from '../mempool.interfaces'; +import config from '../config'; class BackendInfo { private backendInfo: IBackendInfo; @@ -22,7 +23,8 @@ class BackendInfo { this.backendInfo = { hostname: os.hostname(), version: versionInfo.version, - gitCommit: versionInfo.gitCommit + gitCommit: versionInfo.gitCommit, + lightning: config.LIGHTNING.ENABLED }; } diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 046083322..f79786279 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -274,6 +274,7 @@ export interface IBackendInfo { hostname: string; gitCommit: string; version: string; + lightning: boolean; } export interface IDifficultyAdjustment { @@ -337,4 +338,4 @@ export interface IOldestNodes { updatedAt?: number, city?: any, country?: any, -} \ No newline at end of file +} From 2e5d4a6df997fb8003641e3c5868ac0f17b2f64e Mon Sep 17 00:00:00 2001 From: softsimon Date: Wed, 11 Jan 2023 13:37:49 +0400 Subject: [PATCH 05/15] Dark mode inputs --- .../search-form/search-form.component.html | 2 +- .../search-form/search-form.component.scss | 3 -- frontend/src/styles.scss | 32 +++++++++++++++++-- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/frontend/src/app/components/search-form/search-form.component.html b/frontend/src/app/components/search-form/search-form.component.html index 4e38ea6e0..b881c6ea7 100644 --- a/frontend/src/app/components/search-form/search-form.component.html +++ b/frontend/src/app/components/search-form/search-form.component.html @@ -5,7 +5,7 @@
-
diff --git a/frontend/src/app/components/search-form/search-form.component.scss b/frontend/src/app/components/search-form/search-form.component.scss index 7c8161196..d59acadb9 100644 --- a/frontend/src/app/components/search-form/search-form.component.scss +++ b/frontend/src/app/components/search-form/search-form.component.scss @@ -43,9 +43,6 @@ form { @media (min-width: 1200px) { min-width: 300px; } - input { - border: 0px; - } .btn { width: 100px; } diff --git a/frontend/src/styles.scss b/frontend/src/styles.scss index d3d16d12e..b0cdb5ef6 100644 --- a/frontend/src/styles.scss +++ b/frontend/src/styles.scss @@ -115,10 +115,38 @@ body { } .form-control { - color: #495057; + color: #fff; + background-color: #2d3348; + border: 1px solid rgba(17, 19, 31, 0.2); } + .form-control:focus { - color: #000; + color: #fff; + background-color: #2d3348; +} + +.btn-purple { + background-color: #653b9c; + border-color: #653b9c; +} + +.btn-purple:not(:disabled):not(.disabled):active, .btn-purple:not(:disabled):not(.disabled).active, .show > .btn-purple.dropdown-toggle { + color: #fff; + background-color: #4d2d77; + border-color: #472a6e; +} + +.btn-purple:focus, .btn-purple.focus { + color: #fff; + background-color: #533180; + border-color: #4d2d77; + box-shadow: 0 0 0 0.2rem rgb(124 88 171 / 50%); +} + +.btn-purple:hover { + color: #fff; + background-color: #533180; + border-color: #4d2d77; } .form-control.form-control-secondary { From 0d921cf7a63ef775521cff2fd95f2745e8ee632a Mon Sep 17 00:00:00 2001 From: Mononaut Date: Thu, 5 Jan 2023 13:02:53 -0600 Subject: [PATCH 06/15] don't rely on blocks table for cpfp indexing progress --- backend/src/api/blocks.ts | 1 - backend/src/api/database-migration.ts | 7 +++++- backend/src/repositories/BlocksRepository.ts | 24 +++++++++++++++----- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 8292fe241..1376964eb 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -806,7 +806,6 @@ class Blocks { ancestors[vin.txid] = true; }); } - await blocksRepository.$setCPFPIndexed(hash); } } diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 6e0e95699..68087fdb5 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 49; + private static currentVersion = 50; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -442,6 +442,11 @@ class DatabaseMigration { await this.$executeQuery('TRUNCATE TABLE `blocks_audits`'); await this.updateToSchemaVersion(49); } + + if (databaseSchemaVersion < 50 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `blocks` DROP COLUMN `cpfp_indexed`'); + await this.updateToSchemaVersion(50); + } } /** diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 78a8fcce2..5c52a8ba0 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -8,6 +8,8 @@ import HashratesRepository from './HashratesRepository'; import { escape } from 'mysql2'; import BlocksSummariesRepository from './BlocksSummariesRepository'; import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository'; +import bitcoinClient from '../api/bitcoin/bitcoin-client'; +import config from '../config'; class BlocksRepository { /** @@ -667,16 +669,26 @@ class BlocksRepository { */ public async $getCPFPUnindexedBlocks(): Promise { try { - const [rows]: any = await DB.query(`SELECT height, hash FROM blocks WHERE cpfp_indexed = 0 ORDER BY height DESC`); - return rows; + const blockchainInfo = await bitcoinClient.getBlockchainInfo(); + const currentBlockHeight = blockchainInfo.blocks; + const [lastHeightRows]: any = await DB.query(`SELECT MIN(height) AS minHeight from cpfp_clusters`); + const lastHeight = (lastHeightRows.length && lastHeightRows[0].minHeight != null) ? lastHeightRows[0].minHeight : currentBlockHeight; + + let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, blockchainInfo.blocks); + if (indexingBlockAmount <= -1) { + indexingBlockAmount = currentBlockHeight + 1; + } + const firstHeight = Math.max(0, currentBlockHeight - indexingBlockAmount + 1); + + if (firstHeight < lastHeight) { + const [rows]: any = await DB.query(`SELECT height, hash FROM blocks WHERE height BETWEEN ? AND ? ORDER BY height DESC`, [firstHeight, lastHeight]); + return rows; + } } catch (e) { logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e)); throw e; } - } - - public async $setCPFPIndexed(hash: string): Promise { - await DB.query(`UPDATE blocks SET cpfp_indexed = 1 WHERE hash = ?`, [hash]); + return []; } /** From ee95d033ac4386bbfbc99cbca0ad3ff78540a272 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 7 Jan 2023 15:51:31 -0600 Subject: [PATCH 07/15] remove slow cpfp indexing path --- backend/src/api/blocks.ts | 33 +++++++-------------------------- 1 file changed, 7 insertions(+), 26 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 1376964eb..29fac7850 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -753,33 +753,14 @@ class Blocks { public async $indexCPFP(hash: string, height: number): Promise { let transactions; - if (Common.blocksSummariesIndexingEnabled()) { - transactions = await this.$getStrippedBlockTransactions(hash); - const rawBlock = await bitcoinApi.$getRawBlock(hash); - const block = Block.fromBuffer(rawBlock); - const txMap = {}; - for (const tx of block.transactions || []) { - txMap[tx.getId()] = tx; - } - for (const tx of transactions) { - // convert from bitcoinjs to esplora vin format - if (txMap[tx.txid]?.ins) { - tx.vin = txMap[tx.txid].ins.map(vin => { - return { - txid: vin.hash.slice().reverse().toString('hex') - }; - }); - } - } - } else { - const block = await bitcoinClient.getBlock(hash, 2); - transactions = block.tx.map(tx => { - tx.vsize = tx.weight / 4; - tx.fee *= 100_000_000; - return tx; - }); - } + const block = await bitcoinClient.getBlock(hash, 2); + transactions = block.tx.map(tx => { + tx.vsize = tx.weight / 4; + tx.fee *= 100_000_000; + return tx; + }); + let cluster: TransactionStripped[] = []; let ancestors: { [txid: string]: boolean } = {}; for (let i = transactions.length - 1; i >= 0; i--) { From 7793eaecbc193567f9622d9b0c93ab193ed4ccd1 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 9 Jan 2023 08:34:33 -0600 Subject: [PATCH 08/15] fix cpfp indexing rate calculation --- backend/src/api/blocks.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 29fac7850..a782dbacc 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -341,6 +341,7 @@ class Blocks { try { // Get all indexed block hash const unindexedBlocks = await blocksRepository.$getCPFPUnindexedBlocks(); + logger.info(`Indexing cpfp data for ${unindexedBlocks.length} blocks`); if (!unindexedBlocks?.length) { return; @@ -357,7 +358,7 @@ class Blocks { const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer); if (elapsedSeconds > 5) { const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); - const blockPerSeconds = Math.max(1, countThisRun / elapsedSeconds); + const blockPerSeconds = (countThisRun / elapsedSeconds); const progress = Math.round(count / unindexedBlocks.length * 10000) / 100; logger.debug(`Indexing cpfp clusters for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`); timer = new Date().getTime() / 1000; From b50936f001406c3465b7350a3d54cf26c517eddb Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 9 Jan 2023 10:24:12 -0600 Subject: [PATCH 09/15] compact schemas for cpfp tables --- backend/src/api/blocks.ts | 15 ++-- backend/src/api/database-migration.ts | 21 +++++ backend/src/repositories/BlocksRepository.ts | 2 +- backend/src/repositories/CpfpRepository.ts | 86 +++++++++++++++++-- .../src/repositories/TransactionRepository.ts | 53 +++++++----- 5 files changed, 142 insertions(+), 35 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index a782dbacc..2b1f88518 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -22,12 +22,10 @@ import poolsParser from './pools-parser'; import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository'; import cpfpRepository from '../repositories/CpfpRepository'; -import transactionRepository from '../repositories/TransactionRepository'; import mining from './mining/mining'; import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository'; import PricesRepository from '../repositories/PricesRepository'; import priceUpdater from '../tasks/price-updater'; -import { Block } from 'bitcoinjs-lib'; class Blocks { private blocks: BlockExtended[] = []; @@ -753,10 +751,8 @@ class Blocks { } public async $indexCPFP(hash: string, height: number): Promise { - let transactions; - const block = await bitcoinClient.getBlock(hash, 2); - transactions = block.tx.map(tx => { + const transactions = block.tx.map(tx => { tx.vsize = tx.weight / 4; tx.fee *= 100_000_000; return tx; @@ -775,9 +771,12 @@ class Blocks { }); const effectiveFeePerVsize = totalFee / totalVSize; if (cluster.length > 1) { - await cpfpRepository.$saveCluster(height, cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: tx.fee || 0 }; }), effectiveFeePerVsize); - for (const tx of cluster) { - await transactionRepository.$setCluster(tx.txid, cluster[0].txid); + const roundedEffectiveFee = Math.round(effectiveFeePerVsize * 100) / 100; + const equalFee = cluster.reduce((acc, tx) => { + return (acc && Math.round(((tx.fee || 0) / tx.vsize) * 100) / 100 === roundedEffectiveFee); + }, true); + if (!equalFee) { + await cpfpRepository.$saveCluster(cluster[0].txid, height, cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: tx.fee || 0 }; }), effectiveFeePerVsize); } } cluster = []; diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 68087fdb5..6d7901ffa 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -445,6 +445,8 @@ class DatabaseMigration { if (databaseSchemaVersion < 50 && isBitcoin === true) { await this.$executeQuery('ALTER TABLE `blocks` DROP COLUMN `cpfp_indexed`'); + await this.$executeQuery(this.getCreateCompactCPFPTableQuery(), await this.$checkIfTableExists('compact_cpfp_clusters')); + await this.$executeQuery(this.getCreateCompactTransactionsTableQuery(), await this.$checkIfTableExists('compact_transactions')); await this.updateToSchemaVersion(50); } } @@ -918,6 +920,25 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } + private getCreateCompactCPFPTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS compact_cpfp_clusters ( + root binary(32) NOT NULL, + height int(10) NOT NULL, + txs BLOB DEFAULT NULL, + fee_rate float unsigned NOT NULL, + PRIMARY KEY (root), + INDEX (height) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } + + private getCreateCompactTransactionsTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS compact_transactions ( + txid binary(32) NOT NULL, + cluster binary(32) DEFAULT NULL, + PRIMARY KEY (txid) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } + public async $truncateIndexedData(tables: string[]) { const allowedTables = ['blocks', 'hashrates', 'prices']; diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 5c52a8ba0..6cd5d2785 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -671,7 +671,7 @@ class BlocksRepository { try { const blockchainInfo = await bitcoinClient.getBlockchainInfo(); const currentBlockHeight = blockchainInfo.blocks; - const [lastHeightRows]: any = await DB.query(`SELECT MIN(height) AS minHeight from cpfp_clusters`); + const [lastHeightRows]: any = await DB.query(`SELECT MIN(height) AS minHeight from compact_cpfp_clusters`); const lastHeight = (lastHeightRows.length && lastHeightRows[0].minHeight != null) ? lastHeightRows[0].minHeight : currentBlockHeight; let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, blockchainInfo.blocks); diff --git a/backend/src/repositories/CpfpRepository.ts b/backend/src/repositories/CpfpRepository.ts index 563e6ede1..a5f17540a 100644 --- a/backend/src/repositories/CpfpRepository.ts +++ b/backend/src/repositories/CpfpRepository.ts @@ -1,34 +1,72 @@ +import cluster, { Cluster } from 'cluster'; +import { RowDataPacket } from 'mysql2'; import DB from '../database'; import logger from '../logger'; import { Ancestor } from '../mempool.interfaces'; +import transactionRepository from '../repositories/TransactionRepository'; class CpfpRepository { - public async $saveCluster(height: number, txs: Ancestor[], effectiveFeePerVsize: number): Promise { + public async $saveCluster(clusterRoot: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number): Promise { + if (!txs[0]) { + return; + } try { - const txsJson = JSON.stringify(txs); + const packedTxs = Buffer.from(this.pack(txs)); await DB.query( ` - INSERT INTO cpfp_clusters(root, height, txs, fee_rate) - VALUE (?, ?, ?, ?) + INSERT INTO compact_cpfp_clusters(root, height, txs, fee_rate) + VALUE (UNHEX(?), ?, ?, ?) ON DUPLICATE KEY UPDATE height = ?, txs = ?, fee_rate = ? `, - [txs[0].txid, height, txsJson, effectiveFeePerVsize, height, txsJson, effectiveFeePerVsize, height] + [clusterRoot, height, packedTxs, effectiveFeePerVsize, height, packedTxs, effectiveFeePerVsize] ); + for (const tx of txs) { + await transactionRepository.$setCluster(tx.txid, clusterRoot); + } } catch (e: any) { logger.err(`Cannot save cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e)); throw e; } } + public async $getCluster(clusterRoot: string): Promise { + const [clusterRows]: any = await DB.query( + ` + SELECT * + FROM compact_cpfp_clusters + WHERE root = UNHEX(?) + `, + [clusterRoot] + ); + const cluster = clusterRows[0]; + cluster.txs = this.unpack(cluster.txs); + return cluster; + } + public async $deleteClustersFrom(height: number): Promise { logger.info(`Delete newer cpfp clusters from height ${height} from the database`); try { + const [rows] = await DB.query( + ` + SELECT txs, height, root from compact_cpfp_clusters + WHERE height >= ? + `, + [height] + ) as RowDataPacket[][]; + if (rows?.length) { + for (let clusterToDelete of rows) { + const txs = this.unpack(clusterToDelete.txs); + for (let tx of txs) { + await transactionRepository.$removeTransaction(tx.txid); + } + } + } await DB.query( ` - DELETE from cpfp_clusters + DELETE from compact_cpfp_clusters WHERE height >= ? `, [height] @@ -38,6 +76,42 @@ class CpfpRepository { throw e; } } + + public pack(txs: Ancestor[]): ArrayBuffer { + const buf = new ArrayBuffer(44 * txs.length); + const view = new DataView(buf); + txs.forEach((tx, i) => { + const offset = i * 44; + for (let x = 0; x < 32; x++) { + // store txid in little-endian + view.setUint8(offset + (31 - x), parseInt(tx.txid.slice(x * 2, (x * 2) + 2), 16)); + } + view.setUint32(offset + 32, tx.weight); + view.setBigUint64(offset + 36, BigInt(Math.round(tx.fee))); + }); + return buf; + } + + public unpack(buf: Buffer): Ancestor[] { + if (!buf) { + return []; + } + + const arrayBuffer = buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength); + const txs: Ancestor[] = []; + const view = new DataView(arrayBuffer); + for (let offset = 0; offset < arrayBuffer.byteLength; offset += 44) { + const txid = Array.from(new Uint8Array(arrayBuffer, offset, 32)).reverse().map(b => b.toString(16).padStart(2, '0')).join(''); + const weight = view.getUint32(offset + 32); + const fee = Number(view.getBigUint64(offset + 36)); + txs.push({ + txid, + weight, + fee + }); + } + return txs; + } } export default new CpfpRepository(); \ No newline at end of file diff --git a/backend/src/repositories/TransactionRepository.ts b/backend/src/repositories/TransactionRepository.ts index 74debb833..2d05f0e14 100644 --- a/backend/src/repositories/TransactionRepository.ts +++ b/backend/src/repositories/TransactionRepository.ts @@ -1,6 +1,7 @@ import DB from '../database'; import logger from '../logger'; import { Ancestor, CpfpInfo } from '../mempool.interfaces'; +import cpfpRepository from './CpfpRepository'; interface CpfpSummary { txid: string; @@ -12,20 +13,20 @@ interface CpfpSummary { } class TransactionRepository { - public async $setCluster(txid: string, cluster: string): Promise { + public async $setCluster(txid: string, clusterRoot: string): Promise { try { await DB.query( ` - INSERT INTO transactions + INSERT INTO compact_transactions ( txid, cluster ) - VALUE (?, ?) + VALUE (UNHEX(?), UNHEX(?)) ON DUPLICATE KEY UPDATE - cluster = ? + cluster = UNHEX(?) ;`, - [txid, cluster, cluster] + [txid, clusterRoot, clusterRoot] ); } catch (e: any) { logger.err(`Cannot save transaction cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e)); @@ -35,18 +36,19 @@ class TransactionRepository { public async $getCpfpInfo(txid: string): Promise { try { - let query = ` - SELECT * - FROM transactions - LEFT JOIN cpfp_clusters AS cluster ON cluster.root = transactions.cluster - WHERE transactions.txid = ? - `; - const [rows]: any = await DB.query(query, [txid]); - if (rows.length) { - rows[0].txs = JSON.parse(rows[0].txs) as Ancestor[]; - if (rows[0]?.txs?.length) { - return this.convertCpfp(rows[0]); - } + const [txRows]: any = await DB.query( + ` + SELECT HEX(txid) as id, HEX(cluster) as root + FROM compact_transactions + WHERE txid = UNHEX(?) + `, + [txid] + ); + if (txRows.length && txRows[0].root != null) { + const txid = txRows[0].id.toLowerCase(); + const clusterId = txRows[0].root.toLowerCase(); + const cluster = await cpfpRepository.$getCluster(clusterId); + return this.convertCpfp(txid, cluster); } } catch (e) { logger.err('Cannot get transaction cpfp info from db. Reason: ' + (e instanceof Error ? e.message : e)); @@ -54,12 +56,23 @@ class TransactionRepository { } } - private convertCpfp(cpfp: CpfpSummary): CpfpInfo { + public async $removeTransaction(txid: string): Promise { + await DB.query( + ` + DELETE FROM compact_transactions + WHERE txid = UNHEX(?) + `, + [txid] + ); + } + + private convertCpfp(txid, cluster): CpfpInfo { const descendants: Ancestor[] = []; const ancestors: Ancestor[] = []; let matched = false; - for (const tx of cpfp.txs) { - if (tx.txid === cpfp.txid) { + + for (const tx of cluster.txs) { + if (tx.txid === txid) { matched = true; } else if (!matched) { descendants.push(tx); From 01c96f80f9f99fda558ec9df93ca63f7f88b1c6a Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 9 Jan 2023 10:25:25 -0600 Subject: [PATCH 10/15] add cpfp progress marker to avoid reindexing early blocks --- backend/src/api/blocks.ts | 4 +++- backend/src/repositories/CpfpRepository.ts | 28 ++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 2b1f88518..c70bb67f7 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -350,7 +350,7 @@ class Blocks { let countThisRun = 0; let timer = new Date().getTime() / 1000; const startedAt = new Date().getTime() / 1000; - + let lastHeight; for (const block of unindexedBlocks) { // Logging const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer); @@ -365,11 +365,13 @@ class Blocks { await this.$indexCPFP(block.hash, block.height); // Calculate and save CPFP data for transactions in this block + lastHeight = block.height; // Logging count++; countThisRun++; } if (count > 0) { + await cpfpRepository.$insertProgressMarker(lastHeight); logger.notice(`CPFP indexing completed: indexed ${count} blocks`); } else { logger.debug(`CPFP indexing completed: indexed ${count} blocks`); diff --git a/backend/src/repositories/CpfpRepository.ts b/backend/src/repositories/CpfpRepository.ts index a5f17540a..6d091437c 100644 --- a/backend/src/repositories/CpfpRepository.ts +++ b/backend/src/repositories/CpfpRepository.ts @@ -77,6 +77,34 @@ class CpfpRepository { } } + // insert a dummy row to mark that we've indexed as far as this block + public async $insertProgressMarker(height: number): Promise { + try { + const [rows]: any = await DB.query( + ` + SELECT root + FROM compact_cpfp_clusters + WHERE height = ? + `, + [height] + ); + if (!rows?.length) { + const rootBuffer = Buffer.alloc(32); + rootBuffer.writeInt32LE(height); + await DB.query( + ` + INSERT INTO compact_cpfp_clusters(root, height, fee_rate) + VALUE (?, ?, ?) + `, + [rootBuffer, height, 0] + ); + } + } catch (e: any) { + logger.err(`Cannot insert cpfp progress marker. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + public pack(txs: Ancestor[]): ArrayBuffer { const buf = new ArrayBuffer(44 * txs.length); const view = new DataView(buf); From fcd047f30231ff91eb134cc7188bf0e46fedc4cf Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 9 Jan 2023 10:25:56 -0600 Subject: [PATCH 11/15] remove redundant fields from CPFP interfaces --- .../src/repositories/TransactionRepository.ts | 1 - .../transaction/transaction.component.ts | 27 +++++++------------ .../src/app/interfaces/node-api.interface.ts | 1 - 3 files changed, 9 insertions(+), 20 deletions(-) diff --git a/backend/src/repositories/TransactionRepository.ts b/backend/src/repositories/TransactionRepository.ts index 2d05f0e14..fc6af973d 100644 --- a/backend/src/repositories/TransactionRepository.ts +++ b/backend/src/repositories/TransactionRepository.ts @@ -83,7 +83,6 @@ class TransactionRepository { return { descendants, ancestors, - effectiveFeePerVsize: cpfp.fee_rate }; } } diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index 575c00637..81ae2f055 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -131,26 +131,17 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { this.cpfpInfo = null; return; } - if (cpfpInfo.effectiveFeePerVsize) { - this.tx.effectiveFeePerVsize = cpfpInfo.effectiveFeePerVsize; - } else { - const lowerFeeParents = cpfpInfo.ancestors.filter( - (parent) => parent.fee / (parent.weight / 4) < this.tx.feePerVsize - ); - let totalWeight = - this.tx.weight + - lowerFeeParents.reduce((prev, val) => prev + val.weight, 0); - let totalFees = - this.tx.fee + - lowerFeeParents.reduce((prev, val) => prev + val.fee, 0); - if (cpfpInfo?.bestDescendant) { - totalWeight += cpfpInfo?.bestDescendant.weight; - totalFees += cpfpInfo?.bestDescendant.fee; - } + const relatives = [...cpfpInfo.ancestors, ...cpfpInfo.descendants || [cpfpInfo.bestDescendant]]; + let totalWeight = + this.tx.weight + + relatives.reduce((prev, val) => prev + val.weight, 0); + let totalFees = + this.tx.fee + + relatives.reduce((prev, val) => prev + val.fee, 0); + + this.tx.effectiveFeePerVsize = totalFees / (totalWeight / 4); - this.tx.effectiveFeePerVsize = totalFees / (totalWeight / 4); - } if (!this.tx.status.confirmed) { this.stateService.markBlock$.next({ txFeePerVSize: this.tx.effectiveFeePerVsize, diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 2e6b94988..f72886870 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -24,7 +24,6 @@ export interface CpfpInfo { ancestors: Ancestor[]; descendants?: Ancestor[]; bestDescendant?: BestDescendant | null; - effectiveFeePerVsize?: number; } export interface DifficultyAdjustment { From 8de3fd098879d677c9c337465117fcfa5743b705 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 10 Jan 2023 17:13:11 -0600 Subject: [PATCH 12/15] batch db inserts for cpfp data --- backend/src/api/blocks.ts | 16 ++-- backend/src/repositories/CpfpRepository.ts | 82 ++++++++++++++++++- .../src/repositories/TransactionRepository.ts | 24 ++++++ 3 files changed, 113 insertions(+), 9 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index c70bb67f7..69fa208b8 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -760,6 +760,8 @@ class Blocks { return tx; }); + const clusters: any[] = []; + let cluster: TransactionStripped[] = []; let ancestors: { [txid: string]: boolean } = {}; for (let i = transactions.length - 1; i >= 0; i--) { @@ -773,13 +775,12 @@ class Blocks { }); const effectiveFeePerVsize = totalFee / totalVSize; if (cluster.length > 1) { - const roundedEffectiveFee = Math.round(effectiveFeePerVsize * 100) / 100; - const equalFee = cluster.reduce((acc, tx) => { - return (acc && Math.round(((tx.fee || 0) / tx.vsize) * 100) / 100 === roundedEffectiveFee); - }, true); - if (!equalFee) { - await cpfpRepository.$saveCluster(cluster[0].txid, height, cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: tx.fee || 0 }; }), effectiveFeePerVsize); - } + clusters.push({ + root: cluster[0].txid, + height, + txs: cluster.map(tx => { return { txid: tx.txid, weight: tx.vsize * 4, fee: tx.fee || 0 }; }), + effectiveFeePerVsize, + }); } cluster = []; ancestors = {}; @@ -789,6 +790,7 @@ class Blocks { ancestors[vin.txid] = true; }); } + await cpfpRepository.$batchSaveClusters(clusters); } } diff --git a/backend/src/repositories/CpfpRepository.ts b/backend/src/repositories/CpfpRepository.ts index 6d091437c..6fe041763 100644 --- a/backend/src/repositories/CpfpRepository.ts +++ b/backend/src/repositories/CpfpRepository.ts @@ -10,6 +10,15 @@ class CpfpRepository { if (!txs[0]) { return; } + // skip clusters of transactions with the same fees + const roundedEffectiveFee = Math.round(effectiveFeePerVsize * 100) / 100; + const equalFee = txs.reduce((acc, tx) => { + return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee); + }, true); + if (equalFee) { + return; + } + try { const packedTxs = Buffer.from(this.pack(txs)); await DB.query( @@ -23,8 +32,14 @@ class CpfpRepository { `, [clusterRoot, height, packedTxs, effectiveFeePerVsize, height, packedTxs, effectiveFeePerVsize] ); - for (const tx of txs) { - await transactionRepository.$setCluster(tx.txid, clusterRoot); + const maxChunk = 10; + let chunkIndex = 0; + while (chunkIndex < txs.length) { + const chunk = txs.slice(chunkIndex, chunkIndex + maxChunk).map(tx => { + return { txid: tx.txid, cluster: clusterRoot }; + }); + await transactionRepository.$batchSetCluster(chunk); + chunkIndex += maxChunk; } } catch (e: any) { logger.err(`Cannot save cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e)); @@ -32,6 +47,69 @@ class CpfpRepository { } } + public async $batchSaveClusters(clusters: { root: string, height: number, txs: any, effectiveFeePerVsize: number}[]): Promise { + try { + const clusterValues: any[] = []; + const txs: any[] = []; + + for (const cluster of clusters) { + if (cluster.txs?.length > 1) { + const roundedEffectiveFee = Math.round(cluster.effectiveFeePerVsize * 100) / 100; + const equalFee = cluster.txs.reduce((acc, tx) => { + return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee); + }, true); + if (!equalFee) { + clusterValues.push([ + cluster.root, + cluster.height, + Buffer.from(this.pack(cluster.txs)), + cluster.effectiveFeePerVsize + ]); + for (const tx of cluster.txs) { + txs.push({ txid: tx.txid, cluster: cluster.root }); + } + } + } + } + + if (!clusterValues.length) { + return; + } + + const maxChunk = 100; + let chunkIndex = 0; + // insert transactions in batches of up to 100 rows + while (chunkIndex < txs.length) { + const chunk = txs.slice(chunkIndex, chunkIndex + maxChunk); + await transactionRepository.$batchSetCluster(chunk); + chunkIndex += maxChunk; + } + + chunkIndex = 0; + // insert clusters in batches of up to 100 rows + while (chunkIndex < clusterValues.length) { + const chunk = clusterValues.slice(chunkIndex, chunkIndex + maxChunk); + let query = ` + INSERT IGNORE INTO compact_cpfp_clusters(root, height, txs, fee_rate) + VALUES + `; + query += chunk.map(chunk => { + return (' (UNHEX(?), ?, ?, ?)'); + }) + ';'; + const values = chunk.flat(); + await DB.query( + query, + values + ); + chunkIndex += maxChunk; + } + return; + } catch (e: any) { + logger.err(`Cannot save cpfp clusters into db. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + public async $getCluster(clusterRoot: string): Promise { const [clusterRows]: any = await DB.query( ` diff --git a/backend/src/repositories/TransactionRepository.ts b/backend/src/repositories/TransactionRepository.ts index fc6af973d..061617451 100644 --- a/backend/src/repositories/TransactionRepository.ts +++ b/backend/src/repositories/TransactionRepository.ts @@ -34,6 +34,30 @@ class TransactionRepository { } } + public async $batchSetCluster(txs): Promise { + try { + let query = ` + INSERT IGNORE INTO compact_transactions + ( + txid, + cluster + ) + VALUES + `; + query += txs.map(tx => { + return (' (UNHEX(?), UNHEX(?))'); + }) + ';'; + const values = txs.map(tx => [tx.txid, tx.cluster]).flat(); + await DB.query( + query, + values + ); + } catch (e: any) { + logger.err(`Cannot save cpfp transactions into db. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + public async $getCpfpInfo(txid: string): Promise { try { const [txRows]: any = await DB.query( From f0d3bb87c60b3281e970852459d80386ae537fb8 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 10 Jan 2023 17:15:55 -0600 Subject: [PATCH 13/15] handle gaps in indexed CPFP data --- backend/src/api/blocks.ts | 29 +++++++++----------- backend/src/repositories/BlocksRepository.ts | 24 ++++++++++------ backend/src/repositories/CpfpRepository.ts | 13 +++++---- 3 files changed, 35 insertions(+), 31 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 69fa208b8..9ab572401 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -338,10 +338,10 @@ class Blocks { try { // Get all indexed block hash - const unindexedBlocks = await blocksRepository.$getCPFPUnindexedBlocks(); - logger.info(`Indexing cpfp data for ${unindexedBlocks.length} blocks`); + const unindexedBlockHeights = await blocksRepository.$getCPFPUnindexedBlocks(); + logger.info(`Indexing cpfp data for ${unindexedBlockHeights.length} blocks`); - if (!unindexedBlocks?.length) { + if (!unindexedBlockHeights?.length) { return; } @@ -350,32 +350,26 @@ class Blocks { let countThisRun = 0; let timer = new Date().getTime() / 1000; const startedAt = new Date().getTime() / 1000; - let lastHeight; - for (const block of unindexedBlocks) { + for (const height of unindexedBlockHeights) { // Logging + const hash = await bitcoinApi.$getBlockHash(height); const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer); if (elapsedSeconds > 5) { const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); const blockPerSeconds = (countThisRun / elapsedSeconds); - const progress = Math.round(count / unindexedBlocks.length * 10000) / 100; - logger.debug(`Indexing cpfp clusters for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`); + const progress = Math.round(count / unindexedBlockHeights.length * 10000) / 100; + logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor} seconds`); timer = new Date().getTime() / 1000; countThisRun = 0; } - await this.$indexCPFP(block.hash, block.height); // Calculate and save CPFP data for transactions in this block + await this.$indexCPFP(hash, height); // Calculate and save CPFP data for transactions in this block - lastHeight = block.height; // Logging count++; countThisRun++; } - if (count > 0) { - await cpfpRepository.$insertProgressMarker(lastHeight); - logger.notice(`CPFP indexing completed: indexed ${count} blocks`); - } else { - logger.debug(`CPFP indexing completed: indexed ${count} blocks`); - } + logger.notice(`CPFP indexing completed: indexed ${count} blocks`); } catch (e) { logger.err(`CPFP indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`); throw e; @@ -790,7 +784,10 @@ class Blocks { ancestors[vin.txid] = true; }); } - await cpfpRepository.$batchSaveClusters(clusters); + const result = await cpfpRepository.$batchSaveClusters(clusters); + if (!result) { + await cpfpRepository.$insertProgressMarker(height); + } } } diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 6cd5d2785..df98719b9 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -671,19 +671,25 @@ class BlocksRepository { try { const blockchainInfo = await bitcoinClient.getBlockchainInfo(); const currentBlockHeight = blockchainInfo.blocks; - const [lastHeightRows]: any = await DB.query(`SELECT MIN(height) AS minHeight from compact_cpfp_clusters`); - const lastHeight = (lastHeightRows.length && lastHeightRows[0].minHeight != null) ? lastHeightRows[0].minHeight : currentBlockHeight; - - let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, blockchainInfo.blocks); + let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, currentBlockHeight); if (indexingBlockAmount <= -1) { indexingBlockAmount = currentBlockHeight + 1; } - const firstHeight = Math.max(0, currentBlockHeight - indexingBlockAmount + 1); + const minHeight = Math.max(0, currentBlockHeight - indexingBlockAmount + 1); - if (firstHeight < lastHeight) { - const [rows]: any = await DB.query(`SELECT height, hash FROM blocks WHERE height BETWEEN ? AND ? ORDER BY height DESC`, [firstHeight, lastHeight]); - return rows; - } + const [rows]: any[] = await DB.query(` + SELECT height + FROM compact_cpfp_clusters + WHERE height <= ? AND height >= ? + ORDER BY height DESC; + `, [currentBlockHeight, minHeight]); + + const indexedHeights = {}; + rows.forEach((row) => { indexedHeights[row.height] = true; }); + const allHeights: number[] = Array.from(Array(currentBlockHeight - minHeight + 1).keys(), n => n + minHeight).reverse(); + const unindexedHeights = allHeights.filter(x => !indexedHeights[x]); + + return unindexedHeights; } catch (e) { logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e)); throw e; diff --git a/backend/src/repositories/CpfpRepository.ts b/backend/src/repositories/CpfpRepository.ts index 6fe041763..ce7432d5b 100644 --- a/backend/src/repositories/CpfpRepository.ts +++ b/backend/src/repositories/CpfpRepository.ts @@ -6,9 +6,9 @@ import { Ancestor } from '../mempool.interfaces'; import transactionRepository from '../repositories/TransactionRepository'; class CpfpRepository { - public async $saveCluster(clusterRoot: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number): Promise { + public async $saveCluster(clusterRoot: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number): Promise { if (!txs[0]) { - return; + return false; } // skip clusters of transactions with the same fees const roundedEffectiveFee = Math.round(effectiveFeePerVsize * 100) / 100; @@ -16,7 +16,7 @@ class CpfpRepository { return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee); }, true); if (equalFee) { - return; + return false; } try { @@ -41,13 +41,14 @@ class CpfpRepository { await transactionRepository.$batchSetCluster(chunk); chunkIndex += maxChunk; } + return true; } catch (e: any) { logger.err(`Cannot save cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e)); throw e; } } - public async $batchSaveClusters(clusters: { root: string, height: number, txs: any, effectiveFeePerVsize: number}[]): Promise { + public async $batchSaveClusters(clusters: { root: string, height: number, txs: any, effectiveFeePerVsize: number}[]): Promise { try { const clusterValues: any[] = []; const txs: any[] = []; @@ -73,7 +74,7 @@ class CpfpRepository { } if (!clusterValues.length) { - return; + return false; } const maxChunk = 100; @@ -103,7 +104,7 @@ class CpfpRepository { ); chunkIndex += maxChunk; } - return; + return true; } catch (e: any) { logger.err(`Cannot save cpfp clusters into db. Reason: ` + (e instanceof Error ? e.message : e)); throw e; From bd30f2eb12f26ccd5e55d874bb771744e7b29572 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 10 Jan 2023 17:24:18 -0600 Subject: [PATCH 14/15] migrate cpfp data from old to new schemas --- backend/src/api/database-migration.ts | 70 +++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 4 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 6d7901ffa..42f223417 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -2,9 +2,12 @@ import config from '../config'; import DB from '../database'; import logger from '../logger'; import { Common } from './common'; +import blocksRepository from '../repositories/BlocksRepository'; +import cpfpRepository from '../repositories/CpfpRepository'; +import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 50; + private static currentVersion = 52; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -443,11 +446,27 @@ class DatabaseMigration { await this.updateToSchemaVersion(49); } - if (databaseSchemaVersion < 50 && isBitcoin === true) { + if (databaseSchemaVersion < 50) { await this.$executeQuery('ALTER TABLE `blocks` DROP COLUMN `cpfp_indexed`'); + await this.updateToSchemaVersion(50); + } + + if (databaseSchemaVersion < 51) { + await this.$executeQuery('ALTER TABLE `cpfp_clusters` ADD INDEX `height` (`height`)'); + await this.updateToSchemaVersion(51); + } + + if (databaseSchemaVersion < 52) { await this.$executeQuery(this.getCreateCompactCPFPTableQuery(), await this.$checkIfTableExists('compact_cpfp_clusters')); await this.$executeQuery(this.getCreateCompactTransactionsTableQuery(), await this.$checkIfTableExists('compact_transactions')); - await this.updateToSchemaVersion(50); + try { + await this.$convertCompactCpfpTables(); + await this.$executeQuery('DROP TABLE IF EXISTS `cpfp_clusters`'); + await this.$executeQuery('DROP TABLE IF EXISTS `transactions`'); + await this.updateToSchemaVersion(52); + } catch(e) { + logger.warn('' + (e instanceof Error ? e.message : e)); + } } } @@ -925,7 +944,7 @@ class DatabaseMigration { root binary(32) NOT NULL, height int(10) NOT NULL, txs BLOB DEFAULT NULL, - fee_rate float unsigned NOT NULL, + fee_rate float unsigned, PRIMARY KEY (root), INDEX (height) ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; @@ -959,6 +978,49 @@ class DatabaseMigration { logger.warn(`Unable to erase indexed data`); } } + + private async $convertCompactCpfpTables(): Promise { + try { + const batchSize = 250; + const maxHeight = await blocksRepository.$mostRecentBlockHeight() || 0; + const [minHeightRows]: any = await DB.query(`SELECT MIN(height) AS minHeight from cpfp_clusters`); + const minHeight = (minHeightRows.length && minHeightRows[0].minHeight != null) ? minHeightRows[0].minHeight : maxHeight; + let height = maxHeight; + + // Logging + let timer = new Date().getTime() / 1000; + const startedAt = new Date().getTime() / 1000; + + while (height > minHeight) { + const [rows] = await DB.query( + ` + SELECT * from cpfp_clusters + WHERE height <= ? AND height > ? + ORDER BY height + `, + [height, height - batchSize] + ) as RowDataPacket[][]; + if (rows?.length) { + await cpfpRepository.$batchSaveClusters(rows.map(row => { + return { + root: row.root, + height: row.height, + txs: JSON.parse(row.txs), + effectiveFeePerVsize: row.fee_rate, + }; + })); + } + + const elapsed = new Date().getTime() / 1000 - timer; + const runningFor = new Date().getTime() / 1000 - startedAt; + logger.debug(`Migrated cpfp data from block ${height} to ${height - batchSize} in ${elapsed.toFixed(2)} seconds | total elapsed: ${runningFor.toFixed(2)} seconds`); + timer = new Date().getTime() / 1000; + height -= batchSize; + } + } catch (e) { + logger.warn(`Failed to migrate cpfp transaction data`); + } + } } export default new DatabaseMigration(); From 2548d2a5e9add8f72d108c4c125b1a88e051f230 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 11 Jan 2023 08:41:00 -0600 Subject: [PATCH 15/15] fix frontend js error on unconfirmed non-cpfp transactions --- .../app/components/transaction/transaction.component.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index 81ae2f055..e01092189 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -131,8 +131,11 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { this.cpfpInfo = null; return; } - - const relatives = [...cpfpInfo.ancestors, ...cpfpInfo.descendants || [cpfpInfo.bestDescendant]]; + // merge ancestors/descendants + const relatives = [...(cpfpInfo.ancestors || []), ...(cpfpInfo.descendants || [])]; + if (cpfpInfo.bestDescendant && !cpfpInfo.descendants?.length) { + relatives.push(cpfpInfo.bestDescendant); + } let totalWeight = this.tx.weight + relatives.reduce((prev, val) => prev + val.weight, 0);