From c8e967cc0c1563287fb1fc6096b96ac7efcbddb0 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 29 Nov 2024 16:04:54 +0100 Subject: [PATCH 1/5] [blocks] save pools-v2.json hash in blocks table --- backend/src/api/database-migration.ts | 8 +++++- backend/src/api/pools-parser.ts | 9 ------- backend/src/repositories/BlocksRepository.ts | 28 +++++++++++--------- 3 files changed, 22 insertions(+), 23 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 0c17ab9f1..b9d8530bb 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 91; + private static currentVersion = 92; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -775,6 +775,12 @@ class DatabaseMigration { await this.$executeQuery('ALTER TABLE `blocks_audits` ADD INDEX `time` (`time`)'); await this.updateToSchemaVersion(91); } + + // blocks pools-v2.json hash + if (databaseSchemaVersion < 92) { + await this.$executeQuery('ALTER TABLE `blocks` ADD definition_hash varchar(255) NOT NULL DEFAULT "5f32a67401929169f225f5db43c9efa795d1b159"'); + await this.updateToSchemaVersion(92); + } } /** diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index 289389d5e..2fd55d6c5 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -19,15 +19,6 @@ class PoolsParser { 'addresses': '[]', 'slug': 'unknown' }; - private uniqueLogs: string[] = []; - - private uniqueLog(loggerFunction: any, msg: string): void { - if (this.uniqueLogs.includes(msg)) { - return; - } - this.uniqueLogs.push(msg); - loggerFunction(msg); - } public setMiningPools(pools): void { for (const pool of pools) { diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 424a668c7..f673e574b 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -15,6 +15,7 @@ import blocks from '../api/blocks'; import BlocksAuditsRepository from './BlocksAuditsRepository'; import transactionUtils from '../api/transaction-utils'; import { parseDATUMTemplateCreator } from '../utils/bitcoin-script'; +import poolsUpdater from '../tasks/pools-updater'; interface DatabaseBlock { id: string; @@ -114,16 +115,16 @@ class BlocksRepository { try { const query = `INSERT INTO blocks( - height, hash, blockTimestamp, size, - weight, tx_count, coinbase_raw, difficulty, - pool_id, fees, fee_span, median_fee, - reward, version, bits, nonce, - merkle_root, previous_block_hash, avg_fee, avg_fee_rate, - median_timestamp, header, coinbase_address, coinbase_addresses, - coinbase_signature, utxoset_size, utxoset_change, avg_tx_size, - total_inputs, total_outputs, total_input_amt, total_output_amt, - fee_percentiles, segwit_total_txs, segwit_total_size, segwit_total_weight, - median_fee_amt, coinbase_signature_ascii + height, hash, blockTimestamp, size, + weight, tx_count, coinbase_raw, difficulty, + pool_id, fees, fee_span, median_fee, + reward, version, bits, nonce, + merkle_root, previous_block_hash, avg_fee, avg_fee_rate, + median_timestamp, header, coinbase_address, coinbase_addresses, + coinbase_signature, utxoset_size, utxoset_change, avg_tx_size, + total_inputs, total_outputs, total_input_amt, total_output_amt, + fee_percentiles, segwit_total_txs, segwit_total_size, segwit_total_weight, + median_fee_amt, coinbase_signature_ascii, definition_hash ) VALUE ( ?, ?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, @@ -134,7 +135,7 @@ class BlocksRepository { ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ? + ?, ?, ? )`; const poolDbId = await PoolsRepository.$getPoolByUniqueId(block.extras.pool.id); @@ -181,6 +182,7 @@ class BlocksRepository { block.extras.segwitTotalWeight, block.extras.medianFeeAmt, truncatedCoinbaseSignatureAscii, + poolsUpdater.currentSha ]; await DB.query(query, params); @@ -1013,9 +1015,9 @@ class BlocksRepository { public async $savePool(id: string, poolId: number): Promise { try { await DB.query(` - UPDATE blocks SET pool_id = ? + UPDATE blocks SET pool_id = ?, definition_hash = ? WHERE hash = ?`, - [poolId, id] + [poolId, poolsUpdater.currentSha, id] ); } catch (e) { logger.err(`Cannot update block pool. Reason: ` + (e instanceof Error ? e.message : e)); From ac997f3d9e8456548f58b4b29bd253da3a2c8b9d Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 29 Nov 2024 17:13:28 +0100 Subject: [PATCH 2/5] [blocks] add 2 endpoints to retreive pools-v2.json hashes --- backend/src/api/bitcoin/bitcoin.routes.ts | 31 +++++++++++++++++++++++ backend/src/api/blocks.ts | 15 ++++++++++- backend/src/api/database-migration.ts | 1 + backend/src/mempool.interfaces.ts | 2 ++ backend/src/tasks/pools-updater.ts | 2 +- 5 files changed, 49 insertions(+), 2 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index d2d298e09..c95d89d0f 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -21,6 +21,7 @@ import transactionRepository from '../../repositories/TransactionRepository'; import rbfCache from '../rbf-cache'; import { calculateMempoolTxCpfp } from '../cpfp'; import { handleError } from '../../utils/api'; +import poolsUpdater from '../../tasks/pools-updater'; class BitcoinRoutes { public initRoutes(app: Application) { @@ -46,6 +47,8 @@ class BitcoinRoutes { .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary) .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/tx/:txid/audit', this.$getBlockTxAuditSummary) .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', this.getBlockTipHeight) + .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/definition/list', this.getBlockDefinitionHashes) + .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/definition/current', this.getCurrentBlockDefinitionHash) .post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion) .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this)) .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this)) @@ -657,6 +660,34 @@ class BitcoinRoutes { } } + private async getBlockDefinitionHashes(req: Request, res: Response) { + try { + const result = await blocks.$getBlockDefinitionHashes(); + if (!result) { + handleError(req, res, 503, `Service Temporarily Unavailable`); + return; + } + res.setHeader('content-type', 'application/json'); + res.send(result); + } catch (e) { + handleError(req, res, 500, e instanceof Error ? e.message : e); + } + } + + private async getCurrentBlockDefinitionHash(req: Request, res: Response) { + try { + const currentSha = await poolsUpdater.getShaFromDb(); + if (!currentSha) { + handleError(req, res, 503, `Service Temporarily Unavailable`); + return; + } + res.setHeader('content-type', 'text/plain'); + res.send(currentSha); + } catch (e) { + handleError(req, res, 500, e instanceof Error ? e.message : e); + } + } + private getBlockTipHeight(req: Request, res: Response) { try { const result = blocks.getCurrentBlockHeight(); diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index e621056ab..2b468a8f8 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -33,8 +33,8 @@ import AccelerationRepository from '../repositories/AccelerationRepository'; import { calculateFastBlockCpfp, calculateGoodBlockCpfp } from './cpfp'; import mempool from './mempool'; import CpfpRepository from '../repositories/CpfpRepository'; -import accelerationApi from './services/acceleration'; import { parseDATUMTemplateCreator } from '../utils/bitcoin-script'; +import database from '../database'; class Blocks { private blocks: BlockExtended[] = []; @@ -1462,6 +1462,19 @@ class Blocks { // not a fatal error, we'll try again next time the indexer runs } } + + public async $getBlockDefinitionHashes(): Promise { + try { + const [rows]: any = await database.query(`SELECT DISTINCT(definition_hash) FROM blocks`); + if (rows && rows.length) { + return rows.map(r => r.definition_hash); + } + } catch (e) { + // we just return an empty array + } + logger.debug(`Unable to retreive list of blocks.definition_hash from db`); + return []; + } } export default new Blocks(); diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index b9d8530bb..47fd696e4 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -779,6 +779,7 @@ class DatabaseMigration { // blocks pools-v2.json hash if (databaseSchemaVersion < 92) { await this.$executeQuery('ALTER TABLE `blocks` ADD definition_hash varchar(255) NOT NULL DEFAULT "5f32a67401929169f225f5db43c9efa795d1b159"'); + await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `definition_hash` (`definition_hash`)'); await this.updateToSchemaVersion(92); } } diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index dc703af21..e1da4be6f 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -325,6 +325,8 @@ export interface BlockExtension { // Requires coinstatsindex, will be set to NULL otherwise utxoSetSize: number | null; totalInputAmt: number | null; + // pools-v2.json git hash + definitionHash: string | undefined; } /** diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 652383a2a..502e3613f 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -121,7 +121,7 @@ class PoolsUpdater { /** * Fetch our latest pools-v2.json sha from the db */ - private async getShaFromDb(): Promise { + public async getShaFromDb(): Promise { try { const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"'); return (rows.length > 0 ? rows[0].string : null); From 4ff2aad94aeb7edef637cb76cf54f29254db91a3 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 29 Nov 2024 17:55:06 +0100 Subject: [PATCH 3/5] [blocks] return list of block hash filtered by definition hash --- backend/src/api/bitcoin/bitcoin.routes.ts | 28 +++++++++++++++++++---- backend/src/api/blocks.ts | 27 ++++++++++++++++++---- 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index c95d89d0f..eb5cca6f8 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -47,13 +47,15 @@ class BitcoinRoutes { .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary) .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/tx/:txid/audit', this.$getBlockTxAuditSummary) .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', this.getBlockTipHeight) - .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/definition/list', this.getBlockDefinitionHashes) - .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/definition/current', this.getCurrentBlockDefinitionHash) .post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion) .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this)) .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this)) // Temporarily add txs/package endpoint for all backends until esplora supports it .post(config.MEMPOOL.API_URL_PREFIX + 'txs/package', this.$submitPackage) + // Internal routes + .get(config.MEMPOOL.API_URL_PREFIX + 'internal/blocks/definition/list', this.getBlockDefinitionHashes) + .get(config.MEMPOOL.API_URL_PREFIX + 'internal/blocks/definition/current', this.getCurrentBlockDefinitionHash) + .get(config.MEMPOOL.API_URL_PREFIX + 'internal/blocks/:definitionHash', this.getBlocksByDefinitionHash) ; if (config.MEMPOOL.BACKEND !== 'esplora') { @@ -660,7 +662,7 @@ class BitcoinRoutes { } } - private async getBlockDefinitionHashes(req: Request, res: Response) { + private async getBlockDefinitionHashes(req: Request, res: Response): Promise { try { const result = await blocks.$getBlockDefinitionHashes(); if (!result) { @@ -674,7 +676,7 @@ class BitcoinRoutes { } } - private async getCurrentBlockDefinitionHash(req: Request, res: Response) { + private async getCurrentBlockDefinitionHash(req: Request, res: Response): Promise { try { const currentSha = await poolsUpdater.getShaFromDb(); if (!currentSha) { @@ -688,6 +690,24 @@ class BitcoinRoutes { } } + private async getBlocksByDefinitionHash(req: Request, res: Response): Promise { + try { + if (typeof(req.params.definitionHash) !== 'string') { + res.status(400).send('Parameter "hash" must be a valid string'); + return; + } + const blocksHash = await blocks.$getBlocksByDefinitionHash(req.params.definitionHash as string); + if (!blocksHash) { + handleError(req, res, 503, `Service Temporarily Unavailable`); + return; + } + res.setHeader('content-type', 'application/json'); + res.send(blocksHash); + } catch (e) { + handleError(req, res, 500, e instanceof Error ? e.message : e); + } + } + private getBlockTipHeight(req: Request, res: Response) { try { const result = blocks.getCurrentBlockHeight(); diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 2b468a8f8..102601594 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -1463,17 +1463,34 @@ class Blocks { } } - public async $getBlockDefinitionHashes(): Promise { + public async $getBlockDefinitionHashes(): Promise { try { const [rows]: any = await database.query(`SELECT DISTINCT(definition_hash) FROM blocks`); - if (rows && rows.length) { + if (rows && Array.isArray(rows)) { return rows.map(r => r.definition_hash); + } else { + logger.debug(`Unable to retreive list of blocks.definition_hash from db (no result)`); + return null; } } catch (e) { - // we just return an empty array + logger.debug(`Unable to retreive list of blocks.definition_hash from db (exception: ${e})`); + return null; + } + } + + public async $getBlocksByDefinitionHash(definitionHash: string): Promise { + try { + const [rows]: any = await database.query(`SELECT hash FROM blocks WHERE definition_hash = ?`, [definitionHash]); + if (rows && Array.isArray(rows)) { + return rows.map(r => r.hash); + } else { + logger.debug(`Unable to retreive list of blocks for definition hash ${definitionHash} from db (no result)`); + return null; + } + } catch (e) { + logger.debug(`Unable to retreive list of blocks for definition hash ${definitionHash} from db (exception: ${e})`); + return null; } - logger.debug(`Unable to retreive list of blocks.definition_hash from db`); - return []; } } From faa83866fd1d8d09a651d600973612b5952b7e4e Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sun, 12 Jan 2025 12:04:19 +0900 Subject: [PATCH 4/5] [blocks] fix block pools-v2 hash migration --- backend/src/api/database-migration.ts | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 0ca58a785..4f43bd9d2 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -1105,13 +1105,6 @@ class DatabaseMigration { `); } - // blocks pools-v2.json hash - if (databaseSchemaVersion < 95) { - await this.$executeQuery('ALTER TABLE `blocks` ADD definition_hash varchar(255) NOT NULL DEFAULT "5f32a67401929169f225f5db43c9efa795d1b159"'); - await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `definition_hash` (`definition_hash`)'); - await this.updateToSchemaVersion(95); - } - if (config.MEMPOOL.NETWORK !== 'mainnet') { // Apply all the mainnet specific migrations to all other networks // Version 69 @@ -1125,6 +1118,18 @@ class DatabaseMigration { } await this.updateToSchemaVersion(94); } + + // blocks pools-v2.json hash + if (databaseSchemaVersion < 95) { + let poolJsonSha = 'f737d86571d190cf1a1a3cf5fd86b33ba9624254'; + const [poolJsonShaDb]: any[] = await DB.query(`SELECT string FROM state WHERE name = 'pools_json_sha'`); + if (poolJsonShaDb?.length > 0) { + poolJsonSha = poolJsonShaDb[0].string; + } + await this.$executeQuery(`ALTER TABLE blocks ADD definition_hash varchar(255) NOT NULL DEFAULT "${poolJsonSha}"`); + await this.$executeQuery('ALTER TABLE blocks ADD INDEX `definition_hash` (`definition_hash`)'); + await this.updateToSchemaVersion(95); + } } /** From ff4aca83702e595ac12de1c7c717f34f44faeb56 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Wed, 22 Jan 2025 11:26:34 +0900 Subject: [PATCH 5/5] [blocks] update sha in memory/db before re-indexing blocks --- backend/src/tasks/pools-updater.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 502e3613f..38e816d74 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -88,8 +88,8 @@ class PoolsUpdater { try { await DB.query('START TRANSACTION;'); - await poolsParser.migratePoolsJson(); await this.updateDBSha(githubSha); + await poolsParser.migratePoolsJson(); await DB.query('COMMIT;'); } catch (e) { logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, this.tag);