diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index ccdc3dc7c..b56ca4861 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -21,6 +21,7 @@ import transactionRepository from '../../repositories/TransactionRepository'; import rbfCache from '../rbf-cache'; import { calculateMempoolTxCpfp } from '../cpfp'; import { handleError } from '../../utils/api'; +import poolsUpdater from '../../tasks/pools-updater'; const TXID_REGEX = /^[a-f0-9]{64}$/i; const BLOCK_HASH_REGEX = /^[a-f0-9]{64}$/i; @@ -56,6 +57,10 @@ class BitcoinRoutes { .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this)) // Temporarily add txs/package endpoint for all backends until esplora supports it .post(config.MEMPOOL.API_URL_PREFIX + 'txs/package', this.$submitPackage) + // Internal routes + .get(config.MEMPOOL.API_URL_PREFIX + 'internal/blocks/definition/list', this.getBlockDefinitionHashes) + .get(config.MEMPOOL.API_URL_PREFIX + 'internal/blocks/definition/current', this.getCurrentBlockDefinitionHash) + .get(config.MEMPOOL.API_URL_PREFIX + 'internal/blocks/:definitionHash', this.getBlocksByDefinitionHash) ; if (config.MEMPOOL.BACKEND !== 'esplora') { @@ -739,6 +744,52 @@ class BitcoinRoutes { } } + private async getBlockDefinitionHashes(req: Request, res: Response): Promise { + try { + const result = await blocks.$getBlockDefinitionHashes(); + if (!result) { + handleError(req, res, 503, `Service Temporarily Unavailable`); + return; + } + res.setHeader('content-type', 'application/json'); + res.send(result); + } catch (e) { + handleError(req, res, 500, e instanceof Error ? e.message : e); + } + } + + private async getCurrentBlockDefinitionHash(req: Request, res: Response): Promise { + try { + const currentSha = await poolsUpdater.getShaFromDb(); + if (!currentSha) { + handleError(req, res, 503, `Service Temporarily Unavailable`); + return; + } + res.setHeader('content-type', 'text/plain'); + res.send(currentSha); + } catch (e) { + handleError(req, res, 500, e instanceof Error ? e.message : e); + } + } + + private async getBlocksByDefinitionHash(req: Request, res: Response): Promise { + try { + if (typeof(req.params.definitionHash) !== 'string') { + res.status(400).send('Parameter "hash" must be a valid string'); + return; + } + const blocksHash = await blocks.$getBlocksByDefinitionHash(req.params.definitionHash as string); + if (!blocksHash) { + handleError(req, res, 503, `Service Temporarily Unavailable`); + return; + } + res.setHeader('content-type', 'application/json'); + res.send(blocksHash); + } catch (e) { + handleError(req, res, 500, e instanceof Error ? e.message : e); + } + } + private getBlockTipHeight(req: Request, res: Response) { try { const result = blocks.getCurrentBlockHeight(); diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index e621056ab..102601594 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -33,8 +33,8 @@ import AccelerationRepository from '../repositories/AccelerationRepository'; import { calculateFastBlockCpfp, calculateGoodBlockCpfp } from './cpfp'; import mempool from './mempool'; import CpfpRepository from '../repositories/CpfpRepository'; -import accelerationApi from './services/acceleration'; import { parseDATUMTemplateCreator } from '../utils/bitcoin-script'; +import database from '../database'; class Blocks { private blocks: BlockExtended[] = []; @@ -1462,6 +1462,36 @@ class Blocks { // not a fatal error, we'll try again next time the indexer runs } } + + public async $getBlockDefinitionHashes(): Promise { + try { + const [rows]: any = await database.query(`SELECT DISTINCT(definition_hash) FROM blocks`); + if (rows && Array.isArray(rows)) { + return rows.map(r => r.definition_hash); + } else { + logger.debug(`Unable to retreive list of blocks.definition_hash from db (no result)`); + return null; + } + } catch (e) { + logger.debug(`Unable to retreive list of blocks.definition_hash from db (exception: ${e})`); + return null; + } + } + + public async $getBlocksByDefinitionHash(definitionHash: string): Promise { + try { + const [rows]: any = await database.query(`SELECT hash FROM blocks WHERE definition_hash = ?`, [definitionHash]); + if (rows && Array.isArray(rows)) { + return rows.map(r => r.hash); + } else { + logger.debug(`Unable to retreive list of blocks for definition hash ${definitionHash} from db (no result)`); + return null; + } + } catch (e) { + logger.debug(`Unable to retreive list of blocks for definition hash ${definitionHash} from db (exception: ${e})`); + return null; + } + } } export default new Blocks(); diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index dc8c7291a..4f43bd9d2 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 94; + private static currentVersion = 95; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -1118,6 +1118,18 @@ class DatabaseMigration { } await this.updateToSchemaVersion(94); } + + // blocks pools-v2.json hash + if (databaseSchemaVersion < 95) { + let poolJsonSha = 'f737d86571d190cf1a1a3cf5fd86b33ba9624254'; + const [poolJsonShaDb]: any[] = await DB.query(`SELECT string FROM state WHERE name = 'pools_json_sha'`); + if (poolJsonShaDb?.length > 0) { + poolJsonSha = poolJsonShaDb[0].string; + } + await this.$executeQuery(`ALTER TABLE blocks ADD definition_hash varchar(255) NOT NULL DEFAULT "${poolJsonSha}"`); + await this.$executeQuery('ALTER TABLE blocks ADD INDEX `definition_hash` (`definition_hash`)'); + await this.updateToSchemaVersion(95); + } } /** diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index 289389d5e..2fd55d6c5 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -19,15 +19,6 @@ class PoolsParser { 'addresses': '[]', 'slug': 'unknown' }; - private uniqueLogs: string[] = []; - - private uniqueLog(loggerFunction: any, msg: string): void { - if (this.uniqueLogs.includes(msg)) { - return; - } - this.uniqueLogs.push(msg); - loggerFunction(msg); - } public setMiningPools(pools): void { for (const pool of pools) { diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index dc703af21..e1da4be6f 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -325,6 +325,8 @@ export interface BlockExtension { // Requires coinstatsindex, will be set to NULL otherwise utxoSetSize: number | null; totalInputAmt: number | null; + // pools-v2.json git hash + definitionHash: string | undefined; } /** diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 424a668c7..f673e574b 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -15,6 +15,7 @@ import blocks from '../api/blocks'; import BlocksAuditsRepository from './BlocksAuditsRepository'; import transactionUtils from '../api/transaction-utils'; import { parseDATUMTemplateCreator } from '../utils/bitcoin-script'; +import poolsUpdater from '../tasks/pools-updater'; interface DatabaseBlock { id: string; @@ -114,16 +115,16 @@ class BlocksRepository { try { const query = `INSERT INTO blocks( - height, hash, blockTimestamp, size, - weight, tx_count, coinbase_raw, difficulty, - pool_id, fees, fee_span, median_fee, - reward, version, bits, nonce, - merkle_root, previous_block_hash, avg_fee, avg_fee_rate, - median_timestamp, header, coinbase_address, coinbase_addresses, - coinbase_signature, utxoset_size, utxoset_change, avg_tx_size, - total_inputs, total_outputs, total_input_amt, total_output_amt, - fee_percentiles, segwit_total_txs, segwit_total_size, segwit_total_weight, - median_fee_amt, coinbase_signature_ascii + height, hash, blockTimestamp, size, + weight, tx_count, coinbase_raw, difficulty, + pool_id, fees, fee_span, median_fee, + reward, version, bits, nonce, + merkle_root, previous_block_hash, avg_fee, avg_fee_rate, + median_timestamp, header, coinbase_address, coinbase_addresses, + coinbase_signature, utxoset_size, utxoset_change, avg_tx_size, + total_inputs, total_outputs, total_input_amt, total_output_amt, + fee_percentiles, segwit_total_txs, segwit_total_size, segwit_total_weight, + median_fee_amt, coinbase_signature_ascii, definition_hash ) VALUE ( ?, ?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, @@ -134,7 +135,7 @@ class BlocksRepository { ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, - ?, ? + ?, ?, ? )`; const poolDbId = await PoolsRepository.$getPoolByUniqueId(block.extras.pool.id); @@ -181,6 +182,7 @@ class BlocksRepository { block.extras.segwitTotalWeight, block.extras.medianFeeAmt, truncatedCoinbaseSignatureAscii, + poolsUpdater.currentSha ]; await DB.query(query, params); @@ -1013,9 +1015,9 @@ class BlocksRepository { public async $savePool(id: string, poolId: number): Promise { try { await DB.query(` - UPDATE blocks SET pool_id = ? + UPDATE blocks SET pool_id = ?, definition_hash = ? WHERE hash = ?`, - [poolId, id] + [poolId, poolsUpdater.currentSha, id] ); } catch (e) { logger.err(`Cannot update block pool. Reason: ` + (e instanceof Error ? e.message : e)); diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 652383a2a..38e816d74 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -88,8 +88,8 @@ class PoolsUpdater { try { await DB.query('START TRANSACTION;'); - await poolsParser.migratePoolsJson(); await this.updateDBSha(githubSha); + await poolsParser.migratePoolsJson(); await DB.query('COMMIT;'); } catch (e) { logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, this.tag); @@ -121,7 +121,7 @@ class PoolsUpdater { /** * Fetch our latest pools-v2.json sha from the db */ - private async getShaFromDb(): Promise { + public async getShaFromDb(): Promise { try { const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"'); return (rows.length > 0 ? rows[0].string : null);