Merge pull request #5665 from mempool/nymkappa/block-pools-json-hash

[blocks] fetch list of block hash filtered by pools-v2.json sha

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -21,6 +21,7 @@ import transactionRepository from '../../repositories/TransactionRepository';
 import rbfCache from '../rbf-cache';
 import { calculateMempoolTxCpfp } from '../cpfp';
 import { handleError } from '../../utils/api';
+import poolsUpdater from '../../tasks/pools-updater';
 
 const TXID_REGEX = /^[a-f0-9]{64}$/i;
 const BLOCK_HASH_REGEX = /^[a-f0-9]{64}$/i;
@@ -56,6 +57,10 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this))
       // Temporarily add txs/package endpoint for all backends until esplora supports it
       .post(config.MEMPOOL.API_URL_PREFIX + 'txs/package', this.$submitPackage)
+      // Internal routes
+      .get(config.MEMPOOL.API_URL_PREFIX + 'internal/blocks/definition/list', this.getBlockDefinitionHashes)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'internal/blocks/definition/current', this.getCurrentBlockDefinitionHash)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'internal/blocks/:definitionHash', this.getBlocksByDefinitionHash)
       ;
 
       if (config.MEMPOOL.BACKEND !== 'esplora') {
@@ -739,6 +744,52 @@ class BitcoinRoutes {
     }
   }
 
+  private async getBlockDefinitionHashes(req: Request, res: Response): Promise<void> {
+    try {
+      const result = await blocks.$getBlockDefinitionHashes();
+      if (!result) {
+        handleError(req, res, 503, `Service Temporarily Unavailable`);
+        return;
+      }
+      res.setHeader('content-type', 'application/json');
+      res.send(result);
+    } catch (e) {
+      handleError(req, res, 500, e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getCurrentBlockDefinitionHash(req: Request, res: Response): Promise<void> {
+    try {
+      const currentSha = await poolsUpdater.getShaFromDb();
+      if (!currentSha) {
+        handleError(req, res, 503, `Service Temporarily Unavailable`);
+        return;
+      }
+      res.setHeader('content-type', 'text/plain');
+      res.send(currentSha);
+    } catch (e) {
+      handleError(req, res, 500, e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async getBlocksByDefinitionHash(req: Request, res: Response): Promise<void> {
+    try {
+      if (typeof(req.params.definitionHash) !== 'string') {
+        res.status(400).send('Parameter "hash" must be a valid string');
+        return;
+      }
+      const blocksHash = await blocks.$getBlocksByDefinitionHash(req.params.definitionHash as string);
+      if (!blocksHash) {
+        handleError(req, res, 503, `Service Temporarily Unavailable`);
+        return;
+      }
+      res.setHeader('content-type', 'application/json');
+      res.send(blocksHash);
+    } catch (e) {
+      handleError(req, res, 500, e instanceof Error ? e.message : e);
+    }
+  }
+
   private getBlockTipHeight(req: Request, res: Response) {
     try {
       const result = blocks.getCurrentBlockHeight();

backend/src/api/blocks.ts

@@ -33,8 +33,8 @@ import AccelerationRepository from '../repositories/AccelerationRepository';
 import { calculateFastBlockCpfp, calculateGoodBlockCpfp } from './cpfp';
 import mempool from './mempool';
 import CpfpRepository from '../repositories/CpfpRepository';
-import accelerationApi from './services/acceleration';
 import { parseDATUMTemplateCreator } from '../utils/bitcoin-script';
+import database from '../database';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -1462,6 +1462,36 @@ class Blocks {
       // not a fatal error, we'll try again next time the indexer runs
     }
   }
+
+  public async $getBlockDefinitionHashes(): Promise<string[] | null> {
+    try {
+      const [rows]: any = await database.query(`SELECT DISTINCT(definition_hash) FROM blocks`);
+      if (rows && Array.isArray(rows)) {
+        return rows.map(r => r.definition_hash);
+      } else {
+        logger.debug(`Unable to retreive list of blocks.definition_hash from db (no result)`);
+        return null;
+      }
+    } catch (e) {
+      logger.debug(`Unable to retreive list of blocks.definition_hash from db (exception: ${e})`);
+      return null;
+    }
+  }
+
+  public async $getBlocksByDefinitionHash(definitionHash: string): Promise<string[] | null> {
+    try {
+      const [rows]: any = await database.query(`SELECT hash FROM blocks WHERE definition_hash = ?`, [definitionHash]);
+      if (rows && Array.isArray(rows)) {
+        return rows.map(r => r.hash);
+      } else {
+        logger.debug(`Unable to retreive list of blocks for definition hash ${definitionHash} from db (no result)`);
+        return null;
+      }
+    } catch (e) {
+      logger.debug(`Unable to retreive list of blocks for definition hash ${definitionHash} from db (exception: ${e})`);
+      return null;
+    }
+  }
 }
 
 export default new Blocks();

backend/src/api/database-migration.ts

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 94;
+  private static currentVersion = 95;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -1118,6 +1118,18 @@ class DatabaseMigration {
       }
       await this.updateToSchemaVersion(94);
     }
+
+    // blocks pools-v2.json hash
+    if (databaseSchemaVersion < 95) {
+      let poolJsonSha = 'f737d86571d190cf1a1a3cf5fd86b33ba9624254';
+      const [poolJsonShaDb]: any[] = await DB.query(`SELECT string FROM state WHERE name = 'pools_json_sha'`);
+      if (poolJsonShaDb?.length > 0) {
+        poolJsonSha = poolJsonShaDb[0].string;
+      }
+      await this.$executeQuery(`ALTER TABLE blocks ADD definition_hash varchar(255) NOT NULL DEFAULT "${poolJsonSha}"`);
+      await this.$executeQuery('ALTER TABLE blocks ADD INDEX `definition_hash` (`definition_hash`)');
+      await this.updateToSchemaVersion(95);
+    }
   }
 
   /**

backend/src/api/pools-parser.ts

@@ -19,15 +19,6 @@ class PoolsParser {
     'addresses': '[]',
     'slug': 'unknown'
   };
-  private uniqueLogs: string[] = [];
-
-  private uniqueLog(loggerFunction: any, msg: string): void {
-    if (this.uniqueLogs.includes(msg)) {
-      return;
-    }
-    this.uniqueLogs.push(msg);
-    loggerFunction(msg);
-  }
 
   public setMiningPools(pools): void {
     for (const pool of pools) {

backend/src/mempool.interfaces.ts

@@ -325,6 +325,8 @@ export interface BlockExtension {
   // Requires coinstatsindex, will be set to NULL otherwise
   utxoSetSize: number | null;
   totalInputAmt: number | null;
+  // pools-v2.json git hash
+  definitionHash: string | undefined;
 }
 
 /**

backend/src/repositories/BlocksRepository.ts

@@ -15,6 +15,7 @@ import blocks from '../api/blocks';
 import BlocksAuditsRepository from './BlocksAuditsRepository';
 import transactionUtils from '../api/transaction-utils';
 import { parseDATUMTemplateCreator } from '../utils/bitcoin-script';
+import poolsUpdater from '../tasks/pools-updater';
 
 interface DatabaseBlock {
   id: string;
@@ -114,16 +115,16 @@ class BlocksRepository {
 
     try {
       const query = `INSERT INTO blocks(
-        height,             hash,                blockTimestamp,    size,
+        height,             hash,                     blockTimestamp,    size,
-        weight,             tx_count,            coinbase_raw,      difficulty,
+        weight,             tx_count,                 coinbase_raw,      difficulty,
-        pool_id,            fees,                fee_span,          median_fee,
+        pool_id,            fees,                     fee_span,          median_fee,
-        reward,             version,             bits,              nonce,
+        reward,             version,                  bits,              nonce,
-        merkle_root,        previous_block_hash, avg_fee,           avg_fee_rate,
+        merkle_root,        previous_block_hash,      avg_fee,           avg_fee_rate,
-        median_timestamp,   header,              coinbase_address,  coinbase_addresses,
+        median_timestamp,   header,                   coinbase_address,  coinbase_addresses,
-        coinbase_signature, utxoset_size,        utxoset_change,    avg_tx_size,
+        coinbase_signature, utxoset_size,             utxoset_change,    avg_tx_size,
-        total_inputs,       total_outputs,       total_input_amt,   total_output_amt,
+        total_inputs,       total_outputs,            total_input_amt,   total_output_amt,
-        fee_percentiles,    segwit_total_txs,    segwit_total_size, segwit_total_weight,
+        fee_percentiles,    segwit_total_txs,         segwit_total_size, segwit_total_weight,
-        median_fee_amt,     coinbase_signature_ascii
+        median_fee_amt,     coinbase_signature_ascii, definition_hash
       ) VALUE (
         ?, ?, FROM_UNIXTIME(?), ?,
         ?, ?, ?, ?,
@@ -134,7 +135,7 @@ class BlocksRepository {
         ?, ?, ?, ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
-        ?, ?
+        ?, ?, ?
       )`;
 
       const poolDbId = await PoolsRepository.$getPoolByUniqueId(block.extras.pool.id);
@@ -181,6 +182,7 @@ class BlocksRepository {
         block.extras.segwitTotalWeight,
         block.extras.medianFeeAmt,
         truncatedCoinbaseSignatureAscii,
+        poolsUpdater.currentSha
       ];
 
       await DB.query(query, params);
@@ -1013,9 +1015,9 @@ class BlocksRepository {
   public async $savePool(id: string, poolId: number): Promise<void> {
     try {
       await DB.query(`
-        UPDATE blocks SET pool_id = ?
+        UPDATE blocks SET pool_id = ?, definition_hash = ?
         WHERE hash = ?`,
-        [poolId, id]
+        [poolId, poolsUpdater.currentSha, id]
       );
     } catch (e) {
       logger.err(`Cannot update block pool. Reason: ` + (e instanceof Error ? e.message : e));

backend/src/tasks/pools-updater.ts

@@ -88,8 +88,8 @@ class PoolsUpdater {
 
       try {
         await DB.query('START TRANSACTION;');
-        await poolsParser.migratePoolsJson();
         await this.updateDBSha(githubSha);
+        await poolsParser.migratePoolsJson();
         await DB.query('COMMIT;');
       } catch (e) {
         logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, this.tag);
@@ -121,7 +121,7 @@ class PoolsUpdater {
   /**
    * Fetch our latest pools-v2.json sha from the db
    */
-  private async getShaFromDb(): Promise<string | null> {
+  public async getShaFromDb(): Promise<string | null> {
     try {
       const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"');
       return (rows.length > 0 ? rows[0].string : null);