Add block classification indexing

backend/src/api/blocks.ts

@@ -561,6 +561,115 @@ class Blocks {
     logger.debug(`Indexing block audit details completed`);
   }
 
+  /**
+   * [INDEXING] Index transaction classification flags for Goggles
+   */
+  public async $classifyBlocks(): Promise<void> {
+    // classification requires an esplora backend
+    if (config.MEMPOOL.BACKEND !== 'esplora') {
+      return;
+    }
+
+    const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+    const currentBlockHeight = blockchainInfo.blocks;
+
+    const unclassifiedBlocksList = await BlocksSummariesRepository.$getSummariesWithVersion(0);
+    const unclassifiedTemplatesList = await BlocksSummariesRepository.$getTemplatesWithVersion(0);
+
+    // nothing to do
+    if (!unclassifiedBlocksList.length && !unclassifiedTemplatesList.length) {
+      return;
+    }
+
+    let timer = Date.now();
+    let indexedThisRun = 0;
+    let indexedTotal = 0;
+
+    const minHeight = Math.min(
+      unclassifiedBlocksList[unclassifiedBlocksList.length - 1].height ?? Infinity,
+      unclassifiedTemplatesList[unclassifiedTemplatesList.length - 1].height ?? Infinity,
+    );
+    const numToIndex = Math.max(
+      unclassifiedBlocksList.length,
+      unclassifiedTemplatesList.length,
+    );
+
+    const unclassifiedBlocks = {};
+    const unclassifiedTemplates = {};
+    for (const block of unclassifiedBlocksList) {
+      unclassifiedBlocks[block.height] = block.id;
+    }
+    for (const template of unclassifiedTemplatesList) {
+      unclassifiedTemplates[template.height] = template.id;
+    }
+
+    logger.debug(`Classifying blocks and templates from #${currentBlockHeight} to #${minHeight}`, logger.tags.goggles);
+
+    for (let height = currentBlockHeight; height >= 0; height--) {
+      let txs: TransactionExtended[] | null = null;
+      if (unclassifiedBlocks[height]) {
+        const blockHash = unclassifiedBlocks[height];
+        // fetch transactions
+        txs = (await bitcoinApi.$getTxsForBlock(blockHash)).map(tx => transactionUtils.extendTransaction(tx));
+        // add CPFP
+        const cpfpSummary = Common.calculateCpfp(height, txs, true);
+        // classify
+        const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions);
+        BlocksSummariesRepository.$saveTransactions(height, blockHash, classifiedTxs, 1);
+      }
+      if (unclassifiedTemplates[height]) {
+        // classify template
+        const blockHash = unclassifiedTemplates[height];
+        const template = await BlocksSummariesRepository.$getTemplate(blockHash);
+        const alreadyClassified = template?.transactions.reduce((classified, tx) => (classified || tx.flags > 0), false);
+        let classifiedTemplate = template?.transactions || [];
+        if (!alreadyClassified) {
+          const templateTxs: (TransactionExtended | TransactionClassified)[] = [];
+          const blockTxMap: { [txid: string]: TransactionExtended } = {};
+          for (const tx of (txs || [])) {
+            blockTxMap[tx.txid] = tx;
+          }
+          for (const templateTx of (template?.transactions || [])) {
+            let tx: TransactionExtended | null = blockTxMap[templateTx.txid];
+            if (!tx) {
+              try {
+                tx = await transactionUtils.$getTransactionExtended(templateTx.txid, false, true, false);
+              } catch (e) {
+                // transaction probably not found
+              }
+            }
+            templateTxs.push(tx || templateTx);
+          }
+          const cpfpSummary = Common.calculateCpfp(height, txs?.filter(tx => tx.effectiveFeePerVsize != null) as TransactionExtended[], true);
+          // classify
+          const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions);
+          const classifiedTxMap: { [txid: string]: TransactionClassified } = {};
+          for (const tx of classifiedTxs) {
+            classifiedTxMap[tx.txid] = tx;
+          }
+          classifiedTemplate = classifiedTemplate.map(tx => {
+            if (classifiedTxMap[tx.txid]) {
+              tx.flags = classifiedTxMap[tx.txid].flags || 0;
+            }
+            return tx;
+          });
+        }
+        BlocksSummariesRepository.$saveTemplate({ height, template: { id: blockHash, transactions: classifiedTemplate }, version: 1 });
+      }
+
+      // timing & logging
+      indexedThisRun++;
+      indexedTotal++;
+      const elapsedSeconds = (Date.now() - timer) / 1000;
+      if (elapsedSeconds > 5) {
+        const perSecond = indexedThisRun / elapsedSeconds;
+        logger.debug(`Classified #${height}: ${indexedTotal} / ${numToIndex} blocks (${perSecond.toFixed(1)}/s)`);
+        timer = Date.now();
+        indexedThisRun = 0;
+      }
+    }
+  }
+
   /**
    * [INDEXING] Index all blocks metadata for the mining dashboard
    */
@@ -966,6 +1075,7 @@ class Blocks {
 
     let height = blockHeight;
     let summary: BlockSummary;
+    let summaryVersion = 0;
     if (cpfpSummary && !Common.isLiquid()) {
       summary = {
         id: hash,
@@ -980,10 +1090,12 @@ class Blocks {
           };
         }),
       };
+      summaryVersion = 1;
     } else {
       if (config.MEMPOOL.BACKEND === 'esplora') {
         const txs = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx));
         summary = this.summarizeBlockTransactions(hash, txs);
+        summaryVersion = 1;
       } else {
         // Call Core RPC
         const block = await bitcoinClient.getBlock(hash, 2);
@@ -998,7 +1110,7 @@ class Blocks {
 
     // Index the response if needed
     if (Common.blocksSummariesIndexingEnabled() === true) {
-      await BlocksSummariesRepository.$saveTransactions(height, hash, summary.transactions);
+      await BlocksSummariesRepository.$saveTransactions(height, hash, summary.transactions, summaryVersion);
     }
 
     return summary.transactions;
@@ -1114,16 +1226,18 @@ class Blocks {
         if (cleanBlock.fee_amt_percentiles === null) {
 
           let summary;
+          let summaryVersion = 0;
           if (config.MEMPOOL.BACKEND === 'esplora') {
             const txs = (await bitcoinApi.$getTxsForBlock(cleanBlock.hash)).map(tx => transactionUtils.extendTransaction(tx));
             summary = this.summarizeBlockTransactions(cleanBlock.hash, txs);
+            summaryVersion = 1;
           } else {
             // Call Core RPC
             const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
             summary = this.summarizeBlock(block);
           }
 
-          await BlocksSummariesRepository.$saveTransactions(cleanBlock.height, cleanBlock.hash, summary.transactions);
+          await BlocksSummariesRepository.$saveTransactions(cleanBlock.height, cleanBlock.hash, summary.transactions, summaryVersion);
           cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
         }
         if (cleanBlock.fee_amt_percentiles !== null) {