Set block/:hash/summary expiration to 1 month - Support re-org for block summaries

backend/src/api/blocks.ts

@@ -254,16 +254,22 @@ class Blocks {
     try {
       // Get all indexed block hash
       const indexedBlocks = await blocksRepository.$getIndexedBlocks();
-      const indexedBlockSummariesHashes = await BlocksSummariesRepository.$getIndexedSummariesId();
+      const indexedBlockSummariesHashesArray = await BlocksSummariesRepository.$getIndexedSummariesId();
+
+      const indexedBlockSummariesHashes = {}; // Use a map for faster seek during the indexing loop
+      for (const hash of indexedBlockSummariesHashesArray) {
+        indexedBlockSummariesHashes[hash] = true;
+      }
 
       // Logging
-      let totalIndexed = indexedBlockSummariesHashes.length;
+      let newlyIndexed = 0;
+      let totalIndexed = indexedBlockSummariesHashesArray.length;
       let indexedThisRun = 0;
       let timer = new Date().getTime() / 1000;
       const startedAt = new Date().getTime() / 1000;
 
       for (const block of indexedBlocks) {
-        if (indexedBlockSummariesHashes.includes(block.hash)) {
+        if (indexedBlockSummariesHashes[block.hash] === true) {
           continue;
         }
 
@@ -284,7 +290,9 @@ class Blocks {
         // Logging
         indexedThisRun++;
         totalIndexed++;
+        newlyIndexed++;
       }
+      logger.notice(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`);
     } catch (e) {
       logger.err(`Blocks summaries indexing failed. Reason: ${(e instanceof Error ? e.message : e)}`);
     }
@@ -293,10 +301,10 @@ class Blocks {
   /**
    * [INDEXING] Index all blocks metadata for the mining dashboard
    */
-  public async $generateBlockDatabase() {
+  public async $generateBlockDatabase(): Promise<boolean> {
     const blockchainInfo = await bitcoinClient.getBlockchainInfo();
     if (blockchainInfo.blocks !== blockchainInfo.headers) { // Wait for node to sync
-      return;
+      return false;
     }
 
     try {
@@ -364,13 +372,16 @@ class Blocks {
     } catch (e) {
       logger.err('Block indexing failed. Trying again later. Reason: ' + (e instanceof Error ? e.message : e));
       loadingIndicators.setProgress('block-indexing', 100);
-      return;
+      return false;
     }
 
     const chainValid = await BlocksRepository.$validateChain();
     if (!chainValid) {
       indexer.reindex();
+      return false;
     }
+
+    return true;
   }
 
   public async $updateBlocks() {
@@ -435,9 +446,12 @@ class Blocks {
             // We assume there won't be a reorg with more than 10 block depth
             await BlocksRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
             await HashratesRepository.$deleteLastEntries();
+            await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
             for (let i = 10; i >= 0; --i) {
-              await this.$indexBlock(lastBlock['height'] - i);
+              const newBlock = await this.$indexBlock(lastBlock['height'] - i);
+              await this.$getStrippedBlockTransactions(newBlock.id, true, true);
             }
+            logger.info(`Re-indexed 10 blocks and summaries`);
           }
           await blocksRepository.$saveBlockInDatabase(blockExtended);
 
@@ -530,7 +544,9 @@ class Blocks {
     return blockExtended;
   }
 
-  public async $getStrippedBlockTransactions(hash: string, skipMemoryCache: boolean = false, skipDBLookup: boolean = false): Promise<TransactionStripped[]> {
+  public async $getStrippedBlockTransactions(hash: string, skipMemoryCache: boolean = false,
+    skipDBLookup: boolean = false): Promise<TransactionStripped[]>
+  {
     if (skipMemoryCache === false) {
       // Check the memory cache
       const cachedSummary = this.getBlockSummaries().find((b) => b.id === hash);

backend/src/api/database-migration.ts

@@ -521,8 +521,9 @@ class DatabaseMigration {
       height int(10) unsigned NOT NULL,
       id varchar(65) NOT NULL,
       transactions JSON NOT NULL,
-      PRIMARY KEY (id)
+      PRIMARY KEY (id),
-    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; 
+      INDEX (height)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
   public async $truncateIndexedData(tables: string[]) {

backend/src/indexer.ts

@@ -29,7 +29,13 @@ class Indexer {
     this.indexerRunning = true;
 
     try {
-      await blocks.$generateBlockDatabase();
+      const chainValid = await blocks.$generateBlockDatabase();
+      if (chainValid === false) {
+        // Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration
+        this.indexerRunning = false;
+        return;
+      }
+
       await this.$resetHashratesIndexingState();
       await mining.$generateNetworkHashrateHistory();
       await mining.$generatePoolHashrateHistory();

backend/src/repositories/BlocksRepository.ts

@@ -6,6 +6,7 @@ import { prepareBlock } from '../utils/blocks-utils';
 import PoolsRepository from './PoolsRepository';
 import HashratesRepository from './HashratesRepository';
 import { escape } from 'mysql2';
+import BlocksSummariesRepository from './BlocksSummariesRepository';
 
 class BlocksRepository {
   /**
@@ -495,6 +496,7 @@ class BlocksRepository {
         if (blocks[idx].previous_block_hash !== blocks[idx - 1].hash) {
           logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}, re-indexing newer blocks and hashrates`);
           await this.$deleteBlocksFrom(blocks[idx - 1].height);
+          await BlocksSummariesRepository.$deleteBlocksFrom(blocks[idx - 1].height);
           await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800);
           return false;
         }

backend/src/repositories/BlocksSummariesRepository.ts

@@ -40,6 +40,19 @@ class BlocksSummariesRepository {
 
     return [];
   }
+
+  /**
+   * Delete blocks from the database from blockHeight
+   */
+   public async $deleteBlocksFrom(blockHeight: number) {
+    logger.info(`Delete newer blocks summary from height ${blockHeight} from the database`);
+
+    try {
+      await DB.query(`DELETE FROM blocks_summaries where height >= ${blockHeight}`);
+    } catch (e) {
+      logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e));
+    }
+  }
 }
 
 export default new BlocksSummariesRepository();

backend/src/routes.ts

@@ -729,7 +729,7 @@ class Routes {
   public async getStrippedBlockTransactions(req: Request, res: Response) {
     try {
       const transactions = await blocks.$getStrippedBlockTransactions(req.params.hash);
-      res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
       res.json(transactions);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);

frontend/src/app/services/api.service.ts

@@ -1,7 +1,7 @@
 import { Injectable } from '@angular/core';
 import { HttpClient, HttpParams } from '@angular/common/http';
 import { CpfpInfo, OptimizedMempoolStats, AddressInformation, LiquidPegs, ITranslators,
-         PoolsStats, PoolStat, BlockExtended, TransactionStripped, RewardStats } from '../interfaces/node-api.interface';
+  PoolStat, BlockExtended, TransactionStripped, RewardStats } from '../interfaces/node-api.interface';
 import { Observable } from 'rxjs';
 import { StateService } from './state.service';
 import { WebsocketResponse } from '../interfaces/websocket.interface';