Merge pull request #4438 from mempool/mononaut/refactor-difficulty-reindexing

Refactor difficulty reindexing to process blocks in height order

backend/src/api/mining/mining.ts

@@ -15,6 +15,13 @@ import bitcoinApi from '../bitcoin/bitcoin-api-factory';
 import { IEsploraApi } from '../bitcoin/esplora-api.interface';
 import database from '../../database';
 
+interface DifficultyBlock {
+  timestamp: number,
+  height: number,
+  bits: number,
+  difficulty: number,
+}
+
 class Mining {
   private blocksPriceIndexingRunning = false;
   public lastHashrateIndexingDate: number | null = null;
@@ -421,6 +428,7 @@ class Mining {
       indexedHeights[height] = true;
     }
 
+    // gets {time, height, difficulty, bits} of blocks in ascending order of height
     const blocks: any = await BlocksRepository.$getBlocksDifficulty();
     const genesisBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(await bitcoinApi.$getBlockHash(0));
     let currentDifficulty = genesisBlock.difficulty;
@@ -436,41 +444,45 @@ class Mining {
       });
     }
 
-    const oldestConsecutiveBlock = await BlocksRepository.$getOldestConsecutiveBlock();
+    if (!blocks?.length) {
-    if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== -1) {
+      // no blocks in database yet
-      currentBits = oldestConsecutiveBlock.bits;
+      return;
-      currentDifficulty = oldestConsecutiveBlock.difficulty;
     }
 
+    const oldestConsecutiveBlock = this.getOldestConsecutiveBlock(blocks);
+
+    currentBits = oldestConsecutiveBlock.bits;
+    currentDifficulty = oldestConsecutiveBlock.difficulty;
+
     let totalBlockChecked = 0;
     let timer = new Date().getTime() / 1000;
 
     for (const block of blocks) {
+      // skip until the first block after the oldest consecutive block
+      if (block.height <= oldestConsecutiveBlock.height) {
+        continue;
+      }
+
+      // difficulty has changed between two consecutive blocks!
       if (block.bits !== currentBits) {
-        if (indexedHeights[block.height] === true) { // Already indexed
+        // skip if already indexed
-          if (block.height >= oldestConsecutiveBlock.height) {
+        if (indexedHeights[block.height] !== true) {
-            currentDifficulty = block.difficulty;
+          let adjustment = block.difficulty / currentDifficulty;
-            currentBits = block.bits;
+          adjustment = Math.round(adjustment * 1000000) / 1000000; // Remove float point noise
-          }
+
-          continue;          
+          await DifficultyAdjustmentsRepository.$saveAdjustments({
+            time: block.time,
+            height: block.height,
+            difficulty: block.difficulty,
+            adjustment: adjustment,
+          });
+
+          totalIndexed++;
         }
-
+        // update the current difficulty
-        let adjustment = block.difficulty / currentDifficulty;
+        currentDifficulty = block.difficulty;
-        adjustment = Math.round(adjustment * 1000000) / 1000000; // Remove float point noise
+        currentBits = block.bits;
-
+      }
-        await DifficultyAdjustmentsRepository.$saveAdjustments({
-          time: block.time,
-          height: block.height,
-          difficulty: block.difficulty,
-          adjustment: adjustment,
-        });
-
-        totalIndexed++;
-        if (block.height >= oldestConsecutiveBlock.height) {
-          currentDifficulty = block.difficulty;
-          currentBits = block.bits;
-        }
-    }
 
       totalBlockChecked++;
       const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
@@ -633,6 +645,17 @@ class Mining {
       default: return 86400 * scale;
     }
   }
+
+  // Finds the oldest block in a consecutive chain back from the tip
+  // assumes `blocks` is sorted in ascending height order
+  private getOldestConsecutiveBlock(blocks: DifficultyBlock[]): DifficultyBlock {
+    for (let i = blocks.length - 1; i > 0; i--) {
+      if ((blocks[i].height - blocks[i - 1].height) > 1) {
+        return blocks[i];
+      }
+    }
+    return blocks[0];
+  }
 }
 
 export default new Mining();

backend/src/repositories/BlocksRepository.ts

@@ -541,7 +541,7 @@ class BlocksRepository {
    */
   public async $getBlocksDifficulty(): Promise<object[]> {
     try {
-      const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty, bits FROM blocks`);
+      const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty, bits FROM blocks ORDER BY height ASC`);
       return rows;
     } catch (e) {
       logger.err('Cannot get blocks difficulty list from the db. Reason: ' + (e instanceof Error ? e.message : e));