From 5b62966863a8bbcdb4f1aad0ad0042ac5a746599 Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Fri, 9 Jun 2023 13:46:43 -0400
Subject: [PATCH] Add indexer task to backfill audit fee/weight stats

---
 backend/src/api/blocks.ts                     | 35 +++++++++++++++++
 backend/src/indexer.ts                        |  1 +
 .../repositories/BlocksAuditsRepository.ts    | 39 +++++++++++++++++++
 .../repositories/BlocksSummariesRepository.ts | 15 +++++++
 4 files changed, 90 insertions(+)

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index ef4db683e..116a99340 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -457,6 +457,41 @@ class Blocks {
     }
   }
 
+  /**
+   * [INDEXING] Index expected fees & weight for all audited blocks
+   */
+  public async $generateAuditStats(): Promise<void> {
+    const blockIds = await BlocksAuditsRepository.$getBlocksWithoutSummaries();
+    if (!blockIds?.length) {
+      return;
+    }
+    let timer = Date.now();
+    let indexedThisRun = 0;
+    let indexedTotal = 0;
+    logger.debug(`Indexing ${blockIds.length} block audit details`);
+    for (const hash of blockIds) {
+      const summary = await BlocksSummariesRepository.$getTemplate(hash);
+      let totalFees = 0;
+      let totalWeight = 0;
+      for (const tx of summary?.transactions || []) {
+        totalFees += tx.fee;
+        totalWeight += (tx.vsize * 4);
+      }
+      await BlocksAuditsRepository.$setSummary(hash, totalFees, totalWeight);
+
+      indexedThisRun++;
+      indexedTotal++;
+      const elapsedSeconds = (Date.now() - timer) / 1000;
+      if (elapsedSeconds > 5) {
+        const blockPerSeconds = indexedThisRun / elapsedSeconds;
+        logger.debug(`Indexed ${indexedTotal} / ${blockIds.length} block audit details (${blockPerSeconds.toFixed(1)}/s)`);
+        timer = Date.now();
+        indexedThisRun = 0;
+      }
+    }
+    logger.debug(`Indexing block audit details completed`);
+  }
+
   /**
    * [INDEXING] Index all blocks metadata for the mining dashboard
    */
diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts
index 3b16ad155..4b120867f 100644
--- a/backend/src/indexer.ts
+++ b/backend/src/indexer.ts
@@ -134,6 +134,7 @@ class Indexer {
       await mining.$generatePoolHashrateHistory();
       await blocks.$generateBlocksSummariesDatabase();
       await blocks.$generateCPFPDatabase();
+      await blocks.$generateAuditStats();
     } catch (e) {
       this.indexerRunning = false;
       logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e));
diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts
index d755654ea..1fa2b0209 100644
--- a/backend/src/repositories/BlocksAuditsRepository.ts
+++ b/backend/src/repositories/BlocksAuditsRepository.ts
@@ -18,6 +18,19 @@ class BlocksAuditRepositories {
     }
   }
 
+  public async $setSummary(hash: string, expectedFees: number, expectedWeight: number) {
+    try {
+      await DB.query(`
+        UPDATE blocks_audits SET
+        expected_fees = ?,
+        expected_weight = ?
+        WHERE hash = ?
+      `, [expectedFees, expectedWeight, hash]);
+    } catch (e: any) {
+      logger.err(`Cannot update block audit in db. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
+
   public async $getBlocksHealthHistory(div: number, interval: string | null): Promise<any> {
     try {
       let query = `SELECT UNIX_TIMESTAMP(time) as time, height, match_rate FROM blocks_audits`;
@@ -113,6 +126,32 @@ class BlocksAuditRepositories {
       throw e;
     }
   }
+
+  public async $getBlocksWithoutSummaries(): Promise<string[]> {
+    try {
+      const [fromRows]: any[] = await DB.query(`
+        SELECT height
+        FROM blocks_audits
+        WHERE expected_fees IS NULL
+        ORDER BY height DESC
+        LIMIT 1
+      `);
+      if (!fromRows?.length) {
+        return [];
+      }
+      const fromHeight = fromRows[0].height;
+      const [idRows]: any[] = await DB.query(`
+        SELECT hash
+        FROM blocks_audits
+        WHERE height <= ?
+        ORDER BY height DESC
+      `, [fromHeight]);
+      return idRows.map(row => row.hash);
+    } catch (e: any) {
+      logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
 }
 
 export default new BlocksAuditRepositories();
diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts
index 2d2c23d07..09598db03 100644
--- a/backend/src/repositories/BlocksSummariesRepository.ts
+++ b/backend/src/repositories/BlocksSummariesRepository.ts
@@ -50,6 +50,21 @@ class BlocksSummariesRepository {
     }
   }
 
+  public async $getTemplate(id: string): Promise<BlockSummary | undefined> {
+    try {
+      const [templates]: any[] = await DB.query(`SELECT * from blocks_templates WHERE id = ?`, [id]);
+      if (templates.length > 0) {
+        return {
+          id: templates[0].id,
+          transactions: JSON.parse(templates[0].template),
+        };
+      }
+    } catch (e) {
+      logger.err(`Cannot get block template for block id ${id}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+    return undefined;
+  }
+
   public async $getIndexedSummariesId(): Promise<string[]> {
     try {
       const [rows]: any[] = await DB.query(`SELECT id from blocks_summaries`);