From 458f24c9f21a84269c4fd6e67ec57eef84ed4845 Mon Sep 17 00:00:00 2001
From: nymkappa <1612910616@pm.me>
Date: Sat, 18 Feb 2023 11:26:13 +0900
Subject: [PATCH] Compute median fee and fee percentiles in sats

---
 backend/src/api/blocks.ts                     | 18 ++++++--
 backend/src/api/database-migration.ts         |  2 +-
 backend/src/mempool.interfaces.ts             |  4 +-
 backend/src/repositories/BlocksRepository.ts  | 19 +++++++++
 .../repositories/BlocksSummariesRepository.ts | 42 +++++++++++++++++++
 5 files changed, 79 insertions(+), 6 deletions(-)

diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts
index 006d5f055..ccf7bd2f4 100644
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -203,10 +203,13 @@ class Blocks {
       blk.extras.segwitTotalWeight = stats.swtotal_weight;
     }
 
-    blk.extras.feePercentiles = [], // TODO
-    blk.extras.medianFeeAmt = 0; // TODO
     blk.extras.blockTime = 0; // TODO
     blk.extras.orphaned = false; // TODO
+
+    blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id);
+    if (blk.extras.feePercentiles !== null) {
+      blk.extras.medianFeeAmt = blk.extras.feePercentiles[3];
+    }
   
     blk.extras.virtualSize = block.weight / 4.0;
     if (blk.extras.coinbaseTx.vout.length > 0) {
@@ -791,7 +794,6 @@ class Blocks {
           continue;
         }
       }
-
       delete(block.hash);
       delete(block.previous_block_hash);
       delete(block.pool_name);
@@ -800,6 +802,16 @@ class Blocks {
       delete(block.pool_regexes);
       delete(block.median_timestamp);
 
+      // This requires `blocks_summaries` to be available. It takes a very long
+      // time to index this table so we just try to serve the data the best we can
+      if (block.fee_percentiles === null) {
+        block.fee_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id); 
+        if (block.fee_percentiles !== null) {
+          block.median_fee_amt = block.fee_percentiles[3];
+          await blocksRepository.$saveFeePercentilesForBlockId(block.id, block.fee_percentiles);
+        }
+      }
+
       blocks.push(block);
       fromHeight++;
     }
diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts
index 2c6adfd1b..352abfbfe 100644
--- a/backend/src/api/database-migration.ts
+++ b/backend/src/api/database-migration.ts
@@ -772,7 +772,7 @@ class DatabaseMigration {
       ADD total_outputs int unsigned NOT NULL,
       ADD total_output_amt bigint unsigned NOT NULL,
       ADD fee_percentiles longtext NULL,
-      ADD median_fee_amt int unsigned NOT NULL,
+      ADD median_fee_amt int unsigned NULL,
       ADD segwit_total_txs int unsigned NOT NULL,
       ADD segwit_total_size int unsigned NOT NULL,
       ADD segwit_total_weight int unsigned NOT NULL,
diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts
index a1a9e1687..a7e7c4ec6 100644
--- a/backend/src/mempool.interfaces.ts
+++ b/backend/src/mempool.interfaces.ts
@@ -171,8 +171,8 @@ export interface BlockExtension {
   totalInputs?: number;
   totalOutputs?: number;
   totalOutputAmt?: number;
-  medianFeeAmt?: number;
-  feePercentiles?: number[],
+  medianFeeAmt?: number | null;
+  feePercentiles?: number[] | null,
   segwitTotalTxs?: number;
   segwitTotalSize?: number;
   segwitTotalWeight?: number;
diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts
index d7811f601..cc0b43fe9 100644
--- a/backend/src/repositories/BlocksRepository.ts
+++ b/backend/src/repositories/BlocksRepository.ts
@@ -811,6 +811,25 @@ class BlocksRepository {
       throw e;
     }
   }
+
+  /**
+   * Save indexed median fee to avoid recomputing it later
+   * 
+   * @param id 
+   * @param feePercentiles 
+   */
+  public async $saveFeePercentilesForBlockId(id: string, feePercentiles: number[]): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE blocks SET fee_percentiles = ?, median_fee_amt = ?
+        WHERE hash = ?`,
+        [JSON.stringify(feePercentiles), feePercentiles[3], id]
+      );
+    } catch (e) {
+      logger.err(`Cannot update block fee_percentiles. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
 }
 
 export default new BlocksRepository();
diff --git a/backend/src/repositories/BlocksSummariesRepository.ts b/backend/src/repositories/BlocksSummariesRepository.ts
index 1406a1d07..ebc83b7dd 100644
--- a/backend/src/repositories/BlocksSummariesRepository.ts
+++ b/backend/src/repositories/BlocksSummariesRepository.ts
@@ -80,6 +80,48 @@ class BlocksSummariesRepository {
       logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e));
     }
   }
+
+  /**
+   * Get the fee percentiles if the block has already been indexed, [] otherwise
+   * 
+   * @param id 
+   */
+  public async $getFeePercentilesByBlockId(id: string): Promise<number[] | null> {
+    try {
+      const [rows]: any[] = await DB.query(`
+        SELECT transactions
+        FROM blocks_summaries
+        WHERE id = ?`,
+        [id]
+      );
+      if (rows === null || rows.length === 0) {
+        return null;
+      }
+
+      const transactions = JSON.parse(rows[0].transactions);
+      if (transactions === null) {
+        return null;
+      }
+
+      transactions.shift(); // Ignore coinbase
+      transactions.sort((a: any, b: any) => a.fee - b.fee);
+      const fees = transactions.map((t: any) => t.fee);
+
+      return [
+        fees[0], // min
+        fees[Math.max(0, Math.floor(fees.length * 0.1) - 1)], // 10th
+        fees[Math.max(0, Math.floor(fees.length * 0.25) - 1)], // 25th
+        fees[Math.max(0, Math.floor(fees.length * 0.5) - 1)], // median
+        fees[Math.max(0, Math.floor(fees.length * 0.75) - 1)], // 75th
+        fees[Math.max(0, Math.floor(fees.length * 0.9) - 1)], // 90th
+        fees[fees.length - 1], // max
+      ];
+
+    } catch (e) {
+      logger.err(`Cannot get block summaries transactions. Reason: ` + (e instanceof Error ? e.message : e));
+      return null;
+    }
+  }
 }
 
 export default new BlocksSummariesRepository();