Merge branch 'master' into mononaut/block-health-calculation

2023-02-26 17:39:50 +09:00
parent 867c738e5e ab8699de45
commit 63ca463f09
78 changed files with 3438 additions and 1519 deletions
--- a/backend/src/fixtures/mempool-config.template.json
+++ b/backend/src/fixtures/mempool-config.template.json
@@ -3,12 +3,11 @@
    "ENABLED": true,
    "NETWORK": "__MEMPOOL_NETWORK__",
    "BACKEND": "__MEMPOOL_BACKEND__",
-    "ENABLED": true,
    "BLOCKS_SUMMARIES_INDEXING": true,
    "HTTP_PORT": 1,
    "SPAWN_CLUSTER_PROCS": 2,
    "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",
-    "AUTOMATIC_BLOCK_REINDEXING": true,
+    "AUTOMATIC_BLOCK_REINDEXING": false,
    "POLL_RATE_MS": 3,
    "CACHE_DIR": "__MEMPOOL_CACHE_DIR__",
    "CLEAR_PROTECTION_MINUTES": 4,
@@ -28,7 +27,8 @@
    "AUDIT": "__MEMPOOL_AUDIT__",
    "ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__",
    "ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__",
-    "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__"
+    "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__",
+    "MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__"
  },
  "CORE_RPC": {
    "HOST": "__CORE_RPC_HOST__",
--- a/backend/src/tests/config.test.ts
+++ b/backend/src/tests/config.test.ts
@@ -36,11 +36,12 @@ describe('Mempool Backend Config', () => {
        USER_AGENT: 'mempool',
        STDOUT_LOG_MIN_PRIORITY: 'debug',
        POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
-        POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
+        POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
        AUDIT: false,
        ADVANCED_GBT_AUDIT: false,
        ADVANCED_GBT_MEMPOOL: false,
        CPFP_INDEXING: false,
+        MAX_BLOCKS_BULK_QUERY: 0,
      });

      expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
--- a/backend/src/api/bitcoin/bitcoin-api.ts
+++ b/backend/src/api/bitcoin/bitcoin-api.ts
@@ -28,6 +28,7 @@ class BitcoinApi implements AbstractBitcoinApi {
      size: block.size,
      weight: block.weight,
      previousblockhash: block.previousblockhash,
+      medianTime: block.mediantime,
    };
  }

--- a/backend/src/api/bitcoin/bitcoin.routes.ts
+++ b/backend/src/api/bitcoin/bitcoin.routes.ts
@@ -95,6 +95,8 @@ class BitcoinRoutes {
      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions)
      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary)
      .post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this))
      ;

      if (config.MEMPOOL.BACKEND !== 'esplora') {
@@ -402,6 +404,41 @@ class BitcoinRoutes {
    }
  }

+  private async getBlocksByBulk(req: Request, res: Response) {
+    try {
+      if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented
+        return res.status(404).send(`This API is only available for Bitcoin networks`);
+      }
+      if (config.MEMPOOL.MAX_BLOCKS_BULK_QUERY <= 0) {
+        return res.status(404).send(`This API is disabled. Set config.MEMPOOL.MAX_BLOCKS_BULK_QUERY to a positive number to enable it.`);
+      }
+      if (!Common.indexingEnabled()) {
+        return res.status(404).send(`Indexing is required for this API`);
+      }
+
+      const from = parseInt(req.params.from, 10);
+      if (!req.params.from || from < 0) {
+        return res.status(400).send(`Parameter 'from' must be a block height (integer)`);
+      }
+      const to = req.params.to === undefined ? await bitcoinApi.$getBlockHeightTip() : parseInt(req.params.to, 10);
+      if (to < 0) {
+        return res.status(400).send(`Parameter 'to' must be a block height (integer)`);
+      }
+      if (from > to) {
+        return res.status(400).send(`Parameter 'to' must be a higher block height than 'from'`);
+      }
+      if ((to - from + 1) > config.MEMPOOL.MAX_BLOCKS_BULK_QUERY) {
+        return res.status(400).send(`You can only query ${config.MEMPOOL.MAX_BLOCKS_BULK_QUERY} blocks at once.`);
+      }
+
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(await blocks.$getBlocksBetweenHeight(from, to));
+
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
  private async getLegacyBlocks(req: Request, res: Response) {
    try {
      const returnBlocks: IEsploraApi.Block[] = [];
--- a/backend/src/api/bitcoin/esplora-api.interface.ts
+++ b/backend/src/api/bitcoin/esplora-api.interface.ts
@@ -88,6 +88,7 @@ export namespace IEsploraApi {
    size: number;
    weight: number;
    previousblockhash: string;
+    medianTime?: number;
  }

  export interface Address {
--- a/backend/src/api/blocks.ts
+++ b/backend/src/api/blocks.ts
@@ -25,6 +25,7 @@ import mining from './mining/mining';
 import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
 import PricesRepository from '../repositories/PricesRepository';
 import priceUpdater from '../tasks/price-updater';
+import chainTips from './chain-tips';

 class Blocks {
  private blocks: BlockExtended[] = [];
@@ -165,33 +166,80 @@ class Blocks {
   * @returns BlockExtended
   */
  private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
-    const blockExtended: BlockExtended = Object.assign({ extras: {} }, block);
-    blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
-    blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
-    blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig;
-    blockExtended.extras.usd = priceUpdater.latestPrices.USD;
+    const blk: BlockExtended = Object.assign({ extras: {} }, block);
+    blk.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
+    blk.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
+    blk.extras.coinbaseRaw = blk.extras.coinbaseTx.vin[0].scriptsig;
+    blk.extras.usd = priceUpdater.latestPrices.USD;
+    blk.extras.medianTimestamp = block.medianTime;
+    blk.extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height);

    if (block.height === 0) {
-      blockExtended.extras.medianFee = 0; // 50th percentiles
-      blockExtended.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
-      blockExtended.extras.totalFees = 0;
-      blockExtended.extras.avgFee = 0;
-      blockExtended.extras.avgFeeRate = 0;
+      blk.extras.medianFee = 0; // 50th percentiles
+      blk.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
+      blk.extras.totalFees = 0;
+      blk.extras.avgFee = 0;
+      blk.extras.avgFeeRate = 0;
+      blk.extras.utxoSetChange = 0;
+      blk.extras.avgTxSize = 0;
+      blk.extras.totalInputs = 0;
+      blk.extras.totalOutputs = 1;
+      blk.extras.totalOutputAmt = 0;
+      blk.extras.segwitTotalTxs = 0;
+      blk.extras.segwitTotalSize = 0;
+      blk.extras.segwitTotalWeight = 0;
    } else {
-      const stats = await bitcoinClient.getBlockStats(block.id, [
-        'feerate_percentiles', 'minfeerate', 'maxfeerate', 'totalfee', 'avgfee', 'avgfeerate'
-      ]);
-      blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
-      blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
-      blockExtended.extras.totalFees = stats.totalfee;
-      blockExtended.extras.avgFee = stats.avgfee;
-      blockExtended.extras.avgFeeRate = stats.avgfeerate;
+      const stats = await bitcoinClient.getBlockStats(block.id);
+      blk.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
+      blk.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
+      blk.extras.totalFees = stats.totalfee;
+      blk.extras.avgFee = stats.avgfee;
+      blk.extras.avgFeeRate = stats.avgfeerate;
+      blk.extras.utxoSetChange = stats.utxo_increase;
+      blk.extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01;
+      blk.extras.totalInputs = stats.ins;
+      blk.extras.totalOutputs = stats.outs;
+      blk.extras.totalOutputAmt = stats.total_out;
+      blk.extras.segwitTotalTxs = stats.swtxs;
+      blk.extras.segwitTotalSize = stats.swtotal_size;
+      blk.extras.segwitTotalWeight = stats.swtotal_weight;
+    }
+
+    if (Common.blocksSummariesIndexingEnabled()) {
+      blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id);
+      if (blk.extras.feePercentiles !== null) {
+        blk.extras.medianFeeAmt = blk.extras.feePercentiles[3];
+      }
+    }
+  
+    blk.extras.virtualSize = block.weight / 4.0;
+    if (blk.extras.coinbaseTx.vout.length > 0) {
+      blk.extras.coinbaseAddress = blk.extras.coinbaseTx.vout[0].scriptpubkey_address ?? null;
+      blk.extras.coinbaseSignature = blk.extras.coinbaseTx.vout[0].scriptpubkey_asm ?? null;
+      blk.extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(blk.extras.coinbaseTx.vin[0].scriptsig) ?? null;
+    } else {
+      blk.extras.coinbaseAddress = null;
+      blk.extras.coinbaseSignature = null;
+      blk.extras.coinbaseSignatureAscii = null;
+    }
+
+    const header = await bitcoinClient.getBlockHeader(block.id, false);
+    blk.extras.header = header;
+
+    const coinStatsIndex = indexer.isCoreIndexReady('coinstatsindex');
+    if (coinStatsIndex !== null && coinStatsIndex.best_block_height >= block.height) {
+      const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
+      blk.extras.utxoSetSize = txoutset.txouts,
+      blk.extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000);
+    } else {
+      blk.extras.utxoSetSize = null;
+      blk.extras.totalInputAmt = null;
    }

    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
      let pool: PoolTag;
-      if (blockExtended.extras?.coinbaseTx !== undefined) {
-        pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
+      if (blk.extras?.coinbaseTx !== undefined) {
+        pool = await this.$findBlockMiner(blk.extras?.coinbaseTx);
      } else {
        if (config.DATABASE.ENABLED === true) {
          pool = await poolsRepository.$getUnknownPool();
@@ -201,10 +249,10 @@ class Blocks {
      }

      if (!pool) { // We should never have this situation in practise
-        logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` +
+        logger.warn(`Cannot assign pool to block ${blk.height} and 'unknown' pool does not exist. ` +
          `Check your "pools" table entries`);
      } else {
-        blockExtended.extras.pool = {
+        blk.extras.pool = {
          id: pool.id,
          name: pool.name,
          slug: pool.slug,
@@ -214,12 +262,12 @@ class Blocks {
      if (config.MEMPOOL.AUDIT) {
        const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
        if (auditScore != null) {
-          blockExtended.extras.matchRate = auditScore.matchRate;
+          blk.extras.matchRate = auditScore.matchRate;
        }
      }
    }

-    return blockExtended;
+    return blk;
  }

  /**
@@ -500,6 +548,7 @@ class Blocks {
      } else {
        this.currentBlockHeight++;
        logger.debug(`New block found (#${this.currentBlockHeight})!`);
+        await chainTips.updateOrphanedBlocks();
      }

      const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
@@ -688,7 +737,6 @@ class Blocks {
  }

  public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
-
    let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight;
    if (currentHeight > this.currentBlockHeight) {
      limit -= currentHeight - this.currentBlockHeight;
@@ -728,6 +776,113 @@ class Blocks {
    return returnBlocks;
  }

+  /**
+   * Used for bulk block data query
+   * 
+   * @param fromHeight 
+   * @param toHeight 
+   */
+  public async $getBlocksBetweenHeight(fromHeight: number, toHeight: number): Promise<any> {
+    if (!Common.indexingEnabled()) {
+      return [];
+    }
+
+    const blocks: any[] = [];
+
+    while (fromHeight <= toHeight) {
+      let block: any = await blocksRepository.$getBlockByHeight(fromHeight);
+      if (!block) {
+        await this.$indexBlock(fromHeight);
+        block = await blocksRepository.$getBlockByHeight(fromHeight);
+        if (!block) {
+          continue;
+        }
+      }
+
+      // Cleanup fields before sending the response
+      const cleanBlock: any = {
+        height: block.height ?? null,
+        hash: block.id ?? null,
+        timestamp: block.blockTimestamp ?? null,
+        median_timestamp: block.medianTime ?? null,
+        previous_block_hash: block.previousblockhash ?? null,
+        difficulty: block.difficulty ?? null,
+        header: block.header ?? null,
+        version: block.version ?? null,
+        bits: block.bits ?? null,
+        nonce: block.nonce ?? null,
+        size: block.size ?? null,
+        weight: block.weight ?? null,
+        tx_count: block.tx_count ?? null,
+        merkle_root: block.merkle_root ?? null,
+        reward: block.reward ?? null,
+        total_fee_amt: block.fees ?? null,
+        avg_fee_amt: block.avg_fee ?? null,
+        median_fee_amt: block.median_fee_amt ?? null,
+        fee_amt_percentiles: block.fee_percentiles ?? null,
+        avg_fee_rate: block.avg_fee_rate ?? null,
+        median_fee_rate: block.median_fee ?? null,
+        fee_rate_percentiles: block.fee_span ?? null,
+        total_inputs: block.total_inputs ?? null,
+        total_input_amt: block.total_input_amt ?? null,
+        total_outputs: block.total_outputs ?? null,
+        total_output_amt: block.total_output_amt ?? null,
+        segwit_total_txs: block.segwit_total_txs ?? null,
+        segwit_total_size: block.segwit_total_size ?? null,
+        segwit_total_weight: block.segwit_total_weight ?? null,
+        avg_tx_size: block.avg_tx_size ?? null,
+        utxoset_change: block.utxoset_change ?? null,
+        utxoset_size: block.utxoset_size ?? null,
+        coinbase_raw: block.coinbase_raw ?? null,
+        coinbase_address: block.coinbase_address ?? null,
+        coinbase_signature: block.coinbase_signature ?? null,
+        coinbase_signature_ascii: block.coinbase_signature_ascii ?? null,
+        pool_slug: block.pool_slug ?? null,
+      };
+
+      if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) {
+        cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
+        if (cleanBlock.fee_amt_percentiles === null) {
+          const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
+          const summary = this.summarizeBlock(block);
+          await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
+          cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
+        }
+        if (cleanBlock.fee_amt_percentiles !== null) {
+          cleanBlock.median_fee_amt = cleanBlock.fee_amt_percentiles[3];
+        }
+      }
+
+      cleanBlock.fee_amt_percentiles = {
+        'min': cleanBlock.fee_amt_percentiles[0],
+        'perc_10': cleanBlock.fee_amt_percentiles[1],
+        'perc_25': cleanBlock.fee_amt_percentiles[2],
+        'perc_50': cleanBlock.fee_amt_percentiles[3],
+        'perc_75': cleanBlock.fee_amt_percentiles[4],
+        'perc_90': cleanBlock.fee_amt_percentiles[5],
+        'max': cleanBlock.fee_amt_percentiles[6],
+      };
+      cleanBlock.fee_rate_percentiles = {
+        'min': cleanBlock.fee_rate_percentiles[0],
+        'perc_10': cleanBlock.fee_rate_percentiles[1],
+        'perc_25': cleanBlock.fee_rate_percentiles[2],
+        'perc_50': cleanBlock.fee_rate_percentiles[3],
+        'perc_75': cleanBlock.fee_rate_percentiles[4],
+        'perc_90': cleanBlock.fee_rate_percentiles[5],
+        'max': cleanBlock.fee_rate_percentiles[6],
+      };
+
+      // Re-org can happen after indexing so we need to always get the
+      // latest state from core
+      cleanBlock.orphans = chainTips.getOrphanedBlocksAtHeight(cleanBlock.height);
+
+      blocks.push(cleanBlock);
+      fromHeight++;
+    }
+
+    return blocks;
+  }
+
  public async $getBlockAuditSummary(hash: string): Promise<any> {
    let summary;
    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
--- a/backend/src/api/chain-tips.ts
+++ b/backend/src/api/chain-tips.ts
@@ -0,0 +1,57 @@
+import logger from "../logger";
+import bitcoinClient from "./bitcoin/bitcoin-client";
+
+export interface ChainTip {
+  height: number;
+  hash: string;
+  branchlen: number;
+  status: 'invalid' | 'active' | 'valid-fork' | 'valid-headers' | 'headers-only';
+};
+
+export interface OrphanedBlock {
+  height: number;
+  hash: string;
+  status: 'valid-fork' | 'valid-headers' | 'headers-only';
+}
+
+class ChainTips {
+  private chainTips: ChainTip[] = [];
+  private orphanedBlocks: OrphanedBlock[] = [];
+
+  public async updateOrphanedBlocks(): Promise<void> {
+    try {
+      this.chainTips = await bitcoinClient.getChainTips();
+      this.orphanedBlocks = [];
+
+      for (const chain of this.chainTips) {
+        if (chain.status === 'valid-fork' || chain.status === 'valid-headers') {
+          let block = await bitcoinClient.getBlock(chain.hash);
+          while (block && block.confirmations === -1) {
+            this.orphanedBlocks.push({
+              height: block.height,
+              hash: block.hash,
+              status: chain.status
+            });
+            block = await bitcoinClient.getBlock(block.previousblockhash);
+          }
+        }
+      }
+
+      logger.debug(`Updated orphaned blocks cache. Found ${this.orphanedBlocks.length} orphaned blocks`);
+    } catch (e) {
+      logger.err(`Cannot get fetch orphaned blocks. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  public getOrphanedBlocksAtHeight(height: number): OrphanedBlock[] {
+    const orphans: OrphanedBlock[] = [];
+    for (const block of this.orphanedBlocks) {
+      if (block.height === height) {
+        orphans.push(block);
+      }
+    }
+    return orphans;
+  }
+}
+
+export default new ChainTips();
--- a/backend/src/api/database-migration.ts
+++ b/backend/src/api/database-migration.ts
@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';

 class DatabaseMigration {
-  private static currentVersion = 53;
+  private static currentVersion = 56;
  private queryTimeout = 3600_000;
  private statisticsAddedIndexed = false;
  private uniqueLogs: string[] = [];
@@ -62,8 +62,8 @@ class DatabaseMigration {

    if (databaseSchemaVersion <= 2) {
      // Disable some spam logs when they're not relevant
-      this.uniqueLogs.push(this.blocksTruncatedMessage);
-      this.uniqueLogs.push(this.hashratesTruncatedMessage);
+      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+      this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
    }

    logger.debug('MIGRATIONS: Current state.schema_version ' + databaseSchemaVersion);
@@ -86,7 +86,7 @@ class DatabaseMigration {
      try {
        await this.$migrateTableSchemaFromVersion(databaseSchemaVersion);
        if (databaseSchemaVersion === 0) {
-          logger.notice(`MIGRATIONS: OK. Database schema has been properly initialized to version ${DatabaseMigration.currentVersion} (latest version)`);          
+          logger.notice(`MIGRATIONS: OK. Database schema has been properly initialized to version ${DatabaseMigration.currentVersion} (latest version)`);
        } else {
          logger.notice(`MIGRATIONS: OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`);
        }
@@ -300,7 +300,7 @@ class DatabaseMigration {
      await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"');
      await this.updateToSchemaVersion(27);
    }
-    
+
    if (databaseSchemaVersion < 28 && isBitcoin === true) {
      if (config.LIGHTNING.ENABLED) {
        this.uniqueLog(logger.notice, `'lightning_stats' and 'node_stats' tables have been truncated.`);
@@ -464,7 +464,7 @@ class DatabaseMigration {
        await this.$executeQuery('DROP TABLE IF EXISTS `transactions`');
        await this.$executeQuery('DROP TABLE IF EXISTS `cpfp_clusters`');
        await this.updateToSchemaVersion(52);
-      } catch(e) {
+      } catch (e) {
        logger.warn('' + (e instanceof Error ? e.message : e));
      }
    }
@@ -473,6 +473,33 @@ class DatabaseMigration {
      await this.$executeQuery('ALTER TABLE statistics MODIFY mempool_byte_weight bigint(20) UNSIGNED NOT NULL');
      await this.updateToSchemaVersion(53);
    }
+
+    if (databaseSchemaVersion < 54) {
+      this.uniqueLog(logger.notice, `'prices' table has been truncated`);
+      await this.$executeQuery(`TRUNCATE prices`);
+      if (isBitcoin === true) {
+        this.uniqueLog(logger.notice, `'blocks_prices' table has been truncated`);
+        await this.$executeQuery(`TRUNCATE blocks_prices`);
+      }
+      await this.updateToSchemaVersion(54);
+    }
+
+    if (databaseSchemaVersion < 55) {
+      await this.$executeQuery(this.getAdditionalBlocksDataQuery());
+      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+      await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
+      await this.updateToSchemaVersion(55);
+    }
+
+    if (databaseSchemaVersion < 56) {
+      await this.$executeQuery('ALTER TABLE pools ADD unique_id int NOT NULL DEFAULT -1');
+      await this.$executeQuery('TRUNCATE TABLE `blocks`');
+      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+      await this.$executeQuery('DELETE FROM `pools`');
+      await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
+      this.uniqueLog(logger.notice, '`pools` table has been truncated`');
+      await this.updateToSchemaVersion(56);
+    }
  }

  /**
@@ -596,7 +623,7 @@ class DatabaseMigration {
      queries.push(`INSERT INTO state(name, number, string) VALUES ('last_hashrates_indexing', 0, NULL)`);
    }

-    if (version < 9  && isBitcoin === true) {
+    if (version < 9 && isBitcoin === true) {
      queries.push(`INSERT INTO state(name, number, string) VALUES ('last_weekly_hashrates_indexing', 0, NULL)`);
    }

@@ -746,6 +773,28 @@ class DatabaseMigration {
    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
  }

+  private getAdditionalBlocksDataQuery(): string {
+    return `ALTER TABLE blocks
+      ADD median_timestamp timestamp NOT NULL,
+      ADD coinbase_address varchar(100) NULL,
+      ADD coinbase_signature varchar(500) NULL,
+      ADD coinbase_signature_ascii varchar(500) NULL,
+      ADD avg_tx_size double unsigned NOT NULL,
+      ADD total_inputs int unsigned NOT NULL,
+      ADD total_outputs int unsigned NOT NULL,
+      ADD total_output_amt bigint unsigned NOT NULL,
+      ADD fee_percentiles longtext NULL,
+      ADD median_fee_amt int unsigned NULL,
+      ADD segwit_total_txs int unsigned NOT NULL,
+      ADD segwit_total_size int unsigned NOT NULL,
+      ADD segwit_total_weight int unsigned NOT NULL,
+      ADD header varchar(160) NOT NULL,
+      ADD utxoset_change int NOT NULL,
+      ADD utxoset_size int unsigned NULL,
+      ADD total_input_amt bigint unsigned NULL
+    `;
+  }
+
  private getCreateDailyStatsTableQuery(): string {
    return `CREATE TABLE IF NOT EXISTS hashrates (
      hashrate_timestamp timestamp NOT NULL,
@@ -963,26 +1012,16 @@ class DatabaseMigration {
    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
  }

-  public async $truncateIndexedData(tables: string[]) {
-    const allowedTables = ['blocks', 'hashrates', 'prices'];
+  public async $blocksReindexingTruncate(): Promise<void> {
+    logger.warn(`Truncating pools, blocks and hashrates for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`);
+    await Common.sleep$(5000);

-    try {
-      for (const table of tables) {
-        if (!allowedTables.includes(table)) {
-          logger.debug(`Table ${table} cannot to be re-indexed (not allowed)`);
-          continue;
-        }
-
-        await this.$executeQuery(`TRUNCATE ${table}`, true);
-        if (table === 'hashrates') {
-          await this.$executeQuery('UPDATE state set number = 0 where name = "last_hashrates_indexing"', true);
-        }
-        logger.notice(`Table ${table} has been truncated`);
-      }
-    } catch (e) {
-      logger.warn(`Unable to erase indexed data`);
-    }
-  }
+    await this.$executeQuery(`TRUNCATE blocks`);
+    await this.$executeQuery(`TRUNCATE hashrates`);
+    await this.$executeQuery('DELETE FROM `pools`');
+    await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
+    await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`);
+}

  private async $convertCompactCpfpTables(): Promise<void> {
    try {
--- a/backend/src/api/disk-cache.ts
+++ b/backend/src/api/disk-cache.ts
@@ -9,7 +9,7 @@ import { TransactionExtended } from '../mempool.interfaces';
 import { Common } from './common';

 class DiskCache {
-  private cacheSchemaVersion = 1;
+  private cacheSchemaVersion = 2;

  private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
  private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json';
--- a/backend/src/api/mining/mining-routes.ts
+++ b/backend/src/api/mining/mining-routes.ts
@@ -38,7 +38,16 @@ class MiningRoutes {

  private async $getHistoricalPrice(req: Request, res: Response): Promise<void> {
    try {
-      res.status(200).send(await PricesRepository.$getHistoricalPrice());
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      if (req.query.timestamp) {
+        res.status(200).send(await PricesRepository.$getNearestHistoricalPrice(
+          parseInt(<string>req.query.timestamp ?? 0, 10)
+        ));
+      } else {
+        res.status(200).send(await PricesRepository.$getHistoricalPrices());
+      }
    } catch (e) {
      res.status(500).send(e instanceof Error ? e.message : e);
    }
--- a/backend/src/api/mining/mining.ts
+++ b/backend/src/api/mining/mining.ts
@@ -172,7 +172,7 @@ class Mining {
  }

  /**
-   * [INDEXING] Generate weekly mining pool hashrate history
+   * Generate weekly mining pool hashrate history
   */
  public async $generatePoolHashrateHistory(): Promise<void> {
    const now = new Date();
@@ -279,7 +279,7 @@ class Mining {
  }

  /**
-   * [INDEXING] Generate daily hashrate data
+   * Generate daily hashrate data
   */
  public async $generateNetworkHashrateHistory(): Promise<void> {
    // We only run this once a day around midnight
@@ -459,7 +459,7 @@ class Mining {
  /**
   * Create a link between blocks and the latest price at when they were mined
   */
-  public async $indexBlockPrices() {
+  public async $indexBlockPrices(): Promise<void> {
    if (this.blocksPriceIndexingRunning === true) {
      return;
    }
@@ -520,6 +520,41 @@ class Mining {
    this.blocksPriceIndexingRunning = false;
  }

+  /**
+   * Index core coinstatsindex
+   */
+  public async $indexCoinStatsIndex(): Promise<void> {
+    let timer = new Date().getTime() / 1000;
+    let totalIndexed = 0;
+
+    const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+    let currentBlockHeight = blockchainInfo.blocks;
+
+    while (currentBlockHeight > 0) {
+      const indexedBlocks = await BlocksRepository.$getBlocksMissingCoinStatsIndex(
+        currentBlockHeight, currentBlockHeight - 10000);
+        
+      for (const block of indexedBlocks) {
+        const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
+        await BlocksRepository.$updateCoinStatsIndexData(block.hash, txoutset.txouts,
+          Math.round(txoutset.block_info.prevout_spent * 100000000));        
+        ++totalIndexed;
+
+        const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
+        if (elapsedSeconds > 5) {
+          logger.info(`Indexing coinstatsindex data for block #${block.height}. Indexed ${totalIndexed} blocks.`, logger.tags.mining);
+          timer = new Date().getTime() / 1000;
+        }
+      }
+
+      currentBlockHeight -= 10000;
+    }
+
+    if (totalIndexed) {
+      logger.info(`Indexing missing coinstatsindex data completed`, logger.tags.mining);
+    }
+  }
+
  private getDateMidnight(date: Date): Date {
    date.setUTCHours(0);
    date.setUTCMinutes(0);
--- a/backend/src/api/pools-parser.ts
+++ b/backend/src/api/pools-parser.ts
@@ -1,15 +1,8 @@
 import DB from '../database';
 import logger from '../logger';
 import config from '../config';
-import BlocksRepository from '../repositories/BlocksRepository';
-
-interface Pool {
-  name: string;
-  link: string;
-  regexes: string[];
-  addresses: string[];
-  slug: string;
-}
+import PoolsRepository from '../repositories/PoolsRepository';
+import { PoolTag } from '../mempool.interfaces';

 class PoolsParser {
  miningPools: any[] = [];
@@ -20,270 +13,142 @@ class PoolsParser {
    'addresses': '[]',
    'slug': 'unknown'
  };
-  slugWarnFlag = false;
+  private uniqueLogs: string[] = [];
+
+  private uniqueLog(loggerFunction: any, msg: string): void {
+    if (this.uniqueLogs.includes(msg)) {
+      return;
+    }
+    this.uniqueLogs.push(msg);
+    loggerFunction(msg);
+  }
+
+  public setMiningPools(pools): void {
+    for (const pool of pools) {
+      pool.regexes = pool.tags;
+      delete(pool.tags);
+    }
+    this.miningPools = pools;
+  }

  /**
-   * Parse the pools.json file, consolidate the data and dump it into the database
+   * Populate our db with updated mining pool definition
+   * @param pools 
   */
-  public async migratePoolsJson(poolsJson: object): Promise<void> {
-    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
-      return;
-    }
+  public async migratePoolsJson(): Promise<void> {
+    await this.$insertUnknownPool();

-    // First we save every entries without paying attention to pool duplication
-    const poolsDuplicated: Pool[] = [];
-
-    const coinbaseTags = Object.entries(poolsJson['coinbase_tags']);
-    for (let i = 0; i < coinbaseTags.length; ++i) {
-      poolsDuplicated.push({
-        'name': (<Pool>coinbaseTags[i][1]).name,
-        'link': (<Pool>coinbaseTags[i][1]).link,
-        'regexes': [coinbaseTags[i][0]],
-        'addresses': [],
-        'slug': ''
-      });
-    }
-    const addressesTags = Object.entries(poolsJson['payout_addresses']);
-    for (let i = 0; i < addressesTags.length; ++i) {
-      poolsDuplicated.push({
-        'name': (<Pool>addressesTags[i][1]).name,
-        'link': (<Pool>addressesTags[i][1]).link,
-        'regexes': [],
-        'addresses': [addressesTags[i][0]],
-        'slug': ''
-      });
-    }
-
-    // Then, we find unique mining pool names
-    const poolNames: string[] = [];
-    for (let i = 0; i < poolsDuplicated.length; ++i) {
-      if (poolNames.indexOf(poolsDuplicated[i].name) === -1) {
-        poolNames.push(poolsDuplicated[i].name);
+    for (const pool of this.miningPools) {
+      if (!pool.id) {
+        logger.info(`Mining pool ${pool.name} has no unique 'id' defined. Skipping.`);
+        continue;
      }
-    }
-    logger.debug(`Found ${poolNames.length} unique mining pools`, logger.tags.mining);

-    // Get existing pools from the db
-    let existingPools;
-    try {
-      if (config.DATABASE.ENABLED === true) {
-        [existingPools] = await DB.query({ sql: 'SELECT * FROM pools;', timeout: 120000 });
+      const poolDB = await PoolsRepository.$getPoolByUniqueId(pool.id, false);
+      if (!poolDB) {
+        // New mining pool
+        const slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
+        logger.debug(`Inserting new mining pool ${pool.name}`);
+        await PoolsRepository.$insertNewMiningPool(pool, slug);
+        await this.$deleteUnknownBlocks();
      } else {
-        existingPools = [];
-      }
-    } catch (e) {
-      logger.err('Cannot get existing pools from the database, skipping pools.json import', logger.tags.mining);
-      return;
-    }
-
-    this.miningPools = [];
-
-    // Finally, we generate the final consolidated pools data
-    const finalPoolDataAdd: Pool[] = [];
-    const finalPoolDataUpdate: Pool[] = [];
-    const finalPoolDataRename: Pool[] = [];
-    for (let i = 0; i < poolNames.length; ++i) {
-      let allAddresses: string[] = [];
-      let allRegexes: string[] = [];
-      const match = poolsDuplicated.filter((pool: Pool) => pool.name === poolNames[i]);
-
-      for (let y = 0; y < match.length; ++y) {
-        allAddresses = allAddresses.concat(match[y].addresses);
-        allRegexes = allRegexes.concat(match[y].regexes);
-      }
-
-      const finalPoolName = poolNames[i].replace(`'`, `''`); // To support single quote in names when doing db queries
-
-      let slug: string | undefined;
-      try {
-        slug = poolsJson['slugs'][poolNames[i]];
-      } catch (e) {
-        if (this.slugWarnFlag === false) {
-          logger.warn(`pools.json does not seem to contain the 'slugs' object`, logger.tags.mining);
-          this.slugWarnFlag = true;
+        if (poolDB.name !== pool.name) {
+          // Pool has been renamed
+          const newSlug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
+          logger.warn(`Renaming ${poolDB.name} mining pool to ${pool.name}. Slug has been updated. Maybe you want to make a redirection from 'https://mempool.space/mining/pool/${poolDB.slug}' to 'https://mempool.space/mining/pool/${newSlug}`);
+          await PoolsRepository.$renameMiningPool(poolDB.id, newSlug, pool.name);
        }
-      }
-
-      if (slug === undefined) {
-        // Only keep alphanumerical
-        slug = poolNames[i].replace(/[^a-z0-9]/gi, '').toLowerCase();
-        logger.warn(`No slug found for '${poolNames[i]}', generating it => '${slug}'`, logger.tags.mining);
-      }
-
-      const poolObj = {
-        'name': finalPoolName,
-        'link': match[0].link,
-        'regexes': allRegexes,
-        'addresses': allAddresses,
-        'slug': slug
-      };
-
-      const existingPool = existingPools.find((pool) => pool.name === poolNames[i]);
-      if (existingPool !== undefined) {
-        // Check if any data was actually updated
-        const equals = (a, b) =>
-          a.length === b.length &&
-          a.every((v, i) => v === b[i]);
-        if (!equals(JSON.parse(existingPool.addresses), poolObj.addresses) || !equals(JSON.parse(existingPool.regexes), poolObj.regexes)) {
-          finalPoolDataUpdate.push(poolObj);
+        if (poolDB.link !== pool.link) {
+          // Pool link has changed
+          logger.debug(`Updating link for ${pool.name} mining pool`);
+          await PoolsRepository.$updateMiningPoolLink(poolDB.id, pool.link);
        }
-      } else if (config.DATABASE.ENABLED) {
-        // Double check that if we're not just renaming a pool (same address same regex)
-        const [poolToRename]: any[] = await DB.query(`
-          SELECT * FROM pools
-          WHERE addresses = ? OR regexes = ?`,
-          [JSON.stringify(poolObj.addresses), JSON.stringify(poolObj.regexes)]
-        );
-        if (poolToRename && poolToRename.length > 0) {
-          // We're actually renaming an existing pool
-          finalPoolDataRename.push({
-            'name': poolObj.name,
-            'link': poolObj.link,
-            'regexes': allRegexes,
-            'addresses': allAddresses,
-            'slug': slug
-          });
-          logger.debug(`Rename '${poolToRename[0].name}' mining pool to ${poolObj.name}`, logger.tags.mining);
-        } else {
-          logger.debug(`Add '${finalPoolName}' mining pool`, logger.tags.mining);
-          finalPoolDataAdd.push(poolObj);
+        if (JSON.stringify(pool.addresses) !== poolDB.addresses ||
+          JSON.stringify(pool.regexes) !== poolDB.regexes) {
+          // Pool addresses changed or coinbase tags changed
+          logger.notice(`Updating addresses and/or coinbase tags for ${pool.name} mining pool. If 'AUTOMATIC_BLOCK_REINDEXING' is enabled, we will re-index its blocks and 'unknown' blocks`);
+          await PoolsRepository.$updateMiningPoolTags(poolDB.id, pool.addresses, pool.regexes);
+          await this.$deleteBlocksForPool(poolDB);
        }
      }
-
-      this.miningPools.push({
-        'name': finalPoolName,
-        'link': match[0].link,
-        'regexes': JSON.stringify(allRegexes),
-        'addresses': JSON.stringify(allAddresses),
-        'slug': slug
-      });
-    }
-
-    if (config.DATABASE.ENABLED === false) { // Don't run db operations
-      logger.info('Mining pools.json import completed (no database)', logger.tags.mining);
-      return;
-    }
-
-    if (finalPoolDataAdd.length > 0 || finalPoolDataUpdate.length > 0 ||
-      finalPoolDataRename.length > 0
-    ) {    
-      logger.debug(`Update pools table now`, logger.tags.mining);
-
-      // Add new mining pools into the database
-      let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses, slug) VALUES ';
-      for (let i = 0; i < finalPoolDataAdd.length; ++i) {
-        queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}',
-        '${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}',
-        ${JSON.stringify(finalPoolDataAdd[i].slug)}),`;
-      }
-      queryAdd = queryAdd.slice(0, -1) + ';';
-
-      // Updated existing mining pools in the database
-      const updateQueries: string[] = [];
-      for (let i = 0; i < finalPoolDataUpdate.length; ++i) {
-        updateQueries.push(`
-          UPDATE pools
-          SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}',
-          regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}',
-          slug='${finalPoolDataUpdate[i].slug}'
-          WHERE name='${finalPoolDataUpdate[i].name}'
-        ;`);
-      }
-
-      // Rename mining pools
-      const renameQueries: string[] = [];
-      for (let i = 0; i < finalPoolDataRename.length; ++i) {
-        renameQueries.push(`
-          UPDATE pools
-          SET name='${finalPoolDataRename[i].name}', link='${finalPoolDataRename[i].link}',
-            slug='${finalPoolDataRename[i].slug}'
-          WHERE regexes='${JSON.stringify(finalPoolDataRename[i].regexes)}'
-            AND addresses='${JSON.stringify(finalPoolDataRename[i].addresses)}'
-        ;`);
-      }
-
-      try {
-        if (finalPoolDataAdd.length > 0 || updateQueries.length > 0) {
-          await this.$deleteBlocskToReindex(finalPoolDataUpdate);
-        }
-
-        if (finalPoolDataAdd.length > 0) {
-          await DB.query({ sql: queryAdd, timeout: 120000 });
-        }
-        for (const query of updateQueries) {
-          await DB.query({ sql: query, timeout: 120000 });
-        }
-        for (const query of renameQueries) {
-          await DB.query({ sql: query, timeout: 120000 });
-        }
-        await this.insertUnknownPool();
-        logger.info('Mining pools.json import completed', logger.tags.mining);
-      } catch (e) {
-        logger.err(`Cannot import pools in the database`, logger.tags.mining);
-        throw e;
-      }
    }

-    try {
-      await this.insertUnknownPool();
-    } catch (e) {
-      logger.err(`Cannot insert unknown pool in the database`, logger.tags.mining);
-      throw e;
-    }
+    logger.info('Mining pools-v2.json import completed');
  }

  /**
   * Manually add the 'unknown pool'
   */
-  private async insertUnknownPool() {
+  public async $insertUnknownPool(): Promise<void> {
+    if (!config.DATABASE.ENABLED) {
+      return;
+    }
+
    try {
      const [rows]: any[] = await DB.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 });
      if (rows.length === 0) {
        await DB.query({
-          sql: `INSERT INTO pools(name, link, regexes, addresses, slug)
-          VALUES("Unknown", "https://learnmeabitcoin.com/technical/coinbase-transaction", "[]", "[]", "unknown");
+          sql: `INSERT INTO pools(name, link, regexes, addresses, slug, unique_id)
+          VALUES("${this.unknownPool.name}", "${this.unknownPool.link}", "[]", "[]", "${this.unknownPool.slug}", 0);
        `});
      } else {
        await DB.query(`UPDATE pools
-          SET name='Unknown', link='https://learnmeabitcoin.com/technical/coinbase-transaction',
+          SET name='${this.unknownPool.name}', link='${this.unknownPool.link}',
          regexes='[]', addresses='[]',
-          slug='unknown'
-          WHERE name='Unknown'
+          slug='${this.unknownPool.slug}',
+          unique_id=0
+          WHERE slug='${this.unknownPool.slug}'
        `);
      }
    } catch (e) {
-      logger.err('Unable to insert "Unknown" mining pool', logger.tags.mining);
+      logger.err(`Unable to insert or update "Unknown" mining pool. Reason: ${e instanceof Error ? e.message : e}`);
    }
  }

  /**
-   * Delete blocks which needs to be reindexed
+   * Delete indexed blocks for an updated mining pool
+   * 
+   * @param pool 
   */
-   private async $deleteBlocskToReindex(finalPoolDataUpdate: any[]) {
+  private async $deleteBlocksForPool(pool: PoolTag): Promise<void> {
    if (config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING === false) {
      return;
    }

-    const blockCount = await BlocksRepository.$blockCount(null, null);
-    if (blockCount === 0) {
-      return;
-    }
-
-    for (const updatedPool of finalPoolDataUpdate) {
-      const [pool]: any[] = await DB.query(`SELECT id, name from pools where slug = "${updatedPool.slug}"`);
-      if (pool.length > 0) {
-        logger.notice(`Deleting blocks from ${pool[0].name} mining pool for future re-indexing`, logger.tags.mining);
-        await DB.query(`DELETE FROM blocks WHERE pool_id = ${pool[0].id}`);
-      }
-    }
-
-    // Ignore early days of Bitcoin as there were not mining pool yet
-    logger.notice(`Deleting blocks with unknown mining pool from height 130635 for future re-indexing`, logger.tags.mining);
+    // Get oldest blocks mined by the pool and assume pools-v2.json updates only concern most recent years
+    // Ignore early days of Bitcoin as there were no mining pool yet
+    const [oldestPoolBlock]: any[] = await DB.query(`
+      SELECT height
+      FROM blocks
+      WHERE pool_id = ?
+      ORDER BY height
+      LIMIT 1`,
+      [pool.id]
+    );
+    const oldestBlockHeight = oldestPoolBlock.length ?? 0 > 0 ? oldestPoolBlock[0].height : 130635;
    const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
-    await DB.query(`DELETE FROM blocks WHERE pool_id = ${unknownPool[0].id} AND height > 130635`);
+    this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${oldestBlockHeight} for re-indexing`);
+    await DB.query(`
+      DELETE FROM blocks
+      WHERE pool_id = ? AND height >= ${oldestBlockHeight}`,
+      [unknownPool[0].id]
+    );
+    logger.notice(`Deleting blocks from ${pool.name} mining pool for re-indexing`);
+    await DB.query(`
+      DELETE FROM blocks
+      WHERE pool_id = ?`,
+      [pool.id]
+    );
+  }

-    logger.notice(`Truncating hashrates for future re-indexing`, logger.tags.mining);
-    await DB.query(`DELETE FROM hashrates`);
+  private async $deleteUnknownBlocks(): Promise<void> {
+    const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
+    this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height 130635 for re-indexing`);
+    await DB.query(`
+      DELETE FROM blocks
+      WHERE pool_id = ? AND height >= 130635`,
+      [unknownPool[0].id]
+    );
  }
 }

--- a/backend/src/api/transaction-utils.ts
+++ b/backend/src/api/transaction-utils.ts
@@ -14,6 +14,7 @@ class TransactionUtils {
      vout: tx.vout
        .map((vout) => ({
          scriptpubkey_address: vout.scriptpubkey_address,
+          scriptpubkey_asm: vout.scriptpubkey_asm,
          value: vout.value
        }))
        .filter((vout) => vout.value)
--- a/backend/src/config.ts
+++ b/backend/src/config.ts
@@ -32,6 +32,7 @@ interface IConfig {
    ADVANCED_GBT_AUDIT: boolean;
    ADVANCED_GBT_MEMPOOL: boolean;
    CPFP_INDEXING: boolean;
+    MAX_BLOCKS_BULK_QUERY: number;
  };
  ESPLORA: {
    REST_API_URL: string;
@@ -147,12 +148,13 @@ const defaults: IConfig = {
    'USER_AGENT': 'mempool',
    'STDOUT_LOG_MIN_PRIORITY': 'debug',
    'AUTOMATIC_BLOCK_REINDEXING': false,
-    'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
+    'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
    'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
    'AUDIT': false,
    'ADVANCED_GBT_AUDIT': false,
    'ADVANCED_GBT_MEMPOOL': false,
    'CPFP_INDEXING': false,
+    'MAX_BLOCKS_BULK_QUERY': 0,
  },
  'ESPLORA': {
    'REST_API_URL': 'http://127.0.0.1:3000',
--- a/backend/src/database.ts
+++ b/backend/src/database.ts
@@ -24,7 +24,8 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr

  private checkDBFlag() {
    if (config.DATABASE.ENABLED === false) {
-      logger.err('Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue');
+      const stack = new Error().stack;
+      logger.err(`Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue.\nStack trace: ${stack}}`);
    }
  }

--- a/backend/src/index.ts
+++ b/backend/src/index.ts
@@ -36,6 +36,8 @@ import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
 import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher';
 import forensicsService from './tasks/lightning/forensics.service';
 import priceUpdater from './tasks/price-updater';
+import mining from './api/mining/mining';
+import chainTips from './api/chain-tips';
 import { AxiosError } from 'axios';

 class Server {
@@ -82,11 +84,8 @@ class Server {
    if (config.DATABASE.ENABLED) {
      await DB.checkDbConnection();
      try {
-        if (process.env.npm_config_reindex !== undefined) { // Re-index requests
-          const tables = process.env.npm_config_reindex.split(',');
-          logger.warn(`Indexed data for "${process.env.npm_config_reindex}" tables will be erased in 5 seconds (using '--reindex')`);
-          await Common.sleep$(5000);
-          await databaseMigration.$truncateIndexedData(tables);
+        if (process.env.npm_config_reindex_blocks === 'true') { // Re-index requests
+          await databaseMigration.$blocksReindexingTruncate();
        }
        await databaseMigration.$initializeOrMigrateDatabase();
        if (Common.indexingEnabled()) {
@@ -133,6 +132,7 @@ class Server {
    }

    priceUpdater.$run();
+    await chainTips.updateOrphanedBlocks();

    this.setUpHttpApiRoutes();

--- a/backend/src/indexer.ts
+++ b/backend/src/indexer.ts
@@ -8,18 +8,67 @@ import bitcoinClient from './api/bitcoin/bitcoin-client';
 import priceUpdater from './tasks/price-updater';
 import PricesRepository from './repositories/PricesRepository';

+export interface CoreIndex {
+  name: string;
+  synced: boolean;
+  best_block_height: number;
+}
+
 class Indexer {
  runIndexer = true;
  indexerRunning = false;
  tasksRunning: string[] = [];
+  coreIndexes: CoreIndex[] = [];

-  public reindex() {
+  /**
+   * Check which core index is available for indexing
+   */
+  public async checkAvailableCoreIndexes(): Promise<void> {
+    const updatedCoreIndexes: CoreIndex[] = [];
+
+    const indexes: any = await bitcoinClient.getIndexInfo();
+    for (const indexName in indexes) {
+      const newState = {
+        name: indexName,
+        synced: indexes[indexName].synced,
+        best_block_height: indexes[indexName].best_block_height,
+      };
+      logger.info(`Core index '${indexName}' is ${indexes[indexName].synced ? 'synced' : 'not synced'}. Best block height is ${indexes[indexName].best_block_height}`);      
+      updatedCoreIndexes.push(newState);
+
+      if (indexName === 'coinstatsindex' && newState.synced === true) {
+        const previousState = this.isCoreIndexReady('coinstatsindex');
+        // if (!previousState || previousState.synced === false) {
+          this.runSingleTask('coinStatsIndex');
+        // }
+      }
+    }
+
+    this.coreIndexes = updatedCoreIndexes;
+  }
+
+  /**
+   * Return the best block height if a core index is available, or 0 if not
+   * 
+   * @param name 
+   * @returns 
+   */
+  public isCoreIndexReady(name: string): CoreIndex | null {
+    for (const index of this.coreIndexes) {
+      if (index.name === name && index.synced === true) {
+        return index;
+      }
+    }
+    return null;
+  }
+
+  public reindex(): void {
    if (Common.indexingEnabled()) {
      this.runIndexer = true;
    }
  }

-  public async runSingleTask(task: 'blocksPrices') {
+  public async runSingleTask(task: 'blocksPrices' | 'coinStatsIndex'): Promise<void> {
    if (!Common.indexingEnabled()) {
      return;
    }
@@ -28,20 +77,27 @@ class Indexer {
      this.tasksRunning.push(task);
      const lastestPriceId = await PricesRepository.$getLatestPriceId();
      if (priceUpdater.historyInserted === false || lastestPriceId === null) {
-        logger.debug(`Blocks prices indexer is waiting for the price updater to complete`)
+        logger.debug(`Blocks prices indexer is waiting for the price updater to complete`);
        setTimeout(() => {
-          this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
+          this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
          this.runSingleTask('blocksPrices');
        }, 10000);
      } else {
-        logger.debug(`Blocks prices indexer will run now`)
+        logger.debug(`Blocks prices indexer will run now`);
        await mining.$indexBlockPrices();
-        this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
+        this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
      }
    }
+
+    if (task === 'coinStatsIndex' && !this.tasksRunning.includes(task)) {
+      this.tasksRunning.push(task);
+      logger.debug(`Indexing coinStatsIndex now`);
+      await mining.$indexCoinStatsIndex();
+      this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
+    }
  }

-  public async $run() {
+  public async $run(): Promise<void> {
    if (!Common.indexingEnabled() || this.runIndexer === false ||
      this.indexerRunning === true || mempool.hasPriority()
    ) {
@@ -57,7 +113,9 @@ class Indexer {
    this.runIndexer = false;
    this.indexerRunning = true;

-    logger.debug(`Running mining indexer`);
+    logger.info(`Running mining indexer`);
+
+    await this.checkAvailableCoreIndexes();

    try {
      await priceUpdater.$run();
@@ -93,7 +151,7 @@ class Indexer {
    setTimeout(() => this.reindex(), runEvery);
  }

-  async $resetHashratesIndexingState() {
+  async $resetHashratesIndexingState(): Promise<void> {
    try {
      await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0);
      await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0);
--- a/backend/src/mempool.interfaces.ts
+++ b/backend/src/mempool.interfaces.ts
@@ -1,4 +1,5 @@
 import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
+import { OrphanedBlock } from './api/chain-tips';
 import { HeapNode } from "./utils/pairing-heap";

 export interface PoolTag {
@@ -64,6 +65,7 @@ interface VinStrippedToScriptsig {

 interface VoutStrippedToScriptPubkey {
  scriptpubkey_address: string | undefined;
+  scriptpubkey_asm: string | undefined;
  value: number;
 }

@@ -160,6 +162,27 @@ export interface BlockExtension {
  avgFeeRate?: number;
  coinbaseRaw?: string;
  usd?: number | null;
+  medianTimestamp?: number;
+  blockTime?: number;
+  orphans?: OrphanedBlock[] | null;
+  coinbaseAddress?: string | null;
+  coinbaseSignature?: string | null;
+  coinbaseSignatureAscii?: string | null;
+  virtualSize?: number;
+  avgTxSize?: number;
+  totalInputs?: number;
+  totalOutputs?: number;
+  totalOutputAmt?: number;
+  medianFeeAmt?: number | null;
+  feePercentiles?: number[] | null,
+  segwitTotalTxs?: number;
+  segwitTotalSize?: number;
+  segwitTotalWeight?: number;
+  header?: string;
+  utxoSetChange?: number;
+  // Requires coinstatsindex, will be set to NULL otherwise
+  utxoSetSize?: number | null;
+  totalInputAmt?: number | null;
 }

 export interface BlockExtended extends IEsploraApi.Block {
--- a/backend/src/repositories/BlocksRepository.ts
+++ b/backend/src/repositories/BlocksRepository.ts
@@ -18,17 +18,27 @@ class BlocksRepository {
  public async $saveBlockInDatabase(block: BlockExtended) {
    try {
      const query = `INSERT INTO blocks(
-        height,           hash,                blockTimestamp, size,
-        weight,           tx_count,            coinbase_raw,   difficulty,
-        pool_id,          fees,                fee_span,       median_fee,
-        reward,           version,             bits,           nonce,
-        merkle_root,      previous_block_hash, avg_fee,        avg_fee_rate
+        height,             hash,                blockTimestamp,    size,
+        weight,             tx_count,            coinbase_raw,      difficulty,
+        pool_id,            fees,                fee_span,          median_fee,
+        reward,             version,             bits,              nonce,
+        merkle_root,        previous_block_hash, avg_fee,           avg_fee_rate,
+        median_timestamp,   header,              coinbase_address,
+        coinbase_signature, utxoset_size,        utxoset_change,    avg_tx_size,
+        total_inputs,       total_outputs,       total_input_amt,   total_output_amt,
+        fee_percentiles,    segwit_total_txs,    segwit_total_size, segwit_total_weight,
+        median_fee_amt,     coinbase_signature_ascii
      ) VALUE (
        ?, ?, FROM_UNIXTIME(?), ?,
        ?, ?, ?, ?,
        ?, ?, ?, ?,
        ?, ?, ?, ?,
-        ?, ?, ?, ?
+        ?, ?, ?, ?,
+        FROM_UNIXTIME(?), ?, ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?, ?
      )`;

      const params: any[] = [
@@ -52,6 +62,23 @@ class BlocksRepository {
        block.previousblockhash,
        block.extras.avgFee,
        block.extras.avgFeeRate,
+        block.extras.medianTimestamp,
+        block.extras.header,
+        block.extras.coinbaseAddress,
+        block.extras.coinbaseSignature,
+        block.extras.utxoSetSize,
+        block.extras.utxoSetChange,
+        block.extras.avgTxSize,
+        block.extras.totalInputs,
+        block.extras.totalOutputs,
+        block.extras.totalInputAmt,
+        block.extras.totalOutputAmt,
+        block.extras.feePercentiles ? JSON.stringify(block.extras.feePercentiles) : null,
+        block.extras.segwitTotalTxs,
+        block.extras.segwitTotalSize,
+        block.extras.segwitTotalWeight,
+        block.extras.medianFeeAmt,
+        block.extras.coinbaseSignatureAscii,
      ];

      await DB.query(query, params);
@@ -65,6 +92,33 @@ class BlocksRepository {
    }
  }

+  /**
+   * Save newly indexed data from core coinstatsindex
+   * 
+   * @param utxoSetSize 
+   * @param totalInputAmt 
+   */
+  public async $updateCoinStatsIndexData(blockHash: string, utxoSetSize: number,
+    totalInputAmt: number
+  ) : Promise<void> {
+    try {
+      const query = `
+        UPDATE blocks
+        SET utxoset_size = ?, total_input_amt = ?
+        WHERE hash = ?
+      `;
+      const params: any[] = [
+        utxoSetSize,
+        totalInputAmt,
+        blockHash
+      ];
+      await DB.query(query, params);
+    } catch (e: any) {
+      logger.err('Cannot update indexed block coinstatsindex. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
  /**
   * Get all block height that have not been indexed between [startHeight, endHeight]
   */
@@ -310,32 +364,17 @@ class BlocksRepository {
  public async $getBlockByHeight(height: number): Promise<object | null> {
    try {
      const [rows]: any[] = await DB.query(`SELECT
-        blocks.height,
-        hash,
+        blocks.*,
        hash as id,
        UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
-        size,
-        weight,
-        tx_count,
-        coinbase_raw,
-        difficulty,
+        UNIX_TIMESTAMP(blocks.median_timestamp) as medianTime,
        pools.id as pool_id,
        pools.name as pool_name,
        pools.link as pool_link,
        pools.slug as pool_slug,
        pools.addresses as pool_addresses,
        pools.regexes as pool_regexes,
-        fees,
-        fee_span,
-        median_fee,
-        reward,
-        version,
-        bits,
-        nonce,
-        merkle_root,
-        previous_block_hash as previousblockhash,
-        avg_fee,
-        avg_fee_rate
+        previous_block_hash as previousblockhash
        FROM blocks
        JOIN pools ON blocks.pool_id = pools.id
        WHERE blocks.height = ${height}
@@ -346,6 +385,7 @@ class BlocksRepository {
      }

      rows[0].fee_span = JSON.parse(rows[0].fee_span);
+      rows[0].fee_percentiles = JSON.parse(rows[0].fee_percentiles);
      return rows[0];
    } catch (e) {
      logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));
@@ -521,7 +561,7 @@ class BlocksRepository {
        CAST(AVG(blocks.height) as INT) as avgHeight,
        CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
        CAST(AVG(fees) as INT) as avgFees,
-        prices.*
+        prices.USD
        FROM blocks
        JOIN blocks_prices on blocks_prices.height = blocks.height
        JOIN prices on prices.id = blocks_prices.price_id
@@ -550,7 +590,7 @@ class BlocksRepository {
        CAST(AVG(blocks.height) as INT) as avgHeight,
        CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
        CAST(AVG(reward) as INT) as avgRewards,
-        prices.*
+        prices.USD
        FROM blocks
        JOIN blocks_prices on blocks_prices.height = blocks.height
        JOIN prices on prices.id = blocks_prices.price_id
@@ -694,7 +734,6 @@ class BlocksRepository {
      logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e));
      throw e;
    }
-    return [];
  }

  /**
@@ -741,7 +780,7 @@ class BlocksRepository {
    try {
      let query = `INSERT INTO blocks_prices(height, price_id) VALUES`;
      for (const price of blockPrices) {
-        query += ` (${price.height}, ${price.priceId}),`
+        query += ` (${price.height}, ${price.priceId}),`;
      }
      query = query.slice(0, -1);
      await DB.query(query);
@@ -754,6 +793,43 @@ class BlocksRepository {
      }
    }
  }
+
+  /**
+   * Get all indexed blocsk with missing coinstatsindex data
+   */
+  public async $getBlocksMissingCoinStatsIndex(maxHeight: number, minHeight: number): Promise<any> {
+    try {
+      const [blocks] = await DB.query(`
+        SELECT height, hash
+        FROM blocks
+        WHERE height >= ${minHeight} AND height <= ${maxHeight} AND
+          (utxoset_size IS NULL OR total_input_amt IS NULL)
+      `);
+      return blocks;
+    } catch (e) {
+      logger.err(`Cannot get blocks with missing coinstatsindex. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Save indexed median fee to avoid recomputing it later
+   * 
+   * @param id 
+   * @param feePercentiles 
+   */
+  public async $saveFeePercentilesForBlockId(id: string, feePercentiles: number[]): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE blocks SET fee_percentiles = ?, median_fee_amt = ?
+        WHERE hash = ?`,
+        [JSON.stringify(feePercentiles), feePercentiles[3], id]
+      );
+    } catch (e) {
+      logger.err(`Cannot update block fee_percentiles. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
 }

 export default new BlocksRepository();
--- a/backend/src/repositories/BlocksSummariesRepository.ts
+++ b/backend/src/repositories/BlocksSummariesRepository.ts
@@ -80,6 +80,48 @@ class BlocksSummariesRepository {
      logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e));
    }
  }
+
+  /**
+   * Get the fee percentiles if the block has already been indexed, [] otherwise
+   * 
+   * @param id 
+   */
+  public async $getFeePercentilesByBlockId(id: string): Promise<number[] | null> {
+    try {
+      const [rows]: any[] = await DB.query(`
+        SELECT transactions
+        FROM blocks_summaries
+        WHERE id = ?`,
+        [id]
+      );
+      if (rows === null || rows.length === 0) {
+        return null;
+      }
+
+      const transactions = JSON.parse(rows[0].transactions);
+      if (transactions === null) {
+        return null;
+      }
+
+      transactions.shift(); // Ignore coinbase
+      transactions.sort((a: any, b: any) => a.fee - b.fee);
+      const fees = transactions.map((t: any) => t.fee);
+
+      return [
+        fees[0] ?? 0, // min
+        fees[Math.max(0, Math.floor(fees.length * 0.1) - 1)] ?? 0, // 10th
+        fees[Math.max(0, Math.floor(fees.length * 0.25) - 1)] ?? 0, // 25th
+        fees[Math.max(0, Math.floor(fees.length * 0.5) - 1)] ?? 0, // median
+        fees[Math.max(0, Math.floor(fees.length * 0.75) - 1)] ?? 0, // 75th
+        fees[Math.max(0, Math.floor(fees.length * 0.9) - 1)] ?? 0, // 90th
+        fees[fees.length - 1] ?? 0, // max
+      ];
+
+    } catch (e) {
+      logger.err(`Cannot get block summaries transactions. Reason: ` + (e instanceof Error ? e.message : e));
+      return null;
+    }
+  }
 }

 export default new BlocksSummariesRepository();
--- a/backend/src/repositories/PoolsRepository.ts
+++ b/backend/src/repositories/PoolsRepository.ts
@@ -1,4 +1,5 @@
 import { Common } from '../api/common';
+import poolsParser from '../api/pools-parser';
 import config from '../config';
 import DB from '../database';
 import logger from '../logger';
@@ -17,7 +18,11 @@ class PoolsRepository {
   * Get unknown pool tagging info
   */
  public async $getUnknownPool(): Promise<PoolTag> {
-    const [rows] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"');
+    let [rows]: any[] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"');
+    if (rows && rows.length === 0 && config.DATABASE.ENABLED) {
+      await poolsParser.$insertUnknownPool();
+      [rows] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"');
+    }
    return <PoolTag>rows[0];
  }

@@ -59,7 +64,7 @@ class PoolsRepository {
  /**
   * Get basic pool info and block count between two timestamp
   */
-   public async $getPoolsInfoBetween(from: number, to: number): Promise<PoolInfo[]> {
+  public async $getPoolsInfoBetween(from: number, to: number): Promise<PoolInfo[]> {
    const query = `SELECT COUNT(height) as blockCount, pools.id as poolId, pools.name as poolName
      FROM pools
      LEFT JOIN blocks on pools.id = blocks.pool_id AND blocks.blockTimestamp BETWEEN FROM_UNIXTIME(?) AND FROM_UNIXTIME(?)
@@ -75,9 +80,9 @@ class PoolsRepository {
  }

  /**
-   * Get mining pool statistics for one pool
+   * Get a mining pool info
   */
-   public async $getPool(slug: string): Promise<PoolTag | null> {
+  public async $getPool(slug: string, parse: boolean = true): Promise<PoolTag | null> {
    const query = `
      SELECT *
      FROM pools
@@ -90,10 +95,12 @@ class PoolsRepository {
        return null;
      }

-      rows[0].regexes = JSON.parse(rows[0].regexes);
+      if (parse) {
+        rows[0].regexes = JSON.parse(rows[0].regexes);
+      }
      if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
-        rows[0].addresses = []; // pools.json only contains mainnet addresses
-      } else {
+        rows[0].addresses = []; // pools-v2.json only contains mainnet addresses
+      } else if (parse) {
        rows[0].addresses = JSON.parse(rows[0].addresses);
      }

@@ -103,6 +110,116 @@ class PoolsRepository {
      throw e;
    }
  }
+
+  /**
+   * Get a mining pool info by its unique id
+   */
+  public async $getPoolByUniqueId(id: number, parse: boolean = true): Promise<PoolTag | null> {
+    const query = `
+      SELECT *
+      FROM pools
+      WHERE pools.unique_id = ?`;
+
+    try {
+      const [rows]: any[] = await DB.query(query, [id]);
+
+      if (rows.length < 1) {
+        return null;
+      }
+
+      if (parse) {
+        rows[0].regexes = JSON.parse(rows[0].regexes);
+      }
+      if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
+        rows[0].addresses = []; // pools.json only contains mainnet addresses
+      } else if (parse) {
+        rows[0].addresses = JSON.parse(rows[0].addresses);
+      }
+
+      return rows[0];
+    } catch (e) {
+      logger.err('Cannot get pool from db. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Insert a new mining pool in the database
+   * 
+   * @param pool 
+   */
+  public async $insertNewMiningPool(pool: any, slug: string): Promise<void> {
+    try {
+      await DB.query(`
+        INSERT INTO pools
+        SET name = ?, link = ?, addresses = ?, regexes = ?, slug = ?, unique_id = ?`,
+        [pool.name, pool.link, JSON.stringify(pool.addresses), JSON.stringify(pool.regexes), slug, pool.id]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot insert new mining pool into db. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * Rename an existing mining pool
+   * 
+   * @param dbId
+   * @param newSlug
+   * @param newName 
+   */
+  public async $renameMiningPool(dbId: number, newSlug: string, newName: string): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE pools
+        SET slug = ?, name = ?
+        WHERE id = ?`,
+        [newSlug, newName, dbId]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot rename mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * Update an exisiting mining pool link
+   * 
+   * @param dbId 
+   * @param newLink 
+   */
+  public async $updateMiningPoolLink(dbId: number, newLink: string): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE pools
+        SET link = ?
+        WHERE id = ?`,
+        [newLink, dbId]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot update link for mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+
+  }
+
+  /**
+   * Update an existing mining pool addresses or coinbase tags
+   * 
+   * @param dbId 
+   * @param addresses 
+   * @param regexes 
+   */
+  public async $updateMiningPoolTags(dbId: number, addresses: string, regexes: string): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE pools
+        SET addresses = ?, regexes = ?
+        WHERE id = ?`,
+        [JSON.stringify(addresses), JSON.stringify(regexes), dbId]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot update mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
+
 }

 export default new PoolsRepository();
--- a/backend/src/repositories/PricesRepository.ts
+++ b/backend/src/repositories/PricesRepository.ts
@@ -28,6 +28,16 @@ export interface Conversion {
  exchangeRates: ExchangeRates;
 }

+export const MAX_PRICES = {
+  USD: 100000000,
+  EUR: 100000000,
+  GBP: 100000000,
+  CAD: 100000000,
+  CHF: 100000000,
+  AUD: 100000000,
+  JPY: 10000000000,
+};
+
 class PricesRepository {
  public async $savePrices(time: number, prices: IConversionRates): Promise<void> {
    if (prices.USD === 0) {
@@ -36,6 +46,14 @@ class PricesRepository {
      return;
    }

+    // Sanity check
+    for (const currency of Object.keys(prices)) {
+      if (prices[currency] < -1 || prices[currency] > MAX_PRICES[currency]) { // We use -1 to mark a "missing data, so it's a valid entry"
+        logger.info(`Ignore BTC${currency} price of ${prices[currency]}`);
+        prices[currency] = 0;
+      }
+    }
+    
    try {
      await DB.query(`
        INSERT INTO prices(time,             USD, EUR, GBP, CAD, CHF, AUD, JPY)
@@ -86,9 +104,48 @@ class PricesRepository {
    return rates[0];
  }

-  public async $getHistoricalPrice(): Promise<Conversion | null> {
+  public async $getNearestHistoricalPrice(timestamp: number | undefined): Promise<Conversion | null> {
    try {
-      const [rates]: any[] = await DB.query(`SELECT *, UNIX_TIMESTAMP(time) as time FROM prices ORDER BY time DESC`);
+      const [rates]: any[] = await DB.query(`
+        SELECT *, UNIX_TIMESTAMP(time) AS time
+        FROM prices
+        WHERE UNIX_TIMESTAMP(time) < ?
+        ORDER BY time DESC
+        LIMIT 1`,
+        [timestamp]
+      );
+      if (!rates) {
+        throw Error(`Cannot get single historical price from the database`);
+      }
+
+      // Compute fiat exchange rates
+      const latestPrice = await this.$getLatestConversionRates();
+      const exchangeRates: ExchangeRates = {
+        USDEUR: Math.round(latestPrice.EUR / latestPrice.USD * 100) / 100,
+        USDGBP: Math.round(latestPrice.GBP / latestPrice.USD * 100) / 100,
+        USDCAD: Math.round(latestPrice.CAD / latestPrice.USD * 100) / 100,
+        USDCHF: Math.round(latestPrice.CHF / latestPrice.USD * 100) / 100,
+        USDAUD: Math.round(latestPrice.AUD / latestPrice.USD * 100) / 100,
+        USDJPY: Math.round(latestPrice.JPY / latestPrice.USD * 100) / 100,
+      };
+
+      return {
+        prices: rates,
+        exchangeRates: exchangeRates
+      };
+    } catch (e) {
+      logger.err(`Cannot fetch single historical prices from the db. Reason ${e instanceof Error ? e.message : e}`);
+      return null;
+    }
+  }
+
+  public async $getHistoricalPrices(): Promise<Conversion | null> {
+    try {
+      const [rates]: any[] = await DB.query(`
+        SELECT *, UNIX_TIMESTAMP(time) AS time
+        FROM prices
+        ORDER BY time DESC
+      `);
      if (!rates) {
        throw Error(`Cannot get average historical price from the database`);
      }
@@ -109,7 +166,7 @@ class PricesRepository {
        exchangeRates: exchangeRates
      };
    } catch (e) {
-      logger.err(`Cannot fetch averaged historical prices from the db. Reason ${e instanceof Error ? e.message : e}`);
+      logger.err(`Cannot fetch historical prices from the db. Reason ${e instanceof Error ? e.message : e}`);
      return null;
    }
  }
--- a/backend/src/rpc-api/commands.ts
+++ b/backend/src/rpc-api/commands.ts
@@ -88,5 +88,7 @@ module.exports = {
  verifyTxOutProof: 'verifytxoutproof', // bitcoind v0.11.0+
  walletLock: 'walletlock',
  walletPassphrase: 'walletpassphrase',
-  walletPassphraseChange: 'walletpassphrasechange'
-}
+  walletPassphraseChange: 'walletpassphrasechange',
+  getTxoutSetinfo: 'gettxoutsetinfo',
+  getIndexInfo: 'getindexinfo',
+};
--- a/backend/src/tasks/pools-updater.ts
+++ b/backend/src/tasks/pools-updater.ts
@@ -8,7 +8,7 @@ import { SocksProxyAgent } from 'socks-proxy-agent';
 import * as https from 'https';

 /**
- * Maintain the most recent version of pools.json
+ * Maintain the most recent version of pools-v2.json
 */
 class PoolsUpdater {
  lastRun: number = 0;
@@ -31,14 +31,8 @@ class PoolsUpdater {

    this.lastRun = now;

-    if (config.SOCKS5PROXY.ENABLED) {
-      logger.info(`Updating latest mining pools from ${this.poolsUrl} over the Tor network`, logger.tags.mining);
-    } else {
-      logger.info(`Updating latest mining pools from ${this.poolsUrl} over clearnet`, logger.tags.mining);
-    }
-
    try {
-      const githubSha = await this.fetchPoolsSha(); // Fetch pools.json sha from github
+      const githubSha = await this.fetchPoolsSha(); // Fetch pools-v2.json sha from github
      if (githubSha === undefined) {
        return;
      }
@@ -47,32 +41,57 @@ class PoolsUpdater {
        this.currentSha = await this.getShaFromDb();
      }

-      logger.debug(`Pools.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
+      logger.debug(`pools-v2.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
      if (this.currentSha !== undefined && this.currentSha === githubSha) {
        return;
      }

+      // See backend README for more details about the mining pools update process
+      if (this.currentSha !== undefined && // If we don't have any mining pool, download it at least once
+        config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING !== true && // Automatic pools update is disabled
+        !process.env.npm_config_update_pools // We're not manually updating mining pool
+      ) {
+        logger.warn(`Updated mining pools data is available (${githubSha}) but AUTOMATIC_BLOCK_REINDEXING is disabled`);
+        logger.info(`You can update your mining pools using the --update-pools command flag. You may want to clear your nginx cache as well if applicable`);
+        return;
+      }
+
+      const network = config.SOCKS5PROXY.ENABLED ? 'tor' : 'clearnet';
      if (this.currentSha === undefined) {
-        logger.info(`Downloading pools.json for the first time from ${this.poolsUrl}`, logger.tags.mining);
+        logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl} over ${network}`, logger.tags.mining);
      } else {
-        logger.warn(`Pools.json is outdated, fetch latest from ${this.poolsUrl}`, logger.tags.mining);
+        logger.warn(`pools-v2.json is outdated, fetch latest from ${this.poolsUrl} over ${network}`, logger.tags.mining);
      }
      const poolsJson = await this.query(this.poolsUrl);
      if (poolsJson === undefined) {
        return;
      }
-      await poolsParser.migratePoolsJson(poolsJson);
-      await this.updateDBSha(githubSha);
-      logger.notice(`PoolsUpdater completed`, logger.tags.mining);
+      poolsParser.setMiningPools(poolsJson);
+
+      if (config.DATABASE.ENABLED === false) { // Don't run db operations
+        logger.info('Mining pools-v2.json import completed (no database)');
+        return;
+      }
+
+      try {
+        await DB.query('START TRANSACTION;');
+        await poolsParser.migratePoolsJson();
+        await this.updateDBSha(githubSha);
+        await DB.query('COMMIT;');
+      } catch (e) {
+        logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, logger.tags.mining);
+        await DB.query('ROLLBACK;');
+      }
+      logger.notice('PoolsUpdater completed');

    } catch (e) {
      this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week
-      logger.err(`PoolsUpdater failed. Will try again in 24h. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
+      logger.err(`PoolsUpdater failed. Will try again in 24h. Exception: ${JSON.stringify(e)}`, logger.tags.mining);
    }
  }

  /**
-   * Fetch our latest pools.json sha from the db
+   * Fetch our latest pools-v2.json sha from the db
   */
  private async updateDBSha(githubSha: string): Promise<void> {
    this.currentSha = githubSha;
@@ -81,46 +100,46 @@ class PoolsUpdater {
        await DB.query('DELETE FROM state where name="pools_json_sha"');
        await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`);
      } catch (e) {
-        logger.err('Cannot save github pools.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+        logger.err('Cannot save github pools-v2.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
      }
    }
  }

  /**
-   * Fetch our latest pools.json sha from the db
+   * Fetch our latest pools-v2.json sha from the db
   */
  private async getShaFromDb(): Promise<string | undefined> {
    try {
      const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"');
      return (rows.length > 0 ? rows[0].string : undefined);
    } catch (e) {
-      logger.err('Cannot fetch pools.json sha from db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+      logger.err('Cannot fetch pools-v2.json sha from db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
      return undefined;
    }
  }

  /**
-   * Fetch our latest pools.json sha from github
+   * Fetch our latest pools-v2.json sha from github
   */
  private async fetchPoolsSha(): Promise<string | undefined> {
    const response = await this.query(this.treeUrl);

    if (response !== undefined) {
      for (const file of response['tree']) {
-        if (file['path'] === 'pools.json') {
+        if (file['path'] === 'pools-v2.json') {
          return file['sha'];
        }
      }
    }

-    logger.err(`Cannot find "pools.json" in git tree (${this.treeUrl})`, logger.tags.mining);
+    logger.err(`Cannot find "pools-v2.json" in git tree (${this.treeUrl})`, logger.tags.mining);
    return undefined;
  }

  /**
   * Http request wrapper
   */
-  private async query(path): Promise<object | undefined> {
+  private async query(path): Promise<any[] | undefined> {
    type axiosOptions = {
      headers: {
        'User-Agent': string
--- a/backend/src/tasks/price-updater.ts
+++ b/backend/src/tasks/price-updater.ts
@@ -3,7 +3,7 @@ import path from 'path';
 import config from '../config';
 import logger from '../logger';
 import { IConversionRates } from '../mempool.interfaces';
-import PricesRepository from '../repositories/PricesRepository';
+import PricesRepository, { MAX_PRICES } from '../repositories/PricesRepository';
 import BitfinexApi from './price-feeds/bitfinex-api';
 import BitflyerApi from './price-feeds/bitflyer-api';
 import CoinbaseApi from './price-feeds/coinbase-api';
@@ -46,13 +46,13 @@ class PriceUpdater {

  public getEmptyPricesObj(): IConversionRates {
    return {
-      USD: 0,
-      EUR: 0,
-      GBP: 0,
-      CAD: 0,
-      CHF: 0,
-      AUD: 0,
-      JPY: 0,
+      USD: -1,
+      EUR: -1,
+      GBP: -1,
+      CAD: -1,
+      CHF: -1,
+      AUD: -1,
+      JPY: -1,
    };
  }

@@ -115,7 +115,7 @@ class PriceUpdater {
        if (feed.currencies.includes(currency)) {
          try {
            const price = await feed.$fetchPrice(currency);
-            if (price > 0) {
+            if (price > -1 && price < MAX_PRICES[currency]) {
              prices.push(price);
            }
            logger.debug(`${feed.name} BTC/${currency} price: ${price}`, logger.tags.mining);
@@ -239,7 +239,7 @@ class PriceUpdater {

        for (const currency of this.currencies) {
          const price = historicalEntry[time][currency];
-          if (price > 0) {
+          if (price > -1 && price < MAX_PRICES[currency]) {
            grouped[time][currency].push(typeof price === 'string' ? parseInt(price, 10) : price);
          }
        }