Merge branch 'master' into nymkappa/bugfix/node-sockets-lnd

.github/workflows/ci.yml

@@ -9,7 +9,7 @@ jobs:
     if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
     strategy:
       matrix:
-        node: ["16.16.0", "18.5.0"]
+        node: ["16.16.0", "18.14.1"]
         flavor: ["dev", "prod"]
       fail-fast: false
     runs-on: "ubuntu-latest"
@@ -55,7 +55,7 @@ jobs:
     if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
     strategy:
       matrix:
-        node: ["16.15.0", "18.5.0"]
+        node: ["16.16.0", "18.14.1"]
         flavor: ["dev", "prod"]
       fail-fast: false
     runs-on: "ubuntu-latest"

LICENSE

@@ -1,5 +1,5 @@
 The Mempool Open Source Project
-Copyright (c) 2019-2022 The Mempool Open Source Project Developers
+Copyright (c) 2019-2023 The Mempool Open Source Project Developers
 
 This program is free software; you can redistribute it and/or modify it under
 the terms of (at your option) either:

backend/README.md

@@ -160,7 +160,7 @@ npm install -g ts-node nodemon
 Then, run the watcher:
 
 ```
-nodemon src/index.ts --ignore cache/ --ignore pools.json
+nodemon src/index.ts --ignore cache/
 ```
 
 `nodemon` should be in npm's global binary folder. If needed, you can determine where that is with `npm -g bin`.
@@ -219,6 +219,16 @@ Generate block at regular interval (every 10 seconds in this example):
    watch -n 10 "./src/bitcoin-cli -regtest -rpcport=8332 generatetoaddress 1 $address"
    ```
 
+### Mining pools update
+
+By default, mining pools will be not automatically updated regularly (`config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` is set to `false`). 
+
+To manually update your mining pools, you can use the `--update-pools` command line flag when you run the nodejs backend. For example `npm run start --update-pools`. This will trigger the mining pools update and automatically re-index appropriate blocks.
+
+You can enabled the automatic mining pools update by settings `config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` to `true` in your `mempool-config.json`.
+
+When a `coinbase tag` or `coinbase address` change is detected, all blocks tagged to the `unknown` mining pools (starting from height 130635) will be deleted from the `blocks` table. Additionaly, all blocks which were tagged to the pool which has been updated will also be deleted from the `blocks` table. Of course, those blocks will be automatically reindexed.
+
 ### Re-index tables
 
 You can manually force the nodejs backend to drop all data from a specified set of tables for future re-index. This is mostly useful for the mining dashboard and the lightning explorer.
@@ -235,4 +245,4 @@ Feb 13 14:55:27 [63246] WARN: <lightning> Indexed data for "hashrates" tables wi
 Feb 13 14:55:32 [63246] NOTICE: <lightning> Table hashrates has been truncated
 ```
 
-Reference: https://github.com/mempool/mempool/pull/1269
+Reference: https://github.com/mempool/mempool/pull/1269

backend/mempool-config.sample.json

@@ -22,7 +22,7 @@
     "USER_AGENT": "mempool",
     "STDOUT_LOG_MIN_PRIORITY": "debug",
     "AUTOMATIC_BLOCK_REINDEXING": false,
-    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
+    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
     "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
     "AUDIT": false,
     "ADVANCED_GBT_AUDIT": false,

backend/package.json

@@ -27,7 +27,7 @@
     "package": "npm run build && rm -rf package && mv dist package && mv node_modules package && npm run package-rm-build-deps",
     "package-rm-build-deps": "(cd package/node_modules; rm -r typescript @typescript-eslint)",
     "start": "node --max-old-space-size=2048 dist/index.js",
-    "start-production": "node --max-old-space-size=4096 dist/index.js",
+    "start-production": "node --max-old-space-size=16384 dist/index.js",
     "test": "./node_modules/.bin/jest --coverage",
     "lint": "./node_modules/.bin/eslint . --ext .ts",
     "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",

backend/src/__fixtures__/mempool-config.template.json

@@ -3,12 +3,11 @@
     "ENABLED": true,
     "NETWORK": "__MEMPOOL_NETWORK__",
     "BACKEND": "__MEMPOOL_BACKEND__",
-    "ENABLED": true,
     "BLOCKS_SUMMARIES_INDEXING": true,
     "HTTP_PORT": 1,
     "SPAWN_CLUSTER_PROCS": 2,
     "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",
-    "AUTOMATIC_BLOCK_REINDEXING": true,
+    "AUTOMATIC_BLOCK_REINDEXING": false,
     "POLL_RATE_MS": 3,
     "CACHE_DIR": "__MEMPOOL_CACHE_DIR__",
     "CLEAR_PROTECTION_MINUTES": 4,
@@ -28,7 +27,8 @@
     "AUDIT": "__MEMPOOL_AUDIT__",
     "ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__",
     "ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__",
-    "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__"
+    "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__",
+    "MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__"
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",

backend/src/__tests__/config.test.ts

@@ -36,11 +36,12 @@ describe('Mempool Backend Config', () => {
         USER_AGENT: 'mempool',
         STDOUT_LOG_MIN_PRIORITY: 'debug',
         POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
-        POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
+        POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
         AUDIT: false,
         ADVANCED_GBT_AUDIT: false,
         ADVANCED_GBT_MEMPOOL: false,
         CPFP_INDEXING: false,
+        MAX_BLOCKS_BULK_QUERY: 0,
       });
 
       expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });

backend/src/api/audit.ts

@@ -119,7 +119,8 @@ class Audit {
     }
 
     const numCensored = Object.keys(isCensored).length;
-    const score = matches.length > 0 ? (matches.length / (matches.length + numCensored)) : 0;
+    const numMatches = matches.length - 1; // adjust for coinbase tx
+    const score = numMatches > 0 ? (numMatches / (numMatches + numCensored)) : 0;
 
     return {
       censored: Object.keys(isCensored),

backend/src/api/bitcoin/bitcoin-api.ts

@@ -28,6 +28,7 @@ class BitcoinApi implements AbstractBitcoinApi {
       size: block.size,
       weight: block.weight,
       previousblockhash: block.previousblockhash,
+      medianTime: block.mediantime,
     };
   }
 

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -95,6 +95,8 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions)
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary)
       .post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this))
       ;
 
       if (config.MEMPOOL.BACKEND !== 'esplora') {
@@ -402,6 +404,41 @@ class BitcoinRoutes {
     }
   }
 
+  private async getBlocksByBulk(req: Request, res: Response) {
+    try {
+      if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented
+        return res.status(404).send(`This API is only available for Bitcoin networks`);
+      }
+      if (config.MEMPOOL.MAX_BLOCKS_BULK_QUERY <= 0) {
+        return res.status(404).send(`This API is disabled. Set config.MEMPOOL.MAX_BLOCKS_BULK_QUERY to a positive number to enable it.`);
+      }
+      if (!Common.indexingEnabled()) {
+        return res.status(404).send(`Indexing is required for this API`);
+      }
+
+      const from = parseInt(req.params.from, 10);
+      if (!req.params.from || from < 0) {
+        return res.status(400).send(`Parameter 'from' must be a block height (integer)`);
+      }
+      const to = req.params.to === undefined ? await bitcoinApi.$getBlockHeightTip() : parseInt(req.params.to, 10);
+      if (to < 0) {
+        return res.status(400).send(`Parameter 'to' must be a block height (integer)`);
+      }
+      if (from > to) {
+        return res.status(400).send(`Parameter 'to' must be a higher block height than 'from'`);
+      }
+      if ((to - from + 1) > config.MEMPOOL.MAX_BLOCKS_BULK_QUERY) {
+        return res.status(400).send(`You can only query ${config.MEMPOOL.MAX_BLOCKS_BULK_QUERY} blocks at once.`);
+      }
+
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(await blocks.$getBlocksBetweenHeight(from, to));
+
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async getLegacyBlocks(req: Request, res: Response) {
     try {
       const returnBlocks: IEsploraApi.Block[] = [];

backend/src/api/bitcoin/esplora-api.interface.ts

@@ -88,6 +88,7 @@ export namespace IEsploraApi {
     size: number;
     weight: number;
     previousblockhash: string;
+    medianTime?: number;
   }
 
   export interface Address {

backend/src/api/blocks.ts

@@ -25,6 +25,7 @@ import mining from './mining/mining';
 import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
 import PricesRepository from '../repositories/PricesRepository';
 import priceUpdater from '../tasks/price-updater';
+import chainTips from './chain-tips';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -165,33 +166,80 @@ class Blocks {
    * @returns BlockExtended
    */
   private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
-    const blockExtended: BlockExtended = Object.assign({ extras: {} }, block);
-    blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
-    blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
-    blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig;
-    blockExtended.extras.usd = priceUpdater.latestPrices.USD;
+    const blk: BlockExtended = Object.assign({ extras: {} }, block);
+    blk.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
+    blk.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
+    blk.extras.coinbaseRaw = blk.extras.coinbaseTx.vin[0].scriptsig;
+    blk.extras.usd = priceUpdater.latestPrices.USD;
+    blk.extras.medianTimestamp = block.medianTime;
+    blk.extras.orphans = chainTips.getOrphanedBlocksAtHeight(blk.height);
 
     if (block.height === 0) {
-      blockExtended.extras.medianFee = 0; // 50th percentiles
-      blockExtended.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
-      blockExtended.extras.totalFees = 0;
-      blockExtended.extras.avgFee = 0;
-      blockExtended.extras.avgFeeRate = 0;
+      blk.extras.medianFee = 0; // 50th percentiles
+      blk.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
+      blk.extras.totalFees = 0;
+      blk.extras.avgFee = 0;
+      blk.extras.avgFeeRate = 0;
+      blk.extras.utxoSetChange = 0;
+      blk.extras.avgTxSize = 0;
+      blk.extras.totalInputs = 0;
+      blk.extras.totalOutputs = 1;
+      blk.extras.totalOutputAmt = 0;
+      blk.extras.segwitTotalTxs = 0;
+      blk.extras.segwitTotalSize = 0;
+      blk.extras.segwitTotalWeight = 0;
     } else {
-      const stats = await bitcoinClient.getBlockStats(block.id, [
-        'feerate_percentiles', 'minfeerate', 'maxfeerate', 'totalfee', 'avgfee', 'avgfeerate'
-      ]);
-      blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
-      blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
-      blockExtended.extras.totalFees = stats.totalfee;
-      blockExtended.extras.avgFee = stats.avgfee;
-      blockExtended.extras.avgFeeRate = stats.avgfeerate;
+      const stats = await bitcoinClient.getBlockStats(block.id);
+      blk.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
+      blk.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
+      blk.extras.totalFees = stats.totalfee;
+      blk.extras.avgFee = stats.avgfee;
+      blk.extras.avgFeeRate = stats.avgfeerate;
+      blk.extras.utxoSetChange = stats.utxo_increase;
+      blk.extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01;
+      blk.extras.totalInputs = stats.ins;
+      blk.extras.totalOutputs = stats.outs;
+      blk.extras.totalOutputAmt = stats.total_out;
+      blk.extras.segwitTotalTxs = stats.swtxs;
+      blk.extras.segwitTotalSize = stats.swtotal_size;
+      blk.extras.segwitTotalWeight = stats.swtotal_weight;
+    }
+
+    if (Common.blocksSummariesIndexingEnabled()) {
+      blk.extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(block.id);
+      if (blk.extras.feePercentiles !== null) {
+        blk.extras.medianFeeAmt = blk.extras.feePercentiles[3];
+      }
+    }
+  
+    blk.extras.virtualSize = block.weight / 4.0;
+    if (blk.extras.coinbaseTx.vout.length > 0) {
+      blk.extras.coinbaseAddress = blk.extras.coinbaseTx.vout[0].scriptpubkey_address ?? null;
+      blk.extras.coinbaseSignature = blk.extras.coinbaseTx.vout[0].scriptpubkey_asm ?? null;
+      blk.extras.coinbaseSignatureAscii = transactionUtils.hex2ascii(blk.extras.coinbaseTx.vin[0].scriptsig) ?? null;
+    } else {
+      blk.extras.coinbaseAddress = null;
+      blk.extras.coinbaseSignature = null;
+      blk.extras.coinbaseSignatureAscii = null;
+    }
+
+    const header = await bitcoinClient.getBlockHeader(block.id, false);
+    blk.extras.header = header;
+
+    const coinStatsIndex = indexer.isCoreIndexReady('coinstatsindex');
+    if (coinStatsIndex !== null && coinStatsIndex.best_block_height >= block.height) {
+      const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
+      blk.extras.utxoSetSize = txoutset.txouts,
+      blk.extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000);
+    } else {
+      blk.extras.utxoSetSize = null;
+      blk.extras.totalInputAmt = null;
     }
 
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
       let pool: PoolTag;
-      if (blockExtended.extras?.coinbaseTx !== undefined) {
-        pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
+      if (blk.extras?.coinbaseTx !== undefined) {
+        pool = await this.$findBlockMiner(blk.extras?.coinbaseTx);
       } else {
         if (config.DATABASE.ENABLED === true) {
           pool = await poolsRepository.$getUnknownPool();
@@ -201,10 +249,10 @@ class Blocks {
       }
 
       if (!pool) { // We should never have this situation in practise
-        logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` +
+        logger.warn(`Cannot assign pool to block ${blk.height} and 'unknown' pool does not exist. ` +
           `Check your "pools" table entries`);
       } else {
-        blockExtended.extras.pool = {
+        blk.extras.pool = {
           id: pool.id,
           name: pool.name,
           slug: pool.slug,
@@ -214,12 +262,12 @@ class Blocks {
       if (config.MEMPOOL.AUDIT) {
         const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
         if (auditScore != null) {
-          blockExtended.extras.matchRate = auditScore.matchRate;
+          blk.extras.matchRate = auditScore.matchRate;
         }
       }
     }
 
-    return blockExtended;
+    return blk;
   }
 
   /**
@@ -500,6 +548,7 @@ class Blocks {
       } else {
         this.currentBlockHeight++;
         logger.debug(`New block found (#${this.currentBlockHeight})!`);
+        await chainTips.updateOrphanedBlocks();
       }
 
       const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
@@ -688,7 +737,6 @@ class Blocks {
   }
 
   public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
-
     let currentHeight = fromHeight !== undefined ? fromHeight : this.currentBlockHeight;
     if (currentHeight > this.currentBlockHeight) {
       limit -= currentHeight - this.currentBlockHeight;
@@ -728,6 +776,113 @@ class Blocks {
     return returnBlocks;
   }
 
+  /**
+   * Used for bulk block data query
+   * 
+   * @param fromHeight 
+   * @param toHeight 
+   */
+  public async $getBlocksBetweenHeight(fromHeight: number, toHeight: number): Promise<any> {
+    if (!Common.indexingEnabled()) {
+      return [];
+    }
+
+    const blocks: any[] = [];
+
+    while (fromHeight <= toHeight) {
+      let block: any = await blocksRepository.$getBlockByHeight(fromHeight);
+      if (!block) {
+        await this.$indexBlock(fromHeight);
+        block = await blocksRepository.$getBlockByHeight(fromHeight);
+        if (!block) {
+          continue;
+        }
+      }
+
+      // Cleanup fields before sending the response
+      const cleanBlock: any = {
+        height: block.height ?? null,
+        hash: block.id ?? null,
+        timestamp: block.blockTimestamp ?? null,
+        median_timestamp: block.medianTime ?? null,
+        previous_block_hash: block.previousblockhash ?? null,
+        difficulty: block.difficulty ?? null,
+        header: block.header ?? null,
+        version: block.version ?? null,
+        bits: block.bits ?? null,
+        nonce: block.nonce ?? null,
+        size: block.size ?? null,
+        weight: block.weight ?? null,
+        tx_count: block.tx_count ?? null,
+        merkle_root: block.merkle_root ?? null,
+        reward: block.reward ?? null,
+        total_fee_amt: block.fees ?? null,
+        avg_fee_amt: block.avg_fee ?? null,
+        median_fee_amt: block.median_fee_amt ?? null,
+        fee_amt_percentiles: block.fee_percentiles ?? null,
+        avg_fee_rate: block.avg_fee_rate ?? null,
+        median_fee_rate: block.median_fee ?? null,
+        fee_rate_percentiles: block.fee_span ?? null,
+        total_inputs: block.total_inputs ?? null,
+        total_input_amt: block.total_input_amt ?? null,
+        total_outputs: block.total_outputs ?? null,
+        total_output_amt: block.total_output_amt ?? null,
+        segwit_total_txs: block.segwit_total_txs ?? null,
+        segwit_total_size: block.segwit_total_size ?? null,
+        segwit_total_weight: block.segwit_total_weight ?? null,
+        avg_tx_size: block.avg_tx_size ?? null,
+        utxoset_change: block.utxoset_change ?? null,
+        utxoset_size: block.utxoset_size ?? null,
+        coinbase_raw: block.coinbase_raw ?? null,
+        coinbase_address: block.coinbase_address ?? null,
+        coinbase_signature: block.coinbase_signature ?? null,
+        coinbase_signature_ascii: block.coinbase_signature_ascii ?? null,
+        pool_slug: block.pool_slug ?? null,
+      };
+
+      if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) {
+        cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
+        if (cleanBlock.fee_amt_percentiles === null) {
+          const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
+          const summary = this.summarizeBlock(block);
+          await BlocksSummariesRepository.$saveSummary({ height: block.height, mined: summary });
+          cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
+        }
+        if (cleanBlock.fee_amt_percentiles !== null) {
+          cleanBlock.median_fee_amt = cleanBlock.fee_amt_percentiles[3];
+        }
+      }
+
+      cleanBlock.fee_amt_percentiles = {
+        'min': cleanBlock.fee_amt_percentiles[0],
+        'perc_10': cleanBlock.fee_amt_percentiles[1],
+        'perc_25': cleanBlock.fee_amt_percentiles[2],
+        'perc_50': cleanBlock.fee_amt_percentiles[3],
+        'perc_75': cleanBlock.fee_amt_percentiles[4],
+        'perc_90': cleanBlock.fee_amt_percentiles[5],
+        'max': cleanBlock.fee_amt_percentiles[6],
+      };
+      cleanBlock.fee_rate_percentiles = {
+        'min': cleanBlock.fee_rate_percentiles[0],
+        'perc_10': cleanBlock.fee_rate_percentiles[1],
+        'perc_25': cleanBlock.fee_rate_percentiles[2],
+        'perc_50': cleanBlock.fee_rate_percentiles[3],
+        'perc_75': cleanBlock.fee_rate_percentiles[4],
+        'perc_90': cleanBlock.fee_rate_percentiles[5],
+        'max': cleanBlock.fee_rate_percentiles[6],
+      };
+
+      // Re-org can happen after indexing so we need to always get the
+      // latest state from core
+      cleanBlock.orphans = chainTips.getOrphanedBlocksAtHeight(cleanBlock.height);
+
+      blocks.push(cleanBlock);
+      fromHeight++;
+    }
+
+    return blocks;
+  }
+
   public async $getBlockAuditSummary(hash: string): Promise<any> {
     let summary;
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {

backend/src/api/chain-tips.ts

@@ -0,0 +1,57 @@
+import logger from "../logger";
+import bitcoinClient from "./bitcoin/bitcoin-client";
+
+export interface ChainTip {
+  height: number;
+  hash: string;
+  branchlen: number;
+  status: 'invalid' | 'active' | 'valid-fork' | 'valid-headers' | 'headers-only';
+};
+
+export interface OrphanedBlock {
+  height: number;
+  hash: string;
+  status: 'valid-fork' | 'valid-headers' | 'headers-only';
+}
+
+class ChainTips {
+  private chainTips: ChainTip[] = [];
+  private orphanedBlocks: OrphanedBlock[] = [];
+
+  public async updateOrphanedBlocks(): Promise<void> {
+    try {
+      this.chainTips = await bitcoinClient.getChainTips();
+      this.orphanedBlocks = [];
+
+      for (const chain of this.chainTips) {
+        if (chain.status === 'valid-fork' || chain.status === 'valid-headers') {
+          let block = await bitcoinClient.getBlock(chain.hash);
+          while (block && block.confirmations === -1) {
+            this.orphanedBlocks.push({
+              height: block.height,
+              hash: block.hash,
+              status: chain.status
+            });
+            block = await bitcoinClient.getBlock(block.previousblockhash);
+          }
+        }
+      }
+
+      logger.debug(`Updated orphaned blocks cache. Found ${this.orphanedBlocks.length} orphaned blocks`);
+    } catch (e) {
+      logger.err(`Cannot get fetch orphaned blocks. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+  }
+
+  public getOrphanedBlocksAtHeight(height: number): OrphanedBlock[] {
+    const orphans: OrphanedBlock[] = [];
+    for (const block of this.orphanedBlocks) {
+      if (block.height === height) {
+        orphans.push(block);
+      }
+    }
+    return orphans;
+  }
+}
+
+export default new ChainTips();

backend/src/api/database-migration.ts

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 52;
+  private static currentVersion = 56;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -62,8 +62,8 @@ class DatabaseMigration {
 
     if (databaseSchemaVersion <= 2) {
       // Disable some spam logs when they're not relevant
-      this.uniqueLogs.push(this.blocksTruncatedMessage);
-      this.uniqueLogs.push(this.hashratesTruncatedMessage);
+      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+      this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
     }
 
     logger.debug('MIGRATIONS: Current state.schema_version ' + databaseSchemaVersion);
@@ -86,7 +86,7 @@ class DatabaseMigration {
       try {
         await this.$migrateTableSchemaFromVersion(databaseSchemaVersion);
         if (databaseSchemaVersion === 0) {
-          logger.notice(`MIGRATIONS: OK. Database schema has been properly initialized to version ${DatabaseMigration.currentVersion} (latest version)`);          
+          logger.notice(`MIGRATIONS: OK. Database schema has been properly initialized to version ${DatabaseMigration.currentVersion} (latest version)`);
         } else {
           logger.notice(`MIGRATIONS: OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`);
         }
@@ -300,7 +300,7 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"');
       await this.updateToSchemaVersion(27);
     }
-    
+
     if (databaseSchemaVersion < 28 && isBitcoin === true) {
       if (config.LIGHTNING.ENABLED) {
         this.uniqueLog(logger.notice, `'lightning_stats' and 'node_stats' tables have been truncated.`);
@@ -464,10 +464,42 @@ class DatabaseMigration {
         await this.$executeQuery('DROP TABLE IF EXISTS `transactions`');
         await this.$executeQuery('DROP TABLE IF EXISTS `cpfp_clusters`');
         await this.updateToSchemaVersion(52);
-      } catch(e) {
+      } catch (e) {
         logger.warn('' + (e instanceof Error ? e.message : e));
       }
     }
+
+    if (databaseSchemaVersion < 53) {
+      await this.$executeQuery('ALTER TABLE statistics MODIFY mempool_byte_weight bigint(20) UNSIGNED NOT NULL');
+      await this.updateToSchemaVersion(53);
+    }
+
+    if (databaseSchemaVersion < 54) {
+      this.uniqueLog(logger.notice, `'prices' table has been truncated`);
+      await this.$executeQuery(`TRUNCATE prices`);
+      if (isBitcoin === true) {
+        this.uniqueLog(logger.notice, `'blocks_prices' table has been truncated`);
+        await this.$executeQuery(`TRUNCATE blocks_prices`);
+      }
+      await this.updateToSchemaVersion(54);
+    }
+
+    if (databaseSchemaVersion < 55) {
+      await this.$executeQuery(this.getAdditionalBlocksDataQuery());
+      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+      await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
+      await this.updateToSchemaVersion(55);
+    }
+
+    if (databaseSchemaVersion < 56) {
+      await this.$executeQuery('ALTER TABLE pools ADD unique_id int NOT NULL DEFAULT -1');
+      await this.$executeQuery('TRUNCATE TABLE `blocks`');
+      this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
+      await this.$executeQuery('DELETE FROM `pools`');
+      await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
+      this.uniqueLog(logger.notice, '`pools` table has been truncated`');
+      await this.updateToSchemaVersion(56);
+    }
   }
 
   /**
@@ -591,7 +623,7 @@ class DatabaseMigration {
       queries.push(`INSERT INTO state(name, number, string) VALUES ('last_hashrates_indexing', 0, NULL)`);
     }
 
-    if (version < 9  && isBitcoin === true) {
+    if (version < 9 && isBitcoin === true) {
       queries.push(`INSERT INTO state(name, number, string) VALUES ('last_weekly_hashrates_indexing', 0, NULL)`);
     }
 
@@ -741,6 +773,28 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getAdditionalBlocksDataQuery(): string {
+    return `ALTER TABLE blocks
+      ADD median_timestamp timestamp NOT NULL,
+      ADD coinbase_address varchar(100) NULL,
+      ADD coinbase_signature varchar(500) NULL,
+      ADD coinbase_signature_ascii varchar(500) NULL,
+      ADD avg_tx_size double unsigned NOT NULL,
+      ADD total_inputs int unsigned NOT NULL,
+      ADD total_outputs int unsigned NOT NULL,
+      ADD total_output_amt bigint unsigned NOT NULL,
+      ADD fee_percentiles longtext NULL,
+      ADD median_fee_amt int unsigned NULL,
+      ADD segwit_total_txs int unsigned NOT NULL,
+      ADD segwit_total_size int unsigned NOT NULL,
+      ADD segwit_total_weight int unsigned NOT NULL,
+      ADD header varchar(160) NOT NULL,
+      ADD utxoset_change int NOT NULL,
+      ADD utxoset_size int unsigned NULL,
+      ADD total_input_amt bigint unsigned NULL
+    `;
+  }
+
   private getCreateDailyStatsTableQuery(): string {
     return `CREATE TABLE IF NOT EXISTS hashrates (
       hashrate_timestamp timestamp NOT NULL,
@@ -958,26 +1012,16 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
-  public async $truncateIndexedData(tables: string[]) {
-    const allowedTables = ['blocks', 'hashrates', 'prices'];
+  public async $blocksReindexingTruncate(): Promise<void> {
+    logger.warn(`Truncating pools, blocks and hashrates for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`);
+    await Common.sleep$(5000);
 
-    try {
-      for (const table of tables) {
-        if (!allowedTables.includes(table)) {
-          logger.debug(`Table ${table} cannot to be re-indexed (not allowed)`);
-          continue;
-        }
-
-        await this.$executeQuery(`TRUNCATE ${table}`, true);
-        if (table === 'hashrates') {
-          await this.$executeQuery('UPDATE state set number = 0 where name = "last_hashrates_indexing"', true);
-        }
-        logger.notice(`Table ${table} has been truncated`);
-      }
-    } catch (e) {
-      logger.warn(`Unable to erase indexed data`);
-    }
-  }
+    await this.$executeQuery(`TRUNCATE blocks`);
+    await this.$executeQuery(`TRUNCATE hashrates`);
+    await this.$executeQuery('DELETE FROM `pools`');
+    await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
+    await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`);
+}
 
   private async $convertCompactCpfpTables(): Promise<void> {
     try {

backend/src/api/disk-cache.ts

@@ -9,7 +9,7 @@ import { TransactionExtended } from '../mempool.interfaces';
 import { Common } from './common';
 
 class DiskCache {
-  private cacheSchemaVersion = 1;
+  private cacheSchemaVersion = 2;
 
   private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
   private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json';

backend/src/api/explorer/nodes.api.ts

@@ -362,7 +362,13 @@ class NodesApi {
   public async $searchNodeByPublicKeyOrAlias(search: string) {
     try {
       const publicKeySearch = search.replace('%', '') + '%';
-      const aliasSearch = search.replace(/[-_.]/g, ' ').replace(/[^a-zA-Z0-9 ]/g, '').split(' ').map((search) => '+' + search + '*').join(' ');
+      const aliasSearch = search
+        .replace(/[-_.]/g, ' ') // Replace all -_. characters with empty space. Eg: "ln.nicehash" becomes "ln nicehash".  
+        .replace(/[^a-zA-Z0-9 ]/g, '') // Remove all special characters and keep just A to Z, 0 to 9.
+        .split(' ')
+        .filter(key => key.length)
+        .map((search) => '+' + search + '*').join(' ');
+      // %keyword% is wildcard search and can't be indexed so it's slower as the node database grow. keyword% can be indexed but then you can't search for "Nicehash" and get result for ln.nicehash.com. So we use fulltext index for words "ln, nicehash, com" and nicehash* will find it instantly.
       const query = `SELECT public_key, alias, capacity, channels, status FROM nodes WHERE public_key LIKE ? OR MATCH alias_search AGAINST (? IN BOOLEAN MODE) ORDER BY capacity DESC LIMIT 10`;
       const [rows]: any = await DB.query(query, [publicKeySearch, aliasSearch]);
       return rows;

backend/src/api/mining/mining-routes.ts

@@ -1,13 +1,13 @@
 import { Application, Request, Response } from 'express';
 import config from "../../config";
 import logger from '../../logger';
-import audits from '../audit';
 import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
 import BlocksRepository from '../../repositories/BlocksRepository';
 import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository';
 import HashratesRepository from '../../repositories/HashratesRepository';
 import bitcoinClient from '../bitcoin/bitcoin-client';
 import mining from "./mining";
+import PricesRepository from '../../repositories/PricesRepository';
 
 class MiningRoutes {
   public initRoutes(app: Application) {
@@ -32,9 +32,27 @@ class MiningRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/score/:hash', this.$getBlockAuditScore)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/:hash', this.$getBlockAudit)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/timestamp/:timestamp', this.$getHeightFromTimestamp)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'historical-price', this.$getHistoricalPrice)
     ;
   }
 
+  private async $getHistoricalPrice(req: Request, res: Response): Promise<void> {
+    try {
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      if (req.query.timestamp) {
+        res.status(200).send(await PricesRepository.$getNearestHistoricalPrice(
+          parseInt(<string>req.query.timestamp ?? 0, 10)
+        ));
+      } else {
+        res.status(200).send(await PricesRepository.$getHistoricalPrices());
+      }
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async $getPool(req: Request, res: Response): Promise<void> {
     try {
       const stats = await mining.$getPoolStat(req.params.slug);

backend/src/api/mining/mining.ts

@@ -100,6 +100,7 @@ class Mining {
         rank: rank++,
         emptyBlocks: emptyBlocksCount.length > 0 ? emptyBlocksCount[0]['count'] : 0,
         slug: poolInfo.slug,
+        avgMatchRate: poolInfo.avgMatchRate !== null ? Math.round(100 * poolInfo.avgMatchRate) / 100 : null,
       };
       poolsStats.push(poolStat);
     });
@@ -171,7 +172,7 @@ class Mining {
   }
 
   /**
-   * [INDEXING] Generate weekly mining pool hashrate history
+   * Generate weekly mining pool hashrate history
    */
   public async $generatePoolHashrateHistory(): Promise<void> {
     const now = new Date();
@@ -278,7 +279,7 @@ class Mining {
   }
 
   /**
-   * [INDEXING] Generate daily hashrate data
+   * Generate daily hashrate data
    */
   public async $generateNetworkHashrateHistory(): Promise<void> {
     // We only run this once a day around midnight
@@ -458,7 +459,7 @@ class Mining {
   /**
    * Create a link between blocks and the latest price at when they were mined
    */
-  public async $indexBlockPrices() {
+  public async $indexBlockPrices(): Promise<void> {
     if (this.blocksPriceIndexingRunning === true) {
       return;
     }
@@ -519,6 +520,41 @@ class Mining {
     this.blocksPriceIndexingRunning = false;
   }
 
+  /**
+   * Index core coinstatsindex
+   */
+  public async $indexCoinStatsIndex(): Promise<void> {
+    let timer = new Date().getTime() / 1000;
+    let totalIndexed = 0;
+
+    const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+    let currentBlockHeight = blockchainInfo.blocks;
+
+    while (currentBlockHeight > 0) {
+      const indexedBlocks = await BlocksRepository.$getBlocksMissingCoinStatsIndex(
+        currentBlockHeight, currentBlockHeight - 10000);
+        
+      for (const block of indexedBlocks) {
+        const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
+        await BlocksRepository.$updateCoinStatsIndexData(block.hash, txoutset.txouts,
+          Math.round(txoutset.block_info.prevout_spent * 100000000));        
+        ++totalIndexed;
+
+        const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
+        if (elapsedSeconds > 5) {
+          logger.info(`Indexing coinstatsindex data for block #${block.height}. Indexed ${totalIndexed} blocks.`, logger.tags.mining);
+          timer = new Date().getTime() / 1000;
+        }
+      }
+
+      currentBlockHeight -= 10000;
+    }
+
+    if (totalIndexed) {
+      logger.info(`Indexing missing coinstatsindex data completed`, logger.tags.mining);
+    }
+  }
+
   private getDateMidnight(date: Date): Date {
     date.setUTCHours(0);
     date.setUTCMinutes(0);

backend/src/api/pools-parser.ts

@@ -1,15 +1,8 @@
 import DB from '../database';
 import logger from '../logger';
 import config from '../config';
-import BlocksRepository from '../repositories/BlocksRepository';
-
-interface Pool {
-  name: string;
-  link: string;
-  regexes: string[];
-  addresses: string[];
-  slug: string;
-}
+import PoolsRepository from '../repositories/PoolsRepository';
+import { PoolTag } from '../mempool.interfaces';
 
 class PoolsParser {
   miningPools: any[] = [];
@@ -20,270 +13,142 @@ class PoolsParser {
     'addresses': '[]',
     'slug': 'unknown'
   };
-  slugWarnFlag = false;
+  private uniqueLogs: string[] = [];
+
+  private uniqueLog(loggerFunction: any, msg: string): void {
+    if (this.uniqueLogs.includes(msg)) {
+      return;
+    }
+    this.uniqueLogs.push(msg);
+    loggerFunction(msg);
+  }
+
+  public setMiningPools(pools): void {
+    for (const pool of pools) {
+      pool.regexes = pool.tags;
+      delete(pool.tags);
+    }
+    this.miningPools = pools;
+  }
 
   /**
-   * Parse the pools.json file, consolidate the data and dump it into the database
+   * Populate our db with updated mining pool definition
+   * @param pools 
    */
-  public async migratePoolsJson(poolsJson: object): Promise<void> {
-    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
-      return;
-    }
+  public async migratePoolsJson(): Promise<void> {
+    await this.$insertUnknownPool();
 
-    // First we save every entries without paying attention to pool duplication
-    const poolsDuplicated: Pool[] = [];
-
-    const coinbaseTags = Object.entries(poolsJson['coinbase_tags']);
-    for (let i = 0; i < coinbaseTags.length; ++i) {
-      poolsDuplicated.push({
-        'name': (<Pool>coinbaseTags[i][1]).name,
-        'link': (<Pool>coinbaseTags[i][1]).link,
-        'regexes': [coinbaseTags[i][0]],
-        'addresses': [],
-        'slug': ''
-      });
-    }
-    const addressesTags = Object.entries(poolsJson['payout_addresses']);
-    for (let i = 0; i < addressesTags.length; ++i) {
-      poolsDuplicated.push({
-        'name': (<Pool>addressesTags[i][1]).name,
-        'link': (<Pool>addressesTags[i][1]).link,
-        'regexes': [],
-        'addresses': [addressesTags[i][0]],
-        'slug': ''
-      });
-    }
-
-    // Then, we find unique mining pool names
-    const poolNames: string[] = [];
-    for (let i = 0; i < poolsDuplicated.length; ++i) {
-      if (poolNames.indexOf(poolsDuplicated[i].name) === -1) {
-        poolNames.push(poolsDuplicated[i].name);
+    for (const pool of this.miningPools) {
+      if (!pool.id) {
+        logger.info(`Mining pool ${pool.name} has no unique 'id' defined. Skipping.`);
+        continue;
       }
-    }
-    logger.debug(`Found ${poolNames.length} unique mining pools`, logger.tags.mining);
 
-    // Get existing pools from the db
-    let existingPools;
-    try {
-      if (config.DATABASE.ENABLED === true) {
-        [existingPools] = await DB.query({ sql: 'SELECT * FROM pools;', timeout: 120000 });
+      const poolDB = await PoolsRepository.$getPoolByUniqueId(pool.id, false);
+      if (!poolDB) {
+        // New mining pool
+        const slug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
+        logger.debug(`Inserting new mining pool ${pool.name}`);
+        await PoolsRepository.$insertNewMiningPool(pool, slug);
+        await this.$deleteUnknownBlocks();
       } else {
-        existingPools = [];
-      }
-    } catch (e) {
-      logger.err('Cannot get existing pools from the database, skipping pools.json import', logger.tags.mining);
-      return;
-    }
-
-    this.miningPools = [];
-
-    // Finally, we generate the final consolidated pools data
-    const finalPoolDataAdd: Pool[] = [];
-    const finalPoolDataUpdate: Pool[] = [];
-    const finalPoolDataRename: Pool[] = [];
-    for (let i = 0; i < poolNames.length; ++i) {
-      let allAddresses: string[] = [];
-      let allRegexes: string[] = [];
-      const match = poolsDuplicated.filter((pool: Pool) => pool.name === poolNames[i]);
-
-      for (let y = 0; y < match.length; ++y) {
-        allAddresses = allAddresses.concat(match[y].addresses);
-        allRegexes = allRegexes.concat(match[y].regexes);
-      }
-
-      const finalPoolName = poolNames[i].replace(`'`, `''`); // To support single quote in names when doing db queries
-
-      let slug: string | undefined;
-      try {
-        slug = poolsJson['slugs'][poolNames[i]];
-      } catch (e) {
-        if (this.slugWarnFlag === false) {
-          logger.warn(`pools.json does not seem to contain the 'slugs' object`, logger.tags.mining);
-          this.slugWarnFlag = true;
+        if (poolDB.name !== pool.name) {
+          // Pool has been renamed
+          const newSlug = pool.name.replace(/[^a-z0-9]/gi, '').toLowerCase();
+          logger.warn(`Renaming ${poolDB.name} mining pool to ${pool.name}. Slug has been updated. Maybe you want to make a redirection from 'https://mempool.space/mining/pool/${poolDB.slug}' to 'https://mempool.space/mining/pool/${newSlug}`);
+          await PoolsRepository.$renameMiningPool(poolDB.id, newSlug, pool.name);
         }
-      }
-
-      if (slug === undefined) {
-        // Only keep alphanumerical
-        slug = poolNames[i].replace(/[^a-z0-9]/gi, '').toLowerCase();
-        logger.warn(`No slug found for '${poolNames[i]}', generating it => '${slug}'`, logger.tags.mining);
-      }
-
-      const poolObj = {
-        'name': finalPoolName,
-        'link': match[0].link,
-        'regexes': allRegexes,
-        'addresses': allAddresses,
-        'slug': slug
-      };
-
-      const existingPool = existingPools.find((pool) => pool.name === poolNames[i]);
-      if (existingPool !== undefined) {
-        // Check if any data was actually updated
-        const equals = (a, b) =>
-          a.length === b.length &&
-          a.every((v, i) => v === b[i]);
-        if (!equals(JSON.parse(existingPool.addresses), poolObj.addresses) || !equals(JSON.parse(existingPool.regexes), poolObj.regexes)) {
-          finalPoolDataUpdate.push(poolObj);
+        if (poolDB.link !== pool.link) {
+          // Pool link has changed
+          logger.debug(`Updating link for ${pool.name} mining pool`);
+          await PoolsRepository.$updateMiningPoolLink(poolDB.id, pool.link);
         }
-      } else if (config.DATABASE.ENABLED) {
-        // Double check that if we're not just renaming a pool (same address same regex)
-        const [poolToRename]: any[] = await DB.query(`
-          SELECT * FROM pools
-          WHERE addresses = ? OR regexes = ?`,
-          [JSON.stringify(poolObj.addresses), JSON.stringify(poolObj.regexes)]
-        );
-        if (poolToRename && poolToRename.length > 0) {
-          // We're actually renaming an existing pool
-          finalPoolDataRename.push({
-            'name': poolObj.name,
-            'link': poolObj.link,
-            'regexes': allRegexes,
-            'addresses': allAddresses,
-            'slug': slug
-          });
-          logger.debug(`Rename '${poolToRename[0].name}' mining pool to ${poolObj.name}`, logger.tags.mining);
-        } else {
-          logger.debug(`Add '${finalPoolName}' mining pool`, logger.tags.mining);
-          finalPoolDataAdd.push(poolObj);
+        if (JSON.stringify(pool.addresses) !== poolDB.addresses ||
+          JSON.stringify(pool.regexes) !== poolDB.regexes) {
+          // Pool addresses changed or coinbase tags changed
+          logger.notice(`Updating addresses and/or coinbase tags for ${pool.name} mining pool. If 'AUTOMATIC_BLOCK_REINDEXING' is enabled, we will re-index its blocks and 'unknown' blocks`);
+          await PoolsRepository.$updateMiningPoolTags(poolDB.id, pool.addresses, pool.regexes);
+          await this.$deleteBlocksForPool(poolDB);
         }
       }
-
-      this.miningPools.push({
-        'name': finalPoolName,
-        'link': match[0].link,
-        'regexes': JSON.stringify(allRegexes),
-        'addresses': JSON.stringify(allAddresses),
-        'slug': slug
-      });
-    }
-
-    if (config.DATABASE.ENABLED === false) { // Don't run db operations
-      logger.info('Mining pools.json import completed (no database)', logger.tags.mining);
-      return;
-    }
-
-    if (finalPoolDataAdd.length > 0 || finalPoolDataUpdate.length > 0 ||
-      finalPoolDataRename.length > 0
-    ) {    
-      logger.debug(`Update pools table now`, logger.tags.mining);
-
-      // Add new mining pools into the database
-      let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses, slug) VALUES ';
-      for (let i = 0; i < finalPoolDataAdd.length; ++i) {
-        queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}',
-        '${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}',
-        ${JSON.stringify(finalPoolDataAdd[i].slug)}),`;
-      }
-      queryAdd = queryAdd.slice(0, -1) + ';';
-
-      // Updated existing mining pools in the database
-      const updateQueries: string[] = [];
-      for (let i = 0; i < finalPoolDataUpdate.length; ++i) {
-        updateQueries.push(`
-          UPDATE pools
-          SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}',
-          regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}',
-          slug='${finalPoolDataUpdate[i].slug}'
-          WHERE name='${finalPoolDataUpdate[i].name}'
-        ;`);
-      }
-
-      // Rename mining pools
-      const renameQueries: string[] = [];
-      for (let i = 0; i < finalPoolDataRename.length; ++i) {
-        renameQueries.push(`
-          UPDATE pools
-          SET name='${finalPoolDataRename[i].name}', link='${finalPoolDataRename[i].link}',
-            slug='${finalPoolDataRename[i].slug}'
-          WHERE regexes='${JSON.stringify(finalPoolDataRename[i].regexes)}'
-            AND addresses='${JSON.stringify(finalPoolDataRename[i].addresses)}'
-        ;`);
-      }
-
-      try {
-        if (finalPoolDataAdd.length > 0 || updateQueries.length > 0) {
-          await this.$deleteBlocskToReindex(finalPoolDataUpdate);
-        }
-
-        if (finalPoolDataAdd.length > 0) {
-          await DB.query({ sql: queryAdd, timeout: 120000 });
-        }
-        for (const query of updateQueries) {
-          await DB.query({ sql: query, timeout: 120000 });
-        }
-        for (const query of renameQueries) {
-          await DB.query({ sql: query, timeout: 120000 });
-        }
-        await this.insertUnknownPool();
-        logger.info('Mining pools.json import completed', logger.tags.mining);
-      } catch (e) {
-        logger.err(`Cannot import pools in the database`, logger.tags.mining);
-        throw e;
-      }
     }
 
-    try {
-      await this.insertUnknownPool();
-    } catch (e) {
-      logger.err(`Cannot insert unknown pool in the database`, logger.tags.mining);
-      throw e;
-    }
+    logger.info('Mining pools-v2.json import completed');
   }
 
   /**
    * Manually add the 'unknown pool'
    */
-  private async insertUnknownPool() {
+  public async $insertUnknownPool(): Promise<void> {
+    if (!config.DATABASE.ENABLED) {
+      return;
+    }
+
     try {
       const [rows]: any[] = await DB.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 });
       if (rows.length === 0) {
         await DB.query({
-          sql: `INSERT INTO pools(name, link, regexes, addresses, slug)
-          VALUES("Unknown", "https://learnmeabitcoin.com/technical/coinbase-transaction", "[]", "[]", "unknown");
+          sql: `INSERT INTO pools(name, link, regexes, addresses, slug, unique_id)
+          VALUES("${this.unknownPool.name}", "${this.unknownPool.link}", "[]", "[]", "${this.unknownPool.slug}", 0);
         `});
       } else {
         await DB.query(`UPDATE pools
-          SET name='Unknown', link='https://learnmeabitcoin.com/technical/coinbase-transaction',
+          SET name='${this.unknownPool.name}', link='${this.unknownPool.link}',
           regexes='[]', addresses='[]',
-          slug='unknown'
-          WHERE name='Unknown'
+          slug='${this.unknownPool.slug}',
+          unique_id=0
+          WHERE slug='${this.unknownPool.slug}'
         `);
       }
     } catch (e) {
-      logger.err('Unable to insert "Unknown" mining pool', logger.tags.mining);
+      logger.err(`Unable to insert or update "Unknown" mining pool. Reason: ${e instanceof Error ? e.message : e}`);
     }
   }
 
   /**
-   * Delete blocks which needs to be reindexed
+   * Delete indexed blocks for an updated mining pool
+   * 
+   * @param pool 
    */
-   private async $deleteBlocskToReindex(finalPoolDataUpdate: any[]) {
+  private async $deleteBlocksForPool(pool: PoolTag): Promise<void> {
     if (config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING === false) {
       return;
     }
 
-    const blockCount = await BlocksRepository.$blockCount(null, null);
-    if (blockCount === 0) {
-      return;
-    }
-
-    for (const updatedPool of finalPoolDataUpdate) {
-      const [pool]: any[] = await DB.query(`SELECT id, name from pools where slug = "${updatedPool.slug}"`);
-      if (pool.length > 0) {
-        logger.notice(`Deleting blocks from ${pool[0].name} mining pool for future re-indexing`, logger.tags.mining);
-        await DB.query(`DELETE FROM blocks WHERE pool_id = ${pool[0].id}`);
-      }
-    }
-
-    // Ignore early days of Bitcoin as there were not mining pool yet
-    logger.notice(`Deleting blocks with unknown mining pool from height 130635 for future re-indexing`, logger.tags.mining);
+    // Get oldest blocks mined by the pool and assume pools-v2.json updates only concern most recent years
+    // Ignore early days of Bitcoin as there were no mining pool yet
+    const [oldestPoolBlock]: any[] = await DB.query(`
+      SELECT height
+      FROM blocks
+      WHERE pool_id = ?
+      ORDER BY height
+      LIMIT 1`,
+      [pool.id]
+    );
+    const oldestBlockHeight = oldestPoolBlock.length ?? 0 > 0 ? oldestPoolBlock[0].height : 130635;
     const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
-    await DB.query(`DELETE FROM blocks WHERE pool_id = ${unknownPool[0].id} AND height > 130635`);
+    this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${oldestBlockHeight} for re-indexing`);
+    await DB.query(`
+      DELETE FROM blocks
+      WHERE pool_id = ? AND height >= ${oldestBlockHeight}`,
+      [unknownPool[0].id]
+    );
+    logger.notice(`Deleting blocks from ${pool.name} mining pool for re-indexing`);
+    await DB.query(`
+      DELETE FROM blocks
+      WHERE pool_id = ?`,
+      [pool.id]
+    );
+  }
 
-    logger.notice(`Truncating hashrates for future re-indexing`, logger.tags.mining);
-    await DB.query(`DELETE FROM hashrates`);
+  private async $deleteUnknownBlocks(): Promise<void> {
+    const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
+    this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height 130635 for re-indexing`);
+    await DB.query(`
+      DELETE FROM blocks
+      WHERE pool_id = ? AND height >= 130635`,
+      [unknownPool[0].id]
+    );
   }
 }
 

backend/src/api/transaction-utils.ts

@@ -14,6 +14,7 @@ class TransactionUtils {
       vout: tx.vout
         .map((vout) => ({
           scriptpubkey_address: vout.scriptpubkey_address,
+          scriptpubkey_asm: vout.scriptpubkey_asm,
           value: vout.value
         }))
         .filter((vout) => vout.value)

backend/src/config.ts

@@ -32,6 +32,7 @@ interface IConfig {
     ADVANCED_GBT_AUDIT: boolean;
     ADVANCED_GBT_MEMPOOL: boolean;
     CPFP_INDEXING: boolean;
+    MAX_BLOCKS_BULK_QUERY: number;
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -147,12 +148,13 @@ const defaults: IConfig = {
     'USER_AGENT': 'mempool',
     'STDOUT_LOG_MIN_PRIORITY': 'debug',
     'AUTOMATIC_BLOCK_REINDEXING': false,
-    'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
+    'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json',
     'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
     'AUDIT': false,
     'ADVANCED_GBT_AUDIT': false,
     'ADVANCED_GBT_MEMPOOL': false,
     'CPFP_INDEXING': false,
+    'MAX_BLOCKS_BULK_QUERY': 0,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',

backend/src/database.ts

@@ -24,7 +24,8 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr
 
   private checkDBFlag() {
     if (config.DATABASE.ENABLED === false) {
-      logger.err('Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue');
+      const stack = new Error().stack;
+      logger.err(`Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue.\nStack trace: ${stack}}`);
     }
   }
 

backend/src/index.ts

@@ -36,6 +36,7 @@ import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
 import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher';
 import forensicsService from './tasks/lightning/forensics.service';
 import priceUpdater from './tasks/price-updater';
+import chainTips from './api/chain-tips';
 import { AxiosError } from 'axios';
 
 class Server {
@@ -82,11 +83,8 @@ class Server {
     if (config.DATABASE.ENABLED) {
       await DB.checkDbConnection();
       try {
-        if (process.env.npm_config_reindex !== undefined) { // Re-index requests
-          const tables = process.env.npm_config_reindex.split(',');
-          logger.warn(`Indexed data for "${process.env.npm_config_reindex}" tables will be erased in 5 seconds (using '--reindex')`);
-          await Common.sleep$(5000);
-          await databaseMigration.$truncateIndexedData(tables);
+        if (process.env.npm_config_reindex_blocks === 'true') { // Re-index requests
+          await databaseMigration.$blocksReindexingTruncate();
         }
         await databaseMigration.$initializeOrMigrateDatabase();
         if (Common.indexingEnabled()) {
@@ -133,6 +131,7 @@ class Server {
     }
 
     priceUpdater.$run();
+    await chainTips.updateOrphanedBlocks();
 
     this.setUpHttpApiRoutes();
 

backend/src/indexer.ts

@@ -8,18 +8,67 @@ import bitcoinClient from './api/bitcoin/bitcoin-client';
 import priceUpdater from './tasks/price-updater';
 import PricesRepository from './repositories/PricesRepository';
 
+export interface CoreIndex {
+  name: string;
+  synced: boolean;
+  best_block_height: number;
+}
+
 class Indexer {
   runIndexer = true;
   indexerRunning = false;
   tasksRunning: string[] = [];
+  coreIndexes: CoreIndex[] = [];
 
-  public reindex() {
+  /**
+   * Check which core index is available for indexing
+   */
+  public async checkAvailableCoreIndexes(): Promise<void> {
+    const updatedCoreIndexes: CoreIndex[] = [];
+
+    const indexes: any = await bitcoinClient.getIndexInfo();
+    for (const indexName in indexes) {
+      const newState = {
+        name: indexName,
+        synced: indexes[indexName].synced,
+        best_block_height: indexes[indexName].best_block_height,
+      };
+      logger.info(`Core index '${indexName}' is ${indexes[indexName].synced ? 'synced' : 'not synced'}. Best block height is ${indexes[indexName].best_block_height}`);      
+      updatedCoreIndexes.push(newState);
+
+      if (indexName === 'coinstatsindex' && newState.synced === true) {
+        const previousState = this.isCoreIndexReady('coinstatsindex');
+        // if (!previousState || previousState.synced === false) {
+          this.runSingleTask('coinStatsIndex');
+        // }
+      }
+    }
+
+    this.coreIndexes = updatedCoreIndexes;
+  }
+
+  /**
+   * Return the best block height if a core index is available, or 0 if not
+   * 
+   * @param name 
+   * @returns 
+   */
+  public isCoreIndexReady(name: string): CoreIndex | null {
+    for (const index of this.coreIndexes) {
+      if (index.name === name && index.synced === true) {
+        return index;
+      }
+    }
+    return null;
+  }
+
+  public reindex(): void {
     if (Common.indexingEnabled()) {
       this.runIndexer = true;
     }
   }
 
-  public async runSingleTask(task: 'blocksPrices') {
+  public async runSingleTask(task: 'blocksPrices' | 'coinStatsIndex'): Promise<void> {
     if (!Common.indexingEnabled()) {
       return;
     }
@@ -28,20 +77,27 @@ class Indexer {
       this.tasksRunning.push(task);
       const lastestPriceId = await PricesRepository.$getLatestPriceId();
       if (priceUpdater.historyInserted === false || lastestPriceId === null) {
-        logger.debug(`Blocks prices indexer is waiting for the price updater to complete`)
+        logger.debug(`Blocks prices indexer is waiting for the price updater to complete`);
         setTimeout(() => {
-          this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
+          this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
           this.runSingleTask('blocksPrices');
         }, 10000);
       } else {
-        logger.debug(`Blocks prices indexer will run now`)
+        logger.debug(`Blocks prices indexer will run now`);
         await mining.$indexBlockPrices();
-        this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
+        this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
       }
     }
+
+    if (task === 'coinStatsIndex' && !this.tasksRunning.includes(task)) {
+      this.tasksRunning.push(task);
+      logger.debug(`Indexing coinStatsIndex now`);
+      await mining.$indexCoinStatsIndex();
+      this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
+    }
   }
 
-  public async $run() {
+  public async $run(): Promise<void> {
     if (!Common.indexingEnabled() || this.runIndexer === false ||
       this.indexerRunning === true || mempool.hasPriority()
     ) {
@@ -57,7 +113,9 @@ class Indexer {
     this.runIndexer = false;
     this.indexerRunning = true;
 
-    logger.debug(`Running mining indexer`);
+    logger.info(`Running mining indexer`);
+
+    await this.checkAvailableCoreIndexes();
 
     try {
       await priceUpdater.$run();
@@ -93,7 +151,7 @@ class Indexer {
     setTimeout(() => this.reindex(), runEvery);
   }
 
-  async $resetHashratesIndexingState() {
+  async $resetHashratesIndexingState(): Promise<void> {
     try {
       await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0);
       await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0);

backend/src/mempool.interfaces.ts

@@ -1,4 +1,5 @@
 import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
+import { OrphanedBlock } from './api/chain-tips';
 import { HeapNode } from "./utils/pairing-heap";
 
 export interface PoolTag {
@@ -16,6 +17,7 @@ export interface PoolInfo {
   link: string;
   blockCount: number;
   slug: string;
+  avgMatchRate: number | null;
 }
 
 export interface PoolStats extends PoolInfo {
@@ -63,6 +65,7 @@ interface VinStrippedToScriptsig {
 
 interface VoutStrippedToScriptPubkey {
   scriptpubkey_address: string | undefined;
+  scriptpubkey_asm: string | undefined;
   value: number;
 }
 
@@ -159,6 +162,27 @@ export interface BlockExtension {
   avgFeeRate?: number;
   coinbaseRaw?: string;
   usd?: number | null;
+  medianTimestamp?: number;
+  blockTime?: number;
+  orphans?: OrphanedBlock[] | null;
+  coinbaseAddress?: string | null;
+  coinbaseSignature?: string | null;
+  coinbaseSignatureAscii?: string | null;
+  virtualSize?: number;
+  avgTxSize?: number;
+  totalInputs?: number;
+  totalOutputs?: number;
+  totalOutputAmt?: number;
+  medianFeeAmt?: number | null;
+  feePercentiles?: number[] | null,
+  segwitTotalTxs?: number;
+  segwitTotalSize?: number;
+  segwitTotalWeight?: number;
+  header?: string;
+  utxoSetChange?: number;
+  // Requires coinstatsindex, will be set to NULL otherwise
+  utxoSetSize?: number | null;
+  totalInputAmt?: number | null;
 }
 
 export interface BlockExtended extends IEsploraApi.Block {

backend/src/repositories/BlocksRepository.ts

@@ -16,19 +16,32 @@ class BlocksRepository {
    * Save indexed block data in the database
    */
   public async $saveBlockInDatabase(block: BlockExtended) {
+    const truncatedCoinbaseSignature = block?.extras?.coinbaseSignature?.substring(0, 500);
+    const truncatedCoinbaseSignatureAscii = block?.extras?.coinbaseSignatureAscii?.substring(0, 500);
+
     try {
       const query = `INSERT INTO blocks(
-        height,           hash,                blockTimestamp, size,
-        weight,           tx_count,            coinbase_raw,   difficulty,
-        pool_id,          fees,                fee_span,       median_fee,
-        reward,           version,             bits,           nonce,
-        merkle_root,      previous_block_hash, avg_fee,        avg_fee_rate
+        height,             hash,                blockTimestamp,    size,
+        weight,             tx_count,            coinbase_raw,      difficulty,
+        pool_id,            fees,                fee_span,          median_fee,
+        reward,             version,             bits,              nonce,
+        merkle_root,        previous_block_hash, avg_fee,           avg_fee_rate,
+        median_timestamp,   header,              coinbase_address,
+        coinbase_signature, utxoset_size,        utxoset_change,    avg_tx_size,
+        total_inputs,       total_outputs,       total_input_amt,   total_output_amt,
+        fee_percentiles,    segwit_total_txs,    segwit_total_size, segwit_total_weight,
+        median_fee_amt,     coinbase_signature_ascii
       ) VALUE (
         ?, ?, FROM_UNIXTIME(?), ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
-        ?, ?, ?, ?
+        ?, ?, ?, ?,
+        FROM_UNIXTIME(?), ?, ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?, ?
       )`;
 
       const params: any[] = [
@@ -52,6 +65,23 @@ class BlocksRepository {
         block.previousblockhash,
         block.extras.avgFee,
         block.extras.avgFeeRate,
+        block.extras.medianTimestamp,
+        block.extras.header,
+        block.extras.coinbaseAddress,
+        truncatedCoinbaseSignature,
+        block.extras.utxoSetSize,
+        block.extras.utxoSetChange,
+        block.extras.avgTxSize,
+        block.extras.totalInputs,
+        block.extras.totalOutputs,
+        block.extras.totalInputAmt,
+        block.extras.totalOutputAmt,
+        block.extras.feePercentiles ? JSON.stringify(block.extras.feePercentiles) : null,
+        block.extras.segwitTotalTxs,
+        block.extras.segwitTotalSize,
+        block.extras.segwitTotalWeight,
+        block.extras.medianFeeAmt,
+        truncatedCoinbaseSignatureAscii,
       ];
 
       await DB.query(query, params);
@@ -65,6 +95,33 @@ class BlocksRepository {
     }
   }
 
+  /**
+   * Save newly indexed data from core coinstatsindex
+   * 
+   * @param utxoSetSize 
+   * @param totalInputAmt 
+   */
+  public async $updateCoinStatsIndexData(blockHash: string, utxoSetSize: number,
+    totalInputAmt: number
+  ) : Promise<void> {
+    try {
+      const query = `
+        UPDATE blocks
+        SET utxoset_size = ?, total_input_amt = ?
+        WHERE hash = ?
+      `;
+      const params: any[] = [
+        utxoSetSize,
+        totalInputAmt,
+        blockHash
+      ];
+      await DB.query(query, params);
+    } catch (e: any) {
+      logger.err('Cannot update indexed block coinstatsindex. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   /**
    * Get all block height that have not been indexed between [startHeight, endHeight]
    */
@@ -310,32 +367,17 @@ class BlocksRepository {
   public async $getBlockByHeight(height: number): Promise<object | null> {
     try {
       const [rows]: any[] = await DB.query(`SELECT
-        blocks.height,
-        hash,
+        blocks.*,
         hash as id,
         UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
-        size,
-        weight,
-        tx_count,
-        coinbase_raw,
-        difficulty,
+        UNIX_TIMESTAMP(blocks.median_timestamp) as medianTime,
         pools.id as pool_id,
         pools.name as pool_name,
         pools.link as pool_link,
         pools.slug as pool_slug,
         pools.addresses as pool_addresses,
         pools.regexes as pool_regexes,
-        fees,
-        fee_span,
-        median_fee,
-        reward,
-        version,
-        bits,
-        nonce,
-        merkle_root,
-        previous_block_hash as previousblockhash,
-        avg_fee,
-        avg_fee_rate
+        previous_block_hash as previousblockhash
         FROM blocks
         JOIN pools ON blocks.pool_id = pools.id
         WHERE blocks.height = ${height}
@@ -346,6 +388,7 @@ class BlocksRepository {
       }
 
       rows[0].fee_span = JSON.parse(rows[0].fee_span);
+      rows[0].fee_percentiles = JSON.parse(rows[0].fee_percentiles);
       return rows[0];
     } catch (e) {
       logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));
@@ -521,7 +564,7 @@ class BlocksRepository {
         CAST(AVG(blocks.height) as INT) as avgHeight,
         CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
         CAST(AVG(fees) as INT) as avgFees,
-        prices.*
+        prices.USD
         FROM blocks
         JOIN blocks_prices on blocks_prices.height = blocks.height
         JOIN prices on prices.id = blocks_prices.price_id
@@ -550,7 +593,7 @@ class BlocksRepository {
         CAST(AVG(blocks.height) as INT) as avgHeight,
         CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
         CAST(AVG(reward) as INT) as avgRewards,
-        prices.*
+        prices.USD
         FROM blocks
         JOIN blocks_prices on blocks_prices.height = blocks.height
         JOIN prices on prices.id = blocks_prices.price_id
@@ -694,7 +737,6 @@ class BlocksRepository {
       logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e));
       throw e;
     }
-    return [];
   }
 
   /**
@@ -741,7 +783,7 @@ class BlocksRepository {
     try {
       let query = `INSERT INTO blocks_prices(height, price_id) VALUES`;
       for (const price of blockPrices) {
-        query += ` (${price.height}, ${price.priceId}),`
+        query += ` (${price.height}, ${price.priceId}),`;
       }
       query = query.slice(0, -1);
       await DB.query(query);
@@ -754,6 +796,43 @@ class BlocksRepository {
       }
     }
   }
+
+  /**
+   * Get all indexed blocsk with missing coinstatsindex data
+   */
+  public async $getBlocksMissingCoinStatsIndex(maxHeight: number, minHeight: number): Promise<any> {
+    try {
+      const [blocks] = await DB.query(`
+        SELECT height, hash
+        FROM blocks
+        WHERE height >= ${minHeight} AND height <= ${maxHeight} AND
+          (utxoset_size IS NULL OR total_input_amt IS NULL)
+      `);
+      return blocks;
+    } catch (e) {
+      logger.err(`Cannot get blocks with missing coinstatsindex. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Save indexed median fee to avoid recomputing it later
+   * 
+   * @param id 
+   * @param feePercentiles 
+   */
+  public async $saveFeePercentilesForBlockId(id: string, feePercentiles: number[]): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE blocks SET fee_percentiles = ?, median_fee_amt = ?
+        WHERE hash = ?`,
+        [JSON.stringify(feePercentiles), feePercentiles[3], id]
+      );
+    } catch (e) {
+      logger.err(`Cannot update block fee_percentiles. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
 }
 
 export default new BlocksRepository();

backend/src/repositories/BlocksSummariesRepository.ts

@@ -80,6 +80,48 @@ class BlocksSummariesRepository {
       logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e));
     }
   }
+
+  /**
+   * Get the fee percentiles if the block has already been indexed, [] otherwise
+   * 
+   * @param id 
+   */
+  public async $getFeePercentilesByBlockId(id: string): Promise<number[] | null> {
+    try {
+      const [rows]: any[] = await DB.query(`
+        SELECT transactions
+        FROM blocks_summaries
+        WHERE id = ?`,
+        [id]
+      );
+      if (rows === null || rows.length === 0) {
+        return null;
+      }
+
+      const transactions = JSON.parse(rows[0].transactions);
+      if (transactions === null) {
+        return null;
+      }
+
+      transactions.shift(); // Ignore coinbase
+      transactions.sort((a: any, b: any) => a.fee - b.fee);
+      const fees = transactions.map((t: any) => t.fee);
+
+      return [
+        fees[0] ?? 0, // min
+        fees[Math.max(0, Math.floor(fees.length * 0.1) - 1)] ?? 0, // 10th
+        fees[Math.max(0, Math.floor(fees.length * 0.25) - 1)] ?? 0, // 25th
+        fees[Math.max(0, Math.floor(fees.length * 0.5) - 1)] ?? 0, // median
+        fees[Math.max(0, Math.floor(fees.length * 0.75) - 1)] ?? 0, // 75th
+        fees[Math.max(0, Math.floor(fees.length * 0.9) - 1)] ?? 0, // 90th
+        fees[fees.length - 1] ?? 0, // max
+      ];
+
+    } catch (e) {
+      logger.err(`Cannot get block summaries transactions. Reason: ` + (e instanceof Error ? e.message : e));
+      return null;
+    }
+  }
 }
 
 export default new BlocksSummariesRepository();

backend/src/repositories/PoolsRepository.ts

@@ -1,4 +1,5 @@
 import { Common } from '../api/common';
+import poolsParser from '../api/pools-parser';
 import config from '../config';
 import DB from '../database';
 import logger from '../logger';
@@ -17,7 +18,11 @@ class PoolsRepository {
    * Get unknown pool tagging info
    */
   public async $getUnknownPool(): Promise<PoolTag> {
-    const [rows] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"');
+    let [rows]: any[] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"');
+    if (rows && rows.length === 0 && config.DATABASE.ENABLED) {
+      await poolsParser.$insertUnknownPool();
+      [rows] = await DB.query('SELECT id, name, slug FROM pools where name = "Unknown"');
+    }
     return <PoolTag>rows[0];
   }
 
@@ -27,16 +32,25 @@ class PoolsRepository {
   public async $getPoolsInfo(interval: string | null = null): Promise<PoolInfo[]> {
     interval = Common.getSqlInterval(interval);
 
-    let query = `SELECT COUNT(height) as blockCount, pool_id as poolId, pools.name as name, pools.link as link, slug
+    let query = `
+      SELECT
+        COUNT(blocks.height) As blockCount,
+          pool_id AS poolId,
+          pools.name AS name,
+          pools.link AS link,
+          slug,
+          AVG(blocks_audits.match_rate) AS avgMatchRate
       FROM blocks
-      JOIN pools on pools.id = pool_id`;
+      JOIN pools on pools.id = pool_id
+      LEFT JOIN blocks_audits ON blocks_audits.height = blocks.height
+    `;
 
     if (interval) {
       query += ` WHERE blocks.blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
     }
 
     query += ` GROUP BY pool_id
-      ORDER BY COUNT(height) DESC`;
+      ORDER BY COUNT(blocks.height) DESC`;
 
     try {
       const [rows] = await DB.query(query);
@@ -50,7 +64,7 @@ class PoolsRepository {
   /**
    * Get basic pool info and block count between two timestamp
    */
-   public async $getPoolsInfoBetween(from: number, to: number): Promise<PoolInfo[]> {
+  public async $getPoolsInfoBetween(from: number, to: number): Promise<PoolInfo[]> {
     const query = `SELECT COUNT(height) as blockCount, pools.id as poolId, pools.name as poolName
       FROM pools
       LEFT JOIN blocks on pools.id = blocks.pool_id AND blocks.blockTimestamp BETWEEN FROM_UNIXTIME(?) AND FROM_UNIXTIME(?)
@@ -66,9 +80,9 @@ class PoolsRepository {
   }
 
   /**
-   * Get mining pool statistics for one pool
+   * Get a mining pool info
    */
-   public async $getPool(slug: string): Promise<PoolTag | null> {
+  public async $getPool(slug: string, parse: boolean = true): Promise<PoolTag | null> {
     const query = `
       SELECT *
       FROM pools
@@ -81,10 +95,12 @@ class PoolsRepository {
         return null;
       }
 
-      rows[0].regexes = JSON.parse(rows[0].regexes);
+      if (parse) {
+        rows[0].regexes = JSON.parse(rows[0].regexes);
+      }
       if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
-        rows[0].addresses = []; // pools.json only contains mainnet addresses
-      } else {
+        rows[0].addresses = []; // pools-v2.json only contains mainnet addresses
+      } else if (parse) {
         rows[0].addresses = JSON.parse(rows[0].addresses);
       }
 
@@ -94,6 +110,116 @@ class PoolsRepository {
       throw e;
     }
   }
+
+  /**
+   * Get a mining pool info by its unique id
+   */
+  public async $getPoolByUniqueId(id: number, parse: boolean = true): Promise<PoolTag | null> {
+    const query = `
+      SELECT *
+      FROM pools
+      WHERE pools.unique_id = ?`;
+
+    try {
+      const [rows]: any[] = await DB.query(query, [id]);
+
+      if (rows.length < 1) {
+        return null;
+      }
+
+      if (parse) {
+        rows[0].regexes = JSON.parse(rows[0].regexes);
+      }
+      if (['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
+        rows[0].addresses = []; // pools.json only contains mainnet addresses
+      } else if (parse) {
+        rows[0].addresses = JSON.parse(rows[0].addresses);
+      }
+
+      return rows[0];
+    } catch (e) {
+      logger.err('Cannot get pool from db. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Insert a new mining pool in the database
+   * 
+   * @param pool 
+   */
+  public async $insertNewMiningPool(pool: any, slug: string): Promise<void> {
+    try {
+      await DB.query(`
+        INSERT INTO pools
+        SET name = ?, link = ?, addresses = ?, regexes = ?, slug = ?, unique_id = ?`,
+        [pool.name, pool.link, JSON.stringify(pool.addresses), JSON.stringify(pool.regexes), slug, pool.id]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot insert new mining pool into db. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * Rename an existing mining pool
+   * 
+   * @param dbId
+   * @param newSlug
+   * @param newName 
+   */
+  public async $renameMiningPool(dbId: number, newSlug: string, newName: string): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE pools
+        SET slug = ?, name = ?
+        WHERE id = ?`,
+        [newSlug, newName, dbId]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot rename mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * Update an exisiting mining pool link
+   * 
+   * @param dbId 
+   * @param newLink 
+   */
+  public async $updateMiningPoolLink(dbId: number, newLink: string): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE pools
+        SET link = ?
+        WHERE id = ?`,
+        [newLink, dbId]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot update link for mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+
+  }
+
+  /**
+   * Update an existing mining pool addresses or coinbase tags
+   * 
+   * @param dbId 
+   * @param addresses 
+   * @param regexes 
+   */
+  public async $updateMiningPoolTags(dbId: number, addresses: string, regexes: string): Promise<void> {
+    try {
+      await DB.query(`
+        UPDATE pools
+        SET addresses = ?, regexes = ?
+        WHERE id = ?`,
+        [JSON.stringify(addresses), JSON.stringify(regexes), dbId]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot update mining pool id ${dbId}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+  }
+
 }
 
 export default new PoolsRepository();

backend/src/repositories/PricesRepository.ts

@@ -3,6 +3,41 @@ import logger from '../logger';
 import { IConversionRates } from '../mempool.interfaces';
 import priceUpdater from '../tasks/price-updater';
 
+export interface ApiPrice {
+  time?: number,
+  USD: number,
+  EUR: number,
+  GBP: number,
+  CAD: number,
+  CHF: number,
+  AUD: number,
+  JPY: number,
+}
+
+export interface ExchangeRates {
+  USDEUR: number,
+  USDGBP: number,
+  USDCAD: number,
+  USDCHF: number,
+  USDAUD: number,
+  USDJPY: number,
+}
+
+export interface Conversion {
+  prices: ApiPrice[],
+  exchangeRates: ExchangeRates;
+}
+
+export const MAX_PRICES = {
+  USD: 100000000,
+  EUR: 100000000,
+  GBP: 100000000,
+  CAD: 100000000,
+  CHF: 100000000,
+  AUD: 100000000,
+  JPY: 10000000000,
+};
+
 class PricesRepository {
   public async $savePrices(time: number, prices: IConversionRates): Promise<void> {
     if (prices.USD === 0) {
@@ -11,6 +46,14 @@ class PricesRepository {
       return;
     }
 
+    // Sanity check
+    for (const currency of Object.keys(prices)) {
+      if (prices[currency] < -1 || prices[currency] > MAX_PRICES[currency]) { // We use -1 to mark a "missing data, so it's a valid entry"
+        logger.info(`Ignore BTC${currency} price of ${prices[currency]}`);
+        prices[currency] = 0;
+      }
+    }
+    
     try {
       await DB.query(`
         INSERT INTO prices(time,             USD, EUR, GBP, CAD, CHF, AUD, JPY)
@@ -60,6 +103,73 @@ class PricesRepository {
     }
     return rates[0];
   }
+
+  public async $getNearestHistoricalPrice(timestamp: number | undefined): Promise<Conversion | null> {
+    try {
+      const [rates]: any[] = await DB.query(`
+        SELECT *, UNIX_TIMESTAMP(time) AS time
+        FROM prices
+        WHERE UNIX_TIMESTAMP(time) < ?
+        ORDER BY time DESC
+        LIMIT 1`,
+        [timestamp]
+      );
+      if (!rates) {
+        throw Error(`Cannot get single historical price from the database`);
+      }
+
+      // Compute fiat exchange rates
+      const latestPrice = await this.$getLatestConversionRates();
+      const exchangeRates: ExchangeRates = {
+        USDEUR: Math.round(latestPrice.EUR / latestPrice.USD * 100) / 100,
+        USDGBP: Math.round(latestPrice.GBP / latestPrice.USD * 100) / 100,
+        USDCAD: Math.round(latestPrice.CAD / latestPrice.USD * 100) / 100,
+        USDCHF: Math.round(latestPrice.CHF / latestPrice.USD * 100) / 100,
+        USDAUD: Math.round(latestPrice.AUD / latestPrice.USD * 100) / 100,
+        USDJPY: Math.round(latestPrice.JPY / latestPrice.USD * 100) / 100,
+      };
+
+      return {
+        prices: rates,
+        exchangeRates: exchangeRates
+      };
+    } catch (e) {
+      logger.err(`Cannot fetch single historical prices from the db. Reason ${e instanceof Error ? e.message : e}`);
+      return null;
+    }
+  }
+
+  public async $getHistoricalPrices(): Promise<Conversion | null> {
+    try {
+      const [rates]: any[] = await DB.query(`
+        SELECT *, UNIX_TIMESTAMP(time) AS time
+        FROM prices
+        ORDER BY time DESC
+      `);
+      if (!rates) {
+        throw Error(`Cannot get average historical price from the database`);
+      }
+
+      // Compute fiat exchange rates
+      const latestPrice: ApiPrice = rates[0];
+      const exchangeRates: ExchangeRates = {
+        USDEUR: Math.round(latestPrice.EUR / latestPrice.USD * 100) / 100,
+        USDGBP: Math.round(latestPrice.GBP / latestPrice.USD * 100) / 100,
+        USDCAD: Math.round(latestPrice.CAD / latestPrice.USD * 100) / 100,
+        USDCHF: Math.round(latestPrice.CHF / latestPrice.USD * 100) / 100,
+        USDAUD: Math.round(latestPrice.AUD / latestPrice.USD * 100) / 100,
+        USDJPY: Math.round(latestPrice.JPY / latestPrice.USD * 100) / 100,
+      };
+
+      return {
+        prices: rates,
+        exchangeRates: exchangeRates
+      };
+    } catch (e) {
+      logger.err(`Cannot fetch historical prices from the db. Reason ${e instanceof Error ? e.message : e}`);
+      return null;
+    }
+  }
 }
 
 export default new PricesRepository();

backend/src/rpc-api/commands.ts

@@ -88,5 +88,7 @@ module.exports = {
   verifyTxOutProof: 'verifytxoutproof', // bitcoind v0.11.0+
   walletLock: 'walletlock',
   walletPassphrase: 'walletpassphrase',
-  walletPassphraseChange: 'walletpassphrasechange'
-}
+  walletPassphraseChange: 'walletpassphrasechange',
+  getTxoutSetinfo: 'gettxoutsetinfo',
+  getIndexInfo: 'getindexinfo',
+};

backend/src/tasks/pools-updater.ts

@@ -8,7 +8,7 @@ import { SocksProxyAgent } from 'socks-proxy-agent';
 import * as https from 'https';
 
 /**
- * Maintain the most recent version of pools.json
+ * Maintain the most recent version of pools-v2.json
  */
 class PoolsUpdater {
   lastRun: number = 0;
@@ -31,14 +31,8 @@ class PoolsUpdater {
 
     this.lastRun = now;
 
-    if (config.SOCKS5PROXY.ENABLED) {
-      logger.info(`Updating latest mining pools from ${this.poolsUrl} over the Tor network`, logger.tags.mining);
-    } else {
-      logger.info(`Updating latest mining pools from ${this.poolsUrl} over clearnet`, logger.tags.mining);
-    }
-
     try {
-      const githubSha = await this.fetchPoolsSha(); // Fetch pools.json sha from github
+      const githubSha = await this.fetchPoolsSha(); // Fetch pools-v2.json sha from github
       if (githubSha === undefined) {
         return;
       }
@@ -47,32 +41,57 @@ class PoolsUpdater {
         this.currentSha = await this.getShaFromDb();
       }
 
-      logger.debug(`Pools.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
+      logger.debug(`pools-v2.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
       if (this.currentSha !== undefined && this.currentSha === githubSha) {
         return;
       }
 
+      // See backend README for more details about the mining pools update process
+      if (this.currentSha !== undefined && // If we don't have any mining pool, download it at least once
+        config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING !== true && // Automatic pools update is disabled
+        !process.env.npm_config_update_pools // We're not manually updating mining pool
+      ) {
+        logger.warn(`Updated mining pools data is available (${githubSha}) but AUTOMATIC_BLOCK_REINDEXING is disabled`);
+        logger.info(`You can update your mining pools using the --update-pools command flag. You may want to clear your nginx cache as well if applicable`);
+        return;
+      }
+
+      const network = config.SOCKS5PROXY.ENABLED ? 'tor' : 'clearnet';
       if (this.currentSha === undefined) {
-        logger.info(`Downloading pools.json for the first time from ${this.poolsUrl}`, logger.tags.mining);
+        logger.info(`Downloading pools-v2.json for the first time from ${this.poolsUrl} over ${network}`, logger.tags.mining);
       } else {
-        logger.warn(`Pools.json is outdated, fetch latest from ${this.poolsUrl}`, logger.tags.mining);
+        logger.warn(`pools-v2.json is outdated, fetch latest from ${this.poolsUrl} over ${network}`, logger.tags.mining);
       }
       const poolsJson = await this.query(this.poolsUrl);
       if (poolsJson === undefined) {
         return;
       }
-      await poolsParser.migratePoolsJson(poolsJson);
-      await this.updateDBSha(githubSha);
-      logger.notice(`PoolsUpdater completed`, logger.tags.mining);
+      poolsParser.setMiningPools(poolsJson);
+
+      if (config.DATABASE.ENABLED === false) { // Don't run db operations
+        logger.info('Mining pools-v2.json import completed (no database)');
+        return;
+      }
+
+      try {
+        await DB.query('START TRANSACTION;');
+        await poolsParser.migratePoolsJson();
+        await this.updateDBSha(githubSha);
+        await DB.query('COMMIT;');
+      } catch (e) {
+        logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, logger.tags.mining);
+        await DB.query('ROLLBACK;');
+      }
+      logger.notice('PoolsUpdater completed');
 
     } catch (e) {
       this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week
-      logger.err(`PoolsUpdater failed. Will try again in 24h. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining);
+      logger.err(`PoolsUpdater failed. Will try again in 24h. Exception: ${JSON.stringify(e)}`, logger.tags.mining);
     }
   }
 
   /**
-   * Fetch our latest pools.json sha from the db
+   * Fetch our latest pools-v2.json sha from the db
    */
   private async updateDBSha(githubSha: string): Promise<void> {
     this.currentSha = githubSha;
@@ -81,46 +100,46 @@ class PoolsUpdater {
         await DB.query('DELETE FROM state where name="pools_json_sha"');
         await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`);
       } catch (e) {
-        logger.err('Cannot save github pools.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+        logger.err('Cannot save github pools-v2.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
       }
     }
   }
 
   /**
-   * Fetch our latest pools.json sha from the db
+   * Fetch our latest pools-v2.json sha from the db
    */
   private async getShaFromDb(): Promise<string | undefined> {
     try {
       const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"');
       return (rows.length > 0 ? rows[0].string : undefined);
     } catch (e) {
-      logger.err('Cannot fetch pools.json sha from db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
+      logger.err('Cannot fetch pools-v2.json sha from db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining);
       return undefined;
     }
   }
 
   /**
-   * Fetch our latest pools.json sha from github
+   * Fetch our latest pools-v2.json sha from github
    */
   private async fetchPoolsSha(): Promise<string | undefined> {
     const response = await this.query(this.treeUrl);
 
     if (response !== undefined) {
       for (const file of response['tree']) {
-        if (file['path'] === 'pools.json') {
+        if (file['path'] === 'pools-v2.json') {
           return file['sha'];
         }
       }
     }
 
-    logger.err(`Cannot find "pools.json" in git tree (${this.treeUrl})`, logger.tags.mining);
+    logger.err(`Cannot find "pools-v2.json" in git tree (${this.treeUrl})`, logger.tags.mining);
     return undefined;
   }
 
   /**
    * Http request wrapper
    */
-  private async query(path): Promise<object | undefined> {
+  private async query(path): Promise<any[] | undefined> {
     type axiosOptions = {
       headers: {
         'User-Agent': string

backend/src/tasks/price-updater.ts

@@ -3,7 +3,7 @@ import path from 'path';
 import config from '../config';
 import logger from '../logger';
 import { IConversionRates } from '../mempool.interfaces';
-import PricesRepository from '../repositories/PricesRepository';
+import PricesRepository, { MAX_PRICES } from '../repositories/PricesRepository';
 import BitfinexApi from './price-feeds/bitfinex-api';
 import BitflyerApi from './price-feeds/bitflyer-api';
 import CoinbaseApi from './price-feeds/coinbase-api';
@@ -46,13 +46,13 @@ class PriceUpdater {
 
   public getEmptyPricesObj(): IConversionRates {
     return {
-      USD: 0,
-      EUR: 0,
-      GBP: 0,
-      CAD: 0,
-      CHF: 0,
-      AUD: 0,
-      JPY: 0,
+      USD: -1,
+      EUR: -1,
+      GBP: -1,
+      CAD: -1,
+      CHF: -1,
+      AUD: -1,
+      JPY: -1,
     };
   }
 
@@ -115,7 +115,7 @@ class PriceUpdater {
         if (feed.currencies.includes(currency)) {
           try {
             const price = await feed.$fetchPrice(currency);
-            if (price > 0) {
+            if (price > -1 && price < MAX_PRICES[currency]) {
               prices.push(price);
             }
             logger.debug(`${feed.name} BTC/${currency} price: ${price}`, logger.tags.mining);
@@ -239,7 +239,7 @@ class PriceUpdater {
 
         for (const currency of this.currencies) {
           const price = historicalEntry[time][currency];
-          if (price > 0) {
+          if (price > -1 && price < MAX_PRICES[currency]) {
             grouped[time][currency].push(typeof price === 'string' ? parseInt(price, 10) : price);
           }
         }

docker/README.md

@@ -102,15 +102,16 @@ Below we list all settings from `mempool-config.json` and the corresponding over
     "MEMPOOL_BLOCKS_AMOUNT": 8,
     "BLOCKS_SUMMARIES_INDEXING": false,
     "USE_SECOND_NODE_FOR_MINFEE": false,
-    "EXTERNAL_ASSETS": ["https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json"],
+    "EXTERNAL_ASSETS": [],
     "STDOUT_LOG_MIN_PRIORITY": "info",
     "INDEXING_BLOCKS_AMOUNT": false,
     "AUTOMATIC_BLOCK_REINDEXING": false,
-    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
+    "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json",
     "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
     "ADVANCED_GBT_AUDIT": false,
     "ADVANCED_GBT_MEMPOOL": false,
     "CPFP_INDEXING": false,
+    "MAX_BLOCKS_BULK_QUERY": 0,
   },
 ```
 
@@ -141,6 +142,7 @@ Corresponding `docker-compose.yml` overrides:
       MEMPOOL_ADVANCED_GBT_AUDIT: ""
       MEMPOOL_ADVANCED_GBT_MEMPOOL: ""
       MEMPOOL_CPFP_INDEXING: ""
+      MAX_BLOCKS_BULK_QUERY: ""
       ...
 ```
 

docker/backend/mempool-config.json

@@ -25,7 +25,8 @@
     "AUDIT": __MEMPOOL_AUDIT__,
     "ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__,
     "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__,
-    "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__
+    "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__,
+    "MAX_BLOCKS_BULK_QUERY": __MEMPOOL__MAX_BLOCKS_BULK_QUERY__
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",

docker/backend/start.sh

@@ -24,12 +24,13 @@ __MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool}
 __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info}
 __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=false}
 __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false}
-__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json}
+__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools-v2.json}
 __MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master}
 __MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false}
 __MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false}
 __MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false}
 __MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
+__MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
 
 # CORE_RPC
 __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
@@ -142,6 +143,7 @@ sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json
 
 sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json

frontend/src/app/app.constants.ts

@@ -72,7 +72,7 @@ export const chartColors = [
 ];
 
 export const poolsColor = {
-  'unknown': '#9C9C9C',
+  'unknown': '#FDD835',
 };
 
 export const feeLevels = [1, 2, 3, 4, 5, 6, 8, 10, 12, 15, 20, 30, 40, 50, 60, 70, 80, 90, 100, 125, 150, 175, 200,

frontend/src/app/app.module.ts

@@ -7,6 +7,7 @@ import { AppComponent } from './components/app/app.component';
 import { ElectrsApiService } from './services/electrs-api.service';
 import { StateService } from './services/state.service';
 import { CacheService } from './services/cache.service';
+import { PriceService } from './services/price.service';
 import { EnterpriseService } from './services/enterprise.service';
 import { WebsocketService } from './services/websocket.service';
 import { AudioService } from './services/audio.service';
@@ -26,6 +27,7 @@ const providers = [
   ElectrsApiService,
   StateService,
   CacheService,
+  PriceService,
   WebsocketService,
   AudioService,
   SeoService,

frontend/src/app/components/about/about.component.html

@@ -352,7 +352,7 @@
 
   <div class="copyright">
     <div class="title">
-      Copyright &copy; 2019-2022<br>
+      Copyright &copy; 2019-2023<br>
       The Mempool Open Source Project
     </div>
     <p>

frontend/src/app/components/about/about.component.scss

@@ -145,6 +145,13 @@
     }
   }
 
+  .project-translators .wrapper {
+    a img {
+      width: 72px;
+      height: 72px;
+    }
+  }
+
   .copyright {
     text-align: left;
     max-width: 620px;

frontend/src/app/components/amount/amount.component.html

@@ -1,7 +1,19 @@
 <ng-container *ngIf="!noFiat && (viewFiat$ | async) && (conversions$ | async) as conversions; else viewFiatVin">
-  <span class="fiat">{{ addPlus && satoshis >= 0 ? '+' : '' }}{{ (conversions ? conversions[currency] : 0) * satoshis / 100000000 | fiatCurrency : digitsInfo : currency }}</span>
+  <span class="fiat" *ngIf="blockConversion; else noblockconversion">
+    {{ addPlus && satoshis >= 0 ? '+' : '' }}
+    {{
+      (
+        (blockConversion.price[currency] >= 0 ? blockConversion.price[currency] : null) ??
+        (blockConversion.price['USD'] * blockConversion.exchangeRates['USD' + currency]) ?? 0
+      ) * satoshis / 100000000 | fiatCurrency : digitsInfo : currency
+    }}
+  </span>
+  <ng-template #noblockconversion>
+    <span class="fiat">{{ addPlus && satoshis >= 0 ? '+' : '' }}{{ (conversions ? conversions[currency] : 0) * satoshis / 100000000 | fiatCurrency : digitsInfo : currency }}</span>
+  </ng-template>
 </ng-container>
-<ng-template #viewFiatVin>
+
+  <ng-template #viewFiatVin>
   <ng-template [ngIf]="(network === 'liquid' || network === 'liquidtestnet') && (satoshis === undefined || satoshis === null)" [ngIfElse]="default">
     <span i18n="shared.confidential">Confidential</span>
   </ng-template>

frontend/src/app/components/amount/amount.component.ts

@@ -1,6 +1,7 @@
 import { Component, OnInit, OnDestroy, Input, ChangeDetectionStrategy, ChangeDetectorRef } from '@angular/core';
 import { StateService } from '../../services/state.service';
 import { Observable, Subscription } from 'rxjs';
+import { Price } from '../../services/price.service';
 
 @Component({
   selector: 'app-amount',
@@ -21,6 +22,7 @@ export class AmountComponent implements OnInit, OnDestroy {
   @Input() digitsInfo = '1.8-8';
   @Input() noFiat = false;
   @Input() addPlus = false;
+  @Input() blockConversion: Price;
 
   constructor(
     private stateService: StateService,

frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html

@@ -10,7 +10,7 @@
     </div>  
 
     <form [formGroup]="radioGroupForm" class="formRadioGroup" *ngIf="(statsObservable$ | async) as stats">
-      <div class="btn-group btn-group-toggle" name="radioBasic">
+      <div class="btn-group btn-group-toggle" name="radioBasic" [class]="{'disabled': isLoading}">
         <label class="btn btn-primary btn-sm" *ngIf="stats.blockCount >= 144" [class.active]="radioGroupForm.get('dateSpan').value === '24h'">
           <input type="radio" [value]="'24h'" fragment="24h" [routerLink]="['/graphs/mining/block-fee-rates' | relativeUrl]" formControlName="dateSpan"> 24h
         </label>

frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.scss

@@ -78,3 +78,8 @@
     }
   }
 }
+
+.disabled {
+  pointer-events: none;
+  opacity: 0.5;
+}

frontend/src/app/components/block-fees-graph/block-fees-graph.component.html

@@ -10,7 +10,7 @@
     </div>  
 
     <form [formGroup]="radioGroupForm" class="formRadioGroup" *ngIf="(statsObservable$ | async) as stats">
-      <div class="btn-group btn-group-toggle" name="radioBasic">
+      <div class="btn-group btn-group-toggle" name="radioBasic" [class]="{'disabled': isLoading}">
         <label class="btn btn-primary btn-sm" *ngIf="stats.blockCount >= 4320" [class.active]="radioGroupForm.get('dateSpan').value === '1m'">
           <input type="radio" [value]="'1m'" fragment="1m" [routerLink]="['/graphs/mining/block-fees' | relativeUrl]" formControlName="dateSpan"> 1M
         </label>

frontend/src/app/components/block-fees-graph/block-fees-graph.component.scss

@@ -78,3 +78,8 @@
     }
   }
 }
+
+.disabled {
+  pointer-events: none;
+  opacity: 0.5;
+}

frontend/src/app/components/block-fees-graph/block-fees-graph.component.ts

@@ -1,19 +1,17 @@
 import { ChangeDetectionStrategy, Component, Inject, Input, LOCALE_ID, OnInit } from '@angular/core';
 import { EChartsOption, graphic } from 'echarts';
-import { Observable, Subscription } from 'rxjs';
+import { Observable } from 'rxjs';
 import { map, share, startWith, switchMap, tap } from 'rxjs/operators';
 import { ApiService } from '../../services/api.service';
 import { SeoService } from '../../services/seo.service';
-import { formatCurrency, formatNumber, getCurrencySymbol } from '@angular/common';
+import { formatNumber } from '@angular/common';
 import { UntypedFormBuilder, UntypedFormGroup } from '@angular/forms';
-import { download, formatterXAxis, formatterXAxisLabel, formatterXAxisTimeCategory } from '../../shared/graphs.utils';
-import { StateService } from '../../services/state.service';
+import { download, formatterXAxis } from '../../shared/graphs.utils';
 import { StorageService } from '../../services/storage.service';
 import { MiningService } from '../../services/mining.service';
 import { ActivatedRoute } from '@angular/router';
 import { FiatShortenerPipe } from '../../shared/pipes/fiat-shortener.pipe';
 import { FiatCurrencyPipe } from '../../shared/pipes/fiat-currency.pipe';
-import { fiatCurrencies } from '../../app.constants';
 
 @Component({
   selector: 'app-block-fees-graph',
@@ -47,7 +45,6 @@ export class BlockFeesGraphComponent implements OnInit {
   timespan = '';
   chartInstance: any = undefined;
 
-  currencySubscription: Subscription;
   currency: string;
 
   constructor(
@@ -57,21 +54,13 @@ export class BlockFeesGraphComponent implements OnInit {
     private formBuilder: UntypedFormBuilder,
     private storageService: StorageService,
     private miningService: MiningService,
-    private stateService: StateService,
     private route: ActivatedRoute,
     private fiatShortenerPipe: FiatShortenerPipe,
     private fiatCurrencyPipe: FiatCurrencyPipe,
   ) {
     this.radioGroupForm = this.formBuilder.group({ dateSpan: '1y' });
     this.radioGroupForm.controls.dateSpan.setValue('1y');
-
-    this.currencySubscription = this.stateService.fiatCurrency$.subscribe((fiat) => {
-      if (fiat && fiatCurrencies[fiat]?.indexed) {
-        this.currency = fiat;
-      } else {
-        this.currency = 'USD';
-      }
-    });
+    this.currency = 'USD';
   }
 
   ngOnInit(): void {
@@ -92,6 +81,7 @@ export class BlockFeesGraphComponent implements OnInit {
       .pipe(
         startWith(this.radioGroupForm.controls.dateSpan.value),
         switchMap((timespan) => {
+          this.isLoading = true;
           this.storageService.setValue('miningWindowPreference', timespan);
           this.timespan = timespan;
           this.isLoading = true;

frontend/src/app/components/block-overview-graph/block-overview-graph.component.html

@@ -10,5 +10,6 @@
     [cursorPosition]="tooltipPosition"
     [clickable]="!!selectedTx"
     [auditEnabled]="auditHighlighting"
+    [blockConversion]="blockConversion"
   ></app-block-overview-tooltip>
 </div>

frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts

@@ -5,6 +5,7 @@ import BlockScene from './block-scene';
 import TxSprite from './tx-sprite';
 import TxView from './tx-view';
 import { Position } from './sprite-types';
+import { Price } from '../../services/price.service';
 
 @Component({
   selector: 'app-block-overview-graph',
@@ -21,6 +22,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
   @Input() mirrorTxid: string | void;
   @Input() unavailable: boolean = false;
   @Input() auditHighlighting: boolean = false;
+  @Input() blockConversion: Price;
   @Output() txClickEvent = new EventEmitter<TransactionStripped>();
   @Output() txHoverEvent = new EventEmitter<string>();
   @Output() readyEvent = new EventEmitter();

frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html

@@ -16,11 +16,11 @@
       </tr>
       <tr>
         <td class="td-width" i18n="dashboard.latest-transactions.amount">Amount</td>
-        <td><app-amount [satoshis]="value"></app-amount></td>
+        <td><app-amount [blockConversion]="blockConversion" [satoshis]="value"></app-amount></td>
       </tr>
       <tr>
         <td class="td-width" i18n="transaction.fee|Transaction fee">Fee</td>
-        <td>{{ fee | number }} <span class="symbol" i18n="shared.sat|sat">sat</span> &nbsp; <span class="fiat"><app-fiat [value]="fee"></app-fiat></span></td>
+        <td>{{ fee | number }} <span class="symbol" i18n="shared.sat|sat">sat</span> &nbsp; <span class="fiat"><app-fiat [blockConversion]="blockConversion" [value]="fee"></app-fiat></span></td>
       </tr>
       <tr>
         <td class="td-width" i18n="transaction.fee-rate|Transaction fee rate">Fee rate</td>

frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts

@@ -1,6 +1,7 @@
 import { Component, ElementRef, ViewChild, Input, OnChanges, ChangeDetectionStrategy } from '@angular/core';
 import { TransactionStripped } from '../../interfaces/websocket.interface';
 import { Position } from '../../components/block-overview-graph/sprite-types.js';
+import { Price } from '../../services/price.service';
 
 @Component({
   selector: 'app-block-overview-tooltip',
@@ -12,6 +13,7 @@ export class BlockOverviewTooltipComponent implements OnChanges {
   @Input() cursorPosition: Position;
   @Input() clickable: boolean;
   @Input() auditEnabled: boolean = false;
+  @Input() blockConversion: Price;
 
   txid = '';
   fee = 0;

frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.html

@@ -10,7 +10,7 @@
     </div>  
 
     <form [formGroup]="radioGroupForm" class="formRadioGroup" *ngIf="(statsObservable$ | async) as stats">
-      <div class="btn-group btn-group-toggle" name="radioBasic">
+      <div class="btn-group btn-group-toggle" name="radioBasic" [class]="{'disabled': isLoading}">
         <label class="btn btn-primary btn-sm" *ngIf="stats.blockCount >= 144" [class.active]="radioGroupForm.get('dateSpan').value === '24h'">
           <input type="radio" [value]="'24h'" fragment="24h" [routerLink]="['/graphs/mining/block-prediction' | relativeUrl]" formControlName="dateSpan"> 24h
         </label>

frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.scss

@@ -78,3 +78,8 @@
     }
   }
 }
+
+.disabled {
+  pointer-events: none;
+  opacity: 0.5;
+}

frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.html

@@ -11,7 +11,7 @@
     </div>  
   
     <form [formGroup]="radioGroupForm" class="formRadioGroup" *ngIf="(statsObservable$ | async) as stats">
-      <div class="btn-group btn-group-toggle" name="radioBasic">
+      <div class="btn-group btn-group-toggle" name="radioBasic" [class]="{'disabled': isLoading}">
         <label class="btn btn-primary btn-sm" *ngIf="stats.blockCount >= 4320" [class.active]="radioGroupForm.get('dateSpan').value === '1m'">
           <input type="radio" [value]="'1m'" fragment="1m" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]" formControlName="dateSpan"> 1M
         </label>
@@ -31,7 +31,7 @@
           <input type="radio" [value]="'3y'" fragment="3y" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]" formControlName="dateSpan"> 3Y
         </label>
         <label class="btn btn-primary btn-sm" [class.active]="radioGroupForm.get('dateSpan').value === 'all'">
-          <input type="radio" [value]="'all'" fragment="all" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]"> ALL
+          <input type="radio" [value]="'all'" fragment="all" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]" formControlName="dateSpan"> ALL
         </label>
       </div>
     </form>

frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.scss

@@ -78,3 +78,8 @@
     }
   }
 }
+
+.disabled {
+  pointer-events: none;
+  opacity: 0.5;
+}

frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.ts

@@ -1,19 +1,17 @@
 import { ChangeDetectionStrategy, Component, Inject, Input, LOCALE_ID, OnInit } from '@angular/core';
 import { EChartsOption, graphic } from 'echarts';
-import { Observable, Subscription } from 'rxjs';
+import { Observable } from 'rxjs';
 import { map, share, startWith, switchMap, tap } from 'rxjs/operators';
 import { ApiService } from '../../services/api.service';
 import { SeoService } from '../../services/seo.service';
 import { formatNumber } from '@angular/common';
 import { UntypedFormBuilder, UntypedFormGroup } from '@angular/forms';
-import { download, formatterXAxis, formatterXAxisLabel, formatterXAxisTimeCategory } from '../../shared/graphs.utils';
+import { download, formatterXAxis } from '../../shared/graphs.utils';
 import { MiningService } from '../../services/mining.service';
-import { StateService } from '../../services/state.service';
 import { StorageService } from '../../services/storage.service';
 import { ActivatedRoute } from '@angular/router';
 import { FiatShortenerPipe } from '../../shared/pipes/fiat-shortener.pipe';
 import { FiatCurrencyPipe } from '../../shared/pipes/fiat-currency.pipe';
-import { fiatCurrencies } from '../../app.constants';
 
 @Component({
   selector: 'app-block-rewards-graph',
@@ -47,7 +45,6 @@ export class BlockRewardsGraphComponent implements OnInit {
   timespan = '';
   chartInstance: any = undefined;
 
-  currencySubscription: Subscription;
   currency: string;
 
   constructor(
@@ -56,19 +53,12 @@ export class BlockRewardsGraphComponent implements OnInit {
     private apiService: ApiService,
     private formBuilder: UntypedFormBuilder,
     private miningService: MiningService,
-    private stateService: StateService,
     private storageService: StorageService,
     private route: ActivatedRoute,
     private fiatShortenerPipe: FiatShortenerPipe,
     private fiatCurrencyPipe: FiatCurrencyPipe,
   ) {
-    this.currencySubscription = this.stateService.fiatCurrency$.subscribe((fiat) => {
-      if (fiat && fiatCurrencies[fiat]?.indexed) {
-        this.currency = fiat;
-      } else {
-        this.currency = 'USD';
-      }
-    });
+    this.currency = 'USD';
   }
 
   ngOnInit(): void {
@@ -80,7 +70,7 @@ export class BlockRewardsGraphComponent implements OnInit {
     this.route
       .fragment
       .subscribe((fragment) => {
-        if (['3m', '6m', '1y', '2y', '3y', 'all'].indexOf(fragment) > -1) {
+        if (['1m', '3m', '6m', '1y', '2y', '3y', 'all'].indexOf(fragment) > -1) {
           this.radioGroupForm.controls.dateSpan.setValue(fragment, { emitEvent: false });
         }
       });

frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.html

@@ -9,7 +9,7 @@
     </div>  
 
     <form [formGroup]="radioGroupForm" class="formRadioGroup" *ngIf="(blockSizesWeightsObservable$ | async) as stats">
-      <div class="btn-group btn-group-toggle" name="radioBasic">
+      <div class="btn-group btn-group-toggle" name="radioBasic" [class]="{'disabled': isLoading}">
         <label class="btn btn-primary btn-sm" *ngIf="stats.blockCount >= 144" [class.active]="radioGroupForm.get('dateSpan').value === '24h'">
           <input type="radio" [value]="'24h'" fragment="24h" [routerLink]="['/graphs/mining/block-sizes-weights' | relativeUrl]" formControlName="dateSpan"> 24h
         </label>

frontend/src/app/components/block-sizes-weights-graph/block-sizes-weights-graph.component.scss

@@ -78,3 +78,8 @@
     }
   }
 }
+
+.disabled {
+  pointer-events: none;
+  opacity: 0.5;
+}

frontend/src/app/components/block/block-preview.component.html

@@ -8,7 +8,7 @@
         <div class="block-titles">
           <h1 class="title">
             <ng-template [ngIf]="blockHeight === 0"><ng-container i18n="@@2303359202781425764">Genesis</ng-container></ng-template>
-            <ng-template [ngIf]="blockHeight" i18n="shared.block-title">{{ blockHeight }}</ng-template>
+            <ng-template [ngIf]="blockHeight">{{ blockHeight }}</ng-template>
           </h1>
           <div class="blockhash" *ngIf="blockHash">
             <h2 class="truncate right">{{ blockHash.slice(0,32) }}</h2>

frontend/src/app/components/block/block.component.html

@@ -108,6 +108,7 @@
             [blockLimit]="stateService.blockVSize"
             [orientation]="'top'"
             [flip]="false"
+            [blockConversion]="blockConversion"
             (txClickEvent)="onTxClick($event)"
           ></app-block-overview-graph>
           <ng-container *ngTemplateOutlet="emptyBlockInfo"></ng-container>
@@ -124,7 +125,13 @@
       </tr>
       <tr *ngIf="block?.extras?.medianFee != undefined">
         <td class="td-width" i18n="block.median-fee">Median fee</td>
-        <td>~{{ block?.extras?.medianFee | number:'1.0-0' }} <span class="symbol" i18n="shared.sat-vbyte|sat/vB">sat/vB</span> <span class="fiat"><app-fiat [value]="block?.extras?.medianFee * 140" digitsInfo="1.2-2" i18n-ngbTooltip="Transaction fee tooltip" ngbTooltip="Based on average native segwit transaction of 140 vBytes" placement="bottom"></app-fiat></span></td>
+        <td>~{{ block?.extras?.medianFee | number:'1.0-0' }} <span class="symbol" i18n="shared.sat-vbyte|sat/vB">sat/vB</span>
+          <span class="fiat">
+            <app-fiat [blockConversion]="blockConversion" [value]="block?.extras?.medianFee * 140" digitsInfo="1.2-2"
+              i18n-ngbTooltip="Transaction fee tooltip" ngbTooltip="Based on average native segwit transaction of 140 vBytes"
+              placement="bottom"></app-fiat>
+          </span>
+        </td>
       </tr>
       <ng-template [ngIf]="fees !== undefined" [ngIfElse]="loadingFees">
         <tr>
@@ -132,13 +139,13 @@
           <td *ngIf="network !== 'liquid' && network !== 'liquidtestnet'; else liquidTotalFees">
             <app-amount [satoshis]="block.extras.totalFees" digitsInfo="1.2-3" [noFiat]="true"></app-amount>
             <span class="fiat">
-              <app-fiat [value]="block.extras.totalFees" digitsInfo="1.0-0"></app-fiat>
+              <app-fiat [blockConversion]="blockConversion" [value]="block.extras.totalFees" digitsInfo="1.0-0"></app-fiat>
             </span>
           </td>
           <ng-template #liquidTotalFees>
             <td>
               <app-amount [satoshis]="fees * 100000000" digitsInfo="1.2-2" [noFiat]="true"></app-amount>&nbsp; <app-fiat
-                [value]="fees * 100000000" digitsInfo="1.2-2"></app-fiat>
+              [blockConversion]="blockConversion" [value]="fees * 100000000" digitsInfo="1.2-2"></app-fiat>
             </td>
           </ng-template>
         </tr>
@@ -147,7 +154,7 @@
           <td>
             <app-amount [satoshis]="block.extras.reward" digitsInfo="1.2-3" [noFiat]="true"></app-amount>
             <span class="fiat">
-              <app-fiat [value]="(blockSubsidy + fees) * 100000000" digitsInfo="1.0-0"></app-fiat>
+              <app-fiat [blockConversion]="blockConversion" [value]="(blockSubsidy + fees) * 100000000" digitsInfo="1.0-0"></app-fiat>
             </span>
           </td>
         </tr>

frontend/src/app/components/block/block.component.ts

@@ -13,6 +13,7 @@ import { BlockAudit, BlockExtended, TransactionStripped } from '../../interfaces
 import { ApiService } from '../../services/api.service';
 import { BlockOverviewGraphComponent } from '../../components/block-overview-graph/block-overview-graph.component';
 import { detectWebGL } from '../../shared/graphs.utils';
+import { PriceService, Price } from '../../services/price.service';
 
 @Component({
   selector: 'app-block',
@@ -81,6 +82,9 @@ export class BlockComponent implements OnInit, OnDestroy {
   timeLtr: boolean;
   childChangeSubscription: Subscription;
   auditPrefSubscription: Subscription;
+  
+  priceSubscription: Subscription;
+  blockConversion: Price;
 
   @ViewChildren('blockGraphProjected') blockGraphProjected: QueryList<BlockOverviewGraphComponent>;
   @ViewChildren('blockGraphActual') blockGraphActual: QueryList<BlockOverviewGraphComponent>;
@@ -94,7 +98,8 @@ export class BlockComponent implements OnInit, OnDestroy {
     private seoService: SeoService,
     private websocketService: WebsocketService,
     private relativeUrlPipe: RelativeUrlPipe,
-    private apiService: ApiService
+    private apiService: ApiService,
+    private priceService: PriceService,
   ) {
     this.webGlEnabled = detectWebGL();
   }
@@ -432,6 +437,19 @@ export class BlockComponent implements OnInit, OnDestroy {
         }
       }
     });
+
+    if (this.priceSubscription) {
+      this.priceSubscription.unsubscribe();
+    }
+    this.priceSubscription = block$.pipe(
+      switchMap((block) => {
+        return this.priceService.getBlockPrice$(block.timestamp).pipe(
+          tap((price) => {
+            this.blockConversion = price;
+          })
+        );
+      })
+    ).subscribe();
   }
 
   ngAfterViewInit(): void {
@@ -453,6 +471,7 @@ export class BlockComponent implements OnInit, OnDestroy {
     this.auditSubscription?.unsubscribe();
     this.unsubscribeNextBlockSubscriptions();
     this.childChangeSubscription?.unsubscribe();
+    this.priceSubscription?.unsubscribe();
   }
 
   unsubscribeNextBlockSubscriptions() {

frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html

@@ -1,8 +1,8 @@
 <div class="blocks-container blockchain-blocks-container" [class.time-ltr]="timeLtr"
   [style.left]="static ? (offset || 0) + 'px' : null"
-  *ngIf="(loadingBlocks$ | async) === false; else loadingBlocksTemplate">
+  *ngIf="static || (loadingBlocks$ | async) === false; else loadingBlocksTemplate">
   <div *ngFor="let block of blocks; let i = index; trackBy: trackByBlocksFn">
-    <ng-container *ngIf="block && !block.loading && !block.placeholder; else placeholderBlock">
+    <ng-container *ngIf="connected && block && !block.loading && !block.placeholder; else placeholderBlock">
       <div [attr.data-cy]="'bitcoin-block-offset-' + offset + '-index-' + i"
         class="text-center bitcoin-block mined-block blockchain-blocks-offset-{{ offset }}-index-{{ i }}"
         id="bitcoin-block-{{ block.height }}" [ngStyle]="blockStyles[i]"
@@ -14,20 +14,26 @@
             }}</a>
         </div>
         <div class="block-body">
-          <div [attr.data-cy]="'bitcoin-block-offset=' + offset + '-index-' + i + '-fees'" class="fees">
+          <div *ngIf="block?.extras; else emptyfees" [attr.data-cy]="'bitcoin-block-offset=' + offset + '-index-' + i + '-fees'" class="fees">
             ~{{ block?.extras?.medianFee | number:feeRounding }} <ng-container
               i18n="shared.sat-vbyte|sat/vB">sat/vB</ng-container>
           </div>
+          <ng-template #emptyfees>
+            <div [attr.data-cy]="'bitcoin-block-offset=' + offset + '-index-' + i + '-fees'" class="fees">
+              &nbsp;
+            </div>
+          </ng-template>
           <div [attr.data-cy]="'bitcoin-block-' + offset + '-index-' + i + '-fee-span'" class="fee-span"
-            *ngIf="block?.extras?.feeRange">
+            *ngIf="block?.extras?.feeRange; else emptyfeespan">
             {{ block?.extras?.feeRange?.[1] | number:feeRounding }} - {{
             block?.extras?.feeRange[block?.extras?.feeRange?.length - 1] | number:feeRounding }} <ng-container
               i18n="shared.sat-vbyte|sat/vB">sat/vB</ng-container>
           </div>
-          <div [attr.data-cy]="'bitcoin-block-' + offset + '-index-' + i + '-fee-span'" class="fee-span"
-            *ngIf="!block?.extras?.feeRange">
-            &nbsp;
-          </div>
+          <ng-template #emptyfeespan>
+            <div [attr.data-cy]="'bitcoin-block-offset=' + offset + '-index-' + i + '-fees'" class="fee-span">
+              &nbsp;
+            </div>
+          </ng-template>
           <div [attr.data-cy]="'bitcoin-block-' + offset + '-index-' + i + '-total-fees'" *ngIf="showMiningInfo"
             class="block-size">
             <app-amount [satoshis]="block.extras?.totalFees ?? 0" digitsInfo="1.2-3" [noFiat]="true"></app-amount>
@@ -37,10 +43,8 @@
           <div [attr.data-cy]="'bitcoin-block-' + i + '-transactions'" class="transaction-count">
             <ng-container
               *ngTemplateOutlet="block.tx_count === 1 ? transactionsSingular : transactionsPlural; context: {$implicit: block.tx_count | number}"></ng-container>
-            <ng-template #transactionsSingular let-i i18n="shared.transaction-count.singular">{{ i }}
-              transaction</ng-template>
-            <ng-template #transactionsPlural let-i i18n="shared.transaction-count.plural">{{ i }}
-              transactions</ng-template>
+            <ng-template #transactionsSingular let-i i18n="shared.transaction-count.singular">{{ i }} transaction</ng-template>
+            <ng-template #transactionsPlural let-i i18n="shared.transaction-count.plural">{{ i }} transactions</ng-template>
           </div>
           <div [attr.data-cy]="'bitcoin-block-' + offset + '-index-' + i + '-time'" class="time-difference">
             <app-time-since [time]="block.timestamp" [fastRender]="true"></app-time-since></div>
@@ -53,19 +57,19 @@
       </div>
     </ng-container>
     <ng-template #placeholderBlock>
-      <ng-container *ngIf="block && block.placeholder; else loadingBlock">
+      <ng-container *ngIf="block && block.placeholder && connected && !loadingTip; else loadingBlock">
         <div [attr.data-cy]="'bitcoin-block-' + offset + '-index-' + i"
           class="text-center bitcoin-block mined-block placeholder-block blockchain-blocks-{{ i }}"
           id="bitcoin-block-{{ block.height }}" [ngStyle]="blockStyles[i]">
-
+ 
         </div>
       </ng-container>
     </ng-template>
     <ng-template #loadingBlock>
-      <ng-container *ngIf="block && block.loading">
-        <div class="flashing">
+      <ng-container *ngIf="!connected || loadingTip || (block && block.loading)">
+        <div class="flashing loading">
           <div class="text-center bitcoin-block mined-block" id="bitcoin-block-{{ block.height }}"
-            [ngStyle]="blockStyles[i]"></div>
+            [ngStyle]="convertStyleForLoadingBlock(blockStyles[i])"></div>
         </div>
       </ng-container>
     </ng-template>

frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.scss

@@ -137,6 +137,10 @@
   opacity: 1;
 }
 
+.loading .bitcoin-block.mined-block {
+  background: #2d3348;
+}
+
 @keyframes opacityPulse {
   0% {opacity: 0.7;}
   50% {opacity: 1.0;}

frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.ts

@@ -22,6 +22,8 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy {
   @Input() offset: number = 0;
   @Input() height: number = 0;
   @Input() count: number = 8;
+  @Input() loadingTip: boolean = false;
+  @Input() connected: boolean = true;
   
   specialBlocks = specialBlocks;
   network = '';
@@ -288,6 +290,13 @@ export class BlockchainBlocksComponent implements OnInit, OnChanges, OnDestroy {
     };
   }
 
+  convertStyleForLoadingBlock(style) {
+    return {
+      ...style,
+      background: "#2d3348",
+    };
+  }
+
   getStyleForLoadingBlock(index: number, animateEnterFrom: number = 0) {
     const addLeft = animateEnterFrom || 0;
 

frontend/src/app/components/blockchain/blockchain.component.html

@@ -6,7 +6,7 @@
         <app-mempool-blocks [hidden]="pageIndex > 0"></app-mempool-blocks>
         <app-blockchain-blocks [hidden]="pageIndex > 0"></app-blockchain-blocks>
         <ng-container *ngFor="let page of pages; trackBy: trackByPageFn">
-          <app-blockchain-blocks [static]="true" [offset]="page.offset" [height]="page.height" [count]="blocksPerPage"></app-blockchain-blocks>
+          <app-blockchain-blocks [static]="true" [offset]="page.offset" [height]="page.height" [count]="blocksPerPage" [loadingTip]="loadingTip" [connected]="connected"></app-blockchain-blocks>
         </ng-container>
       </div>
       <div id="divider" [hidden]="pageIndex > 0">

frontend/src/app/components/blockchain/blockchain.component.ts

@@ -1,5 +1,5 @@
 import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, Input, OnChanges, SimpleChanges } from '@angular/core';
-import { Subscription } from 'rxjs';
+import { firstValueFrom, Subscription } from 'rxjs';
 import { StateService } from '../../services/state.service';
 
 @Component({
@@ -18,6 +18,9 @@ export class BlockchainComponent implements OnInit, OnDestroy {
   timeLtrSubscription: Subscription;
   timeLtr: boolean = this.stateService.timeLtr.value;
   ltrTransitionEnabled = false;
+  connectionStateSubscription: Subscription;
+  loadingTip: boolean = true;
+  connected: boolean = true;
 
   constructor(
     public stateService: StateService,
@@ -28,10 +31,17 @@ export class BlockchainComponent implements OnInit, OnDestroy {
     this.timeLtrSubscription = this.stateService.timeLtr.subscribe((ltr) => {
       this.timeLtr = !!ltr;
     });
+    this.connectionStateSubscription = this.stateService.connectionState$.subscribe(state => {
+      this.connected = (state === 2);
+    })
+    firstValueFrom(this.stateService.chainTip$).then(tip => {
+      this.loadingTip = false;
+    });
   }
 
   ngOnDestroy() {
     this.timeLtrSubscription.unsubscribe();
+    this.connectionStateSubscription.unsubscribe();
   }
 
   trackByPageFn(index: number, item: { index: number }) {

frontend/src/app/components/hashrate-chart/hashrate-chart.component.html

@@ -31,7 +31,7 @@
     </div>  
 
     <form [formGroup]="radioGroupForm" class="formRadioGroup" *ngIf="(hashrateObservable$ | async) as stats">
-      <div class="btn-group btn-group-toggle" name="radioBasic">
+      <div class="btn-group btn-group-toggle" name="radioBasic" [class]="{'disabled': isLoading}">
         <label class="btn btn-primary btn-sm" *ngIf="stats.blockCount >= 12960" [class.active]="radioGroupForm.get('dateSpan').value === '3m'">
           <input type="radio" [value]="'3m'" fragment="3m" [routerLink]="['/graphs/mining/hashrate-difficulty' | relativeUrl]" [attr.data-cy]="'3m'" formControlName="dateSpan"> 3M
         </label>

frontend/src/app/components/hashrate-chart/hashrate-chart.component.scss

@@ -131,4 +131,9 @@
   display: block;
   max-width: 80px;
   margin: 15px auto 3px;
+}
+
+.disabled {
+  pointer-events: none;
+  opacity: 0.5;
 }

frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.html

@@ -11,7 +11,7 @@
     </div>  
 
     <form [formGroup]="radioGroupForm" class="formRadioGroup" *ngIf="(hashrateObservable$ | async) as stats">
-      <div class="btn-group btn-group-toggle" name="radioBasic">
+      <div class="btn-group btn-group-toggle" name="radioBasic" [class]="{'disabled': isLoading}">
         <label class="btn btn-primary btn-sm" *ngIf="stats.blockCount >= 25920" [class.active]="radioGroupForm.get('dateSpan').value === '6m'">
           <input type="radio" [value]="'6m'" fragment="6m" [routerLink]="['/graphs/mining/pools-dominance' | relativeUrl]" [attr.data-cy]="'6m'" formControlName="dateSpan"> 6M
         </label>

frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.scss

@@ -82,3 +82,8 @@
 .loadingGraphs.widget {
   top: 75%;
 }
+
+.disabled {
+  pointer-events: none;
+  opacity: 0.5;
+}

frontend/src/app/components/hashrates-chart-pools/hashrate-chart-pools.component.ts

@@ -5,7 +5,7 @@ import { delay, map, retryWhen, share, startWith, switchMap, tap } from 'rxjs/op
 import { ApiService } from '../../services/api.service';
 import { SeoService } from '../../services/seo.service';
 import { UntypedFormBuilder, UntypedFormGroup } from '@angular/forms';
-import { poolsColor } from '../../app.constants';
+import { chartColors, poolsColor } from '../../app.constants';
 import { StorageService } from '../../services/storage.service';
 import { MiningService } from '../../services/mining.service';
 import { download } from '../../shared/graphs.utils';
@@ -173,6 +173,7 @@ export class HashrateChartPoolsComponent implements OnInit {
     this.chartOptions = {
       title: title,
       animation: false,
+      color: chartColors.filter(color => color !== '#FDD835'),
       grid: {
         right: this.right,
         left: this.left,

frontend/src/app/components/pool-ranking/pool-ranking.component.html

@@ -40,7 +40,7 @@
     </div>  
     <form [formGroup]="radioGroupForm" class="formRadioGroup"
       *ngIf="!widget && (miningStatsObservable$ | async) as stats">
-      <div class="btn-group btn-group-toggle" name="radioBasic">
+      <div class="btn-group btn-group-toggle" name="radioBasic" [class]="{'disabled': isLoading}">
         <label class="btn btn-primary btn-sm" *ngIf="stats.totalBlockCount >= 144" [class.active]="radioGroupForm.get('dateSpan').value === '24h'">
           <input type="radio" [value]="'24h'" fragment="24h" [routerLink]="['/graphs/mining/pools' | relativeUrl]" [attr.data-cy]="'24h'" formControlName="dateSpan"> 24h
         </label>
@@ -92,6 +92,8 @@
           <th class="" i18n="mining.pool-name">Pool</th>
           <th class="" *ngIf="this.miningWindowPreference === '24h'" i18n="mining.hashrate">Hashrate</th>
           <th class="" i18n="master-page.blocks">Blocks</th>
+          <th *ngIf="auditAvailable" class="health text-right widget" i18n="latest-blocks.avg_health"
+            i18n-ngbTooltip="latest-blocks.avg_health" ngbTooltip="Avg Health" placement="bottom" #health [disableTooltip]="!isEllipsisActive(health)">Avg Health</th>
           <th class="d-none d-md-block" i18n="mining.empty-blocks">Empty blocks</th>
         </tr>
       </thead>
@@ -102,9 +104,23 @@
             <img width="25" height="25" src="{{ pool.logo }}" [alt]="pool.name + ' mining pool logo'" onError="this.src = '/resources/mining-pools/default.svg'">
           </td>
           <td class=""><a [routerLink]="[('/mining/pool/' + pool.slug) | relativeUrl]">{{ pool.name }}</a></td>
-          <td class="" *ngIf="this.miningWindowPreference === '24h' && !isLoading">{{ pool.lastEstimatedHashrate }} {{
+          <td class="" *ngIf="this.miningWindowPreference === '24h'">{{ pool.lastEstimatedHashrate }} {{
             miningStats.miningUnits.hashrateUnit }}</td>
-          <td class="">{{ pool['blockText'] }}</td>
+          <td class="d-flex justify-content-center">
+            {{ pool.blockCount }}<span class="d-none d-md-block">&nbsp;({{ pool.share }}%)</span>
+          </td>
+          <td *ngIf="auditAvailable" class="health text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
+            <a
+              class="health-badge badge"
+              [class.badge-success]="pool.avgMatchRate >= 99"
+              [class.badge-warning]="pool.avgMatchRate >= 75 && pool.avgMatchRate < 99"
+              [class.badge-danger]="pool.avgMatchRate < 75"
+              *ngIf="pool.avgMatchRate != null; else nullHealth"
+            >{{ pool.avgMatchRate }}%</a>
+            <ng-template #nullHealth>
+              <span class="health-badge badge badge-secondary" i18n="unknown">Unknown</span>
+            </ng-template>
+          </td>
           <td class="d-none d-md-block">{{ pool.emptyBlocks }} ({{ pool.emptyBlockRatio }}%)</td>
         </tr>
         <tr style="border-top: 1px solid #555">

frontend/src/app/components/pool-ranking/pool-ranking.component.scss

@@ -139,3 +139,8 @@
   max-width: 80px;
   margin: 15px auto 3px;
 }
+
+.disabled {
+  pointer-events: none;
+  opacity: 0.5;
+}

frontend/src/app/components/pool-ranking/pool-ranking.component.ts

@@ -4,7 +4,6 @@ import { ActivatedRoute, Router } from '@angular/router';
 import { EChartsOption, PieSeriesOption } from 'echarts';
 import { concat, Observable } from 'rxjs';
 import { map, share, startWith, switchMap, tap } from 'rxjs/operators';
-import { SinglePoolStats } from '../../interfaces/node-api.interface';
 import { SeoService } from '../../services/seo.service';
 import { StorageService } from '../..//services/storage.service';
 import { MiningService, MiningStats } from '../../services/mining.service';
@@ -26,6 +25,8 @@ export class PoolRankingComponent implements OnInit {
   miningWindowPreference: string;
   radioGroupForm: UntypedFormGroup;
 
+  auditAvailable = false;
+  indexingAvailable = false;
   isLoading = true;
   chartOptions: EChartsOption = {};
   chartInitOptions = {
@@ -60,6 +61,10 @@ export class PoolRankingComponent implements OnInit {
     this.radioGroupForm = this.formBuilder.group({ dateSpan: this.miningWindowPreference });
     this.radioGroupForm.controls.dateSpan.setValue(this.miningWindowPreference);
 
+    this.indexingAvailable = (this.stateService.env.BASE_MODULE === 'mempool' &&
+      this.stateService.env.MINING_DASHBOARD === true);
+    this.auditAvailable = this.indexingAvailable && this.stateService.env.AUDIT;
+
     this.route
       .fragment
       .subscribe((fragment) => {
@@ -73,6 +78,7 @@ export class PoolRankingComponent implements OnInit {
         .pipe(
           startWith(this.radioGroupForm.controls.dateSpan.value), // (trigger when the page loads)
           tap((value) => {
+            this.isLoading = true;
             this.timespan = value;
             if (!this.widget) {
               this.storageService.setValue('miningWindowPreference', value);
@@ -92,7 +98,6 @@ export class PoolRankingComponent implements OnInit {
       )
       .pipe(
         map(data => {
-          data.pools = data.pools.map((pool: SinglePoolStats) => this.formatPoolUI(pool));
           data['minersLuck'] = (100 * (data.blockCount / 1008)).toFixed(2); // luck 1w
           return data;
         }),
@@ -104,11 +109,6 @@ export class PoolRankingComponent implements OnInit {
       );
   }
 
-  formatPoolUI(pool: SinglePoolStats) {
-    pool['blockText'] = pool.blockCount.toString() + ` (${pool.share}%)`;
-    return pool;
-  }
-
   generatePoolsChartSerieData(miningStats) {
     let poolShareThreshold = 0.5;
     if (isMobile()) {
@@ -219,7 +219,7 @@ export class PoolRankingComponent implements OnInit {
 
     this.chartOptions = {
       animation: false,
-      color: chartColors,
+      color: chartColors.filter(color => color !== '#FDD835'),
       tooltip: {
         trigger: 'item',
         textStyle: {

frontend/src/app/components/qrcode/qrcode.component.html

@@ -1,4 +1,4 @@
 <div class="holder" [ngStyle]="{'width': size, 'height': size}">
   <img *ngIf="imageUrl" [src]="imageUrl">
-  <canvas #canvas></canvas>
+  <canvas #canvas [style]="{'border': border + 'px solid white'}"></canvas>
 </div>

frontend/src/app/components/qrcode/qrcode.component.ts

@@ -12,6 +12,7 @@ export class QrcodeComponent implements AfterViewInit {
   @Input() data: string;
   @Input() size = 125;
   @Input() imageUrl: string;
+  @Input() border = 0;
   @ViewChild('canvas') canvas: ElementRef;
 
   qrcodeObject: any;

frontend/src/app/components/search-form/search-results/search-results.component.html

@@ -1,30 +1,30 @@
 <div class="dropdown-menu show" *ngIf="results" [hidden]="!results.hashQuickMatch && !results.addresses.length && !results.nodes.length && !results.channels.length">
   <ng-template [ngIf]="results.blockHeight">
-    <div class="card-title">Bitcoin Block Height</div>
+    <div class="card-title" i18n="search.bitcoin-block-height">Bitcoin Block Height</div>
     <button (click)="clickItem(0)" [class.active]="0 === activeIdx" type="button" role="option" class="dropdown-item">
-      Go to "{{ results.searchText }}"
+      <ng-container *ngTemplateOutlet="goTo; context: { $implicit: results.searchText }"></ng-container>
     </button>
   </ng-template>
   <ng-template [ngIf]="results.txId">
-    <div class="card-title">Bitcoin Transaction</div>
+    <div class="card-title" i18n="search.bitcoin-transaction">Bitcoin Transaction</div>
     <button (click)="clickItem(0)" [class.active]="0 === activeIdx" type="button" role="option" class="dropdown-item">
-      Go to "{{ results.searchText | shortenString : 13 }}"
+      <ng-container *ngTemplateOutlet="goTo; context: { $implicit: results.searchText | shortenString : 13 }"></ng-container>
     </button>
   </ng-template>
   <ng-template [ngIf]="results.address">
-    <div class="card-title">Bitcoin Address</div>
+    <div class="card-title" i18n="search.bitcoin-address">Bitcoin Address</div>
     <button (click)="clickItem(0)" [class.active]="0 === activeIdx" type="button" role="option" class="dropdown-item">
-      Go to "{{ results.searchText | shortenString : isMobile ? 20 : 30 }}"
+      <ng-container *ngTemplateOutlet="goTo; context: { $implicit: results.searchText | shortenString : isMobile ? 20 : 30 }"></ng-container>
     </button>
   </ng-template>
   <ng-template [ngIf]="results.blockHash">
-    <div class="card-title">Bitcoin Block</div>
+    <div class="card-title" i18n="search.bitcoin-block">Bitcoin Block</div>
     <button (click)="clickItem(0)" [class.active]="0 === activeIdx" type="button" role="option" class="dropdown-item">
-      Go to "{{ results.searchText | shortenString : 13 }}"
+      <ng-container *ngTemplateOutlet="goTo; context: { $implicit: results.searchText | shortenString : 13 }"></ng-container>
     </button>
   </ng-template>
   <ng-template [ngIf]="results.addresses.length">
-    <div class="card-title">Bitcoin Addresses</div>
+    <div class="card-title" i18n="search.bitcoin-addresses">Bitcoin Addresses</div>
     <ng-template ngFor [ngForOf]="results.addresses" let-address let-i="index">
       <button (click)="clickItem(results.hashQuickMatch + i)" [class.active]="(results.hashQuickMatch + i) === activeIdx" type="button" role="option" class="dropdown-item">
         <ngb-highlight [result]="address | shortenString : isMobile ? 25 : 36" [term]="results.searchText"></ngb-highlight>
@@ -32,7 +32,7 @@
     </ng-template>
   </ng-template>
   <ng-template [ngIf]="results.nodes.length">
-    <div class="card-title">Lightning Nodes</div>
+    <div class="card-title" i18n="search.lightning-nodes">Lightning Nodes</div>
     <ng-template ngFor [ngForOf]="results.nodes" let-node let-i="index">
       <button (click)="clickItem(results.hashQuickMatch + results.addresses.length + i)" [class.inactive]="node.status === 0" [class.active]="results.hashQuickMatch + results.addresses.length + i === activeIdx" [routerLink]="['/lightning/node' | relativeUrl, node.public_key]" type="button" role="option" class="dropdown-item">
         <ngb-highlight [result]="node.alias" [term]="results.searchText"></ngb-highlight> &nbsp;<span class="symbol">{{ node.public_key | shortenString : 10 }}</span>
@@ -40,7 +40,7 @@
     </ng-template>
   </ng-template>
   <ng-template [ngIf]="results.channels.length">
-    <div class="card-title">Lightning Channels</div>
+    <div class="card-title" i18n="search.lightning-channels">Lightning Channels</div>
     <ng-template ngFor [ngForOf]="results.channels" let-channel let-i="index">
       <button (click)="clickItem(results.hashQuickMatch + results.addresses.length + results.nodes.length + i)" [class.inactive]="channel.status === 2"  [class.active]="results.hashQuickMatch + results.addresses.length + results.nodes.length + i === activeIdx" type="button" role="option" class="dropdown-item">
         <ngb-highlight [result]="channel.short_id" [term]="results.searchText"></ngb-highlight> &nbsp;<span class="symbol">{{ channel.id }}</span>
@@ -48,3 +48,5 @@
     </ng-template>
   </ng-template>
 </div>
+
+<ng-template #goTo let-x i18n="search.go-to">Go to "{{ x }}"</ng-template>

frontend/src/app/components/start/start.component.ts

@@ -21,6 +21,7 @@ export class StartComponent implements OnInit, OnDestroy {
   timeLtr: boolean = this.stateService.timeLtr.value;
   chainTipSubscription: Subscription;
   chainTip: number = -1;
+  tipIsSet: boolean = false;
   markBlockSubscription: Subscription;
   blockCounterSubscription: Subscription;
   @ViewChild('blockchainContainer') blockchainContainer: ElementRef;
@@ -58,6 +59,7 @@ export class StartComponent implements OnInit, OnDestroy {
     });
     this.chainTipSubscription = this.stateService.chainTip$.subscribe((height) => {
       this.chainTip = height;
+      this.tipIsSet = true;
       this.updatePages();
       if (this.pendingMark != null) {
         this.scrollToBlock(this.pendingMark);
@@ -66,7 +68,7 @@ export class StartComponent implements OnInit, OnDestroy {
     });
     this.markBlockSubscription = this.stateService.markBlock$.subscribe((mark) => {
       if (mark?.blockHeight != null) {
-        if (this.chainTip >=0) {
+        if (this.tipIsSet) {
           if (!this.blockInViewport(mark.blockHeight)) {
             this.scrollToBlock(mark.blockHeight);
           }
@@ -123,7 +125,7 @@ export class StartComponent implements OnInit, OnDestroy {
     this.minScrollWidth = this.firstPageWidth + (this.pageWidth * 2);
 
     if (firstVisibleBlock != null) {
-      this.scrollToBlock(firstVisibleBlock, offset);
+      this.scrollToBlock(firstVisibleBlock, offset + (this.isMobile ? this.blockWidth : 0));
     } else {
       this.updatePages();
     }
@@ -178,8 +180,10 @@ export class StartComponent implements OnInit, OnDestroy {
       setTimeout(() => { this.scrollToBlock(height, blockOffset); }, 50);
       return;
     }
-    const targetHeight = this.isMobile ? height - 1 : height;
-    const viewingPageIndex = this.getPageIndexOf(targetHeight);
+    if (this.isMobile) {
+      blockOffset -= this.blockWidth;
+    }
+    const viewingPageIndex = this.getPageIndexOf(height);
     const pages = [];
     this.pageIndex = Math.max(viewingPageIndex - 1, 0);
     let viewingPage = this.getPageAt(viewingPageIndex);
@@ -189,7 +193,7 @@ export class StartComponent implements OnInit, OnDestroy {
       viewingPage = this.getPageAt(viewingPageIndex);
     }
     const left = viewingPage.offset - this.getConvertedScrollOffset();
-    const blockIndex = viewingPage.height - targetHeight;
+    const blockIndex = viewingPage.height - height;
     const targetOffset = (this.blockWidth * blockIndex) + left;
     let deltaOffset = targetOffset - blockOffset;
 

frontend/src/app/components/transaction/transaction.component.html

@@ -61,12 +61,6 @@
                     </div>
                   </td>
                 </tr>
-                <tr *ngIf="latestBlock && tx.status.block_height <= latestBlock.height - 8">
-                  <td class="td-width" i18n="transaction.included-in-block|Transaction included in block">Included in block</td>
-                  <td>
-                    <a [routerLink]="['/block/' | relativeUrl, tx.status.block_hash]" [state]="{ data: { blockHeight: tx.status.block_height } }">{{ tx.status.block_height }}</a>
-                  </td>
-                </tr>
                 <ng-template [ngIf]="transactionTime > 0">
                   <tr>
                     <td i18n="transaction.confirmed|Transaction Confirmed state">Confirmed</td>
@@ -475,7 +469,7 @@
     <tbody>
       <tr>
         <td class="td-width" i18n="transaction.fee|Transaction fee">Fee</td>
-        <td>{{ tx.fee | number }} <span class="symbol" i18n="shared.sat|sat">sat</span> <span class="fiat"><app-fiat [value]="tx.fee"></app-fiat></span></td>
+        <td>{{ tx.fee | number }} <span class="symbol" i18n="shared.sat|sat">sat</span> <span class="fiat"><app-fiat [blockConversion]="blockConversion" [value]="tx.fee"></app-fiat></span></td>
       </tr>
       <tr>
         <td i18n="transaction.fee-rate|Transaction fee rate">Fee rate</td>

frontend/src/app/components/transaction/transaction.component.ts

@@ -8,10 +8,11 @@ import {
   retryWhen,
   delay,
   map,
-  mergeMap
+  mergeMap,
+  tap
 } from 'rxjs/operators';
 import { Transaction } from '../../interfaces/electrs.interface';
-import { of, merge, Subscription, Observable, Subject, timer, combineLatest, from, throwError } from 'rxjs';
+import { of, merge, Subscription, Observable, Subject, timer, from, throwError } from 'rxjs';
 import { StateService } from '../../services/state.service';
 import { CacheService } from '../../services/cache.service';
 import { WebsocketService } from '../../services/websocket.service';
@@ -21,6 +22,7 @@ import { SeoService } from '../../services/seo.service';
 import { BlockExtended, CpfpInfo } from '../../interfaces/node-api.interface';
 import { LiquidUnblinding } from './liquid-ublinding';
 import { RelativeUrlPipe } from '../../shared/pipes/relative-url/relative-url.pipe';
+import { Price, PriceService } from '../../services/price.service';
 
 @Component({
   selector: 'app-transaction',
@@ -69,7 +71,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
   hideFlow: boolean = this.stateService.hideFlow.value;
   overrideFlowPreference: boolean = null;
   flowEnabled: boolean;
-
+  blockConversion: Price;
   tooltipPosition: { x: number, y: number };
 
   @ViewChild('graphContainer')
@@ -85,7 +87,8 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
     private websocketService: WebsocketService,
     private audioService: AudioService,
     private apiService: ApiService,
-    private seoService: SeoService
+    private seoService: SeoService,
+    private priceService: PriceService,
   ) {}
 
   ngOnInit() {
@@ -323,6 +326,13 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
             }
             this.fetchRbfHistory$.next(this.tx.txid);
           }
+
+          this.priceService.getBlockPrice$(tx.status.block_time, true).pipe(
+            tap((price) => {
+              this.blockConversion = price;
+            })
+          ).subscribe();
+      
           setTimeout(() => { this.applyFragment(); }, 0);
         },
         (error) => {

frontend/src/app/components/transactions-list/transactions-list.component.html

@@ -88,7 +88,7 @@
                   </ng-template>
                   <ng-template #defaultOutput>
                     <span *ngIf="vin.lazy" class="skeleton-loader"></span>
-                    <app-amount *ngIf="vin.prevout" [satoshis]="vin.prevout.value"></app-amount>
+                    <app-amount [blockConversion]="tx.price" *ngIf="vin.prevout" [satoshis]="vin.prevout.value"></app-amount>
                   </ng-template>
                 </td>
               </tr>
@@ -216,7 +216,7 @@
                     </ng-template>
                   </ng-template>
                   <ng-template #defaultOutput>
-                    <app-amount [satoshis]="vout.value"></app-amount>
+                    <app-amount [blockConversion]="tx.price" [satoshis]="vout.value"></app-amount>
                   </ng-template>
                 </td>
                 <td class="arrow-td">
@@ -283,7 +283,9 @@
 
     <div class="summary">
       <div class="float-left mt-2-5" *ngIf="!transactionPage && !tx.vin[0].is_coinbase && tx.fee !== -1">
-        {{ tx.fee / (tx.weight / 4) | feeRounding }} <span class="symbol" i18n="shared.sat-vbyte|sat/vB">sat/vB</span> <span class="d-none d-sm-inline-block">&nbsp;&ndash; {{ tx.fee | number }} <span class="symbol" i18n="shared.sat|sat">sat</span> <span class="fiat"><app-fiat [value]="tx.fee"></app-fiat></span></span>
+        {{ tx.fee / (tx.weight / 4) | feeRounding }} <span class="symbol" i18n="shared.sat-vbyte|sat/vB">sat/vB</span> <span
+          class="d-none d-sm-inline-block">&nbsp;&ndash; {{ tx.fee | number }} <span class="symbol"
+            i18n="shared.sat|sat">sat</span> <span class="fiat"><app-fiat [blockConversion]="tx.price" [value]="tx.fee"></app-fiat></span></span>
       </div>
       <div class="float-left mt-2-5 grey-info-text" *ngIf="tx.fee === -1" i18n="transactions-list.load-to-reveal-fee-info">Show more inputs to reveal fee data</div>
 
@@ -301,12 +303,12 @@
         <button *ngIf="address === ''; else viewingAddress" type="button" class="btn btn-sm btn-primary mt-2 ml-2" (click)="switchCurrency()">
           <ng-template [ngIf]="(network === 'liquid' || network === 'liquidtestnet') && haveBlindedOutputValues(tx)" [ngIfElse]="defaultAmount" i18n="shared.confidential">Confidential</ng-template>
           <ng-template #defaultAmount>
-            <app-amount [satoshis]="getTotalTxOutput(tx)"></app-amount>
+            <app-amount [blockConversion]="tx.price" [satoshis]="getTotalTxOutput(tx)"></app-amount>
           </ng-template>
         </button>
         <ng-template #viewingAddress>
           <button type="button" class="btn btn-sm mt-2 ml-2" (click)="switchCurrency()" [ngClass]="{'btn-success': tx['addressValue'] >= 0, 'btn-danger': tx['addressValue'] < 0}">
-            <app-amount [satoshis]="tx['addressValue']" [addPlus]="true"></app-amount>
+            <app-amount [blockConversion]="tx.price" [satoshis]="tx['addressValue']" [addPlus]="true"></app-amount>
           </button>
         </ng-template>
       </div>

frontend/src/app/components/transactions-list/transactions-list.component.ts

@@ -6,9 +6,10 @@ import { Outspend, Transaction, Vin, Vout } from '../../interfaces/electrs.inter
 import { ElectrsApiService } from '../../services/electrs-api.service';
 import { environment } from '../../../environments/environment';
 import { AssetsService } from '../../services/assets.service';
-import { filter, map, tap, switchMap } from 'rxjs/operators';
+import { filter, map, tap, switchMap, shareReplay } from 'rxjs/operators';
 import { BlockExtended } from '../../interfaces/node-api.interface';
 import { ApiService } from '../../services/api.service';
+import { PriceService } from '../../services/price.service';
 
 @Component({
   selector: 'app-transactions-list',
@@ -50,6 +51,7 @@ export class TransactionsListComponent implements OnInit, OnChanges {
     private apiService: ApiService,
     private assetsService: AssetsService,
     private ref: ChangeDetectorRef,
+    private priceService: PriceService,
   ) { }
 
   ngOnInit(): void {
@@ -147,6 +149,10 @@ export class TransactionsListComponent implements OnInit, OnChanges {
 
           tx['addressValue'] = addressIn - addressOut;
         }
+
+        this.priceService.getBlockPrice$(tx.status.block_time).pipe(
+          tap((price) => tx['price'] = price)
+        ).subscribe();
       });
       const txIds = this.transactions.filter((tx) => !tx._outspends).map((tx) => tx.txid);
       if (txIds.length) {

frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.html

@@ -56,7 +56,7 @@
           </ng-container>
       </ng-container>
       <p *ngIf="line.value == null && line.confidential" i18n="shared.confidential">Confidential</p>
-      <p *ngIf="line.value != null"><app-amount [satoshis]="line.value"></app-amount></p>
+      <p *ngIf="line.value != null"><app-amount [blockConversion]="blockConversion" [satoshis]="line.value"></app-amount></p>
       <p *ngIf="line.type !== 'fee' && line.address" class="address">
         <app-truncate [text]="line.address"></app-truncate>
       </p>

frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts

@@ -1,5 +1,6 @@
-import { Component, ElementRef, ViewChild, Input, OnChanges, ChangeDetectionStrategy } from '@angular/core';
-import { TransactionStripped } from '../../interfaces/websocket.interface';
+import { Component, ElementRef, ViewChild, Input, OnChanges, OnInit } from '@angular/core';
+import { tap } from 'rxjs';
+import { Price, PriceService } from '../../services/price.service';
 
 interface Xput {
   type: 'input' | 'output' | 'fee';
@@ -14,6 +15,7 @@ interface Xput {
   pegin?: boolean;
   pegout?: string;
   confidential?: boolean;
+  timestamp?: number;
 }
 
 @Component({
@@ -27,12 +29,21 @@ export class TxBowtieGraphTooltipComponent implements OnChanges {
   @Input() isConnector: boolean = false;
 
   tooltipPosition = { x: 0, y: 0 };
+  blockConversion: Price;
 
   @ViewChild('tooltip') tooltipElement: ElementRef<HTMLCanvasElement>;
 
-  constructor() {}
+  constructor(private priceService: PriceService) {}
 
   ngOnChanges(changes): void {
+    if (changes.line?.currentValue) {
+      this.priceService.getBlockPrice$(changes.line?.currentValue.timestamp, true).pipe(
+        tap((price) => {
+          this.blockConversion = price;
+        })
+      ).subscribe();
+    }
+
     if (changes.cursorPosition && changes.cursorPosition.currentValue) {
       let x = Math.max(10, changes.cursorPosition.currentValue.x - 50);
       let y = changes.cursorPosition.currentValue.y + 20;

frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts

@@ -29,6 +29,7 @@ interface Xput {
   pegin?: boolean;
   pegout?: string;
   confidential?: boolean;
+  timestamp?: number;
 }
 
 @Component({
@@ -152,6 +153,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
         index: i,
         pegout: v?.pegout?.scriptpubkey_address,
         confidential: (this.isLiquid && v?.value === undefined),
+        timestamp: this.tx.status.block_time
       } as Xput;
     });
 
@@ -171,6 +173,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
         coinbase: v?.is_coinbase,
         pegin: v?.is_pegin,
         confidential: (this.isLiquid && v?.prevout?.value === undefined),
+        timestamp: this.tx.status.block_time
       } as Xput;
     });
 
@@ -196,8 +199,8 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
     this.outputs = this.initLines('out', voutWithFee, totalValue, this.maxStrands);
 
     this.middle = {
-      path: `M ${(this.width / 2) - this.midWidth} ${(this.height / 2) + 0.5} L ${(this.width / 2) + this.midWidth} ${(this.height / 2) + 0.5}`,
-      style: `stroke-width: ${this.combinedWeight + 1}; stroke: ${this.gradient[1]}`
+      path: `M ${(this.width / 2) - this.midWidth} ${(this.height / 2) + 0.25} L ${(this.width / 2) + this.midWidth} ${(this.height / 2) + 0.25}`,
+      style: `stroke-width: ${this.combinedWeight + 0.5}; stroke: ${this.gradient[1]}`
     };
 
     this.hasLine = this.inputs.reduce((line, put) => line || !put.zeroValue, false)
@@ -254,7 +257,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
     const lineParams = weights.map((w, i) => {
       return {
         weight: w,
-        thickness: xputs[i].value === 0 ? this.zeroValueThickness : Math.max(this.minWeight - 1, w) + 1,
+        thickness: xputs[i].value === 0 ? this.zeroValueThickness : Math.min(this.combinedWeight + 0.5, Math.max(this.minWeight - 1, w) + 1),
         offset: 0,
         innerY: 0,
         outerY: 0,
@@ -266,7 +269,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
 
     // bounds of the middle segment
     const innerTop = (this.height / 2) - (this.combinedWeight / 2);
-    const innerBottom = innerTop + this.combinedWeight;
+    const innerBottom = innerTop + this.combinedWeight + 0.5;
     // tracks the visual bottom of the endpoints of the previous line
     let lastOuter = 0;
     let lastInner = innerTop;
@@ -291,7 +294,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
 
       // set the vertical position of the (center of the) outer side of the line
       line.outerY = lastOuter + (line.thickness / 2);
-      line.innerY = Math.min(innerBottom + (line.thickness / 2), Math.max(innerTop + (line.thickness / 2), lastInner + (line.weight / 2)));
+      line.innerY = Math.min(innerBottom - (line.thickness / 2), Math.max(innerTop + (line.thickness / 2), lastInner + (line.weight / 2)));
 
       // special case to center single input/outputs
       if (xputs.length === 1) {

frontend/src/app/docs/api-docs/api-docs.component.html

@@ -10,7 +10,7 @@
       <div class="doc-content">
 
         <div id="disclaimer">
-          <table><tr><td><svg viewBox="0 0 304 304" xmlns="http://www.w3.org/2000/svg"><g fill-rule="evenodd" style="fill:#ffc107;fill-opacity:1"><path d="M135.3 34.474c-15.62 27.306-54.206 95.63-85.21 150.534L9.075 257.583C5.382 264.08 6.76 269.217 7.908 271.7c2.326 5.028 7.29 7.537 11.155 8.215l.78.133 264.698.006-.554-.02c4.152.255 9.664-1.24 12.677-6.194 1.926-3.18 3.31-8.589-1.073-16.278L213.637 114.37l-45.351-79.205c-5.681-9.932-12.272-12.022-16.8-12.022-4.42 0-10.818 1.964-16.181 11.331h-.006zm-69.072 159.94c30.997-54.885 69.563-123.184 85.16-150.446l.186-.297c.2.303.393.582.618.981l45.363 79.22s72.377 126.47 78.569 137.283l-247.618-.007 37.719-66.734" style="fill:#ffc107;fill-opacity:1"/><path d="M152.597 247.445c8.02 0 14.518-6.728 14.518-15.025 0-8.29-6.499-15.018-14.518-15.018-8.031 0-14.529 6.728-14.529 15.018 0 8.297 6.498 15.025 14.53 15.025m-.001-147.18c11.586 0 22.23 10.958 20.977 21.7l-9.922 75.564c-.966 6.601-4.95 11.433-11.055 11.433s-10.102-4.832-11.056-11.433l-9.927-75.564c-1.26-10.742 9.39-21.7 20.983-21.7" style="fill:#ffc107;fill-opacity:1"/></g></svg></td><td><p><b>mempool.space merely provides data about the Bitcoin network.</b> It cannot help you with retrieving funds, confirming your transaction quicker, etc.</p><p>For any such requests, you need to get in touch with the entity that helped make the transaction (wallet software, exchange company, etc).</p></td></tr></table>
+          <table><tr><td><svg viewBox="0 0 304 304" xmlns="http://www.w3.org/2000/svg"><g fill-rule="evenodd" style="fill:#ffc107;fill-opacity:1"><path d="M135.3 34.474c-15.62 27.306-54.206 95.63-85.21 150.534L9.075 257.583C5.382 264.08 6.76 269.217 7.908 271.7c2.326 5.028 7.29 7.537 11.155 8.215l.78.133 264.698.006-.554-.02c4.152.255 9.664-1.24 12.677-6.194 1.926-3.18 3.31-8.589-1.073-16.278L213.637 114.37l-45.351-79.205c-5.681-9.932-12.272-12.022-16.8-12.022-4.42 0-10.818 1.964-16.181 11.331h-.006zm-69.072 159.94c30.997-54.885 69.563-123.184 85.16-150.446l.186-.297c.2.303.393.582.618.981l45.363 79.22s72.377 126.47 78.569 137.283l-247.618-.007 37.719-66.734" style="fill:#ffc107;fill-opacity:1"/><path d="M152.597 247.445c8.02 0 14.518-6.728 14.518-15.025 0-8.29-6.499-15.018-14.518-15.018-8.031 0-14.529 6.728-14.529 15.018 0 8.297 6.498 15.025 14.53 15.025m-.001-147.18c11.586 0 22.23 10.958 20.977 21.7l-9.922 75.564c-.966 6.601-4.95 11.433-11.055 11.433s-10.102-4.832-11.056-11.433l-9.927-75.564c-1.26-10.742 9.39-21.7 20.983-21.7" style="fill:#ffc107;fill-opacity:1"/></g></svg></td><td><p i18n="faq.big-disclaimer"><b>mempool.space merely provides data about the Bitcoin network.</b> It cannot help you with retrieving funds, confirming your transaction quicker, etc.</p><p>For any such requests, you need to get in touch with the entity that helped make the transaction (wallet software, exchange company, etc).</p></td></tr></table>
 
         </div>
 

frontend/src/app/fiat/fiat.component.html

@@ -1 +1,14 @@
-<span class="green-color" *ngIf="(conversions$ | async) as conversions">{{ (conversions ? conversions[currency] : 0) * value / 100000000 | fiatCurrency : digitsInfo : currency }}</span>
+<span class="green-color" *ngIf="blockConversion; else noblockconversion">
+  {{
+    (
+      (blockConversion.price[currency] >= 0 ? blockConversion.price[currency] : null) ??
+      (blockConversion.price['USD'] * blockConversion.exchangeRates['USD' + currency]) ?? 0
+    ) * value / 100000000 | fiatCurrency : digitsInfo : currency
+  }}
+</span>
+
+<ng-template #noblockconversion>
+  <span class="green-color" *ngIf="(conversions$ | async) as conversions">
+    {{ (conversions[currency] ?? conversions['USD'] ?? 0) * value / 100000000 | fiatCurrency : digitsInfo : currency }}
+  </span>
+</ng-template>

frontend/src/app/fiat/fiat.component.ts

@@ -1,5 +1,6 @@
 import { Component, OnInit, ChangeDetectionStrategy, Input, ChangeDetectorRef, OnDestroy } from '@angular/core';
 import { Observable, Subscription } from 'rxjs';
+import { Price } from '../services/price.service';
 import { StateService } from '../services/state.service';
 
 @Component({
@@ -15,6 +16,7 @@ export class FiatComponent implements OnInit, OnDestroy {
 
   @Input() value: number;
   @Input() digitsInfo = '1.2-2';
+  @Input() blockConversion: Price;
 
   constructor(
     private stateService: StateService,

frontend/src/app/interfaces/electrs.interface.ts

@@ -1,3 +1,4 @@
+import { Price } from '../services/price.service';
 import { IChannel } from './node-api.interface';
 
 export interface Transaction {
@@ -23,6 +24,7 @@ export interface Transaction {
   _deduced?: boolean;
   _outspends?: Outspend[];
   _channels?: TransactionChannels;
+  price?: Price;
 }
 
 export interface TransactionChannels {

frontend/src/app/interfaces/node-api.interface.ts

@@ -73,6 +73,7 @@ export interface SinglePoolStats {
   emptyBlockRatio: string;
   logo: string;
   slug: string;
+  avgMatchRate: number;
 }
 export interface PoolsStats {
   blockCount: number;

frontend/src/app/lightning/channel/channel.component.html

@@ -1,25 +1,32 @@
 <div class="container-xl" *ngIf="(channel$ | async) as channel; else skeletonLoader">
-  <h5 class="mb-0" style="color: #ffffff66" i18n="lightning.channel">Lightning channel</h5>
-  <div class="title-container">
-    <h1 class="mb-0">{{ channel.short_id }}</h1>
-    <span class="tx-link">
-      <a [routerLink]="['/lightning/channel' | relativeUrl, channel.id]">{{ channel.id }}</a>
-      <app-clipboard [text]="channel.id"></app-clipboard>
-    </span>
-  </div>
-  <div class="badges mb-2">
-    <span class="badge rounded-pill badge-secondary" *ngIf="channel.status === 0" i18n="status.inactive">Inactive</span>
-    <span class="badge rounded-pill badge-success" *ngIf="channel.status === 1" i18n="status.active">Active</span>
-    <span class="badge rounded-pill badge-danger" *ngIf="channel.status === 2" i18n="status.closed">Closed</span>
-    <app-closing-type *ngIf="channel.closing_reason" [type]="channel.closing_reason"></app-closing-type>
-  </div>
+
+  <ng-container *ngIf="!error">
+    <h5 class="mb-0" style="color: #ffffff66" i18n="lightning.channel">Lightning channel</h5>
+    <div class="title-container">
+      <h1 class="mb-0">{{ channel.short_id }}</h1>
+      <span class="tx-link">
+        <a [routerLink]="['/lightning/channel' | relativeUrl, channel.id]">{{ channel.id }}</a>
+        <app-clipboard [text]="channel.id"></app-clipboard>
+      </span>
+    </div>
+    <div class="badges mb-2">
+      <span class="badge rounded-pill badge-secondary" *ngIf="channel.status === 0" i18n="status.inactive">Inactive</span>
+      <span class="badge rounded-pill badge-success" *ngIf="channel.status === 1" i18n="status.active">Active</span>
+      <span class="badge rounded-pill badge-danger" *ngIf="channel.status === 2" i18n="status.closed">Closed</span>
+      <app-closing-type *ngIf="channel.closing_reason" [type]="channel.closing_reason"></app-closing-type>
+    </div>
+  </ng-container>
 
   <div class="clearfix"></div>
 
+  <div *ngIf="error" class="d-flex flex-column justify-content-around align-items-center mt-5 w-100" style="min-height: 100px">
+    <span class="text-center" i18n="lightning.channel-not-found">No channel found for short id "{{ channel.short_id }}"</span>
+  </div>
+
   <app-nodes-channels-map *ngIf="!error && (channelGeo$ | async) as channelGeo" [style]="'channelpage'"
     [channel]="channelGeo"></app-nodes-channels-map>
 
-  <div class="box">
+  <div class="box" *ngIf="!error">
 
       <div class="row">
         <div class="col-md">
@@ -65,7 +72,7 @@
 
   <br>
 
-  <div class="row row-cols-1 row-cols-md-2">
+  <div class="row row-cols-1 row-cols-md-2" *ngIf="!error">
     <div class="col">
       <app-channel-box [channel]="channel.node_left"></app-channel-box>
       <app-channel-close-box *ngIf="showCloseBoxes(channel)" [channel]="channel" [local]="channel.node_left" [remote]="channel.node_right"></app-channel-close-box>
@@ -104,14 +111,6 @@
 
 <br>
 
-<ng-template [ngIf]="error">
-  <div class="text-center">
-    <span i18n="error.general-loading-data">Error loading data.</span>
-    <br><br>
-    <i>{{ error.status }}: {{ error.error }}</i>
-  </div>
-</ng-template>
-
 <ng-template #skeletonLoader>
   <div class="container-xl">
     <h5 class="mb-0" style="color: #ffffff66" i18n="lightning.channel">Lightning channel</h5>

frontend/src/app/lightning/channel/channel.component.ts

@@ -38,7 +38,9 @@ export class ChannelComponent implements OnInit {
               }),
               catchError((err) => {
                 this.error = err;
-                return of(null);
+                return [{
+                  short_id: params.get('short_id')
+                }];
               })
             );
         }),

frontend/src/app/lightning/channel/closing-type/closing-type.component.ts

@@ -17,19 +17,19 @@ export class ClosingTypeComponent implements OnChanges {
   getLabelFromType(type: number): { label: string; class: string } {
     switch (type) {
       case 1: return { 
-        label: 'Mutually closed',
+        label: $localize`Mutually closed`,
         class: 'success',
       };
       case 2: return {
-        label: 'Force closed',
+        label: $localize`Force closed`,
         class: 'warning',
       };
       case 3: return {
-        label: 'Force closed with penalty',
+        label: $localize`Force closed with penalty`,
         class: 'danger',
       };
       default: return {
-        label: 'Unknown',
+        label: $localize`:@@e5d8bb389c702588877f039d72178f219453a72d:Unknown`,
         class: 'secondary',
       };
     }

frontend/src/app/lightning/channels-statistics/channels-statistics.component.html

@@ -1,9 +1,9 @@
 <div class="widget-toggler">
   <a href="" (click)="switchMode('avg')" class="toggler-option"
-    [ngClass]="{'inactive': mode === 'avg'}"><small>avg</small></a>
+    [ngClass]="{'inactive': mode === 'avg'}"><small i18n="statistics.average-small">avg</small></a>
   <span style="color: #ffffff66; font-size: 8px"> | </span>
   <a href="" (click)="switchMode('med')" class="toggler-option"
-    [ngClass]="{'inactive': mode === 'med'}"><small>med</small></a>
+    [ngClass]="{'inactive': mode === 'med'}"><small i18n="statistics.median-small">med</small></a>
 </div>
 
 <div class="fee-estimation-wrapper" *ngIf="statistics$ | async as statistics; else loadingReward">

frontend/src/app/lightning/node-fee-chart/node-fee-chart.component.ts

@@ -167,7 +167,7 @@ export class NodeFeeChartComponent implements OnInit {
         padding: 10,
         data: [
           {
-            name: 'Outgoing Fees',
+            name: $localize`Outgoing Fees`,
             inactiveColor: 'rgb(110, 112, 121)',
             textStyle: {
               color: 'white',
@@ -175,7 +175,7 @@ export class NodeFeeChartComponent implements OnInit {
             icon: 'roundRect',
           },
           {
-            name: 'Incoming Fees',
+            name: $localize`Incoming Fees`,
             inactiveColor: 'rgb(110, 112, 121)',
             textStyle: {
               color: 'white',
@@ -205,7 +205,7 @@ export class NodeFeeChartComponent implements OnInit {
       series: outgoingData.length === 0 ? undefined : [
         {
           zlevel: 0,
-          name: 'Outgoing Fees',
+          name: $localize`Outgoing Fees`,
           data: outgoingData.map(bucket => ({
             value: bucket.capacity,
             label: bucket.label,
@@ -219,7 +219,7 @@ export class NodeFeeChartComponent implements OnInit {
         },
         {
           zlevel: 0,
-          name: 'Incoming Fees',
+          name: $localize`Incoming Fees`,
           data: incomingData.map(bucket => ({
             value: -bucket.capacity,
             label: bucket.label,

frontend/src/app/lightning/node/node.component.html

@@ -1,20 +1,23 @@
 <div class="container-xl" *ngIf="(node$ | async) as node; else skeletonLoader">
-  <h5 class="mb-0" style="color: #ffffff66" i18n="lightning.node">Lightning node</h5>
-  <div class="title-container mb-2" *ngIf="!error">
-    <h1 class="mb-0 text-truncate">{{ node.alias }}</h1>
-    <span class="tx-link">
-      <span class="node-id">
-        <app-truncate [text]="node.public_key" [lastChars]="8" [link]="['/lightning/node' | relativeUrl, node.public_key]">
-          <app-clipboard [text]="node.public_key"></app-clipboard>
-        </app-truncate>
+
+  <ng-container *ngIf="!error">
+    <h5 class="mb-0" style="color: #ffffff66" i18n="lightning.node">Lightning node</h5>
+    <div class="title-container mb-2">
+      <h1 class="mb-0 text-truncate">{{ node.alias }}</h1>
+      <span class="tx-link">
+        <span class="node-id">
+          <app-truncate [text]="node.public_key" [lastChars]="8" [link]="['/lightning/node' | relativeUrl, node.public_key]">
+            <app-clipboard [text]="node.public_key"></app-clipboard>
+          </app-truncate>
+        </span>
       </span>
-    </span>
-  </div>
+    </div>
+  </ng-container>
 
   <div class="clearfix"></div>
 
   <div *ngIf="error" class="d-flex flex-column justify-content-around align-items-center mt-5 w-100" style="min-height: 100px">
-    <span i18n="lightning.node-not-found">No node found for public key "{{ node.public_key | shortenString : 12}}"</span>
+    <span class="text-center" i18n="lightning.node-not-found">No node found for public key "{{ node.public_key | shortenString : 12}}"</span>
   </div>
 
   <div class="box" *ngIf="!error">

frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts

@@ -1,7 +1,7 @@
 import { ChangeDetectionStrategy, Component, Input, Output, EventEmitter, NgZone, OnInit } from '@angular/core';
 import { SeoService } from '../../services/seo.service';
 import { ApiService } from '../../services/api.service';
-import { Observable, switchMap, tap, zip } from 'rxjs';
+import { delay, Observable, switchMap, tap, zip } from 'rxjs';
 import { AssetsService } from '../../services/assets.service';
 import { ActivatedRoute, ParamMap, Router } from '@angular/router';
 import { RelativeUrlPipe } from '../../shared/pipes/relative-url/relative-url.pipe';
@@ -75,6 +75,7 @@ export class NodesChannelsMap implements OnInit {
     
     this.channelsObservable = this.activatedRoute.paramMap
      .pipe(
+       delay(100),
        switchMap((params: ParamMap) => {
         this.isLoading = true;
         if (this.style === 'channelpage' && this.channel.length === 0 || !this.hasLocation) {

frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.ts

@@ -130,7 +130,7 @@ export class NodesNetworksChartComponent implements OnInit {
         },
         text: $localize`:@@b420668a91f8ebaf6e6409c4ba87f1d45961d2bd:Lightning Nodes Per Network`,
         left: 'center',
-        top: 11,
+        top: 0,
         zlevel: 10,
       };
     }
@@ -227,8 +227,8 @@ export class NodesNetworksChartComponent implements OnInit {
       title: title,
       animation: false,
       grid: {
-        height: this.widget ? 100 : undefined,
-        top: this.widget ? 10 : 40,
+        height: this.widget ? 90 : undefined,
+        top: this.widget ? 20 : 40,
         bottom: this.widget ? 0 : 70,
         right: (isMobile() && this.widget) ? 35 : this.right,
         left: (isMobile() && this.widget) ? 40 :this.left,

frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.ts

@@ -121,7 +121,7 @@ export class LightningStatisticsChartComponent implements OnInit {
         },
         text: $localize`:@@ea8db27e6db64f8b940711948c001a1100e5fe9f:Lightning Network Capacity`,
         left: 'center',
-        top: 11,
+        top: 0,
         zlevel: 10,
       };
     }
@@ -137,8 +137,8 @@ export class LightningStatisticsChartComponent implements OnInit {
         ]),
       ],
       grid: {
-        height: this.widget ? 100 : undefined,
-        top: this.widget ? 10 : 40,
+        height: this.widget ? 90 : undefined,
+        top: this.widget ? 20 : 40,
         bottom: this.widget ? 0 : 70,
         right: (isMobile() && this.widget) ? 35 : this.right,
         left: (isMobile() && this.widget) ? 40 :this.left,

frontend/src/app/services/api.service.ts

@@ -6,6 +6,7 @@ import { Observable } from 'rxjs';
 import { StateService } from './state.service';
 import { WebsocketResponse } from '../interfaces/websocket.interface';
 import { Outspend, Transaction } from '../interfaces/electrs.interface';
+import { Conversion } from './price.service';
 
 @Injectable({
   providedIn: 'root'
@@ -303,4 +304,11 @@ export class ApiService {
         (style     !== undefined ? `?style=${style}` : '')
     );
   }
+
+  getHistoricalPrice$(timestamp: number | undefined): Observable<Conversion> {
+    return this.httpClient.get<Conversion>(
+      this.apiBaseUrl + this.apiBasePath + '/api/v1/historical-price' +
+        (timestamp ? `?timestamp=${timestamp}` : '')
+    );
+  }
 }

frontend/src/app/services/cache.service.ts

@@ -62,7 +62,12 @@ export class CacheService {
       for (let i = 0; i < chunkSize; i++) {
         this.blockLoading[maxHeight - i] = true;
       }
-      const result = await firstValueFrom(this.apiService.getBlocks$(maxHeight));
+      let result;
+      try {
+        result = await firstValueFrom(this.apiService.getBlocks$(maxHeight));
+      } catch (e) {
+        console.log("failed to load blocks: ", e.message);
+      }
       for (let i = 0; i < chunkSize; i++) {
         delete this.blockLoading[maxHeight - i];
       }