Merge pull request #2057 from mempool/simon/transifex-pull-0710

Pull from transifex

.github/workflows/ci.yml

@@ -0,0 +1,94 @@
+name: CI Pipeline for the Backend and Frontend
+
+on:
+  pull_request:
+    types: [opened, review_requested, synchronize]
+
+jobs:
+  backend:
+    if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
+    strategy:
+      matrix:
+        node: ["16.16.0", "18.5.0"]
+        flavor: ["dev", "prod"]
+      fail-fast: false
+    runs-on: "ubuntu-latest"
+
+    name: Backend (${{ matrix.flavor }}) - node ${{ matrix.node }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          path: ${{ matrix.node }}/${{ matrix.flavor }}
+
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Install
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm ci
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+      - name: Install (Prod dependencies only)
+        if: ${{ matrix.flavor == 'prod'}}
+        run: npm ci --omit=dev --omit=optional
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+      - name: Lint
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm run lint
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+      #  - name: Test
+      #    run: npm run test
+
+      - name: Build
+        run: npm run build
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
+
+  frontend:
+    if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
+    strategy:
+      matrix:
+        node: ["16.15.0", "18.5.0"]
+        flavor: ["dev", "prod"]
+      fail-fast: false
+    runs-on: "ubuntu-latest"
+
+    name: Frontend (${{ matrix.flavor }}) - node ${{ matrix.node }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          path: ${{ matrix.node }}/${{ matrix.flavor }}
+
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node }}
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Install (Prod dependencies only)
+        run: npm ci --omit=dev --omit=optional
+        if: ${{ matrix.flavor == 'prod'}}
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend
+
+      - name: Install
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm ci
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend
+
+      - name: Lint
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm run lint
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend
+
+      # - name: Test
+      #   run: npm run test
+
+      - name: Build
+        run: npm run build
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend

.github/workflows/cypress.yml

@@ -1,8 +1,11 @@
 name: Cypress Tests
 
-on: [push, pull_request]
+on:
+  pull_request:
+    types: [ opened, review_requested, synchronize ]
 jobs:
   cypress:
+    if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')"
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false

.gitignore

@@ -2,3 +2,4 @@ sitemap
 data
 docker-compose.yml
 backend/mempool-config.json
+*.swp

.nvmrc

@@ -1 +1 @@
-v16.15.0
+v16.16.0

README.md

@@ -29,5 +29,5 @@ Mempool can be conveniently installed on the following full-node distros:
 Mempool can be installed in other ways too, but we only recommend doing so if you're a developer, have experience managing servers, or otherwise know what you're doing.
 
 - See the [`docker/`](./docker/) directory for instructions on deploying Mempool with Docker.
-- See the [`backend/`](./backend/) and [`frontend/`](./frontend/) directories for manual install instructions oriented for developers and small-scale deployments.
-- See the [`production/`](./production/) directory for guidance on setting up a more serious Mempool instance designed for high performance at scale.
+- See the [`backend/`](./backend/) and [`frontend/`](./frontend/) directories for manual install instructions oriented for developers.
+- See the [`production/`](./production/) directory for guidance on setting up a more serious Mempool instance designed for high performance at scale.

backend/.editorconfig

@@ -0,0 +1,17 @@
+# Editor configuration, see https://editorconfig.org
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 2
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.ts]
+quote_type = single
+
+[*.md]
+max_line_length = off
+trim_trailing_whitespace = false
+

backend/.eslintignore

@@ -0,0 +1,2 @@
+node_modules
+dist

backend/.eslintrc

@@ -0,0 +1,33 @@
+{
+  "root": true,
+  "parser": "@typescript-eslint/parser",
+  "plugins": [
+    "@typescript-eslint"
+  ],
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/eslint-recommended",
+    "plugin:@typescript-eslint/recommended",
+    "prettier"
+  ],
+  "rules": {
+    "@typescript-eslint/ban-ts-comment": 1,
+    "@typescript-eslint/ban-types": 1,
+    "@typescript-eslint/no-empty-function": 1,
+    "@typescript-eslint/no-explicit-any": 1,
+    "@typescript-eslint/no-inferrable-types": 1,
+    "@typescript-eslint/no-namespace": 1,
+    "@typescript-eslint/no-this-alias": 1,
+    "@typescript-eslint/no-var-requires": 1,
+    "no-console": 1,
+    "no-constant-condition": 1,
+    "no-dupe-else-if": 1,
+    "no-empty": 1,
+    "no-prototype-builtins": 1,
+    "no-self-assign": 1,
+    "no-useless-catch": 1,
+    "no-var": 1,
+    "prefer-const": 1,
+    "prefer-rest-params": 1
+  }
+}

backend/.prettierignore

@@ -0,0 +1,2 @@
+node_modules
+package-lock.json

backend/.prettierrc

@@ -0,0 +1,6 @@
+{
+  "endOfLine": "lf",
+  "printWidth": 80,
+  "tabWidth": 2,
+  "trailingComma": "es5"
+}

backend/README.md

@@ -1,6 +1,6 @@
 # Mempool Backend
 
-These instructions are mostly intended for developers, but can be used as a basis for personal or small-scale production setups. 
+These instructions are mostly intended for developers. 
 
 If you choose to use these instructions for a production setup, be aware that you will still probably need to do additional configuration for your specific OS, environment, use-case, etc. We do our best here to provide a good starting point, but only proceed if you know what you're doing. Mempool does not provide support for custom setups.
 
@@ -77,13 +77,13 @@ Query OK, 0 rows affected (0.00 sec)
 
 #### Build
 
-_Make sure to use Node.js 16.15 and npm 7._
+_Make sure to use Node.js 16.10 and npm 7._
 
 Install dependencies with `npm` and build the backend:
 
 ```
 cd backend
-npm install # add --prod for production
+npm install
 npm run build
 ```
 
@@ -159,3 +159,57 @@ nodemon src/index.ts --ignore cache/ --ignore pools.json
 ```
 
 `nodemon` should be in npm's global binary folder. If needed, you can determine where that is with `npm -g bin`.
+
+### Useful Regtest Commands
+
+Helpful link: https://gist.github.com/System-Glitch/cb4e87bf1ae3fec9925725bb3ebe223a
+
+Run bitcoind on regtest:
+   ```
+   bitcoind -regtest -rpcport=8332
+   ```
+
+Create a new wallet, if needed:
+   ```
+   bitcoin-cli -regtest -rpcport=8332 createwallet test
+   ```
+
+Load wallet (this command may take a while if you have lot of UTXOs):
+   ```
+   bitcoin-cli -regtest -rpcport=8332 loadwallet test
+   ```
+
+Get a new address:
+   ```
+   address=$(./src/bitcoin-cli -regtest -rpcport=8332 getnewaddress)
+   ```
+
+Mine blocks to the previously generated address. You need at least 101 blocks before you can spend. This will take some time to execute (~1 min):
+   ```
+   bitcoin-cli -regtest -rpcport=8332 generatetoaddress 101 $address
+   ```
+
+Send 0.1 BTC at 5 sat/vB to another address:
+   ```
+   ./src/bitcoin-cli -named -regtest -rpcport=8332 sendtoaddress address=$(./src/bitcoin-cli -regtest -rpcport=8332 getnewaddress) amount=0.1 fee_rate=5
+   ```
+
+See more example of `sendtoaddress`:
+   ```
+   ./src/bitcoin-cli sendtoaddress # will print the help
+   ```
+
+Mini script to generate transactions with random TX fee-rate (between 1 to 100 sat/vB). It's slow so don't expect to use this to test mempool spam, except if you let it run for a long time, or maybe with multiple regtest nodes connected to each other.
+   ```
+   #!/bin/bash
+   address=$(./src/bitcoin-cli -regtest -rpcport=8332 getnewaddress)
+   for i in {1..1000000}
+   do
+     ./src/bitcoin-cli -regtest -rpcport=8332 -named sendtoaddress address=$address amount=0.01 fee_rate=$(jot -r 1  1 100)
+   done
+   ```
+
+Generate block at regular interval (every 10 seconds in this example):
+   ```
+   watch -n 10 "./src/bitcoin-cli -regtest -rpcport=8332 generatetoaddress 1 $address"
+   ```

backend/mempool-config.sample.json

@@ -13,13 +13,15 @@
     "INITIAL_BLOCKS_AMOUNT": 8,
     "MEMPOOL_BLOCKS_AMOUNT": 8,
     "INDEXING_BLOCKS_AMOUNT": 11000,
+    "BLOCKS_SUMMARIES_INDEXING": false,
     "PRICE_FEED_UPDATE_INTERVAL": 600,
     "USE_SECOND_NODE_FOR_MINFEE": false,
     "EXTERNAL_ASSETS": [],
     "EXTERNAL_MAX_RETRY": 1,
     "EXTERNAL_RETRY_INTERVAL": 0,
     "USER_AGENT": "mempool",
-    "STDOUT_LOG_MIN_PRIORITY": "debug"
+    "STDOUT_LOG_MIN_PRIORITY": "debug",
+    "AUTOMATIC_BLOCK_REINDEXING": false
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",

backend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mempool-backend",
-  "version": "2.4.0",
+  "version": "2.4.1-dev",
   "description": "Bitcoin mempool visualizer and blockchain explorer backend",
   "license": "GNU Affero General Public License v3.0",
   "homepage": "https://mempool.space",
@@ -20,29 +20,37 @@
   ],
   "main": "index.ts",
   "scripts": {
-    "ng": "./node_modules/@angular/cli/bin/ng",
     "tsc": "./node_modules/typescript/bin/tsc",
     "build": "npm run tsc",
     "start": "node --max-old-space-size=2048 dist/index.js",
     "start-production": "node --max-old-space-size=4096 dist/index.js",
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "lint": "./node_modules/.bin/eslint . --ext .ts",
+    "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",
+    "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""
   },
   "dependencies": {
     "@mempool/electrum-client": "^1.1.7",
+    "@types/node": "^16.11.41",
     "axios": "~0.27.2",
     "bitcoinjs-lib": "6.0.1",
     "crypto-js": "^4.0.0",
     "express": "^4.18.0",
     "mysql2": "2.3.3",
     "node-worker-threads-pool": "^1.5.1",
-    "socks-proxy-agent": "^6.2.0",
-    "typescript": "~4.7.2",
-    "ws": "~8.7.0"
+    "socks-proxy-agent": "~7.0.0",
+    "typescript": "~4.7.4",
+    "ws": "~8.8.0"
   },
   "devDependencies": {
     "@types/compression": "^1.7.2",
-    "@types/ws": "~8.5.3",
+    "@types/crypto-js": "^4.1.1",
     "@types/express": "^4.17.13",
-    "tslint": "^6.1.0"
+    "@types/ws": "~8.5.3",
+    "@typescript-eslint/eslint-plugin": "^5.30.5",
+    "@typescript-eslint/parser": "^5.30.5",
+    "eslint": "^8.19.0",
+    "eslint-config-prettier": "^8.5.0",
+    "prettier": "^2.7.1"
   }
 }

backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts

@@ -4,6 +4,7 @@ export interface AbstractBitcoinApi {
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]>;
   $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, lazyPrevouts?: boolean): Promise<IEsploraApi.Transaction>;
   $getBlockHeightTip(): Promise<number>;
+  $getBlockHashTip(): Promise<string>;
   $getTxIdsForBlock(hash: string): Promise<string[]>;
   $getBlockHash(height: number): Promise<string>;
   $getBlockHeader(hash: string): Promise<string>;
@@ -13,6 +14,7 @@ export interface AbstractBitcoinApi {
   $getAddressPrefix(prefix: string): string[];
   $sendRawTransaction(rawTransaction: string): Promise<string>;
   $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]>;
+  $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]>;
 }
 export interface BitcoinRpcCredentials {
   host: string;

backend/src/api/bitcoin/bitcoin-api.interface.ts

@@ -73,6 +73,14 @@ export namespace IBitcoinApi {
     time: number;                    //  (numeric) Same as blocktime
   }
 
+  export interface VerboseBlock extends Block {
+    tx: VerboseTransaction[];        // The transactions in the format of the getrawtransaction RPC. Different from verbosity = 1 "tx" result
+  }
+
+  export interface VerboseTransaction extends Transaction {
+    fee?: number;                   //  (numeric) The transaction fee in BTC, omitted if block undo data is not available
+  }
+
   export interface Vin {
     txid?: string;                   //  (string) The transaction id
     vout?: number;                   //  (string)

backend/src/api/bitcoin/bitcoin-api.ts

@@ -64,6 +64,13 @@ class BitcoinApi implements AbstractBitcoinApi {
       });
   }
 
+  $getBlockHashTip(): Promise<string> {
+    return this.bitcoindClient.getChainTips()
+      .then((result: IBitcoinApi.ChainTips[]) => {
+        return result.find(tip => tip.status === 'active')!.hash;
+      });
+  }
+
   $getTxIdsForBlock(hash: string): Promise<string[]> {
     return this.bitcoindClient.getBlock(hash, 1)
       .then((rpcBlock: IBitcoinApi.Block) => rpcBlock.tx);
@@ -141,6 +148,15 @@ class BitcoinApi implements AbstractBitcoinApi {
     return outSpends;
   }
 
+  async $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]> {
+    const outspends: IEsploraApi.Outspend[][] = [];
+    for (const tx of txId) {
+      const outspend = await this.$getOutspends(tx);
+      outspends.push(outspend);
+    }
+    return outspends;
+  }
+
   $getEstimatedHashrate(blockHeight: number): Promise<number> {
     // 120 is the default block span in Core
     return this.bitcoindClient.getNetworkHashPs(120, blockHeight);

backend/src/api/bitcoin/electrum-api.ts

@@ -1,12 +1,11 @@
 import config from '../../config';
+import Client from '@mempool/electrum-client';
 import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
 import { IEsploraApi } from './esplora-api.interface';
 import { IElectrumApi } from './electrum-api.interface';
 import BitcoinApi from './bitcoin-api';
 import logger from '../../logger';
-import * as ElectrumClient from '@mempool/electrum-client';
-import * as sha256 from 'crypto-js/sha256';
-import * as hexEnc from 'crypto-js/enc-hex';
+import crypto from "crypto-js";
 import loadingIndicators from '../loading-indicators';
 import memoryCache from '../memory-cache';
 
@@ -26,7 +25,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
       onLog: (str) => { logger.debug(str); },
     };
 
-    this.electrumClient = new ElectrumClient(
+    this.electrumClient = new Client(
       config.ELECTRUM.PORT,
       config.ELECTRUM.HOST,
       config.ELECTRUM.TLS_ENABLED ? 'tls' : 'tcp',
@@ -35,7 +34,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
     );
 
     this.electrumClient.initElectrum(electrumConfig, electrumPersistencePolicy)
-      .then(() => {})
+      .then(() => { })
       .catch((err) => {
         logger.err(`Error connecting to Electrum Server at ${config.ELECTRUM.HOST}:${config.ELECTRUM.PORT}`);
       });
@@ -95,7 +94,7 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
   async $getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]> {
     const addressInfo = await this.bitcoindClient.validateAddress(address);
     if (!addressInfo || !addressInfo.isvalid) {
-     return [];
+      return [];
     }
 
     try {
@@ -144,8 +143,8 @@ class BitcoindElectrsApi extends BitcoinApi implements AbstractBitcoinApi {
   }
 
   private encodeScriptHash(scriptPubKey: string): string {
-    const addrScripthash = hexEnc.stringify(sha256(hexEnc.parse(scriptPubKey)));
-    return addrScripthash.match(/.{2}/g).reverse().join('');
+    const addrScripthash = crypto.enc.Hex.stringify(crypto.SHA256(crypto.enc.Hex.parse(scriptPubKey)));
+    return addrScripthash!.match(/.{2}/g)!.reverse().join('');
   }
 
 }

backend/src/api/bitcoin/esplora-api.ts

@@ -25,6 +25,11 @@ class ElectrsApi implements AbstractBitcoinApi {
       .then((response) => response.data);
   }
 
+  $getBlockHashTip(): Promise<string> {
+    return axios.get<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig)
+      .then((response) => response.data);
+  }
+
   $getTxIdsForBlock(hash: string): Promise<string[]> {
     return axios.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
       .then((response) => response.data);
@@ -61,8 +66,18 @@ class ElectrsApi implements AbstractBitcoinApi {
     throw new Error('Method not implemented.');
   }
 
-  $getOutspends(): Promise<IEsploraApi.Outspend[]> {
-    throw new Error('Method not implemented.');
+  $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
+    return axios.get<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig)
+      .then((response) => response.data);
+  }
+
+  async $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]> {
+    const outspends: IEsploraApi.Outspend[][] = [];
+    for (const tx of txId) {
+      const outspend = await this.$getOutspends(tx);
+      outspends.push(outspend);
+    }
+    return outspends;
   }
 }
 

backend/src/api/blocks.ts

@@ -2,11 +2,12 @@ import config from '../config';
 import bitcoinApi from './bitcoin/bitcoin-api-factory';
 import logger from '../logger';
 import memPool from './mempool';
-import { BlockExtended, PoolTag, TransactionExtended, TransactionMinerInfo } from '../mempool.interfaces';
+import { BlockExtended, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
 import { Common } from './common';
 import diskCache from './disk-cache';
 import transactionUtils from './transaction-utils';
 import bitcoinClient from './bitcoin/bitcoin-client';
+import { IBitcoinApi } from './bitcoin/bitcoin-api.interface';
 import { IEsploraApi } from './bitcoin/esplora-api.interface';
 import poolsRepository from '../repositories/PoolsRepository';
 import blocksRepository from '../repositories/BlocksRepository';
@@ -16,12 +17,15 @@ import { prepareBlock } from '../utils/blocks-utils';
 import BlocksRepository from '../repositories/BlocksRepository';
 import HashratesRepository from '../repositories/HashratesRepository';
 import indexer from '../indexer';
-import fiatConversion from './fiat-conversion';
-import RatesRepository from '../repositories/RatesRepository';
 import poolsParser from './pools-parser';
+import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
+import mining from './mining';
+import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
+import difficultyAdjustment from './difficulty-adjustment';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
+  private blockSummaries: BlockSummary[] = [];
   private currentBlockHeight = 0;
   private currentDifficulty = 0;
   private lastDifficultyAdjustmentTime = 0;
@@ -38,6 +42,14 @@ class Blocks {
     this.blocks = blocks;
   }
 
+  public getBlockSummaries(): BlockSummary[] {
+    return this.blockSummaries;
+  }
+
+  public setBlockSummaries(blockSummaries: BlockSummary[]) {
+    this.blockSummaries = blockSummaries;
+  }
+
   public setNewBlockCallback(fn: (block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void) {
     this.newBlockCallbacks.push(fn);
   }
@@ -106,6 +118,27 @@ class Blocks {
     return transactions;
   }
 
+  /**
+   * Return a block summary (list of stripped transactions)
+   * @param block
+   * @returns BlockSummary
+   */
+  private summarizeBlock(block: IBitcoinApi.VerboseBlock): BlockSummary {
+    const stripped = block.tx.map((tx) => {
+      return {
+        txid: tx.txid,
+        vsize: tx.vsize,
+        fee: tx.fee ? Math.round(tx.fee * 100000000) : 0,
+        value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0) * 100000000)
+      };
+    });
+
+    return {
+      id: block.hash,
+      transactions: stripped
+    };
+  }
+
   /**
    * Return a block with additional data (reward, coinbase, fees...)
    * @param block
@@ -135,7 +168,7 @@ class Blocks {
       blockExtended.extras.avgFeeRate = stats.avgfeerate;
     }
 
-    if (['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK)) {
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
       let pool: PoolTag;
       if (blockExtended.extras?.coinbaseTx !== undefined) {
         pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
@@ -212,15 +245,67 @@ class Blocks {
   }
 
   /**
-   * [INDEXING] Index all blocks metadata for the mining dashboard
+   * [INDEXING] Index all blocks summaries for the block txs visualization
    */
-  public async $generateBlockDatabase() {
-    const blockchainInfo = await bitcoinClient.getBlockchainInfo();
-    if (blockchainInfo.blocks !== blockchainInfo.headers) { // Wait for node to sync
+  public async $generateBlocksSummariesDatabase() {
+    if (Common.blocksSummariesIndexingEnabled() === false) {
       return;
     }
 
     try {
+      // Get all indexed block hash
+      const indexedBlocks = await blocksRepository.$getIndexedBlocks();
+      const indexedBlockSummariesHashesArray = await BlocksSummariesRepository.$getIndexedSummariesId();
+
+      const indexedBlockSummariesHashes = {}; // Use a map for faster seek during the indexing loop
+      for (const hash of indexedBlockSummariesHashesArray) {
+        indexedBlockSummariesHashes[hash] = true;
+      }
+
+      // Logging
+      let newlyIndexed = 0;
+      let totalIndexed = indexedBlockSummariesHashesArray.length;
+      let indexedThisRun = 0;
+      let timer = new Date().getTime() / 1000;
+      const startedAt = new Date().getTime() / 1000;
+
+      for (const block of indexedBlocks) {
+        if (indexedBlockSummariesHashes[block.hash] === true) {
+          continue;
+        }
+
+        // Logging
+        const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
+        if (elapsedSeconds > 5) {
+          const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
+          const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds);
+          const progress = Math.round(totalIndexed / indexedBlocks.length * 10000) / 100;
+          const timeLeft = Math.round((indexedBlocks.length - totalIndexed) / blockPerSeconds);
+          logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
+          timer = new Date().getTime() / 1000;
+          indexedThisRun = 0;
+        }
+
+        await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary
+
+        // Logging
+        indexedThisRun++;
+        totalIndexed++;
+        newlyIndexed++;
+      }
+      logger.notice(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`);
+    } catch (e) {
+      logger.err(`Blocks summaries indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
+      throw e;
+    }
+  }
+
+  /**
+   * [INDEXING] Index all blocks metadata for the mining dashboard
+   */
+  public async $generateBlockDatabase(): Promise<boolean> {
+    try {
+      const blockchainInfo = await bitcoinClient.getBlockchainInfo();
       let currentBlockHeight = blockchainInfo.blocks;
 
       let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, blockchainInfo.blocks);
@@ -261,7 +346,7 @@ class Blocks {
           const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
           if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) {
             const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
-            const blockPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
+            const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds);
             const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100;
             const timeLeft = Math.round((indexingBlockAmount - totalIndexed) / blockPerSeconds);
             logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`);
@@ -280,18 +365,15 @@ class Blocks {
 
         currentBlockHeight -= chunkSize;
       }
-      logger.info(`Indexed ${newlyIndexed} blocks`);
+      logger.notice(`Block indexing completed: indexed ${newlyIndexed} blocks`);
       loadingIndicators.setProgress('block-indexing', 100);
     } catch (e) {
-      logger.err('Block indexing failed. Trying again later. Reason: ' + (e instanceof Error ? e.message : e));
+      logger.err('Block indexing failed. Trying again in 10 seconds. Reason: ' + (e instanceof Error ? e.message : e));
       loadingIndicators.setProgress('block-indexing', 100);
-      return;
+      throw e;
     }
 
-    const chainValid = await BlocksRepository.$validateChain();
-    if (!chainValid) {
-      indexer.reindex();
-    }
+    return await BlocksRepository.$validateChain();
   }
 
   public async $updateBlocks() {
@@ -323,7 +405,7 @@ class Blocks {
 
         if (blockHeightTip >= 2016) {
           const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
-          const previousPeriodBlock = await bitcoinApi.$getBlock(previousPeriodBlockHash);
+          const previousPeriodBlock = await bitcoinClient.getBlock(previousPeriodBlockHash)
           this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
           logger.debug(`Initial difficulty adjustment data set.`);
         }
@@ -341,10 +423,12 @@ class Blocks {
       }
 
       const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
-      const block = BitcoinApi.convertBlock(await bitcoinClient.getBlock(blockHash));
+      const verboseBlock = await bitcoinClient.getBlock(blockHash, 2);
+      const block = BitcoinApi.convertBlock(verboseBlock);
       const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
       const transactions = await this.$getTransactionsExtended(blockHash, block.height, false);
       const blockExtended: BlockExtended = await this.$getBlockExtended(block, transactions);
+      const blockSummary: BlockSummary = this.summarizeBlock(verboseBlock);
 
       if (Common.indexingEnabled()) {
         if (!fastForwarded) {
@@ -354,18 +438,35 @@ class Blocks {
             // We assume there won't be a reorg with more than 10 block depth
             await BlocksRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
             await HashratesRepository.$deleteLastEntries();
+            await BlocksSummariesRepository.$deleteBlocksFrom(lastBlock['height'] - 10);
             for (let i = 10; i >= 0; --i) {
-              await this.$indexBlock(lastBlock['height'] - i);
+              const newBlock = await this.$indexBlock(lastBlock['height'] - i);
+              await this.$getStrippedBlockTransactions(newBlock.id, true, true);
             }
+            await mining.$indexDifficultyAdjustments();
+            await DifficultyAdjustmentsRepository.$deleteLastAdjustment();
+            logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`);
+            indexer.reindex();
           }
           await blocksRepository.$saveBlockInDatabase(blockExtended);
+
+          // Save blocks summary for visualization if it's enabled
+          if (Common.blocksSummariesIndexingEnabled() === true) {
+            await this.$getStrippedBlockTransactions(blockExtended.id, true);
+          }
         }
       }
-      if (fiatConversion.ratesInitialized === true && config.DATABASE.ENABLED === true) {
-        await RatesRepository.$saveRate(blockExtended.height, fiatConversion.getConversionRates());
-      }
 
       if (block.height % 2016 === 0) {
+        if (Common.indexingEnabled()) {
+          await DifficultyAdjustmentsRepository.$saveAdjustments({
+            time: block.timestamp,
+            height: block.height,
+            difficulty: block.difficulty,
+            adjustment: Math.round((block.difficulty / this.currentDifficulty) * 1000000) / 1000000, // Remove float point noise
+          });
+        }
+
         this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100;
         this.lastDifficultyAdjustmentTime = block.timestamp;
         this.currentDifficulty = block.difficulty;
@@ -375,6 +476,10 @@ class Blocks {
       if (this.blocks.length > config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4) {
         this.blocks = this.blocks.slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4);
       }
+      this.blockSummaries.push(blockSummary);
+      if (this.blockSummaries.length > config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4) {
+        this.blockSummaries = this.blockSummaries.slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4);
+      }
 
       if (this.newBlockCallbacks.length) {
         this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
@@ -422,13 +527,15 @@ class Blocks {
       }
     }
 
-    const block = await bitcoinApi.$getBlock(hash);
+    let block = await bitcoinClient.getBlock(hash);
 
     // Not Bitcoin network, return the block as it
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
       return block;
     }
 
+    block = prepareBlock(block);
+
     // Bitcoin network, add our custom data on top
     const transactions = await this.$getTransactionsExtended(hash, block.height, true);
     const blockExtended = await this.$getBlockExtended(block, transactions);
@@ -440,48 +547,71 @@ class Blocks {
     return blockExtended;
   }
 
-  public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
-    try {
-      let currentHeight = fromHeight !== undefined ? fromHeight : this.getCurrentBlockHeight();
-      const returnBlocks: BlockExtended[] = [];
-
-      if (currentHeight < 0) {
-        return returnBlocks;
+  public async $getStrippedBlockTransactions(hash: string, skipMemoryCache: boolean = false,
+    skipDBLookup: boolean = false): Promise<TransactionStripped[]>
+  {
+    if (skipMemoryCache === false) {
+      // Check the memory cache
+      const cachedSummary = this.getBlockSummaries().find((b) => b.id === hash);
+      if (cachedSummary) {
+        return cachedSummary.transactions;
       }
-
-      if (currentHeight === 0 && Common.indexingEnabled()) {
-        currentHeight = await blocksRepository.$mostRecentBlockHeight();
-      }
-
-      // Check if block height exist in local cache to skip the hash lookup
-      const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight);
-      let startFromHash: string | null = null;
-      if (blockByHeight) {
-        startFromHash = blockByHeight.id;
-      } else if (!Common.indexingEnabled()) {
-        startFromHash = await bitcoinApi.$getBlockHash(currentHeight);
-      }
-
-      let nextHash = startFromHash;
-      for (let i = 0; i < limit && currentHeight >= 0; i++) {
-        let block = this.getBlocks().find((b) => b.height === currentHeight);
-        if (block) {
-          returnBlocks.push(block);
-        } else if (Common.indexingEnabled()) {
-          block = await this.$indexBlock(currentHeight);
-          returnBlocks.push(block);
-        } else if (nextHash != null) {
-          block = prepareBlock(await bitcoinApi.$getBlock(nextHash));
-          nextHash = block.previousblockhash;
-          returnBlocks.push(block);
-        }
-        currentHeight--;
-      }
-
-      return returnBlocks;
-    } catch (e) {
-      throw e;
     }
+
+    // Check if it's indexed in db
+    if (skipDBLookup === false && Common.blocksSummariesIndexingEnabled() === true) {
+      const indexedSummary = await BlocksSummariesRepository.$getByBlockId(hash);
+      if (indexedSummary !== undefined) {
+        return indexedSummary.transactions;
+      }
+    }
+
+    // Call Core RPC
+    const block = await bitcoinClient.getBlock(hash, 2);
+    const summary = this.summarizeBlock(block);
+
+    // Index the response if needed
+    if (Common.blocksSummariesIndexingEnabled() === true) {
+      await BlocksSummariesRepository.$saveSummary(block.height, summary);
+    }
+
+    return summary.transactions;
+  }
+
+  public async $getBlocks(fromHeight?: number, limit: number = 15): Promise<BlockExtended[]> {
+    let currentHeight = fromHeight !== undefined ? fromHeight : await blocksRepository.$mostRecentBlockHeight();
+    const returnBlocks: BlockExtended[] = [];
+
+    if (currentHeight < 0) {
+      return returnBlocks;
+    }
+
+    // Check if block height exist in local cache to skip the hash lookup
+    const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight);
+    let startFromHash: string | null = null;
+    if (blockByHeight) {
+      startFromHash = blockByHeight.id;
+    } else if (!Common.indexingEnabled()) {
+      startFromHash = await bitcoinApi.$getBlockHash(currentHeight);
+    }
+
+    let nextHash = startFromHash;
+    for (let i = 0; i < limit && currentHeight >= 0; i++) {
+      let block = this.getBlocks().find((b) => b.height === currentHeight);
+      if (block) {
+        returnBlocks.push(block);
+      } else if (Common.indexingEnabled()) {
+        block = await this.$indexBlock(currentHeight);
+        returnBlocks.push(block);
+      } else if (nextHash != null) {
+        block = prepareBlock(await bitcoinClient.getBlock(nextHash));
+        nextHash = block.previousblockhash;
+        returnBlocks.push(block);
+      }
+      currentHeight--;
+    }
+
+    return returnBlocks;
   }
 
   public getLastDifficultyAdjustmentTime(): number {

backend/src/api/common.ts

@@ -114,7 +114,7 @@ export class Common {
       totalFees += tx.bestDescendant.fee;
     }
 
-    tx.effectiveFeePerVsize = Math.max(Common.isLiquid() ? 0.1 : 1, totalFees / (totalWeight / 4));
+    tx.effectiveFeePerVsize = Math.max(0, totalFees / (totalWeight / 4));
     tx.cpfpChecked = true;
 
     return {
@@ -177,4 +177,11 @@ export class Common {
       config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== 0
     );
   }
+
+  static blocksSummariesIndexingEnabled(): boolean {
+    return (
+      Common.indexingEnabled() &&
+      config.MEMPOOL.BLOCKS_SUMMARIES_INDEXING === true
+    );
+  }
 }

backend/src/api/database-migration.ts

@@ -4,11 +4,27 @@ import logger from '../logger';
 import { Common } from './common';
 
 class DatabaseMigration {
-  private static currentVersion = 19;
+  private static currentVersion = 24;
   private queryTimeout = 120000;
   private statisticsAddedIndexed = false;
+  private uniqueLogs: string[] = [];
+
+  private blocksTruncatedMessage = `'blocks' table has been truncated. Re-indexing from scratch.`;
+  private hashratesTruncatedMessage = `'hashrates' table has been truncated. Re-indexing from scratch.`;
 
   constructor() { }
+
+  /**
+   * Avoid printing multiple time the same message
+   */
+  private uniqueLog(loggerFunction: any, msg: string) {
+    if (this.uniqueLogs.includes(msg)) {
+      return;
+    }
+    this.uniqueLogs.push(msg);
+    loggerFunction(msg);
+  }
+
   /**
    * Entry point
    */
@@ -39,6 +55,16 @@ class DatabaseMigration {
       process.exit(-1);
     }
 
+    if (databaseSchemaVersion === 0) {
+      logger.info('Initializing database (first run, clean install)');
+    }
+
+    if (databaseSchemaVersion <= 2) {
+      // Disable some spam logs when they're not relevant
+      this.uniqueLogs.push(this.blocksTruncatedMessage);
+      this.uniqueLogs.push(this.hashratesTruncatedMessage);
+    }
+
     logger.debug('MIGRATIONS: Current state.schema_version ' + databaseSchemaVersion);
     logger.debug('MIGRATIONS: Latest DatabaseMigration.version is ' + DatabaseMigration.currentVersion);
     if (databaseSchemaVersion >= DatabaseMigration.currentVersion) {
@@ -56,10 +82,13 @@ class DatabaseMigration {
     }
 
     if (DatabaseMigration.currentVersion > databaseSchemaVersion) {
-      logger.notice('MIGRATIONS: Upgrading database schema');
       try {
         await this.$migrateTableSchemaFromVersion(databaseSchemaVersion);
-        logger.notice(`MIGRATIONS: OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`);
+        if (databaseSchemaVersion === 0) {
+          logger.notice(`MIGRATIONS: OK. Database schema has been properly initialized to version ${DatabaseMigration.currentVersion} (latest version)`);          
+        } else {
+          logger.notice(`MIGRATIONS: OK. Database schema have been migrated from version ${databaseSchemaVersion} to ${DatabaseMigration.currentVersion} (latest version)`);
+        }
       } catch (e) {
         logger.err('MIGRATIONS: Unable to migrate database, aborting. ' + e);
       }
@@ -89,13 +118,13 @@ class DatabaseMigration {
         await this.$executeQuery(this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
       }
       if (databaseSchemaVersion < 5 && isBitcoin === true) {
-        logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
+        this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
         await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
         await this.$executeQuery('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
       }
 
       if (databaseSchemaVersion < 6 && isBitcoin === true) {
-        logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
+        this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
         await this.$executeQuery('TRUNCATE blocks;');  // Need to re-index
         // Cleanup original blocks fields type
         await this.$executeQuery('ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"');
@@ -122,7 +151,7 @@ class DatabaseMigration {
       }
 
       if (databaseSchemaVersion < 8 && isBitcoin === true) {
-        logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
+        this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
         await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
         await this.$executeQuery('ALTER TABLE `hashrates` DROP INDEX `PRIMARY`');
         await this.$executeQuery('ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
@@ -131,7 +160,7 @@ class DatabaseMigration {
       }
 
       if (databaseSchemaVersion < 9 && isBitcoin === true) {
-        logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
+        this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
         await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
         await this.$executeQuery('ALTER TABLE `state` CHANGE `name` `name` varchar(100)');
         await this.$executeQuery('ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)');
@@ -142,7 +171,7 @@ class DatabaseMigration {
       }
 
       if (databaseSchemaVersion < 11 && isBitcoin === true) {
-        logger.warn(`'blocks' table has been truncated. Re-indexing from scratch.`);
+        this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
         await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
         await this.$executeQuery(`ALTER TABLE blocks
           ADD avg_fee INT UNSIGNED NULL,
@@ -166,14 +195,14 @@ class DatabaseMigration {
       }
 
       if (databaseSchemaVersion < 14 && isBitcoin === true) {
-        logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
+        this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
         await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
         await this.$executeQuery('ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`');
         await this.$executeQuery('ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"');
       }
 
       if (databaseSchemaVersion < 16 && isBitcoin === true) {
-        logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.`);
+        this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
         await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index because we changed timestamps
       }
 
@@ -188,6 +217,37 @@ class DatabaseMigration {
       if (databaseSchemaVersion < 19) {
         await this.$executeQuery(this.getCreateRatesTableQuery(), await this.$checkIfTableExists('rates'));
       }
+
+      if (databaseSchemaVersion < 20 && isBitcoin === true) {
+        await this.$executeQuery(this.getCreateBlocksSummariesTableQuery(), await this.$checkIfTableExists('blocks_summaries'));
+      }
+
+      if (databaseSchemaVersion < 21) {
+        await this.$executeQuery('DROP TABLE IF EXISTS `rates`');
+        await this.$executeQuery(this.getCreatePricesTableQuery(), await this.$checkIfTableExists('prices'));
+      }
+
+      if (databaseSchemaVersion < 22 && isBitcoin === true) {
+        await this.$executeQuery('DROP TABLE IF EXISTS `difficulty_adjustments`');
+        await this.$executeQuery(this.getCreateDifficultyAdjustmentsTableQuery(), await this.$checkIfTableExists('difficulty_adjustments'));
+      }
+
+      if (databaseSchemaVersion < 23) {
+        await this.$executeQuery('TRUNCATE `prices`');
+        await this.$executeQuery('ALTER TABLE `prices` DROP `avg_prices`');
+        await this.$executeQuery('ALTER TABLE `prices` ADD `USD` float DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `prices` ADD `EUR` float DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `prices` ADD `GBP` float DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `prices` ADD `CAD` float DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `prices` ADD `CHF` float DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `prices` ADD `AUD` float DEFAULT "0"');
+        await this.$executeQuery('ALTER TABLE `prices` ADD `JPY` float DEFAULT "0"');
+      }
+
+      if (databaseSchemaVersion < 24 && isBitcoin == true) {
+        await this.$executeQuery('DROP TABLE IF EXISTS `blocks_audits`');
+        await this.$executeQuery(this.getCreateBlocksAuditsTableQuery(), await this.$checkIfTableExists('blocks_audits'));
+      }
     } catch (e) {
       throw e;
     }
@@ -282,6 +342,8 @@ class DatabaseMigration {
     for (const query of this.getMigrationQueriesFromVersion(version)) {
       transactionQueries.push(query);
     }
+
+    logger.notice(`MIGRATIONS: ${version > 0 ? 'Upgrading' : 'Initializing'} database schema version number to ${DatabaseMigration.currentVersion}`);
     transactionQueries.push(this.getUpdateToLatestSchemaVersionQuery());
 
     try {
@@ -305,6 +367,9 @@ class DatabaseMigration {
 
     if (version < 1) {
       if (config.MEMPOOL.NETWORK !== 'liquid' && config.MEMPOOL.NETWORK !== 'liquidtestnet') {
+        if (version > 0) {
+          logger.notice(`MIGRATIONS: Migrating (shifting) statistics table data`);
+        }
         queries.push(this.getShiftStatisticsQuery());
       }
     }
@@ -470,7 +535,7 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
-  private getCreateRatesTableQuery(): string {
+  private getCreateRatesTableQuery(): string { // This table has been replaced by the prices table
     return `CREATE TABLE IF NOT EXISTS rates (
       height int(10) unsigned NOT NULL,
       bisq_rates JSON NOT NULL,
@@ -478,8 +543,50 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getCreateBlocksSummariesTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS blocks_summaries (
+      height int(10) unsigned NOT NULL,
+      id varchar(65) NOT NULL,
+      transactions JSON NOT NULL,
+      PRIMARY KEY (id),
+      INDEX (height)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreatePricesTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS prices (
+      time timestamp NOT NULL,
+      avg_prices JSON NOT NULL,
+      PRIMARY KEY (time)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateDifficultyAdjustmentsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS difficulty_adjustments (
+      time timestamp NOT NULL,
+      height int(10) unsigned NOT NULL,
+      difficulty double unsigned NOT NULL,
+      adjustment float NOT NULL,
+      PRIMARY KEY (height),
+      INDEX (time)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateBlocksAuditsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS blocks_audits (
+      time timestamp NOT NULL,
+      hash varchar(65) NOT NULL,
+      height int(10) unsigned NOT NULL,
+      missing_txs JSON NOT NULL,
+      added_txs JSON NOT NULL,
+      match_rate float unsigned NOT NULL,
+      PRIMARY KEY (hash),
+      INDEX (height)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
   public async $truncateIndexedData(tables: string[]) {
-    const allowedTables = ['blocks', 'hashrates'];
+    const allowedTables = ['blocks', 'hashrates', 'prices'];
 
     try {
       for (const table of tables) {

backend/src/api/disk-cache.ts

@@ -1,6 +1,6 @@
 import * as fs from 'fs';
 const fsPromises = fs.promises;
-import * as cluster from 'cluster';
+import cluster from 'cluster';
 import memPool from './mempool';
 import blocks from './blocks';
 import logger from '../logger';
@@ -19,7 +19,7 @@ class DiskCache {
   constructor() { }
 
   async $saveCacheToDisk(): Promise<void> {
-    if (!cluster.isMaster) {
+    if (!cluster.isPrimary) {
       return;
     }
     if (this.isWritingCache) {
@@ -43,14 +43,15 @@ class DiskCache {
       await fsPromises.writeFile(DiskCache.FILE_NAME, JSON.stringify({
         cacheSchemaVersion: this.cacheSchemaVersion,
         blocks: blocks.getBlocks(),
+        blockSummaries: blocks.getBlockSummaries(),
         mempool: {},
         mempoolArray: mempoolArray.splice(0, chunkSize),
-      }), {flag: 'w'});
+      }), { flag: 'w' });
       for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
         await fsPromises.writeFile(DiskCache.FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
           mempool: {},
           mempoolArray: mempoolArray.splice(0, chunkSize),
-        }), {flag: 'w'});
+        }), { flag: 'w' });
       }
       logger.debug('Mempool and blocks data saved to disk cache');
       this.isWritingCache = false;
@@ -66,7 +67,7 @@ class DiskCache {
       fs.unlinkSync(DiskCache.FILE_NAMES.replace('{number}', i.toString()));
     }
   }
-  
+
   loadMempoolCache() {
     if (!fs.existsSync(DiskCache.FILE_NAME)) {
       return;
@@ -109,6 +110,7 @@ class DiskCache {
 
       memPool.setMempool(data.mempool);
       blocks.setBlocks(data.blocks);
+      blocks.setBlockSummaries(data.blockSummaries || []);
     } catch (e) {
       logger.warn('Failed to parse mempoool and blocks cache. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
     }

backend/src/api/mining.ts

@@ -1,4 +1,4 @@
-import { PoolInfo, PoolStats, RewardStats } from '../mempool.interfaces';
+import { IndexedDifficultyAdjustment, PoolInfo, PoolStats, RewardStats } from '../mempool.interfaces';
 import BlocksRepository from '../repositories/BlocksRepository';
 import PoolsRepository from '../repositories/PoolsRepository';
 import HashratesRepository from '../repositories/HashratesRepository';
@@ -7,11 +7,25 @@ import logger from '../logger';
 import { Common } from './common';
 import loadingIndicators from './loading-indicators';
 import { escape } from 'mysql2';
+import indexer from '../indexer';
+import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
+import config from '../config';
+import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
 
 class Mining {
   constructor() {
   }
 
+  /**
+   * Get historical block predictions match rate
+   */
+   public async $getBlockPredictionsHistory(interval: string | null = null): Promise<any> {
+    return await BlocksAuditsRepository.$getBlockPredictionsHistory(
+      this.getTimeRange(interval),
+      Common.getSqlInterval(interval)
+    );
+  }
+
   /**
    * Get historical block total fee
    */
@@ -159,26 +173,25 @@ class Mining {
    */
   public async $generatePoolHashrateHistory(): Promise<void> {
     const now = new Date();
+    const lastestRunDate = await HashratesRepository.$getLatestRun('last_weekly_hashrates_indexing');
 
-    try {
-      const lastestRunDate = await HashratesRepository.$getLatestRun('last_weekly_hashrates_indexing');
-
-      // Run only if:
-      // * lastestRunDate is set to 0 (node backend restart, reorg)
-      // * we started a new week (around Monday midnight)
-      const runIndexing = lastestRunDate === 0 || now.getUTCDay() === 1 && lastestRunDate !== now.getUTCDate();
-      if (!runIndexing) {
-        return;
-      }
-    } catch (e) {
-      throw e;
+    // Run only if:
+    // * lastestRunDate is set to 0 (node backend restart, reorg)
+    // * we started a new week (around Monday midnight)
+    const runIndexing = lastestRunDate === 0 || now.getUTCDay() === 1 && lastestRunDate !== now.getUTCDate();
+    if (!runIndexing) {
+      return;
     }
 
     try {
+      const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
+
+      const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
+      const genesisTimestamp = genesisBlock.time * 1000;
+
       const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
       const hashrates: any[] = [];
-      const genesisTimestamp = 1231006505000; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
-
+ 
       const lastMonday = new Date(now.setDate(now.getDate() - (now.getDay() + 6) % 7));
       const lastMondayMidnight = this.getDateMidnight(lastMonday);
       let toTimestamp = lastMondayMidnight.getTime();
@@ -193,7 +206,7 @@ class Mining {
       logger.debug(`Indexing weekly mining pool hashrate`);
       loadingIndicators.setProgress('weekly-hashrate-indexing', 0);
 
-      while (toTimestamp > genesisTimestamp) {
+      while (toTimestamp > genesisTimestamp && toTimestamp > oldestConsecutiveBlockTimestamp) {
         const fromTimestamp = toTimestamp - 604800000;
 
         // Skip already indexed weeks
@@ -203,14 +216,6 @@ class Mining {
           continue;
         }
 
-        // Check if we have blocks for the previous week (which mean that the week
-        // we are currently indexing has complete data)
-        const blockStatsPreviousWeek: any = await BlocksRepository.$blockCountBetweenTimestamp(
-          null, (fromTimestamp - 604800000) / 1000, (toTimestamp - 604800000) / 1000);
-        if (blockStatsPreviousWeek.blockCount === 0) { // We are done indexing
-          break;
-        }
-
         const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
           null, fromTimestamp / 1000, toTimestamp / 1000);
         const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
@@ -218,25 +223,27 @@ class Mining {
 
         let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp / 1000, toTimestamp / 1000);
         const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0);
-        pools = pools.map((pool: any) => {
-          pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate;
-          pool.share = (pool.blockCount / totalBlocks);
-          return pool;
-        });
-
-        for (const pool of pools) {
-          hashrates.push({
-            hashrateTimestamp: toTimestamp / 1000,
-            avgHashrate: pool['hashrate'],
-            poolId: pool.poolId,
-            share: pool['share'],
-            type: 'weekly',
+        if (totalBlocks > 0) {
+          pools = pools.map((pool: any) => {
+            pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate;
+            pool.share = (pool.blockCount / totalBlocks);
+            return pool;
           });
-        }
 
-        newlyIndexed += hashrates.length;
-        await HashratesRepository.$saveHashrates(hashrates);
-        hashrates.length = 0;
+          for (const pool of pools) {
+            hashrates.push({
+              hashrateTimestamp: toTimestamp / 1000,
+              avgHashrate: pool['hashrate'] ,
+              poolId: pool.poolId,
+              share: pool['share'],
+              type: 'weekly',
+            });
+          }
+
+          newlyIndexed += hashrates.length;
+          await HashratesRepository.$saveHashrates(hashrates);
+          hashrates.length = 0;
+        }
 
         const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
         if (elapsedSeconds > 1) {
@@ -257,11 +264,12 @@ class Mining {
       }
       await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', new Date().getUTCDate());
       if (newlyIndexed > 0) {
-        logger.info(`Indexed ${newlyIndexed} pools weekly hashrate`);
+        logger.notice(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`);
       }
       loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
     } catch (e) {
       loadingIndicators.setProgress('weekly-hashrate-indexing', 100);
+      logger.err(`Weekly mining pools hashrates indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
       throw e;
     }
   }
@@ -270,20 +278,19 @@ class Mining {
    * [INDEXING] Generate daily hashrate data
    */
   public async $generateNetworkHashrateHistory(): Promise<void> {
-    try {
-      // We only run this once a day around midnight
-      const latestRunDate = await HashratesRepository.$getLatestRun('last_hashrates_indexing');
-      const now = new Date().getUTCDate();
-      if (now === latestRunDate) {
-        return;
-      }
-    } catch (e) {
-      throw e;
+    // We only run this once a day around midnight
+    const latestRunDate = await HashratesRepository.$getLatestRun('last_hashrates_indexing');
+    const now = new Date().getUTCDate();
+    if (now === latestRunDate) {
+      return;
     }
 
+    const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
+
     try {
-      const indexedTimestamp = (await HashratesRepository.$getNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
-      const genesisTimestamp = 1231006505000; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
+      const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
+      const genesisTimestamp = genesisBlock.time * 1000;
+      const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
       const lastMidnight = this.getDateMidnight(new Date());
       let toTimestamp = Math.round(lastMidnight.getTime());
       const hashrates: any[] = [];
@@ -298,27 +305,19 @@ class Mining {
       logger.debug(`Indexing daily network hashrate`);
       loadingIndicators.setProgress('daily-hashrate-indexing', 0);
 
-      while (toTimestamp > genesisTimestamp) {
+      while (toTimestamp > genesisTimestamp && toTimestamp > oldestConsecutiveBlockTimestamp) {
         const fromTimestamp = toTimestamp - 86400000;
 
-        // Skip already indexed weeks
+        // Skip already indexed days
         if (indexedTimestamp.includes(toTimestamp / 1000)) {
           toTimestamp -= 86400000;
           ++totalIndexed;
           continue;
         }
 
-        // Check if we have blocks for the previous day (which mean that the day
-        // we are currently indexing has complete data)
-        const blockStatsPreviousDay: any = await BlocksRepository.$blockCountBetweenTimestamp(
-          null, (fromTimestamp - 86400000) / 1000, (toTimestamp - 86400000) / 1000);
-        if (blockStatsPreviousDay.blockCount === 0) { // We are done indexing
-          break;
-        }
-
         const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
           null, fromTimestamp / 1000, toTimestamp / 1000);
-        const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
+        const lastBlockHashrate = blockStats.blockCount === 0 ? 0 : await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
           blockStats.lastBlockHeight);
 
         hashrates.push({
@@ -354,11 +353,12 @@ class Mining {
       }
 
       // Add genesis block manually
-      if (toTimestamp <= genesisTimestamp && !indexedTimestamp.includes(genesisTimestamp)) {
+      if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && !indexedTimestamp.includes(genesisTimestamp / 1000)) {
         hashrates.push({
-          hashrateTimestamp: genesisTimestamp,
+          hashrateTimestamp: genesisTimestamp / 1000,
           avgHashrate: await bitcoinClient.getNetworkHashPs(1, 1),
-          poolId: null,
+          poolId: 0,
+          share: 1,
           type: 'daily',
         });
       }
@@ -368,15 +368,87 @@ class Mining {
 
       await HashratesRepository.$setLatestRun('last_hashrates_indexing', new Date().getUTCDate());
       if (newlyIndexed > 0) {
-        logger.info(`Indexed ${newlyIndexed} day of network hashrate`);
+        logger.notice(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`);
       }
       loadingIndicators.setProgress('daily-hashrate-indexing', 100);
     } catch (e) {
       loadingIndicators.setProgress('daily-hashrate-indexing', 100);
+      logger.err(`Daily network hashrate indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`);
       throw e;
     }
   }
 
+  /**
+   * Index difficulty adjustments
+   */
+  public async $indexDifficultyAdjustments(): Promise<void> {
+    const indexedHeightsArray = await DifficultyAdjustmentsRepository.$getAdjustmentsHeights();
+    const indexedHeights = {};
+    for (const height of indexedHeightsArray) {
+      indexedHeights[height] = true;
+    }
+
+    const blocks: any = await BlocksRepository.$getBlocksDifficulty();
+    const genesisBlock = await bitcoinClient.getBlock(await bitcoinClient.getBlockHash(0));
+    let currentDifficulty = genesisBlock.difficulty;
+    let totalIndexed = 0;
+
+    if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 && indexedHeights[0] !== true) {
+      await DifficultyAdjustmentsRepository.$saveAdjustments({
+        time: genesisBlock.time,
+        height: 0,
+        difficulty: currentDifficulty,
+        adjustment: 0.0,
+      });
+    }
+
+    const oldestConsecutiveBlock = await BlocksRepository.$getOldestConsecutiveBlock();
+    if (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== -1) {
+      currentDifficulty = oldestConsecutiveBlock.difficulty;
+    }
+
+    let totalBlockChecked = 0;
+    let timer = new Date().getTime() / 1000;
+
+    for (const block of blocks) {
+      if (block.difficulty !== currentDifficulty) {
+        if (indexedHeights[block.height] === true) { // Already indexed
+          if (block.height >= oldestConsecutiveBlock.height) {
+            currentDifficulty = block.difficulty;
+          }
+          continue;          
+        }
+
+        let adjustment = block.difficulty / currentDifficulty;
+        adjustment = Math.round(adjustment * 1000000) / 1000000; // Remove float point noise
+
+        await DifficultyAdjustmentsRepository.$saveAdjustments({
+          time: block.time,
+          height: block.height,
+          difficulty: block.difficulty,
+          adjustment: adjustment,
+        });
+
+        totalIndexed++;
+        if (block.height >= oldestConsecutiveBlock.height) {
+          currentDifficulty = block.difficulty;
+        }
+    }
+
+      totalBlockChecked++;
+      const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer));
+      if (elapsedSeconds > 5) {
+        const progress = Math.round(totalBlockChecked / blocks.length * 100);
+        logger.info(`Indexing difficulty adjustment at block #${block.height} | Progress: ${progress}%`);
+        timer = new Date().getTime() / 1000;
+      }
+    }
+
+    if (totalIndexed > 0) {
+      logger.notice(`Indexed ${totalIndexed} difficulty adjustments`);
+    }
+  }
+
   private getDateMidnight(date: Date): Date {
     date.setUTCHours(0);
     date.setUTCMinutes(0);

backend/src/api/pools-parser.ts

@@ -1,6 +1,7 @@
 import DB from '../database';
 import logger from '../logger';
 import config from '../config';
+import BlocksRepository from '../repositories/BlocksRepository';
 
 interface Pool {
   name: string;
@@ -32,7 +33,6 @@ class PoolsParser {
     // First we save every entries without paying attention to pool duplication
     const poolsDuplicated: Pool[] = [];
 
-    logger.debug('Parse coinbase_tags');
     const coinbaseTags = Object.entries(poolsJson['coinbase_tags']);
     for (let i = 0; i < coinbaseTags.length; ++i) {
       poolsDuplicated.push({
@@ -43,7 +43,6 @@ class PoolsParser {
         'slug': ''
       });
     }
-    logger.debug('Parse payout_addresses');
     const addressesTags = Object.entries(poolsJson['payout_addresses']);
     for (let i = 0; i < addressesTags.length; ++i) {
       poolsDuplicated.push({
@@ -56,7 +55,6 @@ class PoolsParser {
     }
 
     // Then, we find unique mining pool names
-    logger.debug('Identify unique mining pools');
     const poolNames: string[] = [];
     for (let i = 0; i < poolsDuplicated.length; ++i) {
       if (poolNames.indexOf(poolsDuplicated[i].name) === -1) {
@@ -119,8 +117,15 @@ class PoolsParser {
         'slug': slug
       };
 
-      if (existingPools.find((pool) => pool.name === poolNames[i]) !== undefined) {
-        finalPoolDataUpdate.push(poolObj);
+      const existingPool = existingPools.find((pool) => pool.name === poolNames[i]);
+      if (existingPool !== undefined) {
+        // Check if any data was actually updated
+        const equals = (a, b) =>
+          a.length === b.length &&
+          a.every((v, i) => v === b[i]);
+        if (!equals(JSON.parse(existingPool.addresses), poolObj.addresses) || !equals(JSON.parse(existingPool.regexes), poolObj.regexes)) {
+          finalPoolDataUpdate.push(poolObj);
+        }
       } else {
         logger.debug(`Add '${finalPoolName}' mining pool`);
         finalPoolDataAdd.push(poolObj);
@@ -140,40 +145,51 @@ class PoolsParser {
       return;
     }
 
-    logger.debug(`Update pools table now`);
+    if (finalPoolDataAdd.length > 0 || finalPoolDataUpdate.length > 0) {    
+      logger.debug(`Update pools table now`);
 
-    // Add new mining pools into the database
-    let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses, slug) VALUES ';
-    for (let i = 0; i < finalPoolDataAdd.length; ++i) {
-      queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}',
-      '${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}',
-      ${JSON.stringify(finalPoolDataAdd[i].slug)}),`;
-    }
-    queryAdd = queryAdd.slice(0, -1) + ';';
+      // Add new mining pools into the database
+      let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses, slug) VALUES ';
+      for (let i = 0; i < finalPoolDataAdd.length; ++i) {
+        queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}',
+        '${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}',
+        ${JSON.stringify(finalPoolDataAdd[i].slug)}),`;
+      }
+      queryAdd = queryAdd.slice(0, -1) + ';';
 
-    // Updated existing mining pools in the database
-    const updateQueries: string[] = [];
-    for (let i = 0; i < finalPoolDataUpdate.length; ++i) {
-      updateQueries.push(`
-        UPDATE pools
-        SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}',
-        regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}',
-        slug='${finalPoolDataUpdate[i].slug}'
-        WHERE name='${finalPoolDataUpdate[i].name}'
-      ;`);
+      // Updated existing mining pools in the database
+      const updateQueries: string[] = [];
+      for (let i = 0; i < finalPoolDataUpdate.length; ++i) {
+        updateQueries.push(`
+          UPDATE pools
+          SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}',
+          regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}',
+          slug='${finalPoolDataUpdate[i].slug}'
+          WHERE name='${finalPoolDataUpdate[i].name}'
+        ;`);
+      }
+
+      try {
+        await this.$deleteBlocskToReindex(finalPoolDataUpdate);
+
+        if (finalPoolDataAdd.length > 0) {
+          await DB.query({ sql: queryAdd, timeout: 120000 });
+        }
+        for (const query of updateQueries) {
+          await DB.query({ sql: query, timeout: 120000 });
+        }
+        await this.insertUnknownPool();
+        logger.info('Mining pools.json import completed');
+      } catch (e) {
+        logger.err(`Cannot import pools in the database`);
+        throw e;
+      }
     }
 
     try {
-      if (finalPoolDataAdd.length > 0) {
-        await DB.query({ sql: queryAdd, timeout: 120000 });
-      }
-      for (const query of updateQueries) {
-        await DB.query({ sql: query, timeout: 120000 });
-      }
       await this.insertUnknownPool();
-      logger.info('Mining pools.json import completed');
     } catch (e) {
-      logger.err(`Cannot import pools in the database`);
+      logger.err(`Cannot insert unknown pool in the database`);
       throw e;
     }
   }
@@ -201,6 +217,36 @@ class PoolsParser {
       logger.err('Unable to insert "Unknown" mining pool');
     }
   }
+
+  /**
+   * Delete blocks which needs to be reindexed
+   */
+   private async $deleteBlocskToReindex(finalPoolDataUpdate: any[]) {
+    if (config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING === false) {
+      return;
+    }
+
+    const blockCount = await BlocksRepository.$blockCount(null, null);
+    if (blockCount === 0) {
+      return;
+    }
+
+    for (const updatedPool of finalPoolDataUpdate) {
+      const [pool]: any[] = await DB.query(`SELECT id, name from pools where slug = "${updatedPool.slug}"`);
+      if (pool.length > 0) {
+        logger.notice(`Deleting blocks from ${pool[0].name} mining pool for future re-indexing`);
+        await DB.query(`DELETE FROM blocks WHERE pool_id = ${pool[0].id}`);
+      }
+    }
+
+    // Ignore early days of Bitcoin as there were not mining pool yet
+    logger.notice('Deleting blocks with unknown mining pool from height 130635 for future re-indexing');
+    const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
+    await DB.query(`DELETE FROM blocks WHERE pool_id = ${unknownPool[0].id} AND height > 130635`);
+
+    logger.notice('Truncating hashrates for future re-indexing');
+    await DB.query(`DELETE FROM hashrates`);
+  }
 }
 
 export default new PoolsParser();

backend/src/api/websocket-handler.ts

@@ -1,7 +1,9 @@
 import logger from '../logger';
 import * as WebSocket from 'ws';
-import { BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, MempoolBlockDelta,
-  OptimizedStatistic, ILoadingIndicators, IConversionRates } from '../mempool.interfaces';
+import {
+  BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, MempoolBlockDelta,
+  OptimizedStatistic, ILoadingIndicators, IConversionRates
+} from '../mempool.interfaces';
 import blocks from './blocks';
 import memPool from './mempool';
 import backendInfo from './backend-info';
@@ -14,6 +16,7 @@ import transactionUtils from './transaction-utils';
 import rbfCache from './rbf-cache';
 import difficultyAdjustment from './difficulty-adjustment';
 import feeApi from './fee-api';
+import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
 
 class WebsocketHandler {
   private wss: WebSocket.Server | undefined;
@@ -164,7 +167,7 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
-    this.wss.clients.forEach((client: WebSocket) => {
+    this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -179,7 +182,7 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
-    this.wss.clients.forEach((client: WebSocket) => {
+    this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -192,7 +195,7 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
-    this.wss.clients.forEach((client: WebSocket) => {
+    this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -224,7 +227,7 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
-    this.wss.clients.forEach((client: WebSocket) => {
+    this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -255,7 +258,7 @@ class WebsocketHandler {
     memPool.handleRbfTransactions(rbfTransactions);
     const recommendedFees = feeApi.getRecommendedFee();
 
-    this.wss.clients.forEach(async (client: WebSocket) => {
+    this.wss.clients.forEach(async (client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
@@ -414,17 +417,40 @@ class WebsocketHandler {
 
     if (_mempoolBlocks[0]) {
       const matches: string[] = [];
+      const added: string[] = [];
+      const missing: string[] = [];
+
       for (const txId of txIds) {
         if (_mempoolBlocks[0].transactionIds.indexOf(txId) > -1) {
           matches.push(txId);
+        } else {
+          added.push(txId);
         }
         delete _memPool[txId];
       }
 
-      matchRate = Math.round((matches.length / (txIds.length - 1)) * 100);
+      for (const txId of _mempoolBlocks[0].transactionIds) {
+        if (matches.includes(txId) || added.includes(txId)) {
+          continue;
+        }
+        missing.push(txId);
+      }
+
+      matchRate = Math.round((Math.max(0, matches.length - missing.length - added.length) / txIds.length * 100) * 100) / 100;
       mempoolBlocks.updateMempoolBlocks(_memPool);
       mBlocks = mempoolBlocks.getMempoolBlocks();
       mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
+
+      if (Common.indexingEnabled()) {
+        BlocksAuditsRepository.$saveAudit({
+          time: block.timestamp,
+          height: block.height,
+          hash: block.id,
+          addedTxs: added,
+          missingTxs: missing,
+          matchRate: matchRate,
+        });
+      }
     }
 
     if (block.extras) {

backend/src/config.ts

@@ -15,6 +15,7 @@ interface IConfig {
     INITIAL_BLOCKS_AMOUNT: number;
     MEMPOOL_BLOCKS_AMOUNT: number;
     INDEXING_BLOCKS_AMOUNT: number;
+    BLOCKS_SUMMARIES_INDEXING: boolean;
     PRICE_FEED_UPDATE_INTERVAL: number;
     USE_SECOND_NODE_FOR_MINFEE: boolean;
     EXTERNAL_ASSETS: string[];
@@ -22,6 +23,7 @@ interface IConfig {
     EXTERNAL_RETRY_INTERVAL: number;
     USER_AGENT: string;
     STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
+    AUTOMATIC_BLOCK_REINDEXING: boolean;
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -104,6 +106,7 @@ const defaults: IConfig = {
     'INITIAL_BLOCKS_AMOUNT': 8,
     'MEMPOOL_BLOCKS_AMOUNT': 8,
     'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks
+    'BLOCKS_SUMMARIES_INDEXING': false,
     'PRICE_FEED_UPDATE_INTERVAL': 600,
     'USE_SECOND_NODE_FOR_MINFEE': false,
     'EXTERNAL_ASSETS': [],
@@ -111,6 +114,7 @@ const defaults: IConfig = {
     'EXTERNAL_RETRY_INTERVAL': 0,
     'USER_AGENT': 'mempool',
     'STDOUT_LOG_MIN_PRIORITY': 'debug',
+    'AUTOMATIC_BLOCK_REINDEXING': false,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',

backend/src/index.ts

@@ -1,8 +1,8 @@
-import { Express, Request, Response, NextFunction } from 'express';
-import * as express from 'express';
+import express from "express";
+import { Application, Request, Response, NextFunction, Express } from 'express';
 import * as http from 'http';
 import * as WebSocket from 'ws';
-import * as cluster from 'cluster';
+import cluster from 'cluster';
 import axios from 'axios';
 
 import DB from './database';
@@ -27,11 +27,13 @@ import icons from './api/liquid/icons';
 import { Common } from './api/common';
 import poolsUpdater from './tasks/pools-updater';
 import indexer from './indexer';
+import priceUpdater from './tasks/price-updater';
+import BlocksAuditsRepository from './repositories/BlocksAuditsRepository';
 
 class Server {
   private wss: WebSocket.Server | undefined;
   private server: http.Server | undefined;
-  private app: Express;
+  private app: Application;
   private currentBackendRetryInterval = 5;
 
   constructor() {
@@ -42,7 +44,7 @@ class Server {
       return;
     }
 
-    if (cluster.isMaster) {
+    if (cluster.isPrimary) {
       logger.notice(`Mempool Server (Master) is running on port ${config.MEMPOOL.HTTP_PORT} (${backendInfo.getShortCommitHash()})`);
 
       const numCPUs = config.MEMPOOL.SPAWN_CLUSTER_PROCS;
@@ -76,7 +78,7 @@ class Server {
       })
       .use(express.urlencoded({ extended: true }))
       .use(express.text())
-    ;
+      ;
 
     this.server = http.createServer(this.app);
     this.wss = new WebSocket.Server({ server: this.server });
@@ -104,7 +106,7 @@ class Server {
       }
     }
 
-    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isMaster) {
+    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && cluster.isPrimary) {
       statistics.startStatistics();
     }
 
@@ -153,6 +155,7 @@ class Server {
       await blocks.$updateBlocks();
       await memPool.$updateMempool();
       indexer.$run();
+      priceUpdater.$run();
 
       setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
       this.currentBackendRetryInterval = 5;
@@ -195,6 +198,7 @@ class Server {
   setUpHttpApiRoutes() {
     this.app
       .get(config.MEMPOOL.API_URL_PREFIX + 'transaction-times', routes.getTransactionTimes)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'outspends', routes.$getBatchedOutspends)
       .get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', routes.getCpfpInfo)
       .get(config.MEMPOOL.API_URL_PREFIX + 'difficulty-adjustment', routes.getDifficultyChange)
       .get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', routes.getRecommendedFees)
@@ -257,7 +261,7 @@ class Server {
           res.status(500).end();
         }
       })
-    ;
+      ;
 
     if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
       this.app
@@ -282,12 +286,15 @@ class Server {
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug', routes.$getPool)
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/pools/:interval', routes.$getPoolsHistoricalHashrate)
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/hashrate/:interval', routes.$getHistoricalHashrate)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments', routes.$getDifficultyAdjustments)
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/reward-stats/:blockCount', routes.$getRewardStats)
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fees/:interval', routes.$getHistoricalBlockFees)
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', routes.$getHistoricalBlockRewards)
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fee-rates/:interval', routes.$getHistoricalBlockFeeRates)
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', routes.$getHistoricalBlockSizeAndWeight)
-      ;
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments/:interval', routes.$getDifficultyAdjustments)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/predictions/:interval', routes.$getHistoricalBlockPrediction)
+        ;
     }
 
     if (config.BISQ.ENABLED) {
@@ -314,7 +321,8 @@ class Server {
     this.app
       .get(config.MEMPOOL.API_URL_PREFIX + 'blocks', routes.getBlocks.bind(routes))
       .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/:height', routes.getBlocks.bind(routes))
-      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', routes.getBlock);
+      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', routes.getBlock)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', routes.getStrippedBlockTransactions);
 
     if (config.MEMPOOL.BACKEND !== 'esplora') {
       this.app
@@ -328,6 +336,7 @@ class Server {
         .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', routes.getTransactionOutspends)
         .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', routes.getBlockHeader)
         .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', routes.getBlockTipHeight)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/hash', routes.getBlockTipHash)
         .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs', routes.getBlockTransactions)
         .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs/:index', routes.getBlockTransactions)
         .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txids', routes.getTxIdsForBlock)
@@ -336,7 +345,7 @@ class Server {
         .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', routes.getAddressTransactions)
         .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs/chain/:txId', routes.getAddressTransactions)
         .get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', routes.getAddressPrefix)
-      ;
+        ;
     }
 
     if (Common.isLiquid()) {
@@ -345,13 +354,13 @@ class Server {
         .get(config.MEMPOOL.API_URL_PREFIX + 'assets/featured', routes.$getAllFeaturedLiquidAssets)
         .get(config.MEMPOOL.API_URL_PREFIX + 'asset/:assetId/icon', routes.getLiquidIcon)
         .get(config.MEMPOOL.API_URL_PREFIX + 'assets/group/:id', routes.$getAssetGroup)
-      ;
+        ;
     }
 
     if (Common.isLiquid() && config.DATABASE.ENABLED) {
       this.app
         .get(config.MEMPOOL.API_URL_PREFIX + 'liquid/pegs/month', routes.$getElementsPegsByMonth)
-      ;
+        ;
     }
   }
 }

backend/src/indexer.ts

@@ -4,6 +4,7 @@ import mempool from './api/mempool';
 import mining from './api/mining';
 import logger from './logger';
 import HashratesRepository from './repositories/HashratesRepository';
+import bitcoinClient from './api/bitcoin/bitcoin-client';
 
 class Indexer {
   runIndexer = true;
@@ -25,20 +26,45 @@ class Indexer {
       return;
     }
 
+    // Do not attempt to index anything unless Bitcoin Core is fully synced
+    const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+    if (blockchainInfo.blocks !== blockchainInfo.headers) {
+      return;
+    }
+
     this.runIndexer = false;
     this.indexerRunning = true;
 
+    logger.debug(`Running mining indexer`);
+
     try {
-      await blocks.$generateBlockDatabase();
+      const chainValid = await blocks.$generateBlockDatabase();
+      if (chainValid === false) {
+        // Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration
+        logger.warn(`The chain of block hash is invalid, re-indexing invalid data in 10 seconds.`);
+        setTimeout(() => this.reindex(), 10000);
+        this.indexerRunning = false;
+        return;
+      }
+
+      await mining.$indexDifficultyAdjustments();
       await this.$resetHashratesIndexingState();
       await mining.$generateNetworkHashrateHistory();
       await mining.$generatePoolHashrateHistory();
+      await blocks.$generateBlocksSummariesDatabase();
     } catch (e) {
-      this.reindex();
-      logger.err(`Indexer failed, trying again later. Reason: ` + (e instanceof Error ? e.message : e));
+      this.indexerRunning = false;
+      logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e));
+      setTimeout(() => this.reindex(), 10000);
+      this.indexerRunning = false;
+      return;
     }
 
     this.indexerRunning = false;
+
+    const runEvery = 1000 * 3600; // 1 hour
+    logger.debug(`Indexing completed. Next run planned at ${new Date(new Date().getTime() + runEvery).toUTCString()}`);
+    setTimeout(() => this.reindex(), runEvery);
   }
 
   async $resetHashratesIndexingState() {
@@ -47,6 +73,7 @@ class Indexer {
       await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0);
     } catch (e) {
       logger.err(`Cannot reset hashrate indexing timestamps. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
     }
   }
 }

backend/src/mempool.interfaces.ts

@@ -22,6 +22,15 @@ export interface PoolStats extends PoolInfo {
   emptyBlocks: number;
 }
 
+export interface BlockAudit {
+  time: number,
+  height: number,
+  hash: string,
+  missingTxs: string[],
+  addedTxs: string[],
+  matchRate: number,
+}
+
 export interface MempoolBlock {
   blockSize: number;
   blockVSize: number;
@@ -106,6 +115,11 @@ export interface BlockExtended extends IEsploraApi.Block {
   extras: BlockExtension;
 }
 
+export interface BlockSummary {
+  id: string;
+  transactions: TransactionStripped[];
+}
+
 export interface TransactionMinerInfo {
   vin: VinStrippedToScriptsig[];
   vout: VoutStrippedToScriptPubkey[];
@@ -219,6 +233,13 @@ export interface IDifficultyAdjustment {
   timeOffset: number;
 }
 
+export interface IndexedDifficultyAdjustment {
+  time: number; // UNIX timestamp
+  height: number; // Block height
+  difficulty: number;
+  adjustment: number;
+}
+
 export interface RewardStats {
   totalReward: number;
   totalFee: number;

backend/src/repositories/BlocksAuditsRepository.ts

@@ -0,0 +1,51 @@
+import DB from '../database';
+import logger from '../logger';
+import { BlockAudit } from '../mempool.interfaces';
+
+class BlocksAuditRepositories {
+  public async $saveAudit(audit: BlockAudit): Promise<void> {
+    try {
+      await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, match_rate)
+        VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
+          JSON.stringify(audit.addedTxs), audit.matchRate]);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`);
+      } else {
+        logger.err(`Cannot save block audit into db. Reason: ` + (e instanceof Error ? e.message : e));
+        throw e;
+      }
+    }
+  }
+
+  public async $getBlockPredictionsHistory(div: number, interval: string | null): Promise<any> {
+    try {
+      let query = `SELECT UNIX_TIMESTAMP(time) as time, height, match_rate FROM blocks_audits`;
+
+      if (interval !== null) {
+        query += ` WHERE time BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+      }
+
+      query += ` GROUP BY UNIX_TIMESTAMP(time) DIV ${div} ORDER BY height`;
+
+      const [rows] = await DB.query(query);
+      return rows;
+    } catch (e: any) {
+      logger.err(`Cannot fetch block prediction history. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getPredictionsCount(): Promise<number> {
+    try {
+      const [rows] = await DB.query(`SELECT count(hash) as count FROM blocks_audits`);
+      return rows[0].count;
+    } catch (e: any) {
+      logger.err(`Cannot fetch block prediction history. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+}
+
+export default new BlocksAuditRepositories();
+

backend/src/repositories/BlocksRepository.ts

@@ -6,6 +6,8 @@ import { prepareBlock } from '../utils/blocks-utils';
 import PoolsRepository from './PoolsRepository';
 import HashratesRepository from './HashratesRepository';
 import { escape } from 'mysql2';
+import BlocksSummariesRepository from './BlocksSummariesRepository';
+import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository';
 
 class BlocksRepository {
   /**
@@ -380,48 +382,9 @@ class BlocksRepository {
   /**
    * Return blocks difficulty
    */
-  public async $getBlocksDifficulty(interval: string | null): Promise<object[]> {
-    interval = Common.getSqlInterval(interval);
-
-    // :D ... Yeah don't ask me about this one https://stackoverflow.com/a/40303162
-    // Basically, using temporary user defined fields, we are able to extract all
-    // difficulty adjustments from the blocks tables.
-    // This allow use to avoid indexing it in another table.
-    let query = `
-      SELECT
-      *
-      FROM
-      (
-        SELECT
-        UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty, height,
-        IF(@prevStatus = YT.difficulty, @rn := @rn + 1,
-          IF(@prevStatus := YT.difficulty, @rn := 1, @rn := 1)
-        ) AS rn
-        FROM blocks YT
-        CROSS JOIN
-        (
-          SELECT @prevStatus := -1, @rn := 1
-        ) AS var
-    `;
-
-    if (interval) {
-      query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
-    }
-
-    query += `
-        ORDER BY YT.height
-      ) AS t
-      WHERE t.rn = 1
-      ORDER BY t.height
-    `;
-
+  public async $getBlocksDifficulty(): Promise<object[]> {
     try {
-      const [rows]: any[] = await DB.query(query);
-
-      for (const row of rows) {
-        delete row['rn'];
-      }
-
+      const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty FROM blocks`);
       return rows;
     } catch (e) {
       logger.err('Cannot generate difficulty history. Reason: ' + (e instanceof Error ? e.message : e));
@@ -451,26 +414,6 @@ class BlocksRepository {
     }
   }
 
-  /*
-   * Check if the last 10 blocks chain is valid
-   */
-  public async $validateRecentBlocks(): Promise<boolean> {
-    try {
-      const [lastBlocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash FROM blocks ORDER BY height DESC LIMIT 10`);
-
-      for (let i = 0; i < lastBlocks.length - 1; ++i) {
-        if (lastBlocks[i].previous_block_hash !== lastBlocks[i + 1].hash) {
-          logger.warn(`Chain divergence detected at block ${lastBlocks[i].height}, re-indexing most recent data`);
-          return false;
-        }
-      }
-
-      return true;
-    } catch (e) {
-      return true; // Don't do anything if there is a db error
-    }
-  }
-
   /**
    * Check if the chain of block hash is valid and delete data from the stale branch if needed
    */
@@ -493,9 +436,11 @@ class BlocksRepository {
         }
 
         if (blocks[idx].previous_block_hash !== blocks[idx - 1].hash) {
-          logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}, re-indexing newer blocks and hashrates`);
+          logger.warn(`Chain divergence detected at block ${blocks[idx - 1].height}`);
           await this.$deleteBlocksFrom(blocks[idx - 1].height);
+          await BlocksSummariesRepository.$deleteBlocksFrom(blocks[idx - 1].height);
           await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800);
+          await DifficultyAdjustmentsRepository.$deleteAdjustementsFromHeight(blocks[idx - 1].height);
           return false;
         }
         ++idx;
@@ -652,6 +597,37 @@ class BlocksRepository {
       throw e;
     }
   }
+
+  /**
+   * Get a list of blocks that have been indexed
+   */
+  public async $getIndexedBlocks(): Promise<any[]> {
+    try {
+      const [rows]: any = await DB.query(`SELECT height, hash FROM blocks ORDER BY height DESC`);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot generate block size and weight history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  /**
+   * Return the oldest block  from a consecutive chain of block from the most recent one
+   */
+  public async $getOldestConsecutiveBlock(): Promise<any> {
+    try {
+      const [rows]: any = await DB.query(`SELECT height, UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty FROM blocks ORDER BY height DESC`);
+      for (let i = 0; i < rows.length - 1; ++i) {
+        if (rows[i].height - rows[i + 1].height > 1) {
+          return rows[i];
+        }
+      }
+      return rows[rows.length - 1];
+    } catch (e) {
+      logger.err('Cannot generate block size and weight history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
 }
 
 export default new BlocksRepository();

backend/src/repositories/BlocksSummariesRepository.ts

@@ -0,0 +1,59 @@
+import DB from '../database';
+import logger from '../logger';
+import { BlockSummary } from '../mempool.interfaces';
+
+class BlocksSummariesRepository {
+  public async $getByBlockId(id: string): Promise<BlockSummary | undefined> {
+    try {
+      const [summary]: any[] = await DB.query(`SELECT * from blocks_summaries WHERE id = ?`, [id]);
+      if (summary.length > 0) {
+        summary[0].transactions = JSON.parse(summary[0].transactions);
+        return summary[0];
+      }
+    } catch (e) {
+      logger.err(`Cannot get block summary for block id ${id}. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+
+    return undefined;
+  }
+
+  public async $saveSummary(height: number, summary: BlockSummary) {
+    try {
+      await DB.query(`INSERT INTO blocks_summaries VALUE (?, ?, ?)`, [height, summary.id, JSON.stringify(summary.transactions)]);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`Cannot save block summary for ${summary.id} because it has already been indexed, ignoring`);
+      } else {
+        logger.debug(`Cannot save block summary for ${summary.id}. Reason: ${e instanceof Error ? e.message : e}`);
+        throw e;
+      }
+    }
+  }
+
+  public async $getIndexedSummariesId(): Promise<string[]> {
+    try {
+      const [rows]: any[] = await DB.query(`SELECT id from blocks_summaries`);
+      return rows.map(row => row.id);
+    } catch (e) {
+      logger.err(`Cannot get block summaries id list. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+
+    return [];
+  }
+
+  /**
+   * Delete blocks from the database from blockHeight
+   */
+   public async $deleteBlocksFrom(blockHeight: number) {
+    logger.info(`Delete newer blocks summary from height ${blockHeight} from the database`);
+
+    try {
+      await DB.query(`DELETE FROM blocks_summaries where height >= ${blockHeight}`);
+    } catch (e) {
+      logger.err('Cannot delete indexed blocks summaries. Reason: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+}
+
+export default new BlocksSummariesRepository();
+

backend/src/repositories/DifficultyAdjustmentsRepository.ts

@@ -0,0 +1,124 @@
+import { Common } from '../api/common';
+import config from '../config';
+import DB from '../database';
+import logger from '../logger';
+import { IndexedDifficultyAdjustment } from '../mempool.interfaces';
+
+class DifficultyAdjustmentsRepository {
+  public async $saveAdjustments(adjustment: IndexedDifficultyAdjustment): Promise<void> {
+    if (adjustment.height === 1) {
+      return;
+    }
+
+    try {
+      const query = `INSERT INTO difficulty_adjustments(time, height, difficulty, adjustment) VALUE (FROM_UNIXTIME(?), ?, ?, ?)`;
+      const params: any[] = [
+        adjustment.time,
+        adjustment.height,
+        adjustment.difficulty,
+        adjustment.adjustment,
+      ];
+      await DB.query(query, params);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`Cannot save difficulty adjustment at block ${adjustment.height}, already indexed, ignoring`);
+      } else {
+        logger.err(`Cannot save difficulty adjustment at block ${adjustment.height}. Reason: ${e instanceof Error ? e.message : e}`);
+        throw e;
+      }
+    }
+  }
+
+  public async $getAdjustments(interval: string | null, descOrder: boolean = false): Promise<IndexedDifficultyAdjustment[]> {
+    interval = Common.getSqlInterval(interval);
+
+    let query = `SELECT 
+      CAST(AVG(UNIX_TIMESTAMP(time)) as INT) as time,
+      CAST(AVG(height) AS INT) as height,
+      CAST(AVG(difficulty) as DOUBLE) as difficulty,
+      CAST(AVG(adjustment) as DOUBLE) as adjustment
+      FROM difficulty_adjustments`;
+
+    if (interval) {
+      query += ` WHERE time BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+    }
+
+    query += ` GROUP BY UNIX_TIMESTAMP(time) DIV ${86400}`;
+
+    if (descOrder === true) {
+      query += ` ORDER BY height DESC`;
+    } else {
+      query += ` ORDER BY height`;
+    }
+
+    try {
+      const [rows] = await DB.query(query);
+      return rows as IndexedDifficultyAdjustment[];
+    } catch (e) {
+      logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getRawAdjustments(interval: string | null, descOrder: boolean = false): Promise<IndexedDifficultyAdjustment[]> {
+    interval = Common.getSqlInterval(interval);
+
+    let query = `SELECT 
+      UNIX_TIMESTAMP(time) as time,
+      height as height,
+      difficulty as difficulty,
+      adjustment as adjustment
+      FROM difficulty_adjustments`;
+
+    if (interval) {
+      query += ` WHERE time BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+    }
+
+    if (descOrder === true) {
+      query += ` ORDER BY height DESC`;
+    } else {
+      query += ` ORDER BY height`;
+    }
+
+    try {
+      const [rows] = await DB.query(query);
+      return rows as IndexedDifficultyAdjustment[];
+    } catch (e) {
+      logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getAdjustmentsHeights(): Promise<number[]> {
+    try {
+      const [rows]: any[] = await DB.query(`SELECT height FROM difficulty_adjustments`);
+      return rows.map(block => block.height);
+    } catch (e: any) {
+      logger.err(`Cannot get difficulty adjustment block heights. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $deleteAdjustementsFromHeight(height: number): Promise<void> {
+    try {
+      logger.info(`Delete newer difficulty adjustments from height ${height} from the database`);
+      await DB.query(`DELETE FROM difficulty_adjustments WHERE height >= ?`, [height]);
+    } catch (e: any) {
+      logger.err(`Cannot delete difficulty adjustments from the database. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+
+  public async $deleteLastAdjustment(): Promise<void> {
+    try {
+      logger.info(`Delete last difficulty adjustment from the database`);
+      await DB.query(`DELETE FROM difficulty_adjustments ORDER BY time LIMIT 1`);
+    } catch (e: any) {
+      logger.err(`Cannot delete last difficulty adjustment from the database. Reason: ${e instanceof Error ? e.message : e}`);
+      throw e;
+    }
+  }
+}
+
+export default new DifficultyAdjustmentsRepository();
+

backend/src/repositories/HashratesRepository.ts

@@ -29,10 +29,12 @@ class HashratesRepository {
     }
   }
 
-  public async $getNetworkDailyHashrate(interval: string | null): Promise<any[]> {
+  public async $getRawNetworkDailyHashrate(interval: string | null): Promise<any[]> {
     interval = Common.getSqlInterval(interval);
 
-    let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate
+    let query = `SELECT
+      UNIX_TIMESTAMP(hashrate_timestamp) as timestamp,
+      avg_hashrate as avgHashrate
       FROM hashrates`;
 
     if (interval) {
@@ -53,6 +55,33 @@ class HashratesRepository {
     }
   }
 
+  public async $getNetworkDailyHashrate(interval: string | null): Promise<any[]> {
+    interval = Common.getSqlInterval(interval);
+
+    let query = `SELECT
+      CAST(AVG(UNIX_TIMESTAMP(hashrate_timestamp)) as INT) as timestamp,
+      CAST(AVG(avg_hashrate) as DOUBLE) as avgHashrate
+      FROM hashrates`;
+
+    if (interval) {
+      query += ` WHERE hashrate_timestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()
+        AND hashrates.type = 'daily'`;
+    } else {
+      query += ` WHERE hashrates.type = 'daily'`;
+    }
+
+    query += ` GROUP BY UNIX_TIMESTAMP(hashrate_timestamp) DIV ${86400}`;
+    query += ` ORDER by hashrate_timestamp`;
+
+    try {
+      const [rows]: any[] = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   public async $getWeeklyHashrateTimestamps(): Promise<number[]> {
     const query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp
       FROM hashrates
@@ -75,6 +104,9 @@ class HashratesRepository {
     interval = Common.getSqlInterval(interval);
 
     const topPoolsId = (await PoolsRepository.$getPoolsInfo('1w')).map((pool) => pool.poolId);
+    if (topPoolsId.length === 0) {
+      return [];
+    }
 
     let query = `SELECT UNIX_TIMESTAMP(hashrate_timestamp) as timestamp, avg_hashrate as avgHashrate, share, pools.name as poolName
       FROM hashrates

backend/src/repositories/PricesRepository.ts

@@ -0,0 +1,42 @@
+import DB from '../database';
+import logger from '../logger';
+import { Prices } from '../tasks/price-updater';
+
+class PricesRepository {
+  public async $savePrices(time: number, prices: Prices): Promise<void> {
+    if (prices.USD === -1) {
+      // Some historical price entries have not USD prices, so we just ignore them to avoid future UX issues
+      // As of today there are only 4 (on 2013-09-05, 2013-09-19, 2013-09-12 and 2013-09-26) so that's fine
+      return;
+    }
+
+    try {
+      await DB.query(`
+        INSERT INTO prices(time,             USD, EUR, GBP, CAD, CHF, AUD, JPY)
+        VALUE             (FROM_UNIXTIME(?), ?,   ?,   ?,   ?,   ?,   ?,   ?  )`,
+        [time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot save exchange rate into db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getOldestPriceTime(): Promise<number> {
+    const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time LIMIT 1`);
+    return oldestRow[0] ? oldestRow[0].time : 0;
+  }
+
+  public async $getLatestPriceTime(): Promise<number> {
+    const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
+    return oldestRow[0] ? oldestRow[0].time : 0;
+  }
+
+  public async $getPricesTimes(): Promise<number[]> {
+    const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1`);
+    return times.map(time => time.time);
+  }
+}
+
+export default new PricesRepository();
+

backend/src/repositories/RatesRepository.ts

@@ -1,21 +0,0 @@
-import DB from '../database';
-import logger from '../logger';
-import { IConversionRates } from '../mempool.interfaces';
-
-class RatesRepository {
-  public async $saveRate(height: number, rates: IConversionRates) {
-    try {
-      await DB.query(`INSERT INTO rates(height, bisq_rates) VALUE (?, ?)`, [height, JSON.stringify(rates)]);
-    } catch (e: any) {
-      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
-        logger.debug(`Rate already exists for block ${height}, ignoring`);
-      } else {
-        logger.err(`Cannot save exchange rate into db for block ${height} Reason: ` + (e instanceof Error ? e.message : e));
-        throw e;
-      }
-    }
-  }
-}
-
-export default new RatesRepository();
-

backend/src/routes.ts

@@ -26,6 +26,8 @@ import mining from './api/mining';
 import BlocksRepository from './repositories/BlocksRepository';
 import HashratesRepository from './repositories/HashratesRepository';
 import difficultyAdjustment from './api/difficulty-adjustment';
+import DifficultyAdjustmentsRepository from './repositories/DifficultyAdjustmentsRepository';
+import BlocksAuditsRepository from './repositories/BlocksAuditsRepository';
 
 class Routes {
   constructor() {}
@@ -120,6 +122,30 @@ class Routes {
     res.json(times);
   }
 
+  public async $getBatchedOutspends(req: Request, res: Response) {
+    if (!Array.isArray(req.query.txId)) {
+      res.status(500).send('Not an array');
+      return;
+    }
+    if (req.query.txId.length > 50) {
+      res.status(400).send('Too many txids requested');
+      return;
+    }
+    const txIds: string[] = [];
+    for (const _txId in req.query.txId) {
+      if (typeof req.query.txId[_txId] === 'string') {
+        txIds.push(req.query.txId[_txId].toString());
+      }
+    }
+
+    try {
+      const batchedOutspends = await bitcoinApi.$getBatchedOutspends(txIds);
+      res.json(batchedOutspends);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   public getCpfpInfo(req: Request, res: Response) {
     if (!/^[a-fA-F0-9]{64}$/.test(req.params.txId)) {
       res.status(501).send(`Invalid transaction ID.`);
@@ -629,7 +655,7 @@ class Routes {
 
     try {
       const hashrates = await HashratesRepository.$getNetworkDailyHashrate(req.params.interval);
-      const difficulty = await BlocksRepository.$getBlocksDifficulty(req.params.interval);
+      const difficulty = await DifficultyAdjustmentsRepository.$getAdjustments(req.params.interval, false);
       const blockCount = await BlocksRepository.$blockCount(null, null);
       res.header('Pragma', 'public');
       res.header('Cache-control', 'public');
@@ -706,6 +732,32 @@ class Routes {
     }
   }
 
+  public async $getDifficultyAdjustments(req: Request, res: Response) {
+    try {
+      const difficulty = await DifficultyAdjustmentsRepository.$getRawAdjustments(req.params.interval, true);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      res.json(difficulty.map(adj => [adj.time, adj.height, adj.difficulty, adj.adjustment]));
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  public async $getHistoricalBlockPrediction(req: Request, res: Response) {
+    try {
+      const blockPredictions = await mining.$getBlockPredictionsHistory(req.params.interval);
+      const blockCount = await BlocksAuditsRepository.$getPredictionsCount();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(blockPredictions.map(prediction => [prediction.time, prediction.height, prediction.match_rate]));
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   public async getBlock(req: Request, res: Response) {
     try {
       const block = await blocks.$getBlock(req.params.hash);
@@ -726,9 +778,19 @@ class Routes {
     }
   }
 
+  public async getStrippedBlockTransactions(req: Request, res: Response) {
+    try {
+      const transactions = await blocks.$getStrippedBlockTransactions(req.params.hash);
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
+      res.json(transactions);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   public async getBlocks(req: Request, res: Response) {
     try {
-      if (['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
+      if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { // Bitcoin
         const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
         res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
         res.json(await blocks.$getBlocks(height, 15));
@@ -891,6 +953,16 @@ class Routes {
     }
   }
 
+  public async getBlockTipHash(req: Request, res: Response) {
+    try {
+      const result = await bitcoinApi.$getBlockHashTip();
+      res.setHeader('content-type', 'text/plain');
+      res.send(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   public async getTxIdsForBlock(req: Request, res: Response) {
     try {
       const result = await bitcoinApi.$getTxIdsForBlock(req.params.hash);

backend/src/tasks/pools-updater.ts

@@ -13,6 +13,8 @@ import * as https from 'https';
 class PoolsUpdater {
   lastRun: number = 0;
   currentSha: any = undefined;
+  poolsUrl: string = 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json';
+  treeUrl: string = 'https://api.github.com/repos/mempool/mining-pools/git/trees/master';
 
   constructor() {
   }
@@ -32,11 +34,10 @@ class PoolsUpdater {
 
     this.lastRun = now;
 
-    logger.info('Updating latest mining pools from Github');
     if (config.SOCKS5PROXY.ENABLED) {
-      logger.info('List of public pools will be queried over the Tor network');
+      logger.info(`Updating latest mining pools from ${this.poolsUrl} over the Tor network`);
     } else {
-      logger.info('List of public pools will be queried over clearnet');
+      logger.info(`Updating latest mining pools from ${this.poolsUrl} over clearnet`);
     }
 
     try {
@@ -46,7 +47,7 @@ class PoolsUpdater {
       }
 
       if (config.DATABASE.ENABLED === true) {
-        this.currentSha = await this.getShaFromDb();      
+        this.currentSha = await this.getShaFromDb();
       }
 
       logger.debug(`Pools.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
@@ -54,8 +55,12 @@ class PoolsUpdater {
         return;
       }
 
-      logger.warn('Pools.json is outdated, fetch latest from github');
-      const poolsJson = await this.query('https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json');
+      if (this.currentSha === undefined) {
+        logger.info(`Downloading pools.json for the first time from ${this.poolsUrl}`);
+      } else {
+        logger.warn(`Pools.json is outdated, fetch latest from ${this.poolsUrl}`);
+      }
+      const poolsJson = await this.query(this.poolsUrl);
       if (poolsJson === undefined) {
         return;
       }
@@ -65,7 +70,7 @@ class PoolsUpdater {
 
     } catch (e) {
       this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week
-      logger.err('PoolsUpdater failed. Will try again in 24h. Reason: '  + (e instanceof Error ? e.message : e));
+      logger.err('PoolsUpdater failed. Will try again in 24h. Reason: ' + (e instanceof Error ? e.message : e));
     }
   }
 
@@ -79,7 +84,7 @@ class PoolsUpdater {
         await DB.query('DELETE FROM state where name="pools_json_sha"');
         await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`);
       } catch (e) {
-        logger.err('Cannot save github pools.json sha into the db. Reason: '  + (e instanceof Error ? e.message : e));
+        logger.err('Cannot save github pools.json sha into the db. Reason: ' + (e instanceof Error ? e.message : e));
       }
     }
   }
@@ -92,7 +97,7 @@ class PoolsUpdater {
       const [rows]: any[] = await DB.query('SELECT string FROM state WHERE name="pools_json_sha"');
       return (rows.length > 0 ? rows[0].string : undefined);
     } catch (e) {
-      logger.err('Cannot fetch pools.json sha from db. Reason: '  + (e instanceof Error ? e.message : e));
+      logger.err('Cannot fetch pools.json sha from db. Reason: ' + (e instanceof Error ? e.message : e));
       return undefined;
     }
   }
@@ -101,7 +106,7 @@ class PoolsUpdater {
    * Fetch our latest pools.json sha from github
    */
   private async fetchPoolsSha(): Promise<string | undefined> {
-    const response = await this.query('https://api.github.com/repos/mempool/mining-pools/git/trees/master');
+    const response = await this.query(this.treeUrl);
 
     if (response !== undefined) {
       for (const file of response['tree']) {
@@ -111,7 +116,7 @@ class PoolsUpdater {
       }
     }
 
-    logger.err('Cannot to find latest pools.json sha from github api response');
+    logger.err(`Cannot find "pools.json" in git tree (${this.treeUrl})`);
     return undefined;
   }
 
@@ -125,7 +130,7 @@ class PoolsUpdater {
       };
       timeout: number;
       httpsAgent?: https.Agent;
-    }
+    };
     const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
     const axiosOptions: axiosOptions = {
       headers: {
@@ -135,7 +140,7 @@ class PoolsUpdater {
     };
     let retry = 0;
 
-    while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
+    while (retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
       try {
         if (config.SOCKS5PROXY.ENABLED) {
           const socksOptions: any = {
@@ -156,14 +161,14 @@ class PoolsUpdater {
 
           axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
         }
-        
+
         const data: AxiosResponse = await axios.get(path, axiosOptions);
         if (data.statusText === 'error' || !data.data) {
-          throw new Error(`Could not fetch data from Github, Error: ${data.status}`);
+          throw new Error(`Could not fetch data from ${path}, Error: ${data.status}`);
         }
         return data.data;
       } catch (e) {
-        logger.err('Could not connect to Github. Reason: '  + (e instanceof Error ? e.message : e));
+        logger.err('Could not connect to Github. Reason: ' + (e instanceof Error ? e.message : e));
         retry++;
       }
       await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);

backend/src/tasks/price-feeds/bitfinex-api.ts

@@ -0,0 +1,43 @@
+import { query } from '../../utils/axios-query';
+import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater';
+
+class BitfinexApi implements PriceFeed {
+  public name: string = 'Bitfinex';
+  public currencies: string[] = ['USD', 'EUR', 'GPB', 'JPY'];
+
+  public url: string = 'https://api.bitfinex.com/v1/pubticker/BTC';
+  public urlHist: string = 'https://api-pub.bitfinex.com/v2/candles/trade:{GRANULARITY}:tBTC{CURRENCY}/hist';
+
+  constructor() {
+  }
+
+  public async $fetchPrice(currency): Promise<number> {
+    const response = await query(this.url + currency);
+    return response ? parseInt(response['last_price'], 10) : -1;
+  }
+
+  public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
+    const priceHistory: PriceHistory = {};
+
+    for (const currency of currencies) {
+      if (this.currencies.includes(currency) === false) {
+        continue;
+      }
+
+      const response = await query(this.urlHist.replace('{GRANULARITY}', '1h').replace('{CURRENCY}', currency));
+      const pricesRaw = response ? response : [];
+
+      for (const price of pricesRaw as any[]) {
+        const time = Math.round(price[0] / 1000);
+        if (priceHistory[time] === undefined) {
+          priceHistory[time] = priceUpdater.getEmptyPricesObj();
+        }
+        priceHistory[time][currency] = price[2];
+      }
+    }
+
+    return priceHistory;
+  }
+}
+
+export default BitfinexApi;

backend/src/tasks/price-feeds/bitflyer-api.ts

@@ -0,0 +1,24 @@
+import { query } from '../../utils/axios-query';
+import { PriceFeed, PriceHistory } from '../price-updater';
+
+class BitflyerApi implements PriceFeed {
+  public name: string = 'Bitflyer';
+  public currencies: string[] = ['USD', 'EUR', 'JPY'];
+
+  public url: string = 'https://api.bitflyer.com/v1/ticker?product_code=BTC_';
+  public urlHist: string = '';
+
+  constructor() {
+  }
+
+  public async $fetchPrice(currency): Promise<number> {
+    const response = await query(this.url + currency);
+    return response ? parseInt(response['ltp'], 10) : -1;
+  }
+
+  public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
+    return [];
+  }
+}
+
+export default BitflyerApi;

backend/src/tasks/price-feeds/coinbase-api.ts

@@ -0,0 +1,42 @@
+import { query } from '../../utils/axios-query';
+import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater';
+
+class CoinbaseApi implements PriceFeed {
+  public name: string = 'Coinbase';
+  public currencies: string[] = ['USD', 'EUR', 'GBP'];
+
+  public url: string = 'https://api.coinbase.com/v2/prices/spot?currency=';
+  public urlHist: string = 'https://api.exchange.coinbase.com/products/BTC-{CURRENCY}/candles?granularity={GRANULARITY}';
+
+  constructor() {
+  }
+
+  public async $fetchPrice(currency): Promise<number> {
+    const response = await query(this.url + currency);
+    return response ? parseInt(response['data']['amount'], 10) : -1;
+  }
+
+  public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
+    const priceHistory: PriceHistory = {};
+
+    for (const currency of currencies) {
+      if (this.currencies.includes(currency) === false) {
+        continue;
+      }
+
+      const response = await query(this.urlHist.replace('{GRANULARITY}', '3600').replace('{CURRENCY}', currency));
+      const pricesRaw = response ? response : [];
+
+      for (const price of pricesRaw as any[]) {
+        if (priceHistory[price[0]] === undefined) {
+          priceHistory[price[0]] = priceUpdater.getEmptyPricesObj();
+        }
+        priceHistory[price[0]][currency] = price[4];
+      }
+    }
+
+    return priceHistory;
+  }
+}
+
+export default CoinbaseApi;

backend/src/tasks/price-feeds/ftx-api.ts

@@ -0,0 +1,43 @@
+import { query } from '../../utils/axios-query';
+import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater';
+
+class FtxApi implements PriceFeed {
+  public name: string = 'FTX';
+  public currencies: string[] = ['USD', 'BRZ', 'EUR', 'JPY', 'AUD'];
+
+  public url: string = 'https://ftx.com/api/markets/BTC/';
+  public urlHist: string = 'https://ftx.com/api/markets/BTC/{CURRENCY}/candles?resolution={GRANULARITY}';
+
+  constructor() {
+  }
+
+  public async $fetchPrice(currency): Promise<number> {
+    const response = await query(this.url + currency);
+    return response ? parseInt(response['result']['last'], 10) : -1;
+  }
+
+  public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
+    const priceHistory: PriceHistory = {};
+
+    for (const currency of currencies) {
+      if (this.currencies.includes(currency) === false) {
+        continue;
+      }
+
+      const response = await query(this.urlHist.replace('{GRANULARITY}', '3600').replace('{CURRENCY}', currency));
+      const pricesRaw = response ? response['result'] : [];
+
+      for (const price of pricesRaw as any[]) {
+        const time = Math.round(price['time'] / 1000);
+        if (priceHistory[time] === undefined) {
+          priceHistory[time] = priceUpdater.getEmptyPricesObj();
+        }
+        priceHistory[time][currency] = price['close'];
+      }
+    }
+
+    return priceHistory;
+  }
+}
+
+export default FtxApi;

backend/src/tasks/price-feeds/gemini-api.ts

@@ -0,0 +1,43 @@
+import { query } from '../../utils/axios-query';
+import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater';
+
+class GeminiApi implements PriceFeed {
+  public name: string = 'Gemini';
+  public currencies: string[] = ['USD', 'EUR', 'GBP', 'SGD'];
+
+  public url: string = 'https://api.gemini.com/v1/pubticker/BTC';
+  public urlHist: string = 'https://api.gemini.com/v2/candles/BTC{CURRENCY}/{GRANULARITY}';
+
+  constructor() {
+  }
+
+  public async $fetchPrice(currency): Promise<number> {
+    const response = await query(this.url + currency);
+    return response ? parseInt(response['last'], 10) : -1;
+  }
+
+  public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
+    const priceHistory: PriceHistory = {};
+
+    for (const currency of currencies) {
+      if (this.currencies.includes(currency) === false) {
+        continue;
+      }
+
+      const response = await query(this.urlHist.replace('{GRANULARITY}', '1hr').replace('{CURRENCY}', currency));
+      const pricesRaw = response ? response : [];
+
+      for (const price of pricesRaw as any[]) {
+        const time = Math.round(price[0] / 1000);
+        if (priceHistory[time] === undefined) {
+          priceHistory[time] = priceUpdater.getEmptyPricesObj();
+        }
+        priceHistory[time][currency] = price[4];
+      }
+    }
+
+    return priceHistory;
+  }
+}
+
+export default GeminiApi;

backend/src/tasks/price-feeds/kraken-api.ts

@@ -0,0 +1,95 @@
+import logger from '../../logger';
+import PricesRepository from '../../repositories/PricesRepository';
+import { query } from '../../utils/axios-query';
+import priceUpdater, { PriceFeed, PriceHistory } from '../price-updater';
+
+class KrakenApi implements PriceFeed {
+  public name: string = 'Kraken';
+  public currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
+
+  public url: string = 'https://api.kraken.com/0/public/Ticker?pair=XBT';
+  public urlHist: string = 'https://api.kraken.com/0/public/OHLC?interval={GRANULARITY}&pair=XBT';
+
+  constructor() {
+  }
+
+  private getTicker(currency) {
+    let ticker = `XXBTZ${currency}`;
+    if (['CHF', 'AUD'].includes(currency)) {
+      ticker = `XBT${currency}`;
+    }
+    return ticker;
+  }
+
+  public async $fetchPrice(currency): Promise<number> {
+    const response = await query(this.url + currency);
+    return response ? parseInt(response['result'][this.getTicker(currency)]['c'][0], 10) : -1;
+  }
+
+  public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
+    const priceHistory: PriceHistory = {};
+
+    for (const currency of currencies) {
+      if (this.currencies.includes(currency) === false) {
+        continue;
+      }
+
+      const response = await query(this.urlHist.replace('{GRANULARITY}', '60') + currency);
+      const pricesRaw = response ? response['result'][this.getTicker(currency)] : [];
+
+      for (const price of pricesRaw) {
+        if (priceHistory[price[0]] === undefined) {
+          priceHistory[price[0]] = priceUpdater.getEmptyPricesObj();
+        }
+        priceHistory[price[0]][currency] = price[4];
+      }
+    }
+
+    return priceHistory;
+  }
+
+  /**
+   * Fetch weekly price and save it into the database
+   */
+  public async $insertHistoricalPrice(): Promise<void> {
+    const existingPriceTimes = await PricesRepository.$getPricesTimes();
+
+    // EUR weekly price history goes back to timestamp 1378339200 (September 5, 2013)
+    // USD weekly price history goes back to timestamp 1380758400 (October 3, 2013)
+    // GBP weekly price history goes back to timestamp 1415232000 (November 6, 2014)
+    // JPY weekly price history goes back to timestamp 1415232000 (November 6, 2014)
+    // CAD weekly price history goes back to timestamp 1436400000 (July 9, 2015)
+    // CHF weekly price history goes back to timestamp 1575504000 (December 5, 2019)
+    // AUD weekly price history goes back to timestamp 1591833600 (June 11, 2020)
+
+    const priceHistory: any = {}; // map: timestamp -> Prices
+
+    for (const currency of this.currencies) {
+      const response = await query(this.urlHist.replace('{GRANULARITY}', '10080') + currency);
+      const priceHistoryRaw = response ? response['result'][this.getTicker(currency)] : [];
+
+      for (const price of priceHistoryRaw) {
+        if (existingPriceTimes.includes(parseInt(price[0]))) {
+          continue;
+        }
+
+        // prices[0] = kraken price timestamp
+        // prices[4] = closing price
+        if (priceHistory[price[0]] === undefined) {
+          priceHistory[price[0]] = priceUpdater.getEmptyPricesObj();
+        }
+        priceHistory[price[0]][currency] = price[4];
+      }
+    }
+
+    for (const time in priceHistory) {
+      await PricesRepository.$savePrices(parseInt(time, 10), priceHistory[time]);
+    }
+
+    if (Object.keys(priceHistory).length > 0) {
+      logger.notice(`Inserted ${Object.keys(priceHistory).length} Kraken EUR, USD, GBP, JPY, CAD, CHF and AUD weekly price history into db`);
+    }
+  }
+}
+
+export default KrakenApi;

backend/src/tasks/price-feeds/mtgox-weekly.json

@@ -0,0 +1,762 @@
+[
+  {
+    "ct": 1279497600,
+    "c": "0.08584"
+  },
+  {
+    "ct": 1280102400,
+    "c": "0.0505"
+  },
+  {
+    "ct": 1280707200,
+    "c": "0.0611"
+  },
+  {
+    "ct": 1281312000,
+    "c": "0.0609"
+  },
+  {
+    "ct": 1281916800,
+    "c": "0.06529"
+  },
+  {
+    "ct": 1282521600,
+    "c": "0.066"
+  },
+  {
+    "ct": 1283126400,
+    "c": "0.064"
+  },
+  {
+    "ct": 1283731200,
+    "c": "0.06165"
+  },
+  {
+    "ct": 1284336000,
+    "c": "0.0615"
+  },
+  {
+    "ct": 1284940800,
+    "c": "0.0627"
+  },
+  {
+    "ct": 1285545600,
+    "c": "0.0622"
+  },
+  {
+    "ct": 1286150400,
+    "c": "0.06111"
+  },
+  {
+    "ct": 1286755200,
+    "c": "0.0965"
+  },
+  {
+    "ct": 1287360000,
+    "c": "0.102"
+  },
+  {
+    "ct": 1287964800,
+    "c": "0.11501"
+  },
+  {
+    "ct": 1288569600,
+    "c": "0.1925"
+  },
+  {
+    "ct": 1289174400,
+    "c": "0.34"
+  },
+  {
+    "ct": 1289779200,
+    "c": "0.27904"
+  },
+  {
+    "ct": 1290384000,
+    "c": "0.27675"
+  },
+  {
+    "ct": 1290988800,
+    "c": "0.27"
+  },
+  {
+    "ct": 1291593600,
+    "c": "0.19"
+  },
+  {
+    "ct": 1292198400,
+    "c": "0.2189"
+  },
+  {
+    "ct": 1292803200,
+    "c": "0.2401"
+  },
+  {
+    "ct": 1293408000,
+    "c": "0.263"
+  },
+  {
+    "ct": 1294012800,
+    "c": "0.29997"
+  },
+  {
+    "ct": 1294617600,
+    "c": "0.323"
+  },
+  {
+    "ct": 1295222400,
+    "c": "0.38679"
+  },
+  {
+    "ct": 1295827200,
+    "c": "0.4424"
+  },
+  {
+    "ct": 1296432000,
+    "c": "0.4799"
+  },
+  {
+    "ct": 1297036800,
+    "c": "0.8968"
+  },
+  {
+    "ct": 1297641600,
+    "c": "1.05"
+  },
+  {
+    "ct": 1298246400,
+    "c": "0.865"
+  },
+  {
+    "ct": 1298851200,
+    "c": "0.89"
+  },
+  {
+    "ct": 1299456000,
+    "c": "0.8999"
+  },
+  {
+    "ct": 1300060800,
+    "c": "0.89249"
+  },
+  {
+    "ct": 1300665600,
+    "c": "0.75218"
+  },
+  {
+    "ct": 1301270400,
+    "c": "0.82754"
+  },
+  {
+    "ct": 1301875200,
+    "c": "0.779"
+  },
+  {
+    "ct": 1302480000,
+    "c": "0.7369"
+  },
+  {
+    "ct": 1303084800,
+    "c": "1.1123"
+  },
+  {
+    "ct": 1303689600,
+    "c": "1.6311"
+  },
+  {
+    "ct": 1304294400,
+    "c": "3.03311"
+  },
+  {
+    "ct": 1304899200,
+    "c": "3.8659"
+  },
+  {
+    "ct": 1305504000,
+    "c": "6.98701"
+  },
+  {
+    "ct": 1306108800,
+    "c": "6.6901"
+  },
+  {
+    "ct": 1306713600,
+    "c": "8.4"
+  },
+  {
+    "ct": 1307318400,
+    "c": "16.7"
+  },
+  {
+    "ct": 1307923200,
+    "c": "18.5464"
+  },
+  {
+    "ct": 1308528000,
+    "c": "17.51"
+  },
+  {
+    "ct": 1309132800,
+    "c": "16.45001"
+  },
+  {
+    "ct": 1309737600,
+    "c": "15.44049"
+  },
+  {
+    "ct": 1310342400,
+    "c": "14.879"
+  },
+  {
+    "ct": 1310947200,
+    "c": "13.16"
+  },
+  {
+    "ct": 1311552000,
+    "c": "13.98001"
+  },
+  {
+    "ct": 1312156800,
+    "c": "13.35"
+  },
+  {
+    "ct": 1312761600,
+    "c": "7.9"
+  },
+  {
+    "ct": 1313366400,
+    "c": "10.7957"
+  },
+  {
+    "ct": 1313971200,
+    "c": "11.31125"
+  },
+  {
+    "ct": 1314576000,
+    "c": "9.07011"
+  },
+  {
+    "ct": 1315180800,
+    "c": "8.17798"
+  },
+  {
+    "ct": 1315785600,
+    "c": "5.86436"
+  },
+  {
+    "ct": 1316390400,
+    "c": "5.2"
+  },
+  {
+    "ct": 1316995200,
+    "c": "5.33"
+  },
+  {
+    "ct": 1317600000,
+    "c": "5.02701"
+  },
+  {
+    "ct": 1318204800,
+    "c": "4.10288"
+  },
+  {
+    "ct": 1318809600,
+    "c": "3.5574"
+  },
+  {
+    "ct": 1319414400,
+    "c": "3.12657"
+  },
+  {
+    "ct": 1320019200,
+    "c": "3.27"
+  },
+  {
+    "ct": 1320624000,
+    "c": "2.95959"
+  },
+  {
+    "ct": 1321228800,
+    "c": "2.99626"
+  },
+  {
+    "ct": 1321833600,
+    "c": "2.2"
+  },
+  {
+    "ct": 1322438400,
+    "c": "2.47991"
+  },
+  {
+    "ct": 1323043200,
+    "c": "2.82809"
+  },
+  {
+    "ct": 1323648000,
+    "c": "3.2511"
+  },
+  {
+    "ct": 1324252800,
+    "c": "3.193"
+  },
+  {
+    "ct": 1324857600,
+    "c": "4.225"
+  },
+  {
+    "ct": 1325462400,
+    "c": "5.26766"
+  },
+  {
+    "ct": 1326067200,
+    "c": "7.11358"
+  },
+  {
+    "ct": 1326672000,
+    "c": "7.00177"
+  },
+  {
+    "ct": 1327276800,
+    "c": "6.3097"
+  },
+  {
+    "ct": 1327881600,
+    "c": "5.38191"
+  },
+  {
+    "ct": 1328486400,
+    "c": "5.68881"
+  },
+  {
+    "ct": 1329091200,
+    "c": "5.51468"
+  },
+  {
+    "ct": 1329696000,
+    "c": "4.38669"
+  },
+  {
+    "ct": 1330300800,
+    "c": "4.922"
+  },
+  {
+    "ct": 1330905600,
+    "c": "4.8201"
+  },
+  {
+    "ct": 1331510400,
+    "c": "4.90901"
+  },
+  {
+    "ct": 1332115200,
+    "c": "5.27943"
+  },
+  {
+    "ct": 1332720000,
+    "c": "4.55001"
+  },
+  {
+    "ct": 1333324800,
+    "c": "4.81922"
+  },
+  {
+    "ct": 1333929600,
+    "c": "4.79253"
+  },
+  {
+    "ct": 1334534400,
+    "c": "4.96892"
+  },
+  {
+    "ct": 1335139200,
+    "c": "5.20352"
+  },
+  {
+    "ct": 1335744000,
+    "c": "4.90441"
+  },
+  {
+    "ct": 1336348800,
+    "c": "5.04991"
+  },
+  {
+    "ct": 1336953600,
+    "c": "4.92996"
+  },
+  {
+    "ct": 1337558400,
+    "c": "5.09002"
+  },
+  {
+    "ct": 1338163200,
+    "c": "5.13896"
+  },
+  {
+    "ct": 1338768000,
+    "c": "5.2051"
+  },
+  {
+    "ct": 1339372800,
+    "c": "5.46829"
+  },
+  {
+    "ct": 1339977600,
+    "c": "6.16382"
+  },
+  {
+    "ct": 1340582400,
+    "c": "6.35002"
+  },
+  {
+    "ct": 1341187200,
+    "c": "6.62898"
+  },
+  {
+    "ct": 1341792000,
+    "c": "6.79898"
+  },
+  {
+    "ct": 1342396800,
+    "c": "7.62101"
+  },
+  {
+    "ct": 1343001600,
+    "c": "8.4096"
+  },
+  {
+    "ct": 1343606400,
+    "c": "8.71027"
+  },
+  {
+    "ct": 1344211200,
+    "c": "10.86998"
+  },
+  {
+    "ct": 1344816000,
+    "c": "11.6239"
+  },
+  {
+    "ct": 1345420800,
+    "c": "7.98"
+  },
+  {
+    "ct": 1346025600,
+    "c": "10.61"
+  },
+  {
+    "ct": 1346630400,
+    "c": "10.2041"
+  },
+  {
+    "ct": 1347235200,
+    "c": "11.02"
+  },
+  {
+    "ct": 1347840000,
+    "c": "11.87"
+  },
+  {
+    "ct": 1348444800,
+    "c": "12.19331"
+  },
+  {
+    "ct": 1349049600,
+    "c": "12.4"
+  },
+  {
+    "ct": 1349654400,
+    "c": "11.8034"
+  },
+  {
+    "ct": 1350259200,
+    "c": "11.7389"
+  },
+  {
+    "ct": 1350864000,
+    "c": "11.63107"
+  },
+  {
+    "ct": 1351468800,
+    "c": "10.69998"
+  },
+  {
+    "ct": 1352073600,
+    "c": "10.80011"
+  },
+  {
+    "ct": 1352678400,
+    "c": "10.84692"
+  },
+  {
+    "ct": 1353283200,
+    "c": "11.65961"
+  },
+  {
+    "ct": 1353888000,
+    "c": "12.4821"
+  },
+  {
+    "ct": 1354492800,
+    "c": "12.50003"
+  },
+  {
+    "ct": 1355097600,
+    "c": "13.388"
+  },
+  {
+    "ct": 1355702400,
+    "c": "13.30002"
+  },
+  {
+    "ct": 1356307200,
+    "c": "13.31202"
+  },
+  {
+    "ct": 1356912000,
+    "c": "13.45001"
+  },
+  {
+    "ct": 1357516800,
+    "c": "13.5199"
+  },
+  {
+    "ct": 1358121600,
+    "c": "14.11601"
+  },
+  {
+    "ct": 1358726400,
+    "c": "15.7"
+  },
+  {
+    "ct": 1359331200,
+    "c": "17.95"
+  },
+  {
+    "ct": 1359936000,
+    "c": "20.59"
+  },
+  {
+    "ct": 1360540800,
+    "c": "23.96975"
+  },
+  {
+    "ct": 1361145600,
+    "c": "26.8146"
+  },
+  {
+    "ct": 1361750400,
+    "c": "29.88999"
+  },
+  {
+    "ct": 1362355200,
+    "c": "34.49999"
+  },
+  {
+    "ct": 1362960000,
+    "c": "46"
+  },
+  {
+    "ct": 1363564800,
+    "c": "47.4"
+  },
+  {
+    "ct": 1364169600,
+    "c": "71.93"
+  },
+  {
+    "ct": 1364774400,
+    "c": "93.03001"
+  },
+  {
+    "ct": 1365379200,
+    "c": "162.30102"
+  },
+  {
+    "ct": 1365984000,
+    "c": "89.99999"
+  },
+  {
+    "ct": 1366588800,
+    "c": "119.2"
+  },
+  {
+    "ct": 1367193600,
+    "c": "134.44444"
+  },
+  {
+    "ct": 1367798400,
+    "c": "115.98"
+  },
+  {
+    "ct": 1368403200,
+    "c": "114.82002"
+  },
+  {
+    "ct": 1369008000,
+    "c": "122.49999"
+  },
+  {
+    "ct": 1369612800,
+    "c": "133.5"
+  },
+  {
+    "ct": 1370217600,
+    "c": "122.5"
+  },
+  {
+    "ct": 1370822400,
+    "c": "100.43743"
+  },
+  {
+    "ct": 1371427200,
+    "c": "99.9"
+  },
+  {
+    "ct": 1372032000,
+    "c": "107.90001"
+  },
+  {
+    "ct": 1372636800,
+    "c": "97.51"
+  },
+  {
+    "ct": 1373241600,
+    "c": "76.5"
+  },
+  {
+    "ct": 1373846400,
+    "c": "94.41986"
+  },
+  {
+    "ct": 1374451200,
+    "c": "91.998"
+  },
+  {
+    "ct": 1375056000,
+    "c": "98.78008"
+  },
+  {
+    "ct": 1375660800,
+    "c": "105.12"
+  },
+  {
+    "ct": 1376265600,
+    "c": "105"
+  },
+  {
+    "ct": 1376870400,
+    "c": "113.38"
+  },
+  {
+    "ct": 1377475200,
+    "c": "122.11102"
+  },
+  {
+    "ct": 1378080000,
+    "c": "146.01003"
+  },
+  {
+    "ct": 1378684800,
+    "c": "126.31501"
+  },
+  {
+    "ct": 1379289600,
+    "c": "138.3002"
+  },
+  {
+    "ct": 1379894400,
+    "c": "134.00001"
+  },
+  {
+    "ct": 1380499200,
+    "c": "143.88402"
+  },
+  {
+    "ct": 1381104000,
+    "c": "137.8"
+  },
+  {
+    "ct": 1381708800,
+    "c": "147.53"
+  },
+  {
+    "ct": 1382313600,
+    "c": "186.1"
+  },
+  {
+    "ct": 1382918400,
+    "c": "207.0001"
+  },
+  {
+    "ct": 1383523200,
+    "c": "224.01001"
+  },
+  {
+    "ct": 1384128000,
+    "c": "336.33101"
+  },
+  {
+    "ct": 1384732800,
+    "c": "528"
+  },
+  {
+    "ct": 1385337600,
+    "c": "795"
+  },
+  {
+    "ct": 1385942400,
+    "c": "1004.42392"
+  },
+  {
+    "ct": 1386547200,
+    "c": "804.5"
+  },
+  {
+    "ct": 1387152000,
+    "c": "919.985"
+  },
+  {
+    "ct": 1387756800,
+    "c": "639.48"
+  },
+  {
+    "ct": 1388361600,
+    "c": "786.98"
+  },
+  {
+    "ct": 1388966400,
+    "c": "1015"
+  },
+  {
+    "ct": 1389571200,
+    "c": "940"
+  },
+  {
+    "ct": 1390176000,
+    "c": "954.995"
+  },
+  {
+    "ct": 1390780800,
+    "c": "1007.98999"
+  },
+  {
+    "ct": 1391385600,
+    "c": "954"
+  },
+  {
+    "ct": 1391990400,
+    "c": "659.49776"
+  },
+  {
+    "ct": 1392595200,
+    "c": "299.702"
+  },
+  {
+    "ct": 1393200000,
+    "c": "310.00001"
+  },
+  {
+    "ct": 1393804800,
+    "c": "135"
+  }
+]

backend/src/tasks/price-updater.ts

@@ -0,0 +1,258 @@
+import * as fs from 'fs';
+import config from '../config';
+import logger from '../logger';
+import PricesRepository from '../repositories/PricesRepository';
+import BitfinexApi from './price-feeds/bitfinex-api';
+import BitflyerApi from './price-feeds/bitflyer-api';
+import CoinbaseApi from './price-feeds/coinbase-api';
+import FtxApi from './price-feeds/ftx-api';
+import GeminiApi from './price-feeds/gemini-api';
+import KrakenApi from './price-feeds/kraken-api';
+
+export interface PriceFeed {
+  name: string;
+  url: string;
+  urlHist: string;
+  currencies: string[];
+
+  $fetchPrice(currency): Promise<number>;
+  $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory>;
+}
+
+export interface PriceHistory {
+  [timestamp: number]: Prices;
+}
+
+export interface Prices {
+  USD: number;
+  EUR: number;
+  GBP: number;
+  CAD: number;
+  CHF: number;
+  AUD: number;
+  JPY: number;
+}
+
+class PriceUpdater {
+  historyInserted: boolean = false;
+  lastRun: number = 0;
+  lastHistoricalRun: number = 0;
+  running: boolean = false;
+  feeds: PriceFeed[] = [];
+  currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY'];
+  latestPrices: Prices;
+
+  constructor() {
+    this.latestPrices = this.getEmptyPricesObj();
+
+    this.feeds.push(new BitflyerApi()); // Does not have historical endpoint
+    this.feeds.push(new FtxApi());
+    this.feeds.push(new KrakenApi());
+    this.feeds.push(new CoinbaseApi());
+    this.feeds.push(new BitfinexApi());
+    this.feeds.push(new GeminiApi());
+  }
+
+  public getEmptyPricesObj(): Prices {
+    return {
+      USD: -1,
+      EUR: -1,
+      GBP: -1,
+      CAD: -1,
+      CHF: -1,
+      AUD: -1,
+      JPY: -1,
+    };
+  }
+
+  public async $run(): Promise<void> {
+    if (this.running === true) {
+      return;
+    }
+    this.running = true;
+
+    if ((Math.round(new Date().getTime() / 1000) - this.lastHistoricalRun) > 3600 * 24) {
+      // Once a day, look for missing prices (could happen due to network connectivity issues)
+      this.historyInserted = false;
+    }
+
+    try {
+      if (this.historyInserted === false && config.DATABASE.ENABLED === true) {
+        await this.$insertHistoricalPrices();
+      } else {
+        await this.$updatePrice();
+      }
+    } catch (e) {
+      logger.err(`Cannot save BTC prices in db. Reason: ${e instanceof Error ? e.message : e}`);
+    }
+
+    this.running = false;
+  }
+
+  /**
+   * Fetch last BTC price from exchanges, average them, and save it in the database once every hour
+   */
+  private async $updatePrice(): Promise<void> {
+    if (this.lastRun === 0 && config.DATABASE.ENABLED === true) {
+      this.lastRun = await PricesRepository.$getLatestPriceTime();
+    }
+
+    if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < 3600) {
+      // Refresh only once every hour
+      return;
+    }
+
+    const previousRun = this.lastRun;
+    this.lastRun = new Date().getTime() / 1000;
+
+    for (const currency of this.currencies) {
+      let prices: number[] = [];
+
+      for (const feed of this.feeds) {
+        // Fetch prices from API which supports `currency`
+        if (feed.currencies.includes(currency)) {
+          try {
+            const price = await feed.$fetchPrice(currency);
+            if (price > 0) {
+              prices.push(price);
+            }
+            logger.debug(`${feed.name} BTC/${currency} price: ${price}`);
+          } catch (e) {
+            logger.debug(`Could not fetch BTC/${currency} price at ${feed.name}. Reason: ${(e instanceof Error ? e.message : e)}`);
+          }
+        }
+      }
+      if (prices.length === 1) {
+        logger.debug(`Only ${prices.length} feed available for BTC/${currency} price`);
+      }
+
+      // Compute average price, non weighted
+      prices = prices.filter(price => price > 0);
+      this.latestPrices[currency] = Math.round((prices.reduce((partialSum, a) => partialSum + a, 0)) / prices.length);
+    }
+
+    logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`);
+
+    if (config.DATABASE.ENABLED === true) {
+      // Save everything in db
+      try {
+        const p = 60 * 60 * 1000; // milliseconds in an hour
+        const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042
+        await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices);
+      } catch (e) {
+        this.lastRun = previousRun + 5 * 60;
+        logger.err(`Cannot save latest prices into db. Trying again in 5 minutes. Reason: ${(e instanceof Error ? e.message : e)}`);
+      }
+    }
+
+    this.lastRun = new Date().getTime() / 1000;
+  }
+
+  /**
+   * Called once by the database migration to initialize historical prices data (weekly)
+   * We use MtGox weekly price from July 19, 2010 to September 30, 2013
+   * We use Kraken weekly price from October 3, 2013 up to last month
+   * We use Kraken hourly price for the past month
+   */
+  private async $insertHistoricalPrices(): Promise<void> {
+    const existingPriceTimes = await PricesRepository.$getPricesTimes();
+
+    // Insert MtGox weekly prices
+    const pricesJson: any[] = JSON.parse(fs.readFileSync('./src/tasks/price-feeds/mtgox-weekly.json').toString());
+    const prices = this.getEmptyPricesObj();
+    let insertedCount: number = 0;
+    for (const price of pricesJson) {
+      if (existingPriceTimes.includes(price['ct'])) {
+        continue;
+      }
+
+      // From 1380758400 we will use Kraken price as it follows closely MtGox, but was not affected as much
+      // by the MtGox exchange collapse a few months later
+      if (price['ct'] > 1380758400) {
+        break;
+      }
+      prices.USD = price['c'];
+      await PricesRepository.$savePrices(price['ct'], prices);
+      ++insertedCount;
+    }
+    if (insertedCount > 0) {
+      logger.notice(`Inserted ${insertedCount} MtGox USD weekly price history into db`);
+    }
+
+    // Insert Kraken weekly prices
+    await new KrakenApi().$insertHistoricalPrice();
+
+    // Insert missing recent hourly prices
+    await this.$insertMissingRecentPrices();
+
+    this.historyInserted = true;
+    this.lastHistoricalRun = new Date().getTime();
+  }
+
+  /**
+   * Find missing hourly prices and insert them in the database
+   * It has a limited backward range and it depends on which API are available
+   */
+  private async $insertMissingRecentPrices(): Promise<void> {
+    const existingPriceTimes = await PricesRepository.$getPricesTimes();
+
+    logger.info(`Fetching hourly price history from exchanges and saving missing ones into the database, this may take a while`);
+
+    const historicalPrices: PriceHistory[] = [];
+
+    // Fetch all historical hourly prices
+    for (const feed of this.feeds) {
+      try {
+        historicalPrices.push(await feed.$fetchRecentHourlyPrice(this.currencies));
+      } catch (e) {
+        logger.err(`Cannot fetch hourly historical price from ${feed.name}. Ignoring this feed. Reason: ${e instanceof Error ? e.message : e}`);
+      }
+    }
+
+    // Group them by timestamp and currency, for example
+    // grouped[123456789]['USD'] = [1, 2, 3, 4];
+    const grouped: Object = {};
+    for (const historicalEntry of historicalPrices) {
+      for (const time in historicalEntry) {
+        if (existingPriceTimes.includes(parseInt(time, 10))) {
+          continue;
+        }
+
+        if (grouped[time] === undefined) {
+          grouped[time] = {
+            USD: [], EUR: [], GBP: [], CAD: [], CHF: [], AUD: [], JPY: []
+          };
+        }
+
+        for (const currency of this.currencies) {
+          const price = historicalEntry[time][currency];
+          if (price > 0) {
+            grouped[time][currency].push(parseInt(price, 10));
+          }
+        }
+      }
+    }
+
+    // Average prices and insert everything into the db
+    let totalInserted = 0;
+    for (const time in grouped) {
+      const prices: Prices = this.getEmptyPricesObj();
+      for (const currency in grouped[time]) {
+        if (grouped[time][currency].length === 0) {
+          continue;
+        }
+        prices[currency] = Math.round((grouped[time][currency].reduce(
+          (partialSum, a) => partialSum + a, 0)
+        ) / grouped[time][currency].length);
+      }
+      await PricesRepository.$savePrices(parseInt(time, 10), prices);
+      ++totalInserted;
+    }
+
+    if (totalInserted > 0) {
+      logger.notice(`Inserted ${totalInserted} hourly historical prices into the db`);
+    }
+  }
+}
+
+export default new PriceUpdater();

backend/src/utils/axios-query.ts

@@ -0,0 +1,63 @@
+import axios, { AxiosResponse } from 'axios';
+import { SocksProxyAgent } from 'socks-proxy-agent';
+import backendInfo from '../api/backend-info';
+import config from '../config';
+import logger from '../logger';
+import * as https from 'https';
+
+export async function query(path): Promise<object | undefined> {
+ type axiosOptions = {
+   headers: {
+     'User-Agent': string
+   };
+   timeout: number;
+   httpsAgent?: https.Agent;
+ };
+ const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
+ const axiosOptions: axiosOptions = {
+   headers: {
+     'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+   },
+   timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
+ };
+ let retry = 0;
+
+ while (retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
+   try {
+     if (config.SOCKS5PROXY.ENABLED) {
+       const socksOptions: any = {
+         agentOptions: {
+           keepAlive: true,
+         },
+         hostname: config.SOCKS5PROXY.HOST,
+         port: config.SOCKS5PROXY.PORT
+       };
+
+       if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+         socksOptions.username = config.SOCKS5PROXY.USERNAME;
+         socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+       } else {
+         // Retry with different tor circuits https://stackoverflow.com/a/64960234
+         socksOptions.username = `circuit${retry}`;
+       }
+
+       axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
+     }
+
+     const data: AxiosResponse = await axios.get(path, axiosOptions);
+     if (data.statusText === 'error' || !data.data) {
+       throw new Error(`Could not fetch data from ${path}, Error: ${data.status}`);
+     }
+     return data.data;
+   } catch (e) {
+     logger.warn(`Could not connect to ${path} (Attempt ${retry + 1}/${config.MEMPOOL.EXTERNAL_MAX_RETRY}). Reason: ` + (e instanceof Error ? e.message : e));
+     retry++;
+   }
+   if (retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
+     await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
+   }
+ }
+
+ logger.err(`Could not connect to ${path}. All ${config.MEMPOOL.EXTERNAL_MAX_RETRY} attempts failed`);
+ return undefined;
+}

backend/src/utils/blocks-utils.ts

@@ -3,14 +3,14 @@ import { BlockExtended } from '../mempool.interfaces';
 export function prepareBlock(block: any): BlockExtended {
   return <BlockExtended>{
     id: block.id ?? block.hash, // hash for indexed block
-    timestamp: block.timestamp ?? block.blockTimestamp, // blockTimestamp for indexed block
+    timestamp: block.timestamp ?? block.time ?? block.blockTimestamp, // blockTimestamp for indexed block
     height: block.height,
     version: block.version,
-    bits: block.bits,
+    bits: (typeof block.bits === 'string' ? parseInt(block.bits, 16): block.bits),
     nonce: block.nonce,
     difficulty: block.difficulty,
-    merkle_root: block.merkle_root,
-    tx_count: block.tx_count,
+    merkle_root: block.merkle_root ?? block.merkleroot,
+    tx_count: block.tx_count ?? block.nTx,
     size: block.size,
     weight: block.weight,
     previousblockhash: block.previousblockhash,

backend/tsconfig.json

@@ -1,5 +1,6 @@
 {
   "compilerOptions": {
+    "types": ["node"],
     "module": "commonjs",
     "target": "esnext",
     "lib": ["es2019", "dom"],
@@ -11,7 +12,8 @@
     "typeRoots": [
       "node_modules/@types"
     ],
-    "allowSyntheticDefaultImports": true
+    "allowSyntheticDefaultImports": true,
+    "esModuleInterop": true
   },
   "include": [
     "src/**/*.ts"

backend/tslint.json

@@ -1,137 +0,0 @@
-{
-  "rules": {
-    "arrow-return-shorthand": true,
-    "callable-types": true,
-    "class-name": true,
-    "comment-format": [
-      true,
-      "check-space"
-    ],
-    "curly": true,
-    "deprecation": {
-      "severity": "warn"
-    },
-    "eofline": true,
-    "forin": false,
-    "import-blacklist": [
-      true,
-      "rxjs",
-      "rxjs/Rx"
-    ],
-    "import-spacing": true,
-    "indent": [
-      true,
-      "spaces"
-    ],
-    "interface-over-type-literal": true,
-    "label-position": true,
-    "max-line-length": [
-      true,
-      140
-    ],
-    "member-access": false,
-    "member-ordering": [
-      true,
-      {
-        "order": [
-          "static-field",
-          "instance-field",
-          "static-method",
-          "instance-method"
-        ]
-      }
-    ],
-    "no-arg": true,
-    "no-bitwise": true,
-    "no-console": [
-      true,
-      "debug",
-      "info",
-      "time",
-      "timeEnd",
-      "trace"
-    ],
-    "no-construct": true,
-    "no-debugger": true,
-    "no-duplicate-super": true,
-    "no-empty": false,
-    "no-empty-interface": true,
-    "no-eval": true,
-    "no-inferrable-types": false,
-    "no-misused-new": true,
-    "no-non-null-assertion": true,
-    "no-shadowed-variable": true,
-    "no-string-literal": false,
-    "no-string-throw": true,
-    "no-switch-case-fall-through": true,
-    "no-trailing-whitespace": true,
-    "no-unnecessary-initializer": true,
-    "no-unused-expression": true,
-    "no-use-before-declare": true,
-    "no-var-keyword": true,
-    "object-literal-sort-keys": false,
-    "one-line": [
-      true,
-      "check-open-brace",
-      "check-catch",
-      "check-else",
-      "check-whitespace"
-    ],
-    "prefer-const": true,
-    "quotemark": [
-      true,
-      "single"
-    ],
-    "radix": true,
-    "semicolon": [
-      true,
-      "always"
-    ],
-    "triple-equals": [
-      true,
-      "allow-null-check"
-    ],
-    "typedef-whitespace": [
-      true,
-      {
-        "call-signature": "nospace",
-        "index-signature": "nospace",
-        "parameter": "nospace",
-        "property-declaration": "nospace",
-        "variable-declaration": "nospace"
-      }
-    ],
-    "unified-signatures": true,
-    "variable-name": false,
-    "whitespace": [
-      true,
-      "check-branch",
-      "check-decl",
-      "check-operator",
-      "check-separator",
-      "check-type"
-    ],
-    "directive-selector": [
-      true,
-      "attribute",
-      "app",
-      "camelCase"
-    ],
-    "component-selector": [
-      true,
-      "element",
-      "app",
-      "kebab-case"
-    ],
-    "no-output-on-prefix": true,
-    "use-input-property-decorator": true,
-    "use-output-property-decorator": true,
-    "use-host-property-decorator": true,
-    "no-input-rename": true,
-    "no-output-rename": true,
-    "use-life-cycle-interface": true,
-    "use-pipe-transform-interface": true,
-    "component-class-suffix": true,
-    "directive-class-suffix": true
-  }
-}

contributors/erikarvstedt.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of July 6, 2022.
+
+Signed: erikarvstedt

docker/README.md

@@ -4,6 +4,8 @@ This directory contains the Dockerfiles used to build and release the official i
 
 If you are looking to use these Docker images to deploy your own instance of Mempool, note that they only containerize Mempool's frontend and backend. You will still need to deploy and configure Bitcoin Core and an Electrum Server separately, along with any other utilities specific to your use case (e.g., a reverse proxy, etc). Such configuration is mostly beyond the scope of the Mempool project, so please only proceed if you know what you're doing.
 
+See a video guide of this installation method by k3tan [on BitcoinTV.com](https://bitcointv.com/w/8fpAx6rf5CQ16mMhospwjg).
+
 Jump to a section in this doc:
 - [Configure with Bitcoin Core Only](#configure-with-bitcoin-core-only)
 - [Configure with Bitcoin Core + Electrum Server](#configure-with-bitcoin-core--electrum-server)
@@ -233,7 +235,7 @@ Corresponding `docker-compose.yml` overrides:
       DATABASE_HOST: ""
       DATABASE_PORT: ""
       DATABASE_DATABASE: ""
-      DATABASE_USERAME: ""
+      DATABASE_USERNAME: ""
       DATABASE_PASSWORD: ""
       ...
 ```

docker/backend/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:16.15.0-buster-slim AS builder
+FROM node:16.16.0-buster-slim AS builder
 
 ARG commitHash
 ENV DOCKER_COMMIT_HASH=${commitHash}
@@ -8,10 +8,10 @@ COPY . .
 
 RUN apt-get update
 RUN apt-get install -y build-essential python3 pkg-config
-RUN npm install
+RUN npm install --omit=dev --omit=optional
 RUN npm run build
 
-FROM node:16.15.0-buster-slim
+FROM node:16.16.0-buster-slim
 
 WORKDIR /backend
 

docker/backend/mempool-config.json

@@ -19,7 +19,9 @@
     "EXTERNAL_RETRY_INTERVAL": __MEMPOOL_EXTERNAL_RETRY_INTERVAL__,
     "USER_AGENT": "__MEMPOOL_USER_AGENT__",
     "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
-    "INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__
+    "INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__,
+    "BLOCKS_SUMMARIES_INDEXING": __MEMPOOL_BLOCKS_SUMMARIES_INDEXING__,
+    "AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",

docker/backend/start.sh

@@ -14,6 +14,7 @@ __MEMPOOL_BLOCK_WEIGHT_UNITS__=${MEMPOOL_BLOCK_WEIGHT_UNITS:=4000000}
 __MEMPOOL_INITIAL_BLOCKS_AMOUNT__=${MEMPOOL_INITIAL_BLOCKS_AMOUNT:=8}
 __MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_MEMPOOL_BLOCKS_AMOUNT:=8}
 __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=11000}
+__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__=${MEMPOOL_BLOCKS_SUMMARIES_INDEXING:=false}
 __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=600}
 __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__=${MEMPOOL_USE_SECOND_NODE_FOR_MINFEE:=false}
 __MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[]}
@@ -21,6 +22,8 @@ __MEMPOOL_EXTERNAL_MAX_RETRY__=${MEMPOOL_EXTERNAL_MAX_RETRY:=1}
 __MEMPOOL_EXTERNAL_RETRY_INTERVAL__=${MEMPOOL_EXTERNAL_RETRY_INTERVAL:=0}
 __MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool}
 __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info}
+__MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=false}
+__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false}
 
 # CORE_RPC
 __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
@@ -101,6 +104,7 @@ sed -i "s/__MEMPOOL_BLOCK_WEIGHT_UNITS__/${__MEMPOOL_BLOCK_WEIGHT_UNITS__}/g" me
 sed -i "s/__MEMPOOL_INITIAL_BLOCKS_AMOUNT__/${__MEMPOOL_INITIAL_BLOCKS_AMOUNT__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__/${__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json
+sed -i "s/__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__/${__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__/${__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__/${__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__}/g" mempool-config.json
 sed -i "s!__MEMPOOL_EXTERNAL_ASSETS__!${__MEMPOOL_EXTERNAL_ASSETS__}!g" mempool-config.json
@@ -108,6 +112,8 @@ sed -i "s!__MEMPOOL_EXTERNAL_MAX_RETRY__!${__MEMPOOL_EXTERNAL_MAX_RETRY__}!g" me
 sed -i "s!__MEMPOOL_EXTERNAL_RETRY_INTERVAL__!${__MEMPOOL_EXTERNAL_RETRY_INTERVAL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_USER_AGENT__!${__MEMPOOL_USER_AGENT__}!g" mempool-config.json
 sed -i "s/__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__/${__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__}/g" mempool-config.json
+sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json
+sed -i "s/__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__/${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}/g" mempool-config.json
 
 sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json

docker/frontend/Dockerfile

@@ -1,4 +1,4 @@
-FROM node:16.15.0-buster-slim AS builder
+FROM node:16.16.0-buster-slim AS builder
 
 ARG commitHash
 ENV DOCKER_COMMIT_HASH=${commitHash}
@@ -8,7 +8,7 @@ WORKDIR /build
 COPY . .
 RUN apt-get update
 RUN apt-get install -y build-essential rsync
-RUN npm i
+RUN npm install --omit=dev --omit=optional
 RUN npm run build
 
 FROM nginx:1.17.8-alpine

frontend/.editorconfig

@@ -8,6 +8,10 @@ indent_size = 2
 insert_final_newline = true
 trim_trailing_whitespace = true
 
+[*.ts]
+quote_type = single
+
 [*.md]
 max_line_length = off
 trim_trailing_whitespace = false
+

frontend/.eslintignore

@@ -0,0 +1,3 @@
+node_modules
+dist
+frontend

frontend/.eslintrc

@@ -0,0 +1,34 @@
+{
+  "root": true,
+  "parser": "@typescript-eslint/parser",
+  "plugins": [
+    "@typescript-eslint"
+  ],
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/eslint-recommended",
+    "plugin:@typescript-eslint/recommended"
+  ],
+  "rules": {
+    "@typescript-eslint/ban-ts-comment": 1,
+    "@typescript-eslint/ban-types": 1,
+    "@typescript-eslint/no-empty-function": 1,
+    "@typescript-eslint/no-explicit-any": 1,
+    "@typescript-eslint/no-inferrable-types": 1,
+    "@typescript-eslint/no-namespace": 1,
+    "@typescript-eslint/no-this-alias": 1,
+    "@typescript-eslint/no-var-requires": 1,
+    "no-case-declarations": 1,
+    "no-console": 1,
+    "no-constant-condition": 1,
+    "no-dupe-else-if": 1,
+    "no-empty": 1,
+    "no-extra-boolean-cast": 1,
+    "no-prototype-builtins": 1,
+    "no-self-assign": 1,
+    "no-useless-catch": 1,
+    "no-var": 1,
+    "prefer-const": 1,
+    "prefer-rest-params": 1
+  }
+}

frontend/.prettierignore

@@ -0,0 +1,2 @@
+node_modules
+package-lock.json

frontend/.prettierrc

@@ -0,0 +1,6 @@
+{
+  "endOfLine": "lf",
+  "printWidth": 80,
+  "tabWidth": 2,
+  "trailingComma": "es5"
+}

frontend/README.md

@@ -34,7 +34,7 @@ $ npm run config:defaults:bisq
 
 ### 3. Run the Frontend
 
-_Make sure to use Node.js 16.15 and npm 7._
+_Make sure to use Node.js 16.10 and npm 7._
 
 Install project dependencies and run the frontend server:
 
@@ -71,13 +71,13 @@ Set up the [Mempool backend](../backend/) first, if you haven't already.
 
 ### 1. Build the Frontend
 
-_Node.js 16 and npm 7 are recommended._
+_Make sure to use Node.js 16.10 and npm 7._
 
 Build the frontend:
 
 ```
 cd frontend
-npm install # add --prod for production
+npm install
 npm run build
 ```
 

frontend/angular.json

@@ -248,23 +248,6 @@
             "browserTarget": "mempool:build"
           }
         },
-        "test": {
-          "builder": "@angular-devkit/build-angular:karma",
-          "options": {
-            "main": "src/test.ts",
-            "polyfills": "src/polyfills.ts",
-            "tsConfig": "tsconfig.spec.json",
-            "karmaConfig": "karma.conf.js",
-            "assets": [
-              "src/favicon.ico",
-              "src/resources"
-            ],
-            "styles": [
-              "src/styles.scss"
-            ],
-            "scripts": []
-          }
-        },
         "e2e": {
           "builder": "@cypress/schematic:cypress",
           "options": {

frontend/cypress/e2e/mainnet/mining.spec.ts

@@ -0,0 +1,171 @@
+const baseModule = Cypress.env("BASE_MODULE");
+
+describe('Mainnet - Mining Features', () => {
+  beforeEach(() => {
+    //https://github.com/cypress-io/cypress/issues/14459
+    if (Cypress.browser.family === 'chromium') {
+      Cypress.automation('remote:debugger:protocol', {
+        command: 'Network.enable',
+        params: {}
+      });
+      Cypress.automation('remote:debugger:protocol', {
+        command: 'Network.setCacheDisabled',
+        params: { cacheDisabled: true }
+      });
+    }
+  });
+
+  if (baseModule === 'mempool') {
+
+    describe('Miner page', () => {
+      beforeEach(() => {
+        cy.intercept('/api/v1/mining/pool/**').as('pool');
+        cy.intercept('/api/v1/mining/hashrate/pools/**').as('hashrate');
+        cy.intercept('/api/tx/**').as('tx');
+        cy.intercept('/api/v1/outpends/**').as('outspends');
+      });
+      it('loads the mining pool page from the dashboard', () => {
+        cy.visit('/mining');
+        cy.waitForSkeletonGone();
+        cy.get('[data-cy="bitcoin-block-0-pool"]').click().then(() => {
+          cy.waitForSkeletonGone();
+          cy.wait('@pool');
+          cy.url().should('match', /\/mining\/pool\/(\w+)/);
+        });
+      });
+
+      it('loads the mining pool page from the blocks page', () => {
+        cy.visit('/mining');
+        cy.waitForSkeletonGone();
+        cy.get('[data-cy="bitcoin-block-0-height"]').click().then(() => {
+          cy.waitForSkeletonGone();
+          cy.get('[data-cy="block-details-miner-badge"]').click().then(() => {
+            cy.waitForSkeletonGone();
+            cy.wait('@pool');
+            cy.url().should('match', /\/mining\/pool\/(\w+)/);
+          });
+        });
+      });
+    });
+
+    describe('Mining Dashboard Landing page widgets', () => {
+
+      beforeEach(() => {
+        cy.visit('/mining');
+        cy.waitForSkeletonGone();
+      });
+
+      it('shows the mempool blocks', () => {
+        cy.get('[data-cy="mempool-block-0-fees"]').invoke('text').should('match', /~(.*) sat\/vB/);
+        cy.get('[data-cy="mempool-block-0-fee-span"]').invoke('text').should('match', /(.*) - (.*) sat\/vB/);
+        cy.get('[data-cy="mempool-block-0-total-fees"]').invoke('text').should('match', /(.*) BTC/);
+        cy.get('[data-cy="mempool-block-0-transaction-count"]').invoke('text').should('match', /(.*) transactions/);
+        cy.get('[data-cy="mempool-block-0-time"]').invoke('text').should('match', /In ~(.*) minutes/);
+      });
+
+      it('shows the mined blocks', () => {
+        cy.get('[data-cy="bitcoin-block-0-height"]').invoke('text').should('match', /(\d)/);
+        cy.get('[data-cy="bitcoin-block-0-fees"]').invoke('text').should('match', /~(.*) sat\/vB/);
+        cy.get('[data-cy="bitcoin-block-0-fee-span"]').invoke('text').should('match', /(.*) - (.*) sat\/vB/);
+        cy.get('[data-cy="bitcoin-block-0-total-fees"]').invoke('text').should('match', /(.*) BTC/);
+        cy.get('[data-cy="bitcoin-block-0-transactions"]').invoke('text').should('match', /(.*) transactions/);
+        cy.get('[data-cy="bitcoin-block-0-time"]').invoke('text').should('match', /((.*) ago|Just now)/);
+        cy.get('[data-cy="bitcoin-block-0-pool"]').invoke('text').should('match', /(\w)/);
+      });
+
+      it('shows the reward stats for the last 144 blocks', () => {
+        cy.get('[data-cy="reward-stats"]');
+      });
+
+      it('shows the difficulty adjustment stats', () => {
+        cy.get('[data-cy="difficulty-adjustment"]');
+      });
+
+      it('shows the latest blocks', () => {
+        cy.get('[data-cy="latest-blocks"]');
+      });
+
+      it('shows the pools pie chart', () => {
+        cy.get('[data-cy="pool-distribution"]');
+      });
+
+      it('shows the hashrate graph', () => {
+        cy.get('[data-cy="hashrate-graph"]');
+      });
+      it('shows the latest blocks', () => {
+        cy.get('[data-cy="latest-blocks"]');
+      });
+
+      it('shows the latest adjustments', () => {
+        cy.get('[data-cy="difficulty-adjustments-table"]');
+      });
+    });
+
+    describe.only('mining graphs', () => {
+      describe('pools ranking', () => {
+        it('loads the graph', () => {
+          cy.visit('/graphs/mining/pools');
+          cy.waitForSkeletonGone();
+          cy.waitForPageIdle();
+          cy.get('.spinner-border').should('not.exist');
+        });
+      });
+
+      describe('pools dominance', () => {
+        it('loads the graph', () => {
+          cy.visit('/graphs/mining/pools-dominance');
+          cy.waitForSkeletonGone();
+          cy.waitForPageIdle();
+          cy.get('.spinner-border').should('not.exist');
+        });
+      });
+
+      describe('hashrate & difficulty', () => {
+        it('loads the graph', () => {
+          cy.visit('/graphs/mining/hashrate-difficulty');
+          cy.waitForSkeletonGone();
+          cy.waitForPageIdle();
+          cy.get('.spinner-border').should('not.exist');
+        });
+      });
+
+      describe('block fee rates', () => {
+        it('loads the graph', () => {
+          cy.visit('/graphs/mining/block-fee-rates');
+          cy.waitForSkeletonGone();
+          cy.waitForPageIdle();
+          cy.get('.spinner-border').should('not.exist');
+        });
+      });
+
+      describe('block fees', () => {
+        it('loads the graph', () => {
+          cy.visit('/graphs/mining/block-fees');
+          cy.waitForSkeletonGone();
+          cy.waitForPageIdle();
+          cy.get('.spinner-border').should('not.exist');
+        });
+      });
+
+      describe('block rewards', () => {
+        it('loads the graph', () => {
+          cy.visit('/graphs/mining/block-rewards');
+          cy.waitForSkeletonGone();
+          cy.waitForPageIdle();
+          cy.get('.spinner-border').should('not.exist');
+        });
+      });
+
+      describe('block sizes and weights', () => {
+        it('loads the graph', () => {
+          cy.visit('/graphs/mining/block-sizes-weights');
+          cy.waitForSkeletonGone();
+          cy.waitForPageIdle();
+          cy.get('.spinner-border').should('not.exist');
+        });
+      });
+    });
+  } else {
+    it.skip(`Tests cannot be run on the selected BASE_MODULE ${baseModule}`);
+  }
+});

frontend/karma.conf.js

@@ -1,32 +0,0 @@
-// Karma configuration file, see link for more information
-// https://karma-runner.github.io/1.0/config/configuration-file.html
-
-module.exports = function (config) {
-  config.set({
-    basePath: '',
-    frameworks: ['jasmine', '@angular-devkit/build-angular'],
-    plugins: [
-      require('karma-jasmine'),
-      require('karma-chrome-launcher'),
-      require('karma-jasmine-html-reporter'),
-      require('karma-coverage-istanbul-reporter'),
-      require('@angular-devkit/build-angular/plugins/karma')
-    ],
-    client: {
-      clearContext: false // leave Jasmine Spec Runner output visible in browser
-    },
-    coverageIstanbulReporter: {
-      dir: require('path').join(__dirname, './coverage/mempool'),
-      reports: ['html', 'lcovonly', 'text-summary'],
-      fixWebpackSourcePaths: true
-    },
-    reporters: ['progress', 'kjhtml'],
-    port: 9876,
-    colors: true,
-    logLevel: config.LOG_INFO,
-    autoWatch: true,
-    browsers: ['Chrome'],
-    singleRun: false,
-    restartOnFileChange: true
-  });
-};

frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mempool-frontend",
-  "version": "2.4.0",
+  "version": "2.4.1-dev",
   "description": "Bitcoin mempool visualizer and blockchain explorer backend",
   "license": "GNU Affero General Public License v3.0",
   "homepage": "https://mempool.space",
@@ -20,20 +20,20 @@
   ],
   "main": "index.ts",
   "scripts": {
-    "ng": "./node_modules/@angular/cli/bin/ng",
+    "ng": "./node_modules/@angular/cli/bin/ng.js",
     "tsc": "./node_modules/typescript/bin/tsc",
     "i18n-extract-from-source": "./node_modules/@angular/cli/bin/ng extract-i18n --out-file ./src/locale/messages.xlf",
     "i18n-pull-from-transifex": "tx pull -a --parallel --minimum-perc 1 --force",
-    "serve": "npm run generate-config && ng serve -c local",
-    "serve:stg": "npm run generate-config && ng serve -c staging",
-    "serve:local-prod": "npm run generate-config && ng serve -c local-prod",
-    "serve:local-staging": "npm run generate-config && ng serve -c local-staging",
-    "start": "npm run generate-config && npm run sync-assets-dev && ng serve -c local",
-    "start:stg": "npm run generate-config && npm run sync-assets-dev && ng serve -c staging",
-    "start:local-prod": "npm run generate-config && npm run sync-assets-dev && ng serve -c local-prod",
-    "start:local-staging": "npm run generate-config && npm run sync-assets-dev && ng serve -c local-staging",
-    "start:mixed": "npm run generate-config && npm run sync-assets-dev && ng serve -c mixed",
-    "build": "npm run generate-config && ng build --configuration production --localize && npm run sync-assets && npm run build-mempool.js",
+    "serve": "npm run generate-config && npm run ng -- serve -c local",
+    "serve:stg": "npm run generate-config && npm run ng -- serve -c staging",
+    "serve:local-prod": "npm run generate-config && npm run ng -- serve -c local-prod",
+    "serve:local-staging": "npm run generate-config && npm run ng -- serve -c local-staging",
+    "start": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local",
+    "start:stg": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c staging",
+    "start:local-prod": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local-prod",
+    "start:local-staging": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local-staging",
+    "start:mixed": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c mixed",
+    "build": "npm run generate-config && npm run ng -- build --configuration production --localize && npm run sync-assets && npm run build-mempool.js",
     "sync-assets": "node sync-assets.js && rsync -av ./dist/mempool/browser/en-US/resources ./dist/mempool/browser/resources",
     "sync-assets-dev": "node sync-assets.js dev",
     "generate-config": "node generate-config.js",
@@ -41,17 +41,19 @@
     "build-mempool-js": "browserify -p tinyify ./node_modules/@mempool/mempool.js/lib/index.js --standalone mempoolJS > ./dist/mempool/browser/en-US/mempool.js",
     "build-mempool-bisq-js": "browserify -p tinyify ./node_modules/@mempool/mempool.js/lib/index-bisq.js --standalone bisqJS > ./dist/mempool/browser/en-US/bisq.js",
     "build-mempool-liquid-js": "browserify -p tinyify ./node_modules/@mempool/mempool.js/lib/index-liquid.js --standalone liquidJS > ./dist/mempool/browser/en-US/liquid.js",
-    "test": "ng test",
-    "lint": "ng lint",
-    "e2e": "npm run generate-config && ng e2e",
+    "test": "npm run ng -- test",
+    "lint": "./node_modules/.bin/eslint . --ext .ts",
+    "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",
+    "prettier": "prettier --write \"src/app/**/*.{js,json,css,scss,less,md,ts,html,component.html}\"",
+    "e2e": "npm run generate-config && npm run ng -- e2e",
     "e2e:ci": "npm run cypress:run:ci",
     "config:defaults:mempool": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true LIQUID_TESTNET_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 BASE_MODULE=mempool BLOCK_WEIGHT_UNITS=4000000 && npm run generate-config",
     "config:defaults:liquid": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true LIQUID_TESTNET_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 BASE_MODULE=liquid BLOCK_WEIGHT_UNITS=300000 && npm run generate-config",
     "config:defaults:bisq": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 BASE_MODULE=bisq BLOCK_WEIGHT_UNITS=4000000 && npm run generate-config",
-    "dev:ssr": "npm run generate-config && ng run mempool:serve-ssr",
+    "dev:ssr": "npm run generate-config && npm run ng -- run mempool:serve-ssr",
     "serve:ssr": "node server.run.js",
-    "build:ssr": "npm run build && ng run mempool:server:production && ./node_modules/typescript/bin/tsc server.run.ts",
-    "prerender": "ng run mempool:prerender",
+    "build:ssr": "npm run build && npm run ng -- run mempool:server:production && npm run tsc -- server.run.ts",
+    "prerender": "npm run ng -- run mempool:prerender",
     "cypress:open": "cypress open",
     "cypress:run": "cypress run",
     "cypress:run:record": "cypress run --record",
@@ -77,7 +79,6 @@
     "@fortawesome/fontawesome-common-types": "~6.1.1",
     "@fortawesome/fontawesome-svg-core": "~6.1.1",
     "@fortawesome/free-solid-svg-icons": "~6.1.1",
-    "@juggle/resize-observer": "^3.3.1",
     "@mempool/mempool.js": "2.3.0",
     "@ng-bootstrap/ng-bootstrap": "^11.0.0",
     "@nguniversal/express-engine": "~13.1.1",
@@ -88,8 +89,7 @@
     "domino": "^2.1.6",
     "echarts": "~5.3.2",
     "express": "^4.17.1",
-    "lightweight-charts": "^3.3.0",
-    "ngx-bootrap-multiselect": "^2.0.0",
+    "lightweight-charts": "~3.8.0",
     "ngx-echarts": "8.0.1",
     "ngx-infinite-scroll": "^10.0.1",
     "qrcode": "1.5.0",
@@ -104,28 +104,24 @@
     "@angular/language-service": "~13.3.10",
     "@nguniversal/builders": "~13.1.1",
     "@types/express": "^4.17.0",
-    "@types/jasmine": "~4.0.3",
-    "@types/jasminewd2": "~2.0.10",
     "@types/node": "^12.11.1",
-    "codelyzer": "~6.0.2",
-    "http-proxy-middleware": "^1.0.5",
-    "jasmine-core": "~4.1.0",
-    "jasmine-spec-reporter": "~7.0.0",
-    "karma": "~6.3.19",
-    "karma-chrome-launcher": "~3.1.0",
-    "karma-coverage": "~2.0.3",
-    "karma-jasmine": "~5.0.0",
-    "karma-jasmine-html-reporter": "^1.5.0",
-    "ts-node": "~8.3.0",
-    "tslint": "~6.1.0",
+    "@typescript-eslint/eslint-plugin": "^5.30.5",
+    "@typescript-eslint/parser": "^5.30.5",
+    "eslint": "^8.19.0",
+    "http-proxy-middleware": "~2.0.6",
+    "prettier": "^2.7.1",
+    "ts-node": "~10.8.1",
     "typescript": "~4.6.4"
   },
   "optionalDependencies": {
-    "@cypress/schematic": "^1.3.0",
-    "cypress": "^10.0.2",
-    "cypress-fail-on-console-error": "^2.1.3",
-    "cypress-wait-until": "^1.7.1",
-    "mock-socket": "^9.0.3",
-    "start-server-and-test": "^1.12.6"
+    "@cypress/schematic": "~2.0.0",
+    "cypress": "^10.3.0",
+    "cypress-fail-on-console-error": "~3.0.0",
+    "cypress-wait-until": "^1.7.2",
+    "mock-socket": "~9.1.4",
+    "start-server-and-test": "~1.14.0"
+  },
+  "scarfSettings": {
+    "enabled": false
   }
 }

frontend/src/app/app-routing.module.ts

@@ -171,7 +171,7 @@ let routes: Routes = [
           {
             path: 'block',
             component: StartComponent,
-              children: [
+            children: [
               {
                 path: ':id',
                 component: BlockComponent
@@ -258,7 +258,7 @@ let routes: Routes = [
       {
         path: 'block',
         component: StartComponent,
-          children: [
+        children: [
           {
             path: ':id',
             component: BlockComponent
@@ -361,7 +361,7 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
             {
               path: 'block',
               component: StartComponent,
-                children: [
+              children: [
                 {
                   path: ':id',
                   component: BlockComponent
@@ -465,7 +465,7 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
         {
           path: 'block',
           component: StartComponent,
-            children: [
+          children: [
             {
               path: ':id',
               component: BlockComponent

frontend/src/app/bisq/bisq-transactions/bisq-transactions.component.html

@@ -3,7 +3,7 @@
 
   <div class="d-block float-right" id="filter">
     <form [formGroup]="radioGroupForm">
-      <ngx-bootrap-multiselect [options]="txTypeOptions" [settings]="txTypeDropdownSettings" [texts]="txTypeDropdownTexts" formControlName="txTypes"></ngx-bootrap-multiselect>
+      <ngx-bootstrap-multiselect [options]="txTypeOptions" [settings]="txTypeDropdownSettings" [texts]="txTypeDropdownTexts" formControlName="txTypes"></ngx-bootstrap-multiselect>
     </form>
   </div>
 

frontend/src/app/bisq/bisq-transactions/bisq-transactions.component.ts

@@ -7,7 +7,7 @@ import { BisqApiService } from '../bisq-api.service';
 import { SeoService } from 'src/app/services/seo.service';
 import { FormGroup, FormBuilder } from '@angular/forms';
 import { Router, ActivatedRoute } from '@angular/router';
-import { IMultiSelectOption, IMultiSelectSettings, IMultiSelectTexts } from 'ngx-bootrap-multiselect';
+import { IMultiSelectOption, IMultiSelectSettings, IMultiSelectTexts } from 'src/app/components/ngx-bootstrap-multiselect/types'
 import { WebsocketService } from 'src/app/services/websocket.service';
 
 @Component({

frontend/src/app/bisq/bisq-transfers/bisq-transfers.component.ts

@@ -31,7 +31,7 @@ export class BisqTransfersComponent implements OnInit, OnChanges {
   }
 
   ngOnChanges() {
-    this.totalOutput = this.tx.outputs.filter((output) => output.isVerified).reduce((acc, output) => acc + output.bsqAmount, 0);;
+    this.totalOutput = this.tx.outputs.filter((output) => output.isVerified).reduce((acc, output) => acc + output.bsqAmount, 0);
   }
 
   switchCurrency() {

frontend/src/app/bisq/bisq.module.ts

@@ -1,7 +1,6 @@
 import { NgModule } from '@angular/core';
 import { BisqRoutingModule } from './bisq.routing.module';
 import { SharedModule } from '../shared/shared.module';
-import { NgxBootstrapMultiselectModule } from 'ngx-bootrap-multiselect';
 
 import { LightweightChartsComponent } from './lightweight-charts/lightweight-charts.component';
 import { LightweightChartsAreaComponent } from './lightweight-charts-area/lightweight-charts-area.component';
@@ -24,6 +23,10 @@ import { BisqStatsComponent } from './bisq-stats/bisq-stats.component';
 import { BsqAmountComponent } from './bsq-amount/bsq-amount.component';
 import { BisqTradesComponent } from './bisq-trades/bisq-trades.component';
 import { CommonModule } from '@angular/common';
+import { AutofocusDirective } from '../components/ngx-bootstrap-multiselect/autofocus.directive';
+import { MultiSelectSearchFilter } from '../components/ngx-bootstrap-multiselect/search-filter.pipe';
+import { OffClickDirective } from '../components/ngx-bootstrap-multiselect/off-click.directive';
+import { NgxDropdownMultiselectComponent } from '../components/ngx-bootstrap-multiselect/ngx-bootstrap-multiselect.component';
 
 @NgModule({
   declarations: [
@@ -44,16 +47,21 @@ import { CommonModule } from '@angular/common';
     BisqMarketComponent,
     BisqTradesComponent,
     BisqMainDashboardComponent,
+    NgxDropdownMultiselectComponent,
+    AutofocusDirective,
+    OffClickDirective,
   ],
   imports: [
     CommonModule,
     BisqRoutingModule,
     SharedModule,
     FontAwesomeModule,
-    NgxBootstrapMultiselectModule,
   ],
   providers: [
     BisqApiService,
+    MultiSelectSearchFilter,
+    AutofocusDirective,
+    OffClickDirective,
   ]
 })
 export class BisqModule {

frontend/src/app/bisq/lightweight-charts-area/lightweight-charts-area.component.ts

@@ -1,5 +1,5 @@
 import { createChart, CrosshairMode, isBusinessDay } from 'lightweight-charts';
-import { ChangeDetectionStrategy, Component, ElementRef, Input, OnChanges, OnDestroy, OnInit, SimpleChanges } from '@angular/core';
+import { ChangeDetectionStrategy, Component, ElementRef, HostListener, Input, OnChanges, OnDestroy, OnInit, SimpleChanges } from '@angular/core';
 
 @Component({
   selector: 'app-lightweight-charts-area',
@@ -25,6 +25,15 @@ export class LightweightChartsAreaComponent implements OnInit, OnChanges, OnDest
     private element: ElementRef,
   ) { }
 
+  @HostListener('window:resize', ['$event'])
+  resizeCanvas(): void {
+    this.width = this.element.nativeElement.parentElement.offsetWidth;
+    this.chart.applyOptions({
+      width: this.width,
+      height: this.height,
+    });
+  }
+
   ngOnInit() {
     this.width = this.element.nativeElement.parentElement.offsetWidth;
     this.container = document.createElement('div');

frontend/src/app/bisq/lightweight-charts/lightweight-charts.component.ts

@@ -1,5 +1,5 @@
 import { createChart, CrosshairMode } from 'lightweight-charts';
-import { ChangeDetectionStrategy, Component, ElementRef, Input, OnChanges, OnDestroy, OnInit, SimpleChanges } from '@angular/core';
+import { ChangeDetectionStrategy, Component, ElementRef, HostListener, Input, OnChanges, OnDestroy, OnInit, SimpleChanges } from '@angular/core';
 
 @Component({
   selector: 'app-lightweight-charts',
@@ -21,6 +21,14 @@ export class LightweightChartsComponent implements OnInit, OnChanges, OnDestroy
     private element: ElementRef,
   ) { }
 
+  @HostListener('window:resize', ['$event'])
+  resizeCanvas(): void {
+    this.chart.applyOptions({
+      width: this.element.nativeElement.parentElement.offsetWidth,
+      height: this.height,
+    });
+  }
+
   ngOnInit() {
     this.chart = createChart(this.element.nativeElement, {
       width: this.element.nativeElement.parentElement.offsetWidth,

frontend/src/app/components/address-labels/address-labels.component.spec.ts

@@ -1,25 +0,0 @@
-import { async, ComponentFixture, TestBed } from '@angular/core/testing';
-
-import { AddressLabelsComponent } from './address-labels.component';
-
-describe('AddressLabelsComponent', () => {
-  let component: AddressLabelsComponent;
-  let fixture: ComponentFixture<AddressLabelsComponent>;
-
-  beforeEach(async(() => {
-    TestBed.configureTestingModule({
-      declarations: [ AddressLabelsComponent ]
-    })
-    .compileComponents();
-  }));
-
-  beforeEach(() => {
-    fixture = TestBed.createComponent(AddressLabelsComponent);
-    component = fixture.componentInstance;
-    fixture.detectChanges();
-  });
-
-  it('should create', () => {
-    expect(component).toBeTruthy();
-  });
-});

frontend/src/app/components/address/address.component.html

@@ -130,7 +130,7 @@
       <span i18n="address.error.loading-address-data">Error loading address data.</span>
       <br>
       <ng-template #displayServerError><i class="small">({{ error.error }})</i></ng-template>
-      <ng-template [ngIf]="error.status === 413 || error.status === 405" [ngIfElse]="displayServerError">
+      <ng-template [ngIf]="error.status === 413 || error.status === 405 || error.status === 504" [ngIfElse]="displayServerError">
         <ng-container i18n="Electrum server limit exceeded error">
           <i>There many transactions on this address, more than your backend can handle. See more on <a href="/docs/faq#address-lookup-issues">setting up a stronger backend</a>.</i>
           <br><br>

frontend/src/app/components/amount/amount.component.spec.ts

@@ -1,25 +0,0 @@
-import { async, ComponentFixture, TestBed } from '@angular/core/testing';
-
-import { AmountComponent } from './amount.component';
-
-describe('AmountComponent', () => {
-  let component: AmountComponent;
-  let fixture: ComponentFixture<AmountComponent>;
-
-  beforeEach(async(() => {
-    TestBed.configureTestingModule({
-      declarations: [ AmountComponent ]
-    })
-    .compileComponents();
-  }));
-
-  beforeEach(() => {
-    fixture = TestBed.createComponent(AmountComponent);
-    component = fixture.componentInstance;
-    fixture.detectChanges();
-  });
-
-  it('should create', () => {
-    expect(component).toBeTruthy();
-  });
-});

frontend/src/app/components/app/app.component.spec.ts

@@ -1,35 +0,0 @@
-import { TestBed, async } from '@angular/core/testing';
-import { RouterTestingModule } from '@angular/router/testing';
-import { AppComponent } from './app.component';
-
-describe('AppComponent', () => {
-  beforeEach(async(() => {
-    TestBed.configureTestingModule({
-      imports: [
-        RouterTestingModule
-      ],
-      declarations: [
-        AppComponent
-      ],
-    }).compileComponents();
-  }));
-
-  it('should create the app', () => {
-    const fixture = TestBed.createComponent(AppComponent);
-    const app = fixture.debugElement.componentInstance;
-    expect(app).toBeTruthy();
-  });
-
-  it(`should have as title 'mempool'`, () => {
-    const fixture = TestBed.createComponent(AppComponent);
-    const app = fixture.debugElement.componentInstance;
-    expect(app.title).toEqual('mempool');
-  });
-
-  it('should render title in a h1 tag', () => {
-    const fixture = TestBed.createComponent(AppComponent);
-    fixture.detectChanges();
-    const compiled = fixture.debugElement.nativeElement;
-    expect(compiled.querySelector('h1').textContent).toContain('Welcome to mempool!');
-  });
-});

frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.html

@@ -9,34 +9,34 @@
     <form [formGroup]="radioGroupForm" class="formRadioGroup" *ngIf="(statsObservable$ | async) as stats">
       <div class="btn-group btn-group-toggle" ngbRadioGroup name="radioBasic" formControlName="dateSpan">
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 144">
-          <input ngbButton type="radio" [value]="'24h'" fragment="24h"> 24h
+          <input ngbButton type="radio" [value]="'24h'" fragment="24h" [routerLink]="['/graphs/mining/block-fee-rates' | relativeUrl]"> 24h
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 432">
-          <input ngbButton type="radio" [value]="'3d'" fragment="3d"> 3D
+          <input ngbButton type="radio" [value]="'3d'" fragment="3d" [routerLink]="['/graphs/mining/block-fee-rates' | relativeUrl]"> 3D
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 1008">
-          <input ngbButton type="radio" [value]="'1w'" fragment="1w"> 1W
+          <input ngbButton type="radio" [value]="'1w'" fragment="1w" [routerLink]="['/graphs/mining/block-fee-rates' | relativeUrl]"> 1W
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 4320">
-          <input ngbButton type="radio" [value]="'1m'" fragment="1m"> 1M
+          <input ngbButton type="radio" [value]="'1m'" fragment="1m" [routerLink]="['/graphs/mining/block-fee-rates' | relativeUrl]"> 1M
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 12960">
-          <input ngbButton type="radio" [value]="'3m'" fragment="3m"> 3M
+          <input ngbButton type="radio" [value]="'3m'" fragment="3m" [routerLink]="['/graphs/mining/block-fee-rates' | relativeUrl]"> 3M
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 25920">
-          <input ngbButton type="radio" [value]="'6m'" fragment="6m"> 6M
+          <input ngbButton type="radio" [value]="'6m'" fragment="6m" [routerLink]="['/graphs/mining/block-fee-rates' | relativeUrl]"> 6M
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 52560">
-          <input ngbButton type="radio" [value]="'1y'" fragment="1y"> 1Y
+          <input ngbButton type="radio" [value]="'1y'" fragment="1y" [routerLink]="['/graphs/mining/block-fee-rates' | relativeUrl]"> 1Y
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 105120">
-          <input ngbButton type="radio" [value]="'2y'" fragment="2y"> 2Y
+          <input ngbButton type="radio" [value]="'2y'" fragment="2y" [routerLink]="['/graphs/mining/block-fee-rates' | relativeUrl]"> 2Y
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 157680">
-          <input ngbButton type="radio" [value]="'3y'" fragment="3y"> 3Y
+          <input ngbButton type="radio" [value]="'3y'" fragment="3y" [routerLink]="['/graphs/mining/block-fee-rates' | relativeUrl]"> 3Y
         </label>
-        <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount > 157680">
-          <input ngbButton type="radio" [value]="'all'" fragment="all"> ALL
+        <label ngbButtonLabel class="btn-primary btn-sm">
+          <input ngbButton type="radio" [value]="'all'" fragment="all" [routerLink]="['/graphs/mining/block-fee-rates' | relativeUrl]"> ALL
         </label>
       </div>
     </form>

frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.scss

@@ -79,57 +79,3 @@
     }
   }
 }
-
-.pool-distribution {
-  min-height: 56px;
-  display: block;
-  @media (min-width: 485px) {
-    display: flex;
-    flex-direction: row;
-  }
-  h5 {
-    margin-bottom: 10px;
-  }
-  .item {
-    width: 50%;
-    display: inline-block;
-    margin: 0px auto 20px;
-    &:nth-child(2) {
-      order: 2;
-      @media (min-width: 485px) {
-        order: 3;
-      }
-    }
-    &:nth-child(3) {
-      order: 3;
-      @media (min-width: 485px) {
-        order: 2;
-        display: block;
-      }
-      @media (min-width: 768px) {
-        display: none;
-      }
-      @media (min-width: 992px) {
-        display: block;
-      }
-    }
-    .card-title {
-      font-size: 1rem;
-      color: #4a68b9;
-    }
-    .card-text {
-      font-size: 18px;
-      span {
-        color: #ffffff66;
-        font-size: 12px;
-      }
-    }
-  }
-}
-
-.skeleton-loader {
-  width: 100%;
-  display: block;
-  max-width: 80px;
-  margin: 15px auto 3px;
-}

frontend/src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.ts

@@ -12,7 +12,7 @@ import { MiningService } from 'src/app/services/mining.service';
 import { selectPowerOfTen } from 'src/app/bitcoin.utils';
 import { RelativeUrlPipe } from 'src/app/shared/pipes/relative-url/relative-url.pipe';
 import { StateService } from 'src/app/services/state.service';
-import { Router } from '@angular/router';
+import { ActivatedRoute, Router } from '@angular/router';
 
 @Component({
   selector: 'app-block-fee-rates-graph',
@@ -56,6 +56,7 @@ export class BlockFeeRatesGraphComponent implements OnInit {
     private stateService: StateService,
     private router: Router,
     private zone: NgZone,
+    private route: ActivatedRoute,
   ) {
     this.radioGroupForm = this.formBuilder.group({ dateSpan: '1y' });
     this.radioGroupForm.controls.dateSpan.setValue('1y');
@@ -67,9 +68,17 @@ export class BlockFeeRatesGraphComponent implements OnInit {
     this.radioGroupForm = this.formBuilder.group({ dateSpan: this.miningWindowPreference });
     this.radioGroupForm.controls.dateSpan.setValue(this.miningWindowPreference);
 
+    this.route
+      .fragment
+      .subscribe((fragment) => {
+        if (['24h', '3d', '1w', '1m', '3m', '6m', '1y', '2y', '3y', 'all'].indexOf(fragment) > -1) {
+          this.radioGroupForm.controls.dateSpan.setValue(fragment, { emitEvent: false });
+        }
+      });
+
     this.statsObservable$ = this.radioGroupForm.get('dateSpan').valueChanges
       .pipe(
-        startWith(this.miningWindowPreference),
+        startWith(this.radioGroupForm.controls.dateSpan.value),
         switchMap((timespan) => {
           this.storageService.setValue('miningWindowPreference', timespan);
           this.timespan = timespan;
@@ -165,21 +174,20 @@ export class BlockFeeRatesGraphComponent implements OnInit {
           align: 'left',
         },
         borderColor: '#000',
-        formatter: function (data) {
+        formatter: function(data) {
           if (data.length <= 0) {
             return '';
           }
-          let tooltip = `<b style="color: white; margin-left: 2px">
-            ${formatterXAxis(this.locale, this.timespan, parseInt(data[0].axisValue, 10))}</b><br>`;
+          let tooltip = `<b style="color: white; margin-left: 2px">${formatterXAxis(this.locale, this.timespan, parseInt(data[0].axisValue, 10))}</b><br>`;
 
           for (const pool of data.reverse()) {
             tooltip += `${pool.marker} ${pool.seriesName}: ${pool.data[1]} sats/vByte<br>`;
           }
 
           if (['24h', '3d'].includes(this.timespan)) {
-            tooltip += `<small>At block: ${data[0].data[2]}</small>`;
+            tooltip += `<small>` + $localize`At block: ${data[0].data[2]}` + `</small>`;
           } else {
-            tooltip += `<small>Around block ${data[0].data[2]}</small>`;
+            tooltip += `<small>` + $localize`Around block: ${data[0].data[2]}` + `</small>`;
           }
 
           return tooltip;

frontend/src/app/components/block-fees-graph/block-fees-graph.component.html

@@ -9,34 +9,34 @@
     <form [formGroup]="radioGroupForm" class="formRadioGroup" *ngIf="(statsObservable$ | async) as stats">
       <div class="btn-group btn-group-toggle" ngbRadioGroup name="radioBasic" formControlName="dateSpan">
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 144">
-          <input ngbButton type="radio" [value]="'24h'" fragment="24h"> 24h
+          <input ngbButton type="radio" [value]="'24h'" fragment="24h" [routerLink]="['/graphs/mining/block-fees' | relativeUrl]"> 24h
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 432">
-          <input ngbButton type="radio" [value]="'3d'" fragment="3d"> 3D
+          <input ngbButton type="radio" [value]="'3d'" fragment="3d" [routerLink]="['/graphs/mining/block-fees' | relativeUrl]"> 3D
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 1008">
-          <input ngbButton type="radio" [value]="'1w'" fragment="1w"> 1W
+          <input ngbButton type="radio" [value]="'1w'" fragment="1w" [routerLink]="['/graphs/mining/block-fees' | relativeUrl]"> 1W
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 4320">
-          <input ngbButton type="radio" [value]="'1m'" fragment="1m"> 1M
+          <input ngbButton type="radio" [value]="'1m'" fragment="1m" [routerLink]="['/graphs/mining/block-fees' | relativeUrl]"> 1M
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 12960">
-          <input ngbButton type="radio" [value]="'3m'" fragment="3m"> 3M
+          <input ngbButton type="radio" [value]="'3m'" fragment="3m" [routerLink]="['/graphs/mining/block-fees' | relativeUrl]"> 3M
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 25920">
-          <input ngbButton type="radio" [value]="'6m'" fragment="6m"> 6M
+          <input ngbButton type="radio" [value]="'6m'" fragment="6m" [routerLink]="['/graphs/mining/block-fees' | relativeUrl]"> 6M
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 52560">
-          <input ngbButton type="radio" [value]="'1y'" fragment="1y"> 1Y
+          <input ngbButton type="radio" [value]="'1y'" fragment="1y" [routerLink]="['/graphs/mining/block-fees' | relativeUrl]"> 1Y
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 105120">
-          <input ngbButton type="radio" [value]="'2y'" fragment="2y"> 2Y
+          <input ngbButton type="radio" [value]="'2y'" fragment="2y" [routerLink]="['/graphs/mining/block-fees' | relativeUrl]"> 2Y
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 157680">
-          <input ngbButton type="radio" [value]="'3y'" fragment="3y"> 3Y
+          <input ngbButton type="radio" [value]="'3y'" fragment="3y" [routerLink]="['/graphs/mining/block-fees' | relativeUrl]"> 3Y
         </label>
-        <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount > 157680">
-          <input ngbButton type="radio" [value]="'all'" fragment="all"> ALL
+        <label ngbButtonLabel class="btn-primary btn-sm">
+          <input ngbButton type="radio" [value]="'all'" fragment="all" [routerLink]="['/graphs/mining/block-fees' | relativeUrl]"> ALL
         </label>
       </div>
     </form>

frontend/src/app/components/block-fees-graph/block-fees-graph.component.scss

@@ -79,57 +79,3 @@
     }
   }
 }
-
-.pool-distribution {
-  min-height: 56px;
-  display: block;
-  @media (min-width: 485px) {
-    display: flex;
-    flex-direction: row;
-  }
-  h5 {
-    margin-bottom: 10px;
-  }
-  .item {
-    width: 50%;
-    display: inline-block;
-    margin: 0px auto 20px;
-    &:nth-child(2) {
-      order: 2;
-      @media (min-width: 485px) {
-        order: 3;
-      }
-    }
-    &:nth-child(3) {
-      order: 3;
-      @media (min-width: 485px) {
-        order: 2;
-        display: block;
-      }
-      @media (min-width: 768px) {
-        display: none;
-      }
-      @media (min-width: 992px) {
-        display: block;
-      }
-    }
-    .card-title {
-      font-size: 1rem;
-      color: #4a68b9;
-    }
-    .card-text {
-      font-size: 18px;
-      span {
-        color: #ffffff66;
-        font-size: 12px;
-      }
-    }
-  }
-}
-
-.skeleton-loader {
-  width: 100%;
-  display: block;
-  max-width: 80px;
-  margin: 15px auto 3px;
-}

frontend/src/app/components/block-fees-graph/block-fees-graph.component.ts

@@ -6,9 +6,10 @@ import { ApiService } from 'src/app/services/api.service';
 import { SeoService } from 'src/app/services/seo.service';
 import { formatNumber } from '@angular/common';
 import { FormBuilder, FormGroup } from '@angular/forms';
-import { download, formatterXAxisLabel } from 'src/app/shared/graphs.utils';
+import { download, formatterXAxis, formatterXAxisLabel } from 'src/app/shared/graphs.utils';
 import { StorageService } from 'src/app/services/storage.service';
 import { MiningService } from 'src/app/services/mining.service';
+import { ActivatedRoute } from '@angular/router';
 
 @Component({
   selector: 'app-block-fees-graph',
@@ -48,7 +49,8 @@ export class BlockFeesGraphComponent implements OnInit {
     private apiService: ApiService,
     private formBuilder: FormBuilder,
     private storageService: StorageService,
-    private miningService: MiningService
+    private miningService: MiningService,
+    private route: ActivatedRoute,
   ) {
     this.radioGroupForm = this.formBuilder.group({ dateSpan: '1y' });
     this.radioGroupForm.controls.dateSpan.setValue('1y');
@@ -60,9 +62,17 @@ export class BlockFeesGraphComponent implements OnInit {
     this.radioGroupForm = this.formBuilder.group({ dateSpan: this.miningWindowPreference });
     this.radioGroupForm.controls.dateSpan.setValue(this.miningWindowPreference);
 
+    this.route
+      .fragment
+      .subscribe((fragment) => {
+        if (['24h', '3d', '1w', '1m', '3m', '6m', '1y', '2y', '3y', 'all'].indexOf(fragment) > -1) {
+          this.radioGroupForm.controls.dateSpan.setValue(fragment, { emitEvent: false });
+        }
+      });
+
     this.statsObservable$ = this.radioGroupForm.get('dateSpan').valueChanges
       .pipe(
-        startWith(this.miningWindowPreference),
+        startWith(this.radioGroupForm.controls.dateSpan.value),
         switchMap((timespan) => {
           this.storageService.setValue('miningWindowPreference', timespan);
           this.timespan = timespan;
@@ -71,7 +81,7 @@ export class BlockFeesGraphComponent implements OnInit {
             .pipe(
               tap((response) => {
                 this.prepareChartOptions({
-                  blockFees: response.body.map(val => [val.timestamp * 1000, val.avgFees / 100000000]),
+                  blockFees: response.body.map(val => [val.timestamp * 1000, val.avgFees / 100000000, val.avgHeight]),
                 });
                 this.isLoading = false;
               }),
@@ -119,12 +129,17 @@ export class BlockFeesGraphComponent implements OnInit {
         },
         borderColor: '#000',
         formatter: (ticks) => {
-          const tick = ticks[0];
-          const feesString = `${tick.marker} ${tick.seriesName}: ${formatNumber(tick.data[1], this.locale, '1.3-3')} BTC`;
-          return `
-            <b style="color: white; margin-left: 18px">${tick.axisValueLabel}</b><br>
-            <span>${feesString}</span>
-          `;
+          let tooltip = `<b style="color: white; margin-left: 2px">${formatterXAxis(this.locale, this.timespan, parseInt(ticks[0].axisValue, 10))}</b><br>`;
+          tooltip += `${ticks[0].marker} ${ticks[0].seriesName}: ${formatNumber(ticks[0].data[1], this.locale, '1.3-3')} BTC`;
+          tooltip += `<br>`;
+
+          if (['24h', '3d'].includes(this.timespan)) {
+            tooltip += `<small>` + $localize`At block: ${ticks[0].data[2]}` + `</small>`;
+          } else {
+            tooltip += `<small>` + $localize`Around block: ${ticks[0].data[2]}` + `</small>`;
+          }
+
+          return tooltip;
         }
       },
       xAxis: {

frontend/src/app/components/block-overview-graph/block-overview-graph.component.html

@@ -0,0 +1,12 @@
+<div class="block-overview-graph">
+  <canvas class="block-overview-canvas" [class.clickable]="!!hoverTx" #blockCanvas></canvas>
+  <div class="loader-wrapper" [class.hidden]="!isLoading">
+    <div class="spinner-border ml-3 loading" role="status"></div>
+  </div>
+
+  <app-block-overview-tooltip
+    [tx]="selectedTx || hoverTx"
+    [cursorPosition]="tooltipPosition"
+    [clickable]="!!selectedTx"
+  ></app-block-overview-tooltip>
+</div>

frontend/src/app/components/block-overview-graph/block-overview-graph.component.scss

@@ -1,4 +1,4 @@
-.mempool-block-overview {
+.block-overview-graph {
   position: relative;
   width: 100%;
   padding-bottom: 100%;
@@ -8,17 +8,25 @@
   align-items: center;
 }
 
-.block-overview {
+
+.block-overview-canvas {
   position: absolute;
   left: 0;
   right: 0;
   top: 0;
   bottom: 0;
+  width: 100%;
+  height: 100%;
   overflow: hidden;
+
+  &.clickable {
+    cursor: pointer;
+  }
 }
 
 .loader-wrapper {
   position: absolute;
+  background: #181b2d7f;
   left: 0;
   right: 0;
   top: 0;
@@ -27,6 +35,7 @@
   justify-content: center;
   align-items: center;
   transition: opacity 500ms 500ms;
+  pointer-events: none;
 
   &.hidden {
     opacity: 0;

frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts

@@ -0,0 +1,445 @@
+import { Component, ElementRef, ViewChild, HostListener, Input, Output, EventEmitter, NgZone, AfterViewInit, OnDestroy } from '@angular/core';
+import { MempoolBlockDelta, TransactionStripped } from 'src/app/interfaces/websocket.interface';
+import { WebsocketService } from 'src/app/services/websocket.service';
+import { FastVertexArray } from './fast-vertex-array';
+import BlockScene from './block-scene';
+import TxSprite from './tx-sprite';
+import TxView from './tx-view';
+import { Position } from './sprite-types';
+
+@Component({
+  selector: 'app-block-overview-graph',
+  templateUrl: './block-overview-graph.component.html',
+  styleUrls: ['./block-overview-graph.component.scss'],
+})
+export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy {
+  @Input() isLoading: boolean;
+  @Input() resolution: number;
+  @Input() blockLimit: number;
+  @Input() orientation = 'left';
+  @Input() flip = true;
+  @Output() txClickEvent = new EventEmitter<TransactionStripped>();
+
+  @ViewChild('blockCanvas')
+  canvas: ElementRef<HTMLCanvasElement>;
+
+  gl: WebGLRenderingContext;
+  animationFrameRequest: number;
+  animationHeartBeat: number;
+  displayWidth: number;
+  displayHeight: number;
+  cssWidth: number;
+  cssHeight: number;
+  shaderProgram: WebGLProgram;
+  vertexArray: FastVertexArray;
+  running: boolean;
+  scene: BlockScene;
+  hoverTx: TxView | void;
+  selectedTx: TxView | void;
+  tooltipPosition: Position;
+
+  constructor(
+    readonly ngZone: NgZone,
+    readonly elRef: ElementRef,
+  ) {
+    this.vertexArray = new FastVertexArray(512, TxSprite.dataSize);
+  }
+
+  ngAfterViewInit(): void {
+    this.canvas.nativeElement.addEventListener('webglcontextlost', this.handleContextLost, false);
+    this.canvas.nativeElement.addEventListener('webglcontextrestored', this.handleContextRestored, false);
+    this.gl = this.canvas.nativeElement.getContext('webgl');
+    this.initCanvas();
+
+    this.resizeCanvas();
+  }
+
+  ngOnDestroy(): void {
+    if (this.animationFrameRequest) {
+      cancelAnimationFrame(this.animationFrameRequest);
+      clearTimeout(this.animationHeartBeat);
+    }
+  }
+
+  clear(direction): void {
+    this.exit(direction);
+    this.hoverTx = null;
+    this.selectedTx = null;
+    this.start();
+  }
+
+  destroy(): void {
+    if (this.scene) {
+      this.scene.destroy();
+      this.start();
+    }
+  }
+
+  // initialize the scene without any entry transition
+  setup(transactions: TransactionStripped[]): void {
+    if (this.scene) {
+      this.scene.setup(transactions);
+      this.start();
+    }
+  }
+
+  enter(transactions: TransactionStripped[], direction: string): void {
+    if (this.scene) {
+      this.scene.enter(transactions, direction);
+      this.start();
+    }
+  }
+
+  exit(direction: string): void {
+    if (this.scene) {
+      this.scene.exit(direction);
+      this.start();
+    }
+  }
+
+  replace(transactions: TransactionStripped[], direction: string, sort: boolean = true): void {
+    if (this.scene) {
+      this.scene.replace(transactions || [], direction, sort);
+      this.start();
+    }
+  }
+
+  update(add: TransactionStripped[], remove: string[], direction: string = 'left', resetLayout: boolean = false): void {
+    if (this.scene) {
+      this.scene.update(add, remove, direction, resetLayout);
+      this.start();
+    }
+  }
+
+  initCanvas(): void {
+    this.gl.clearColor(0.0, 0.0, 0.0, 0.0);
+    this.gl.clear(this.gl.COLOR_BUFFER_BIT);
+
+    const shaderSet = [
+      {
+        type: this.gl.VERTEX_SHADER,
+        src: vertShaderSrc
+      },
+      {
+        type: this.gl.FRAGMENT_SHADER,
+        src: fragShaderSrc
+      }
+    ];
+
+    this.shaderProgram = this.buildShaderProgram(shaderSet);
+
+    this.gl.useProgram(this.shaderProgram);
+
+    // Set up alpha blending
+    this.gl.enable(this.gl.BLEND);
+    this.gl.blendFunc(this.gl.ONE, this.gl.ONE_MINUS_SRC_ALPHA);
+
+    const glBuffer = this.gl.createBuffer();
+    this.gl.bindBuffer(this.gl.ARRAY_BUFFER, glBuffer);
+
+    /* SET UP SHADER ATTRIBUTES */
+    Object.keys(attribs).forEach((key, i) => {
+      attribs[key].pointer = this.gl.getAttribLocation(this.shaderProgram, key);
+      this.gl.enableVertexAttribArray(attribs[key].pointer);
+    });
+
+    this.start();
+  }
+
+  handleContextLost(event): void {
+    event.preventDefault();
+    cancelAnimationFrame(this.animationFrameRequest);
+    this.animationFrameRequest = null;
+    this.running = false;
+  }
+
+  handleContextRestored(event): void {
+    this.initCanvas();
+  }
+
+  @HostListener('window:resize', ['$event'])
+  resizeCanvas(): void {
+    this.cssWidth = this.canvas.nativeElement.offsetParent.clientWidth;
+    this.cssHeight = this.canvas.nativeElement.offsetParent.clientHeight;
+    this.displayWidth = window.devicePixelRatio * this.cssWidth;
+    this.displayHeight = window.devicePixelRatio * this.cssHeight;
+    this.canvas.nativeElement.width = this.displayWidth;
+    this.canvas.nativeElement.height = this.displayHeight;
+    if (this.gl) {
+      this.gl.viewport(0, 0, this.displayWidth, this.displayHeight);
+    }
+    if (this.scene) {
+      this.scene.resize({ width: this.displayWidth, height: this.displayHeight });
+      this.start();
+    } else {
+      this.scene = new BlockScene({ width: this.displayWidth, height: this.displayHeight, resolution: this.resolution,
+        blockLimit: this.blockLimit, orientation: this.orientation, flip: this.flip, vertexArray: this.vertexArray });
+      this.start();
+    }
+  }
+
+  compileShader(src, type): WebGLShader {
+    const shader = this.gl.createShader(type);
+
+    this.gl.shaderSource(shader, src);
+    this.gl.compileShader(shader);
+
+    if (!this.gl.getShaderParameter(shader, this.gl.COMPILE_STATUS)) {
+      console.log(`Error compiling ${type === this.gl.VERTEX_SHADER ? 'vertex' : 'fragment'} shader:`);
+      console.log(this.gl.getShaderInfoLog(shader));
+    }
+    return shader;
+  }
+
+  buildShaderProgram(shaderInfo): WebGLProgram {
+    const program = this.gl.createProgram();
+
+    shaderInfo.forEach((desc) => {
+      const shader = this.compileShader(desc.src, desc.type);
+      if (shader) {
+        this.gl.attachShader(program, shader);
+      }
+    });
+
+    this.gl.linkProgram(program);
+
+    if (!this.gl.getProgramParameter(program, this.gl.LINK_STATUS)) {
+      console.log('Error linking shader program:');
+      console.log(this.gl.getProgramInfoLog(program));
+    }
+
+    return program;
+  }
+
+  start(): void {
+    this.running = true;
+    this.ngZone.runOutsideAngular(() => this.doRun());
+  }
+
+  doRun(): void {
+    if (this.animationFrameRequest) {
+      cancelAnimationFrame(this.animationFrameRequest);
+    }
+    this.animationFrameRequest = requestAnimationFrame(() => this.run());
+  }
+
+  run(now?: DOMHighResTimeStamp): void {
+    if (!now) {
+      now = performance.now();
+    }
+    // skip re-render if there's no change to the scene
+    if (this.scene) {
+      /* SET UP SHADER UNIFORMS */
+      // screen dimensions
+      this.gl.uniform2f(this.gl.getUniformLocation(this.shaderProgram, 'screenSize'), this.displayWidth, this.displayHeight);
+      // frame timestamp
+      this.gl.uniform1f(this.gl.getUniformLocation(this.shaderProgram, 'now'), now);
+
+      if (this.vertexArray.dirty) {
+        /* SET UP SHADER ATTRIBUTES */
+        Object.keys(attribs).forEach((key, i) => {
+          this.gl.vertexAttribPointer(attribs[key].pointer,
+          attribs[key].count,  // number of primitives in this attribute
+          this.gl[attribs[key].type],  // type of primitive in this attribute (e.g. gl.FLOAT)
+          false, // never normalised
+          stride,   // distance between values of the same attribute
+          attribs[key].offset);  // offset of the first value
+        });
+
+        const pointArray = this.vertexArray.getVertexData();
+
+        if (pointArray.length) {
+          this.gl.bufferData(this.gl.ARRAY_BUFFER, pointArray, this.gl.DYNAMIC_DRAW);
+          this.gl.drawArrays(this.gl.TRIANGLES, 0, pointArray.length / TxSprite.vertexSize);
+        }
+        this.vertexArray.dirty = false;
+      } else {
+        const pointArray = this.vertexArray.getVertexData();
+        if (pointArray.length) {
+          this.gl.drawArrays(this.gl.TRIANGLES, 0, pointArray.length / TxSprite.vertexSize);
+        }
+      }
+    }
+
+    /* LOOP */
+    if (this.running && this.scene && now <= (this.scene.animateUntil + 500)) {
+      this.doRun();
+    } else {
+      if (this.animationHeartBeat) {
+        clearTimeout(this.animationHeartBeat);
+      }
+      this.animationHeartBeat = window.setTimeout(() => {
+        this.start();
+      }, 1000);
+    }
+  }
+
+  @HostListener('document:click', ['$event'])
+  clickAway(event) {
+    if (!this.elRef.nativeElement.contains(event.target)) {
+      const currentPreview = this.selectedTx || this.hoverTx;
+      if (currentPreview && this.scene) {
+        this.scene.setHover(currentPreview, false);
+        this.start();
+      }
+      this.hoverTx = null;
+      this.selectedTx = null;
+    }
+  }
+
+  @HostListener('pointerup', ['$event'])
+  onClick(event) {
+    if (event.target === this.canvas.nativeElement && event.pointerType === 'touch') {
+      this.setPreviewTx(event.offsetX, event.offsetY, true);
+    } else if (event.target === this.canvas.nativeElement) {
+      this.onTxClick(event.offsetX, event.offsetY);
+    }
+  }
+
+  @HostListener('pointermove', ['$event'])
+  onPointerMove(event) {
+    if (event.target === this.canvas.nativeElement) {
+      this.setPreviewTx(event.offsetX, event.offsetY, false);
+    }
+  }
+
+  @HostListener('pointerleave', ['$event'])
+  onPointerLeave(event) {
+    if (event.pointerType !== 'touch') {
+      this.setPreviewTx(-1, -1, true);
+    }
+  }
+
+  setPreviewTx(cssX: number, cssY: number, clicked: boolean = false) {
+    const x = cssX * window.devicePixelRatio;
+    const y = cssY * window.devicePixelRatio;
+    if (this.scene && (!this.selectedTx || clicked)) {
+      this.tooltipPosition = {
+        x: cssX,
+        y: cssY
+      };
+      const selected = this.scene.getTxAt({ x, y });
+      const currentPreview = this.selectedTx || this.hoverTx;
+
+      if (selected !== currentPreview) {
+        if (currentPreview && this.scene) {
+          this.scene.setHover(currentPreview, false);
+          this.start();
+        }
+        if (selected) {
+          if (selected && this.scene) {
+            this.scene.setHover(selected, true);
+            this.start();
+          }
+          if (clicked) {
+            this.selectedTx = selected;
+          } else {
+            this.hoverTx = selected;
+          }
+        } else {
+          if (clicked) {
+            this.selectedTx = null;
+          }
+          this.hoverTx = null;
+        }
+      } else if (clicked) {
+        if (selected === this.selectedTx) {
+          this.hoverTx = this.selectedTx;
+          this.selectedTx = null;
+        } else {
+          this.selectedTx = selected;
+        }
+      }
+    }
+  }
+
+  onTxClick(cssX: number, cssY: number) {
+    const x = cssX * window.devicePixelRatio;
+    const y = cssY * window.devicePixelRatio;
+    const selected = this.scene.getTxAt({ x, y });
+    if (selected && selected.txid) {
+      this.txClickEvent.emit(selected);
+    }
+  }
+}
+
+// WebGL shader attributes
+const attribs = {
+  offset: { type: 'FLOAT', count: 2, pointer: null, offset: 0 },
+  posX: { type: 'FLOAT', count: 4, pointer: null, offset: 0 },
+  posY: { type: 'FLOAT', count: 4, pointer: null, offset: 0 },
+  posR: { type: 'FLOAT', count: 4, pointer: null, offset: 0 },
+  colR: { type: 'FLOAT', count: 4, pointer: null, offset: 0 },
+  colG: { type: 'FLOAT', count: 4, pointer: null, offset: 0 },
+  colB: { type: 'FLOAT', count: 4, pointer: null, offset: 0 },
+  colA: { type: 'FLOAT', count: 4, pointer: null, offset: 0 }
+};
+// Calculate the number of bytes per vertex based on specified attributes
+const stride = Object.values(attribs).reduce((total, attrib) => {
+  return total + (attrib.count * 4);
+}, 0);
+// Calculate vertex attribute offsets
+for (let i = 0, offset = 0; i < Object.keys(attribs).length; i++) {
+  const attrib = Object.values(attribs)[i];
+  attrib.offset = offset;
+  offset += (attrib.count * 4);
+}
+
+const vertShaderSrc = `
+varying lowp vec4 vColor;
+
+// each attribute contains [x: startValue, y: endValue, z: startTime, w: rate]
+// shader interpolates between start and end values at the given rate, from the given time
+
+attribute vec2 offset;
+attribute vec4 posX;
+attribute vec4 posY;
+attribute vec4 posR;
+attribute vec4 colR;
+attribute vec4 colG;
+attribute vec4 colB;
+attribute vec4 colA;
+
+uniform vec2 screenSize;
+uniform float now;
+
+float smootherstep(float x) {
+  x = clamp(x, 0.0, 1.0);
+  float ix = 1.0 - x;
+  x = x * x;
+  return x / (x + ix * ix);
+}
+
+float interpolateAttribute(vec4 attr) {
+  float d = (now - attr.z) * attr.w;
+  float delta = smootherstep(d);
+  return mix(attr.x, attr.y, delta);
+}
+
+void main() {
+  vec4 screenTransform = vec4(2.0 / screenSize.x, 2.0 / screenSize.y, -1.0, -1.0);
+  // vec4 screenTransform = vec4(1.0 / screenSize.x, 1.0 / screenSize.y, -0.5, -0.5);
+
+  float radius = interpolateAttribute(posR);
+  vec2 position = vec2(interpolateAttribute(posX), interpolateAttribute(posY)) + (radius * offset);
+
+  gl_Position = vec4(position * screenTransform.xy + screenTransform.zw, 1.0, 1.0);
+
+  float red = interpolateAttribute(colR);
+  float green = interpolateAttribute(colG);
+  float blue = interpolateAttribute(colB);
+  float alpha = interpolateAttribute(colA);
+
+  vColor = vec4(red, green, blue, alpha);
+}
+`;
+
+const fragShaderSrc = `
+varying lowp vec4 vColor;
+
+void main() {
+  gl_FragColor = vColor;
+  // premultiply alpha
+  gl_FragColor.rgb *= gl_FragColor.a;
+}
+`;

frontend/src/app/components/block-overview-graph/block-scene.ts

@@ -0,0 +1,810 @@
+import { FastVertexArray } from './fast-vertex-array';
+import TxView from './tx-view';
+import { TransactionStripped } from 'src/app/interfaces/websocket.interface';
+import { Position, Square, ViewUpdateParams } from './sprite-types';
+
+export default class BlockScene {
+  scene: { count: number, offset: { x: number, y: number}};
+  vertexArray: FastVertexArray;
+  txs: { [key: string]: TxView };
+  orientation: string;
+  flip: boolean;
+  width: number;
+  height: number;
+  gridWidth: number;
+  gridHeight: number;
+  gridSize: number;
+  vbytesPerUnit: number;
+  unitPadding: number;
+  unitWidth: number;
+  initialised: boolean;
+  layout: BlockLayout;
+  animateUntil = 0;
+  dirty: boolean;
+
+  constructor({ width, height, resolution, blockLimit, orientation, flip, vertexArray }:
+      { width: number, height: number, resolution: number, blockLimit: number,
+        orientation: string, flip: boolean, vertexArray: FastVertexArray }
+  ) {
+    this.init({ width, height, resolution, blockLimit, orientation, flip, vertexArray });
+  }
+
+  resize({ width = this.width, height = this.height }: { width?: number, height?: number}): void {
+    this.width = width;
+    this.height = height;
+    this.gridSize = this.width / this.gridWidth;
+    this.unitPadding =  width / 500;
+    this.unitWidth = this.gridSize - (this.unitPadding * 2);
+
+    this.dirty = true;
+    if (this.initialised && this.scene) {
+      this.updateAll(performance.now(), 50);
+    }
+  }
+
+  // Destroy the current layout and clean up graphics sprites without any exit animation
+  destroy(): void {
+    Object.values(this.txs).forEach(tx => tx.destroy());
+    this.txs = {};
+    this.layout = null;
+  }
+
+  // set up the scene with an initial set of transactions, without any transition animation
+  setup(txs: TransactionStripped[]) {
+    // clean up any old transactions
+    Object.values(this.txs).forEach(tx => {
+      tx.destroy();
+      delete this.txs[tx.txid];
+    });
+    this.layout = new BlockLayout({ width: this.gridWidth, height: this.gridHeight });
+    txs.forEach(tx => {
+      const txView = new TxView(tx, this.vertexArray);
+      this.txs[tx.txid] = txView;
+      this.place(txView);
+      this.saveGridToScreenPosition(txView);
+      this.applyTxUpdate(txView, {
+        display: {
+          position: txView.screenPosition,
+          color: txView.getColor()
+        },
+        duration: 0
+      });
+    });
+  }
+
+  // Animate new block entering scene
+  enter(txs: TransactionStripped[], direction) {
+    this.replace(txs, direction);
+  }
+
+  // Animate block leaving scene
+  exit(direction: string): void {
+    const startTime = performance.now();
+    const removed = this.removeBatch(Object.keys(this.txs), startTime, direction);
+
+    // clean up sprites
+    setTimeout(() => {
+      removed.forEach(tx => {
+        tx.destroy();
+      });
+    }, 2000);
+  }
+
+  // Reset layout and replace with new set of transactions
+  replace(txs: TransactionStripped[], direction: string = 'left', sort: boolean = true): void {
+    const startTime = performance.now();
+    const nextIds = {};
+    const remove = [];
+    txs.forEach(tx => {
+      nextIds[tx.txid] = true;
+    });
+    Object.keys(this.txs).forEach(txid => {
+      if (!nextIds[txid]) {
+        remove.push(txid);
+      }
+    });
+    txs.forEach(tx => {
+      if (!this.txs[tx.txid]) {
+        this.txs[tx.txid] = new TxView(tx, this.vertexArray);
+      }
+    });
+
+    const removed = this.removeBatch(remove, startTime, direction);
+
+    // clean up sprites
+    setTimeout(() => {
+      removed.forEach(tx => {
+        tx.destroy();
+      });
+    }, 1000);
+
+    this.layout = new BlockLayout({ width: this.gridWidth, height: this.gridHeight });
+
+    if (sort) {
+      Object.values(this.txs).sort(feeRateDescending).forEach(tx => {
+        this.place(tx);
+      });
+    } else {
+      txs.forEach(tx => {
+        this.place(this.txs[tx.txid]);
+      });
+    }
+
+    this.updateAll(startTime, 200, direction);
+  }
+
+  update(add: TransactionStripped[], remove: string[], direction: string = 'left', resetLayout: boolean = false): void {
+    const startTime = performance.now();
+    const removed = this.removeBatch(remove, startTime, direction);
+
+    // clean up sprites
+    setTimeout(() => {
+      removed.forEach(tx => {
+        tx.destroy();
+      });
+    }, 1000);
+
+    if (resetLayout) {
+      add.forEach(tx => {
+        if (!this.txs[tx.txid]) {
+          this.txs[tx.txid] = new TxView(tx, this.vertexArray);
+        }
+      });
+      this.layout = new BlockLayout({ width: this.gridWidth, height: this.gridHeight });
+      Object.values(this.txs).sort(feeRateDescending).forEach(tx => {
+        this.place(tx);
+      });
+    } else {
+      // try to insert new txs directly
+      const remaining = [];
+      add.map(tx => new TxView(tx, this.vertexArray)).sort(feeRateDescending).forEach(tx => {
+        if (!this.tryInsertByFee(tx)) {
+          remaining.push(tx);
+        }
+      });
+      this.placeBatch(remaining);
+      this.layout.applyGravity();
+    }
+
+    this.updateAll(startTime, 100, direction);
+  }
+
+  // return the tx at this screen position, if any
+  getTxAt(position: Position): TxView | void {
+    if (this.layout) {
+      const gridPosition = this.screenToGrid(position);
+      return this.layout.getTx(gridPosition);
+    } else {
+      return null;
+    }
+  }
+
+  setHover(tx: TxView, value: boolean): void {
+    this.animateUntil = Math.max(this.animateUntil, tx.setHover(value));
+  }
+
+  private init({ width, height, resolution, blockLimit, orientation, flip, vertexArray }:
+      { width: number, height: number, resolution: number, blockLimit: number,
+        orientation: string, flip: boolean, vertexArray: FastVertexArray }
+  ): void {
+    this.orientation = orientation;
+    this.flip = flip;
+    this.vertexArray = vertexArray;
+
+    this.scene = {
+      count: 0,
+      offset: {
+        x: 0,
+        y: 0
+      }
+    };
+
+    // Set the scale of the visualization (with a 5% margin)
+    this.vbytesPerUnit = blockLimit / Math.pow(resolution / 1.02, 2);
+    this.gridWidth = resolution;
+    this.gridHeight = resolution;
+    this.resize({ width, height });
+    this.layout = new BlockLayout({ width: this.gridWidth, height: this.gridHeight });
+
+    this.txs = {};
+
+    this.initialised = true;
+    this.dirty = true;
+  }
+
+  private applyTxUpdate(tx: TxView, update: ViewUpdateParams): void {
+    this.animateUntil = Math.max(this.animateUntil, tx.update(update));
+  }
+
+  private updateTx(tx: TxView, startTime: number, delay: number, direction: string = 'left'): void {
+    if (tx.dirty || this.dirty) {
+      this.saveGridToScreenPosition(tx);
+      this.setTxOnScreen(tx, startTime, delay, direction);
+    }
+  }
+
+  private setTxOnScreen(tx: TxView, startTime: number, delay: number = 50, direction: string = 'left'): void {
+    if (!tx.initialised) {
+      const txColor = tx.getColor();
+      this.applyTxUpdate(tx, {
+        display: {
+          position: {
+            x: tx.screenPosition.x + (direction === 'right' ? -this.width : (direction === 'left' ? this.width : 0)) * 1.4,
+            y: tx.screenPosition.y + (direction === 'up' ? -this.height : (direction === 'down' ? this.height : 0)) * 1.4,
+            s: tx.screenPosition.s
+          },
+          color: txColor,
+        },
+        start: startTime,
+        delay: 0,
+      });
+      this.applyTxUpdate(tx, {
+        display: {
+          position: tx.screenPosition,
+          color: txColor
+        },
+        duration: 1000,
+        start: startTime,
+        delay,
+      });
+    } else {
+      this.applyTxUpdate(tx, {
+        display: {
+          position: tx.screenPosition
+        },
+        duration: 1000,
+        minDuration: 500,
+        start: startTime,
+        delay,
+        adjust: true
+      });
+    }
+  }
+
+  private updateAll(startTime: number, delay: number = 50, direction: string = 'left'): void {
+    this.scene.count = 0;
+    const ids = this.getTxList();
+    startTime = startTime || performance.now();
+    for (const id of ids) {
+      this.updateTx(this.txs[id], startTime, delay, direction);
+    }
+    this.dirty = false;
+  }
+
+  private remove(id: string, startTime: number, direction: string = 'left'): TxView | void {
+    const tx = this.txs[id];
+    if (tx) {
+      this.layout.remove(tx);
+      this.applyTxUpdate(tx, {
+        display: {
+          position: {
+            x: tx.screenPosition.x + (direction === 'right' ? this.width : (direction === 'left' ? -this.width : 0)) * 1.4,
+            y: tx.screenPosition.y + (direction === 'up' ? this.height : (direction === 'down' ? -this.height : 0)) * 1.4,
+          }
+        },
+        duration: 1000,
+        start: startTime,
+        delay: 50
+      });
+    }
+    delete this.txs[id];
+    return tx;
+  }
+
+  private getTxList(): string[] {
+    return Object.keys(this.txs);
+  }
+
+  private saveGridToScreenPosition(tx: TxView): void {
+    tx.screenPosition = this.gridToScreen(tx.gridPosition);
+  }
+
+  // convert grid coordinates to screen coordinates
+  private gridToScreen(position: Square | void): Square {
+    if (position) {
+      const slotSize = (position.s * this.gridSize);
+      const squareSize = slotSize - (this.unitPadding * 2);
+
+      // The grid is laid out notionally left-to-right, bottom-to-top,
+      // so we rotate and/or flip the y axis to match the target configuration.
+      //
+      // e.g. for flip = true, orientation = 'left':
+      //
+      //    grid                             screen
+      //  ________        ________          ________
+      // |        |      |        |        |       a|
+      // |        | flip |        | rotate |     c  |
+      // |  c     |  --> |     c  |  -->   |        |
+      // |a______b|      |b______a|        |_______b|
+
+      let x = (this.gridSize * position.x) + (slotSize / 2);
+      let y = (this.gridSize * position.y) + (slotSize / 2);
+      let t;
+      if (this.flip) {
+        x = this.width - x;
+      }
+      switch (this.orientation) {
+        case 'left':
+          t = x;
+          x = this.width - y;
+          y = t;
+          break;
+        case 'right':
+          t = x;
+          x = y;
+          y = t;
+          break;
+        case 'bottom':
+          y = this.height - y;
+          break;
+      }
+      return {
+        x: x + this.unitPadding - (slotSize / 2),
+        y: y + this.unitPadding - (slotSize / 2),
+        s: squareSize
+      };
+    } else {
+      return { x: 0, y: 0, s: 0 };
+    }
+  }
+
+  private screenToGrid(position: Position): Position {
+    let x = position.x;
+    let y = this.height - position.y;
+    let t;
+
+    switch (this.orientation) {
+      case 'left':
+        t = x;
+        x = y;
+        y = this.width - t;
+        break;
+      case 'right':
+        t = x;
+        x = y;
+        y = t;
+        break;
+      case 'bottom':
+        y = this.height - y;
+        break;
+    }
+    if (this.flip) {
+      x = this.width - x;
+    }
+    return {
+      x: Math.floor(x / this.gridSize),
+      y: Math.floor(y / this.gridSize)
+    };
+  }
+
+  // calculates and returns the size of the tx in multiples of the grid size
+  private txSize(tx: TxView): number {
+    const scale = Math.max(1, Math.round(Math.sqrt(tx.vsize / this.vbytesPerUnit)));
+    return Math.min(this.gridWidth, Math.max(1, scale)); // bound between 1 and the max displayable size (just in case!)
+  }
+
+  private place(tx: TxView): void {
+    const size = this.txSize(tx);
+    this.layout.insert(tx, size);
+  }
+
+  private tryInsertByFee(tx: TxView): boolean {
+    const size = this.txSize(tx);
+    const position = this.layout.tryInsertByFee(tx, size);
+    if (position) {
+      this.txs[tx.txid] = tx;
+      return true;
+    } else {
+      return false;
+    }
+  }
+
+  // Add a list of transactions to the layout,
+  // keeping everything approximately sorted by feerate.
+  private placeBatch(txs: TxView[]): void {
+    if (txs.length) {
+      // grab the new tx with the highest fee rate
+      txs = txs.sort(feeRateDescending);
+      const maxSize = 2 * txs.reduce((max, tx) => {
+        return Math.max(this.txSize(tx), max);
+      }, 1);
+
+      // find a reasonable place for it in the layout
+      const root = this.layout.getReplacementRoot(txs[0].feerate, maxSize);
+
+      // extract a sub tree of transactions from the layout, rooted at that point
+      const popped = this.layout.popTree(root.x, root.y, maxSize);
+      // combine those with the new transactions and sort
+      txs = txs.concat(popped);
+      txs = txs.sort(feeRateDescending);
+
+      // insert everything back into the layout
+      txs.forEach(tx => {
+        this.txs[tx.txid] = tx;
+        this.place(tx);
+      });
+    }
+  }
+
+  private removeBatch(ids: string[], startTime: number, direction: string = 'left'): TxView[] {
+    if (!startTime) {
+      startTime = performance.now();
+    }
+    return ids.map(id => {
+      return this.remove(id, startTime, direction);
+    }).filter(tx => tx != null) as TxView[];
+  }
+}
+
+
+class Slot {
+  l: number;
+  r: number;
+  w: number;
+
+  constructor(l: number, r: number) {
+    this.l = l;
+    this.r = r;
+    this.w = r - l;
+  }
+
+  intersects(slot: Slot): boolean {
+    return !((slot.r <= this.l) || (slot.l >= this.r));
+  }
+
+  subtract(slot: Slot): Slot[] | void {
+    if (this.intersects(slot)) {
+      // from middle
+      if (slot.l > this.l && slot.r < this.r) {
+        return [
+          new Slot(this.l, slot.l),
+          new Slot(slot.r, this.r)
+        ];
+      } // totally covered
+      else if (slot.l <= this.l && slot.r >= this.r) {
+        return [];
+      } // from left side
+      else if (slot.l <= this.l) {
+        if (slot.r === this.r) {
+          return [];
+        } else {
+          return [new Slot(slot.r, this.r)];
+        }
+      } // from right side
+      else if (slot.r >= this.r) {
+        if (slot.l === this.l) {
+          return [];
+        } else {
+          return [new Slot(this.l, slot.l)];
+        }
+      }
+    } else {
+      return [this];
+    }
+  }
+}
+
+class TxSlot extends Slot {
+  tx: TxView;
+
+  constructor(l: number, r: number, tx: TxView) {
+    super(l, r);
+    this.tx = tx;
+  }
+}
+
+class Row {
+  y: number;
+  w: number;
+  filled: TxSlot[];
+  slots: Slot[];
+
+
+  constructor(y: number, width: number) {
+    this.y = y;
+    this.w = width;
+    this.filled = [];
+    this.slots = [new Slot(0, this.w)];
+  }
+
+  // insert a transaction w/ given width into row starting at position x
+  insert(x: number, w: number, tx: TxView): void {
+    const newSlot = new TxSlot(x, x + w, tx);
+    // insert into filled list
+    let index = this.filled.findIndex((slot) => (slot.l >= newSlot.r));
+    if (index < 0) {
+      index = this.filled.length;
+    }
+    this.filled.splice(index || 0, 0, newSlot);
+    // subtract from overlapping slots
+    for (let i = 0; i < this.slots.length; i++) {
+      if (newSlot.intersects(this.slots[i])) {
+        const diff = this.slots[i].subtract(newSlot);
+        if (diff) {
+          this.slots.splice(i, 1, ...diff);
+          i += diff.length - 1;
+        }
+      }
+    }
+  }
+
+  remove(x: number, w: number): void {
+    const txIndex = this.filled.findIndex((slot) => (slot.l === x) );
+    this.filled.splice(txIndex, 1);
+
+    const newSlot = new Slot(x, x + w);
+    let slotIndex = this.slots.findIndex((slot) => (slot.l >= newSlot.r) );
+    if (slotIndex < 0) {
+      slotIndex = this.slots.length;
+    }
+    this.slots.splice(slotIndex || 0, 0, newSlot);
+    this.normalize();
+  }
+
+  // merge any contiguous empty slots
+  private normalize(): void {
+    for (let i = 0; i < this.slots.length - 1; i++) {
+      if (this.slots[i].r === this.slots[i + 1].l) {
+        this.slots[i].r = this.slots[i + 1].r;
+        this.slots[i].w += this.slots[i + 1].w;
+        this.slots.splice(i + 1, 1);
+        i--;
+      }
+    }
+  }
+
+  txAt(x: number): TxView | void {
+    let i = 0;
+    while (i < this.filled.length && this.filled[i].l <= x) {
+      if (this.filled[i].l <= x && this.filled[i].r > x) {
+        return this.filled[i].tx;
+      }
+      i++;
+    }
+  }
+
+  getSlotsBetween(left: number, right: number): TxSlot[] {
+    const range = new Slot(left, right);
+    return this.filled.filter(slot => {
+      return slot.intersects(range);
+    });
+  }
+
+  slotAt(x: number): Slot | void {
+    let i = 0;
+    while (i < this.slots.length && this.slots[i].l <= x) {
+      if (this.slots[i].l <= x && this.slots[i].r > x) {
+        return this.slots[i];
+      }
+      i++;
+    }
+  }
+
+  getAvgFeerate(): number {
+    let count = 0;
+    let total = 0;
+    this.filled.forEach(slot => {
+      if (slot.tx) {
+        count += slot.w;
+        total += (slot.tx.feerate * slot.w);
+      }
+    });
+    return total / count;
+  }
+}
+
+class BlockLayout {
+  width: number;
+  height: number;
+  rows: Row[];
+  txPositions: { [key: string]: Square };
+  txs: { [key: string]: TxView };
+
+  constructor({ width, height }: { width: number, height: number }) {
+    this.width = width;
+    this.height = height;
+    this.rows = [new Row(0, this.width)];
+    this.txPositions = {};
+    this.txs = {};
+  }
+
+  getRow(position: Square): Row {
+    return this.rows[position.y];
+  }
+
+  getTx(position: Square): TxView | void {
+    if (this.getRow(position)) {
+      return this.getRow(position).txAt(position.x);
+    }
+  }
+
+  addRow(): void {
+    this.rows.push(new Row(this.rows.length, this.width));
+  }
+
+  remove(tx: TxView) {
+    const position = this.txPositions[tx.txid];
+    if (position) {
+      for (let y = position.y; y < position.y + position.s && y < this.rows.length; y++) {
+        this.rows[y].remove(position.x, position.s);
+      }
+    }
+    delete this.txPositions[tx.txid];
+    delete this.txs[tx.txid];
+  }
+
+  insert(tx: TxView, width: number): Square {
+    const fit = this.fit(tx, width);
+
+    // insert the tx into rows at that position
+    for (let y = fit.y; y < fit.y + width; y++) {
+      if (y >= this.rows.length) {
+        this.addRow();
+      }
+      this.rows[y].insert(fit.x, width, tx);
+    }
+    const position = { x: fit.x, y: fit.y, s: width };
+    this.txPositions[tx.txid] = position;
+    this.txs[tx.txid] = tx;
+    tx.applyGridPosition(position);
+    return position;
+  }
+
+  // Find the first slot large enough to hold a transaction of this size
+  fit(tx: TxView, width: number): Square {
+    let fit;
+    for (let y = 0; y < this.rows.length && !fit; y++) {
+      fit = this.findFit(0, this.width, y, y, width);
+    }
+    // fall back to placing tx in a new row at the top of the layout
+    if (!fit) {
+      fit = { x: 0, y: this.rows.length };
+    }
+    return fit;
+  }
+
+  // recursively check rows to see if there's space for a tx (depth-first)
+  // left/right: initial column boundaries to check
+  // row: current row to check
+  // start: starting row
+  // size: size of space needed
+  findFit(left: number, right: number, row: number, start: number, size: number): Square {
+    if ((row - start) >= size || row >= this.rows.length) {
+      return { x: left, y: start };
+    }
+    for (const slot of this.rows[row].slots) {
+      const l = Math.max(left, slot.l);
+      const r = Math.min(right, slot.r);
+      if (r - l >= size) {
+        const fit = this.findFit(l, r, row + 1, start, size);
+        if (fit) {
+          return fit;
+        }
+      }
+    }
+  }
+
+  // insert only if the tx fits into a fee-appropriate position
+  tryInsertByFee(tx: TxView, size: number): Square | void {
+    const fit = this.fit(tx, size);
+
+    if (this.checkRowFees(fit.y, tx.feerate)) {
+      // insert the tx into rows at that position
+      for (let y = fit.y; y < fit.y + size; y++) {
+        if (y >= this.rows.length) {
+          this.addRow();
+        }
+        this.rows[y].insert(fit.x, size, tx);
+      }
+      const position = { x: fit.x, y: fit.y, s: size };
+      this.txPositions[tx.txid] = position;
+      this.txs[tx.txid] = tx;
+      tx.applyGridPosition(position);
+      return position;
+    }
+  }
+
+  // Return the first slot with a lower feerate
+  getReplacementRoot(feerate: number, width: number): Square {
+    let slot;
+    for (let row = 0; row <= this.rows.length; row++) {
+      if (this.rows[row].slots.length > 0) {
+        return { x: this.rows[row].slots[0].l, y: row };
+      } else {
+        slot = this.rows[row].filled.find(x => {
+          return x.tx.feerate < feerate;
+        });
+        if (slot) {
+          return { x: Math.min(slot.l, this.width - width), y: row };
+        }
+      }
+    }
+    return { x: 0, y: this.rows.length };
+  }
+
+  // remove and return all transactions in a subtree of the layout
+  popTree(x: number, y: number, width: number) {
+    const selected: { [key: string]: TxView } = {};
+    let left = x;
+    let right = x + width;
+    let prevWidth = right - left;
+    let prevFee = Infinity;
+    // scan rows upwards within a channel bounded by 'left' and 'right'
+    for (let row = y; row < this.rows.length; row++) {
+      let rowMax = 0;
+      const slots = this.rows[row].getSlotsBetween(left, right);
+      // check each slot in this row overlapping the search channel
+      slots.forEach(slot => {
+        // select the associated transaction
+        selected[slot.tx.txid] = slot.tx;
+        rowMax = Math.max(rowMax, slot.tx.feerate);
+        // widen the search channel to accommodate this slot if necessary
+        if (slot.w > prevWidth) {
+          left = slot.l;
+          right = slot.r;
+          // if this slot's tx has a higher feerate than the max in the previous row
+          // (i.e. it's out of position)
+          // select all txs overlapping the slot's full width in some rows *below*
+          // to free up space for this tx to sink down to its proper position
+          if (slot.tx.feerate > prevFee) {
+            let count = 0;
+            // keep scanning back down until we find a full row of higher-feerate txs
+            for (let echo = row - 1; echo >= 0 && count < slot.w; echo--) {
+              const echoSlots = this.rows[echo].getSlotsBetween(slot.l, slot.r);
+              count = 0;
+              echoSlots.forEach(echoSlot => {
+                selected[echoSlot.tx.txid] = echoSlot.tx;
+                if (echoSlot.tx.feerate >= slot.tx.feerate) {
+                  count += echoSlot.w;
+                }
+              });
+            }
+          }
+        }
+      });
+      prevWidth = right - left;
+      prevFee = rowMax;
+    }
+
+    const txList = Object.values(selected);
+
+    txList.forEach(tx => {
+      this.remove(tx);
+    });
+    return txList;
+  }
+
+  // Check if this row has high enough avg fees
+  // for a tx with this feerate to make sense here
+  checkRowFees(row: number, targetFee: number): boolean {
+    // first row is always fine
+    if (row === 0 || !this.rows[row]) {
+      return true;
+    }
+    return (this.rows[row].getAvgFeerate() > (targetFee * 0.9));
+  }
+
+  // drop any free-floating transactions down into empty spaces
+  applyGravity(): void {
+    Object.entries(this.txPositions).sort(([keyA, posA], [keyB, posB]) => {
+      return posA.y - posB.y || posA.x - posB.x;
+    }).forEach(([txid, position]) => {
+      // see how far this transaction can fall
+      let dropTo = position.y;
+      while (dropTo > 0 && !this.rows[dropTo - 1].getSlotsBetween(position.x, position.x + position.s).length) {
+        dropTo--;
+      }
+      // if it can fall at all
+      if (dropTo < position.y) {
+        // remove and reinsert in the row we found
+        const tx = this.txs[txid];
+        this.remove(tx);
+        this.insert(tx, position.s);
+      }
+    });
+  }
+}
+
+function feeRateDescending(a: TxView, b: TxView) {
+  return b.feerate - a.feerate;
+}

frontend/src/app/components/block-overview-graph/fast-vertex-array.ts

@@ -18,6 +18,7 @@ export class FastVertexArray {
   data: Float32Array;
   freeSlots: number[];
   lastSlot: number;
+  dirty = false;
 
   constructor(length, stride) {
     this.length = length;
@@ -27,6 +28,7 @@ export class FastVertexArray {
     this.data = new Float32Array(this.length * this.stride);
     this.freeSlots = [];
     this.lastSlot = 0;
+    this.dirty = true;
   }
 
   insert(sprite: TxSprite): number {
@@ -44,6 +46,7 @@ export class FastVertexArray {
     }
     this.sprites[position] = sprite;
     return position;
+    this.dirty = true;
   }
 
   remove(index: number): void {
@@ -54,14 +57,17 @@ export class FastVertexArray {
     if (this.length > 2048 && this.count < (this.length * 0.4)) {
       this.compact();
     }
+    this.dirty = true;
   }
 
   setData(index: number, dataChunk: number[]): void {
     this.data.set(dataChunk, (index * this.stride));
+    this.dirty = true;
   }
 
   clearData(index: number): void {
     this.data.fill(0, (index * this.stride), ((index + 1) * this.stride));
+    this.dirty = true;
   }
 
   getData(index: number): Float32Array {
@@ -73,6 +79,7 @@ export class FastVertexArray {
     const newData = new Float32Array(this.length * this.stride);
     newData.set(this.data);
     this.data = newData;
+    this.dirty = true;
   }
 
   compact(): void {
@@ -97,6 +104,7 @@ export class FastVertexArray {
       this.freeSlots = [];
       this.lastSlot = i;
     }
+    this.dirty = true;
   }
 
   getVertexData(): Float32Array {

frontend/src/app/components/block-overview-graph/tx-view.ts

@@ -82,8 +82,10 @@ export default class TxView implements TransactionStripped {
     delay: additional milliseconds to wait before starting
     jitter: if set, adds a random amount to the delay,
     adjust: if true, modify an in-progress transition instead of replacing it
+
+    returns minimum transition end time
   */
-  update(params: ViewUpdateParams): void {
+  update(params: ViewUpdateParams): number {
     if (params.jitter) {
       params.delay += (Math.random() * params.jitter);
     }
@@ -96,6 +98,7 @@ export default class TxView implements TransactionStripped {
       );
       // apply any pending hover event
       if (this.hover) {
+        params.duration = Math.max(params.duration, hoverTransitionTime);
         this.sprite.update({
           ...this.hoverColor,
           duration: hoverTransitionTime,
@@ -109,10 +112,12 @@ export default class TxView implements TransactionStripped {
       );
     }
     this.dirty = false;
+    return (params.start || performance.now()) + (params.delay || 0) + (params.duration || 0);
   }
 
   // Temporarily override the tx color
-  setHover(hoverOn: boolean, color: Color | void = defaultHoverColor): void {
+  // returns minimum transition end time
+  setHover(hoverOn: boolean, color: Color | void = defaultHoverColor): number {
     if (hoverOn) {
       this.hover = true;
       this.hoverColor = color;
@@ -131,12 +136,12 @@ export default class TxView implements TransactionStripped {
       }
     }
     this.dirty = false;
+    return performance.now() + hoverTransitionTime;
   }
 
   getColor(): Color {
-    let feeLevelIndex = feeLevels.slice().reverse().findIndex((feeLvl) => (this.feerate || 1) >= feeLvl);
-    feeLevelIndex = feeLevelIndex >= 0 ? feeLevels.length - feeLevelIndex : feeLevelIndex;
-    return hexToColor(mempoolFeeColors[feeLevelIndex - 1] || mempoolFeeColors[mempoolFeeColors.length - 1]);
+    const feeLevelIndex = feeLevels.findIndex((feeLvl) => Math.max(1, this.feerate) < feeLvl) - 1;
+    return hexToColor(mempoolFeeColors[feeLevelIndex] || mempoolFeeColors[mempoolFeeColors.length - 1]);
   }
 }
 

frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html

@@ -0,0 +1,37 @@
+<div
+  #tooltip
+  class="block-overview-tooltip"
+  [class.clickable]="clickable"
+  [style.visibility]="tx ? 'visible' : 'hidden'"
+  [style.left]="tooltipPosition.x + 'px'"
+  [style.top]="tooltipPosition.y + 'px'"
+>
+  <table>
+    <tbody>
+      <tr>
+        <td class="td-width" i18n="shared.transaction">Transaction</td>
+        <td>
+          <a [routerLink]="['/tx/' | relativeUrl, txid]">{{ txid | shortenString : 16}}</a>
+        </td>
+      </tr>
+      <tr>
+        <td class="td-width" i18n="dashboard.latest-transactions.amount">Amount</td>
+        <td><app-amount [satoshis]="value"></app-amount></td>
+      </tr>
+      <tr>
+        <td class="td-width" i18n="transaction.fee|Transaction fee">Fee</td>
+        <td>{{ fee | number }} <span class="symbol" i18n="shared.sat|sat">sat</span> &nbsp; <span class="fiat"><app-fiat [value]="fee"></app-fiat></span></td>
+      </tr>
+      <tr>
+        <td class="td-width" i18n="transaction.fee-rate|Transaction fee rate">Fee rate</td>
+        <td>
+          {{ feeRate | feeRounding }} <span class="symbol" i18n="shared.sat-vbyte|sat/vB">sat/vB</span>
+        </td>
+      </tr>
+      <tr>
+        <td class="td-width" i18n="transaction.vsize|Transaction Virtual Size">Virtual size</td>
+        <td [innerHTML]="'&lrm;' + (vsize | vbytes: 2)"></td>
+      </tr>
+    </tbody>
+  </table>
+</div>

frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.scss

@@ -0,0 +1,22 @@
+.block-overview-tooltip {
+  position: absolute;
+  background: rgba(#11131f, 0.95);
+  border-radius: 4px;
+  box-shadow: 1px 1px 10px rgba(0,0,0,0.5);
+  color: #b1b1b1;
+  display: flex;
+  flex-direction: column;
+  justify-content: space-between;
+  padding: 10px 15px;
+  text-align: left;
+  min-width: 320px;
+  pointer-events: none;
+
+  &.clickable {
+    pointer-events: all;
+  }
+}
+
+.td-width {
+  padding-right: 10px;
+}

frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts

@@ -0,0 +1,53 @@
+import { Component, ElementRef, ViewChild, Input, OnChanges, ChangeDetectionStrategy } from '@angular/core';
+import { TransactionStripped } from 'src/app/interfaces/websocket.interface';
+import { Position } from 'src/app/components/block-overview-graph/sprite-types.js';
+
+@Component({
+  selector: 'app-block-overview-tooltip',
+  templateUrl: './block-overview-tooltip.component.html',
+  styleUrls: ['./block-overview-tooltip.component.scss'],
+})
+export class BlockOverviewTooltipComponent implements OnChanges {
+  @Input() tx: TransactionStripped | void;
+  @Input() cursorPosition: Position;
+  @Input() clickable: boolean;
+
+  txid = '';
+  fee = 0;
+  value = 0;
+  vsize = 1;
+  feeRate = 0;
+
+  tooltipPosition: Position = { x: 0, y: 0 };
+
+  @ViewChild('tooltip') tooltipElement: ElementRef<HTMLCanvasElement>;
+
+  constructor() {}
+
+  ngOnChanges(changes): void {
+    if (changes.cursorPosition && changes.cursorPosition.currentValue) {
+      let x = changes.cursorPosition.currentValue.x + 10;
+      let y = changes.cursorPosition.currentValue.y + 10;
+      if (this.tooltipElement) {
+        const elementBounds = this.tooltipElement.nativeElement.getBoundingClientRect();
+        const parentBounds = this.tooltipElement.nativeElement.offsetParent.getBoundingClientRect();
+        if ((parentBounds.left + x + elementBounds.width) > parentBounds.right) {
+          x = Math.max(0, parentBounds.width - elementBounds.width - 10);
+        }
+        if (y + elementBounds.height > parentBounds.height) {
+          y = y - elementBounds.height - 20;
+        }
+      }
+      this.tooltipPosition = { x, y };
+    }
+
+    if (changes.tx) {
+      const tx = changes.tx.currentValue || {};
+      this.txid = tx.txid || '';
+      this.fee = tx.fee || 0;
+      this.value = tx.value || 0;
+      this.vsize = tx.vsize || 1;
+      this.feeRate = this.fee / this.vsize;
+    }
+  }
+}

frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.html

@@ -0,0 +1,52 @@
+<app-indexing-progress></app-indexing-progress>
+
+<div class="full-container">
+  <div class="card-header mb-0 mb-md-4">
+    <span i18n="mining.block-prediction-accuracy">Block Prediction Accuracy</span>
+      <button class="btn" style="margin: 0 0 4px 0px" (click)="onSaveChart()">
+      <fa-icon [icon]="['fas', 'download']" [fixedWidth]="true"></fa-icon>
+    </button>
+    <form [formGroup]="radioGroupForm" class="formRadioGroup" *ngIf="(statsObservable$ | async) as stats">
+      <div class="btn-group btn-group-toggle" ngbRadioGroup name="radioBasic" formControlName="dateSpan">
+        <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 144">
+          <input ngbButton type="radio" [value]="'24h'" fragment="24h" [routerLink]="['/graphs/mining/block-prediction' | relativeUrl]"> 24h
+        </label>
+        <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 432">
+          <input ngbButton type="radio" [value]="'3d'" fragment="3d" [routerLink]="['/graphs/mining/block-prediction' | relativeUrl]"> 3D
+        </label>
+        <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 1008">
+          <input ngbButton type="radio" [value]="'1w'" fragment="1w" [routerLink]="['/graphs/mining/block-prediction' | relativeUrl]"> 1W
+        </label>
+        <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 4320">
+          <input ngbButton type="radio" [value]="'1m'" fragment="1m" [routerLink]="['/graphs/mining/block-prediction' | relativeUrl]"> 1M
+        </label>
+        <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 12960">
+          <input ngbButton type="radio" [value]="'3m'" fragment="3m" [routerLink]="['/graphs/mining/block-prediction' | relativeUrl]"> 3M
+        </label>
+        <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 25920">
+          <input ngbButton type="radio" [value]="'6m'" fragment="6m" [routerLink]="['/graphs/mining/block-prediction' | relativeUrl]"> 6M
+        </label>
+        <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 52560">
+          <input ngbButton type="radio" [value]="'1y'" fragment="1y" [routerLink]="['/graphs/mining/block-prediction' | relativeUrl]"> 1Y
+        </label>
+        <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 105120">
+          <input ngbButton type="radio" [value]="'2y'" fragment="2y" [routerLink]="['/graphs/mining/block-prediction' | relativeUrl]"> 2Y
+        </label>
+        <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 157680">
+          <input ngbButton type="radio" [value]="'3y'" fragment="3y" [routerLink]="['/graphs/mining/block-prediction' | relativeUrl]"> 3Y
+        </label>
+        <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount > 157680">
+          <input ngbButton type="radio" [value]="'all'" fragment="all" [routerLink]="['/graphs/mining/block-prediction' | relativeUrl]"> ALL
+        </label>
+      </div>
+    </form>
+  </div>
+
+  <div class="chart" echarts [initOpts]="chartInitOptions" [options]="chartOptions"
+    (chartInit)="onChartInit($event)">
+  </div>
+  <div class="text-center loadingGraphs" *ngIf="isLoading">
+    <div class="spinner-border text-light"></div>
+  </div>
+
+</div>

frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.scss

@@ -0,0 +1,81 @@
+.card-header {
+  border-bottom: 0;
+  font-size: 18px;
+  @media (min-width: 465px) {
+    font-size: 20px;
+  }
+}
+
+.main-title {
+  position: relative;
+  color: #ffffff91;
+  margin-top: -13px;
+  font-size: 10px;
+  text-transform: uppercase;
+  font-weight: 500;
+  text-align: center;
+  padding-bottom: 3px;
+}
+
+.full-container {
+  padding: 0px 15px;
+  width: 100%;
+  min-height: 500px;
+  height: calc(100% - 150px);
+  @media (max-width: 992px) {
+    height: 100%;
+    padding-bottom: 100px;
+  };
+}
+
+.chart {
+  width: 100%;
+  height: 100%;
+  padding-bottom: 20px;
+  padding-right: 10px;
+  @media (max-width: 992px) {
+    padding-bottom: 25px;
+  }
+  @media (max-width: 829px) {
+    padding-bottom: 50px;
+  }
+  @media (max-width: 767px) {
+    padding-bottom: 25px;
+  }
+  @media (max-width: 629px) {
+    padding-bottom: 55px;
+  }
+  @media (max-width: 567px) {
+    padding-bottom: 55px;
+  }
+}
+.chart-widget {
+  width: 100%;
+  height: 100%;
+  max-height: 270px;
+}
+
+.formRadioGroup {
+  margin-top: 6px;
+  display: flex;
+  flex-direction: column;
+  @media (min-width: 991px) {
+    position: relative;
+    top: -65px;
+  }
+  @media (min-width: 830px) and (max-width: 991px) {
+    position: relative;
+    top: 0px;
+  }
+  @media (min-width: 830px) {
+    flex-direction: row;
+    float: right;
+    margin-top: 0px;
+  }
+  .btn-sm {
+    font-size: 9px;
+    @media (min-width: 830px) {
+      font-size: 14px;
+    }
+  }
+}

frontend/src/app/components/block-prediction-graph/block-prediction-graph.component.ts

@@ -0,0 +1,289 @@
+import { ChangeDetectionStrategy, Component, Inject, Input, LOCALE_ID, NgZone, OnInit } from '@angular/core';
+import { EChartsOption } from 'echarts';
+import { Observable } from 'rxjs';
+import { map, share, startWith, switchMap, tap } from 'rxjs/operators';
+import { ApiService } from 'src/app/services/api.service';
+import { SeoService } from 'src/app/services/seo.service';
+import { formatNumber } from '@angular/common';
+import { FormBuilder, FormGroup } from '@angular/forms';
+import { download, formatterXAxis, formatterXAxisLabel, formatterXAxisTimeCategory } from 'src/app/shared/graphs.utils';
+import { StorageService } from 'src/app/services/storage.service';
+import { ActivatedRoute, Router } from '@angular/router';
+import { RelativeUrlPipe } from 'src/app/shared/pipes/relative-url/relative-url.pipe';
+import { StateService } from 'src/app/services/state.service';
+
+@Component({
+  selector: 'app-block-prediction-graph',
+  templateUrl: './block-prediction-graph.component.html',
+  styleUrls: ['./block-prediction-graph.component.scss'],
+  styles: [`
+    .loadingGraphs {
+      position: absolute;
+      top: 50%;
+      left: calc(50% - 15px);
+      z-index: 100;
+    }
+  `],
+  changeDetection: ChangeDetectionStrategy.OnPush,
+})
+export class BlockPredictionGraphComponent implements OnInit {
+  @Input() right: number | string = 45;
+  @Input() left: number | string = 75;
+
+  miningWindowPreference: string;
+  radioGroupForm: FormGroup;
+
+  chartOptions: EChartsOption = {};
+  chartInitOptions = {
+    renderer: 'svg',
+  };
+
+  statsObservable$: Observable<any>;
+  isLoading = true;
+  formatNumber = formatNumber;
+  timespan = '';
+  chartInstance: any = undefined;
+
+  constructor(
+    @Inject(LOCALE_ID) public locale: string,
+    private seoService: SeoService,
+    private apiService: ApiService,
+    private formBuilder: FormBuilder,
+    private storageService: StorageService,
+    private zone: NgZone,
+    private route: ActivatedRoute,
+    private stateService: StateService,
+    private router: Router,
+  ) {
+    this.radioGroupForm = this.formBuilder.group({ dateSpan: '1y' });
+    this.radioGroupForm.controls.dateSpan.setValue('1y');
+  }
+
+  ngOnInit(): void {
+    this.seoService.setTitle($localize`:@@d7d5fcf50179ad70c938491c517efb82de2c8146:Block Prediction Accuracy`);
+    this.miningWindowPreference = '24h';//this.miningService.getDefaultTimespan('24h');
+    this.radioGroupForm = this.formBuilder.group({ dateSpan: this.miningWindowPreference });
+    this.radioGroupForm.controls.dateSpan.setValue(this.miningWindowPreference);
+
+    this.route
+      .fragment
+      .subscribe((fragment) => {
+        if (['24h', '3d', '1w', '1m', '3m', '6m', '1y', '2y', '3y', 'all'].indexOf(fragment) > -1) {
+          this.radioGroupForm.controls.dateSpan.setValue(fragment, { emitEvent: false });
+        }
+      });
+
+    this.statsObservable$ = this.radioGroupForm.get('dateSpan').valueChanges
+      .pipe(
+        startWith(this.radioGroupForm.controls.dateSpan.value),
+        switchMap((timespan) => {
+          this.storageService.setValue('miningWindowPreference', timespan);
+          this.timespan = timespan;
+          this.isLoading = true;
+          return this.apiService.getHistoricalBlockPrediction$(timespan)
+            .pipe(
+              tap((response) => {
+                this.prepareChartOptions(response.body);
+                this.isLoading = false;
+              }),
+              map((response) => {
+                return {
+                  blockCount: parseInt(response.headers.get('x-total-count'), 10),
+                };
+              }),
+            );
+        }),
+        share()
+      );
+  }
+
+  prepareChartOptions(data) {
+    this.chartOptions = {
+      animation: false,
+      grid: {
+        top: 30,
+        bottom: 80,
+        right: this.right,
+        left: this.left,
+      },
+      tooltip: {
+        show: !this.isMobile(),
+        trigger: 'axis',
+        axisPointer: {
+          type: 'line'
+        },
+        backgroundColor: 'rgba(17, 19, 31, 1)',
+        borderRadius: 4,
+        shadowColor: 'rgba(0, 0, 0, 0.5)',
+        textStyle: {
+          color: '#b1b1b1',
+          align: 'left',
+        },
+        borderColor: '#000',
+        formatter: (ticks) => {
+          let tooltip = `<b style="color: white; margin-left: 2px">${formatterXAxis(this.locale, this.timespan, parseInt(ticks[0].axisValue, 10) * 1000)}</b><br>`;
+          tooltip += `${ticks[0].marker} ${ticks[0].seriesName}: ${formatNumber(ticks[0].data.value, this.locale, '1.2-2')}%<br>`;
+
+          if (['24h', '3d'].includes(this.timespan)) {
+            tooltip += `<small>` + $localize`At block: ${ticks[0].data.block}` + `</small>`;
+          } else {
+            tooltip += `<small>` + $localize`Around block: ${ticks[0].data.block}` + `</small>`;
+          }
+
+          return tooltip;
+        }
+      },
+      xAxis: {
+        name: formatterXAxisLabel(this.locale, this.timespan),
+        nameLocation: 'middle',
+        nameTextStyle: {
+          padding: [10, 0, 0, 0],
+        },
+        type: 'category',
+        boundaryGap: false,
+        axisLine: { onZero: true },
+        axisLabel: {
+          formatter: val => formatterXAxisTimeCategory(this.locale, this.timespan, parseInt(val, 10)),
+          align: 'center',
+          fontSize: 11,
+          lineHeight: 12,
+          hideOverlap: true,
+          padding: [0, 5],
+        },
+        data: data.map(prediction => prediction[0])
+      },
+      yAxis: [
+        {
+          type: 'value',
+          axisLabel: {
+            color: 'rgb(110, 112, 121)',
+            formatter: (val) => {
+              return `${val}%`;
+            }
+          },
+          splitLine: {
+            lineStyle: {
+              type: 'dotted',
+              color: '#ffffff66',
+              opacity: 0.25,
+            }
+          },
+        },
+      ],
+      series: [
+        {
+          zlevel: 0,
+          name: $localize`Match rate`,
+          data: data.map(prediction => ({
+            value: prediction[2],
+            block: prediction[1],
+            itemStyle: {
+              color: this.getPredictionColor(prediction[2])
+            }
+          })),
+          type: 'bar',
+          barWidth: '90%',
+        },
+      ],
+      dataZoom: [{
+        type: 'inside',
+        realtime: true,
+        zoomLock: true,
+        maxSpan: 100,
+        minSpan: 5,
+        moveOnMouseMove: false,
+      }, {
+        showDetail: false,
+        show: true,
+        type: 'slider',
+        brushSelect: false,
+        realtime: true,
+        left: 20,
+        right: 15,
+        selectedDataBackground: {
+          lineStyle: {
+            color: '#fff',
+            opacity: 0.45,
+          },
+          areaStyle: {
+            opacity: 0,
+          }
+        },
+      }],
+    };
+  }
+
+  colorGradient(fadeFraction, rgbColor1, rgbColor2, rgbColor3) {
+    let color1 = rgbColor1;
+    let color2 = rgbColor2;
+    let fade = fadeFraction;
+
+    // Do we have 3 colors for the gradient? Need to adjust the params.
+    if (rgbColor3) {
+      fade = fade * 2;
+
+      // Find which interval to use and adjust the fade percentage
+      if (fade >= 1) {
+        fade -= 1;
+        color1 = rgbColor2;
+        color2 = rgbColor3;
+      }
+    }
+
+    const diffRed = color2.red - color1.red;
+    const diffGreen = color2.green - color1.green;
+    const diffBlue = color2.blue - color1.blue;
+
+    const gradient = {
+      red: Math.floor(color1.red + (diffRed * fade)),
+      green: Math.floor(color1.green + (diffGreen * fade)),
+      blue: Math.floor(color1.blue + (diffBlue * fade)),
+    };
+
+    return 'rgb(' + gradient.red + ',' + gradient.green + ',' + gradient.blue + ')';
+  }
+
+  getPredictionColor(matchRate) {
+    return this.colorGradient(
+      Math.pow((100 - matchRate) / 100, 0.5),
+      {red: 67, green: 171, blue: 71},
+      {red: 253, green: 216, blue: 53},
+      {red: 244, green: 0, blue: 0},
+    );
+  }
+
+  onChartInit(ec) {
+    this.chartInstance = ec;
+
+    this.chartInstance.on('click', (e) => {
+      this.zone.run(() => {
+        if (['24h', '3d'].includes(this.timespan)) {
+          const url = new RelativeUrlPipe(this.stateService).transform(`/block/${e.data.block}`);
+          this.router.navigate([url]);
+        }
+      });
+    });
+  }
+
+  isMobile() {
+    return (window.innerWidth <= 767.98);
+  }
+
+  onSaveChart() {
+    // @ts-ignore
+    const prevBottom = this.chartOptions.grid.bottom;
+    const now = new Date();
+    // @ts-ignore
+    this.chartOptions.grid.bottom = 40;
+    this.chartOptions.backgroundColor = '#11131f';
+    this.chartInstance.setOption(this.chartOptions);
+    download(this.chartInstance.getDataURL({
+      pixelRatio: 2,
+      excludeComponents: ['dataZoom'],
+    }), `block-fees-${this.timespan}-${Math.round(now.getTime() / 1000)}.svg`);
+    // @ts-ignore
+    this.chartOptions.grid.bottom = prevBottom;
+    this.chartOptions.backgroundColor = 'none';
+    this.chartInstance.setOption(this.chartOptions);
+  }
+}

frontend/src/app/components/block-rewards-graph/block-rewards-graph.component.html

@@ -10,34 +10,34 @@
     <form [formGroup]="radioGroupForm" class="formRadioGroup" *ngIf="(statsObservable$ | async) as stats">
       <div class="btn-group btn-group-toggle" ngbRadioGroup name="radioBasic" formControlName="dateSpan">
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 144">
-          <input ngbButton type="radio" [value]="'24h'" fragment="24h"> 24h
+          <input ngbButton type="radio" [value]="'24h'" fragment="24h" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]"> 24h
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 432">
-          <input ngbButton type="radio" [value]="'3d'" fragment="3d"> 3D
+          <input ngbButton type="radio" [value]="'3d'" fragment="3d" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]"> 3D
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 1008">
-          <input ngbButton type="radio" [value]="'1w'" fragment="1w"> 1W
+          <input ngbButton type="radio" [value]="'1w'" fragment="1w" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]"> 1W
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 4320">
-          <input ngbButton type="radio" [value]="'1m'" fragment="1m"> 1M
+          <input ngbButton type="radio" [value]="'1m'" fragment="1m" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]"> 1M
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 12960">
-          <input ngbButton type="radio" [value]="'3m'" fragment="3m"> 3M
+          <input ngbButton type="radio" [value]="'3m'" fragment="3m" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]"> 3M
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 25920">
-          <input ngbButton type="radio" [value]="'6m'" fragment="6m"> 6M
+          <input ngbButton type="radio" [value]="'6m'" fragment="6m" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]"> 6M
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 52560">
-          <input ngbButton type="radio" [value]="'1y'" fragment="1y"> 1Y
+          <input ngbButton type="radio" [value]="'1y'" fragment="1y" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]"> 1Y
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 105120">
-          <input ngbButton type="radio" [value]="'2y'" fragment="2y"> 2Y
+          <input ngbButton type="radio" [value]="'2y'" fragment="2y" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]"> 2Y
         </label>
         <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount >= 157680">
-          <input ngbButton type="radio" [value]="'3y'" fragment="3y"> 3Y
+          <input ngbButton type="radio" [value]="'3y'" fragment="3y" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]"> 3Y
         </label>
-        <label ngbButtonLabel class="btn-primary btn-sm" *ngIf="stats.blockCount > 157680">
-          <input ngbButton type="radio" [value]="'all'" fragment="all"> ALL
+        <label ngbButtonLabel class="btn-primary btn-sm">
+          <input ngbButton type="radio" [value]="'all'" fragment="all" [routerLink]="['/graphs/mining/block-rewards' | relativeUrl]"> ALL
         </label>
       </div>
     </form>