Merge branch 'master' into nymkappa/bugfix/404-ftx-not-found

.github/workflows/on-tag.yml

@@ -68,24 +68,24 @@ jobs:
         run: echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin
 
       - name: Checkout project
-        uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
+        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791 # v2.5.0
 
       - name: Init repo for Dockerization
         run: docker/init.sh "$TAG"
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
+        uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v2.1.0
         id: qemu
 
       - name: Setup Docker buildx action
-        uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
+        uses: docker/setup-buildx-action@8c0edbc76e98fa90f69d9a2c020dcb50019dc325 # v2.2.1
         id: buildx
 
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }}
 
       - name: Cache Docker layers
-        uses: actions/cache@661fd3eb7f2f20d8c7c84bc2b0509efd7a826628 # v2
+        uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # v3.0.11
         id: cache
         with:
           path: /tmp/.buildx-cache

.gitignore

@@ -3,3 +3,5 @@ data
 docker-compose.yml
 backend/mempool-config.json
 *.swp
+frontend/src/resources/config.template.js
+frontend/src/resources/config.js

backend/mempool-config.sample.json

@@ -2,6 +2,7 @@
   "MEMPOOL": {
     "NETWORK": "mainnet",
     "BACKEND": "electrum",
+    "ENABLED": true,
     "HTTP_PORT": 8999,
     "SPAWN_CLUSTER_PROCS": 0,
     "API_URL_PREFIX": "/api/v1/",
@@ -23,7 +24,8 @@
     "STDOUT_LOG_MIN_PRIORITY": "debug",
     "AUTOMATIC_BLOCK_REINDEXING": false,
     "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
-    "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master"
+    "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
+    "ADVANCED_TRANSACTION_SELECTION": false
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",

backend/src/__fixtures__/mempool-config.template.json

@@ -1,7 +1,9 @@
 {
   "MEMPOOL": {
+    "ENABLED": true,
     "NETWORK": "__MEMPOOL_NETWORK__",
     "BACKEND": "__MEMPOOL_BACKEND__",
+    "ENABLED": true,
     "BLOCKS_SUMMARIES_INDEXING": true,
     "HTTP_PORT": 1,
     "SPAWN_CLUSTER_PROCS": 2,
@@ -23,7 +25,8 @@
     "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
     "INDEXING_BLOCKS_AMOUNT": 14,
     "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__",
-    "POOLS_JSON_URL": "__POOLS_JSON_URL__"
+    "POOLS_JSON_URL": "__POOLS_JSON_URL__",
+    "ADVANCED_TRANSACTION_SELECTION": "__ADVANCED_TRANSACTION_SELECTION__"
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",

backend/src/__tests__/config.test.ts

@@ -13,6 +13,7 @@ describe('Mempool Backend Config', () => {
       const config = jest.requireActual('../config').default;
 
       expect(config.MEMPOOL).toStrictEqual({
+        ENABLED: true,
         NETWORK: 'mainnet',
         BACKEND: 'none',
         BLOCKS_SUMMARIES_INDEXING: false,
@@ -36,7 +37,8 @@ describe('Mempool Backend Config', () => {
         USER_AGENT: 'mempool',
         STDOUT_LOG_MIN_PRIORITY: 'debug',
         POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
-        POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json'
+        POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
+        ADVANCED_TRANSACTION_SELECTION: false,
       });
 
       expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });

backend/src/api/audit.ts

@@ -1,5 +1,10 @@
-import logger from '../logger';
+import config from '../config';
-import { BlockExtended, TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
+import bitcoinApi from './bitcoin/bitcoin-api-factory';
+import { Common } from './common';
+import { TransactionExtended, MempoolBlockWithTransactions, AuditScore } from '../mempool.interfaces';
+import blocksRepository from '../repositories/BlocksRepository';
+import blocksAuditsRepository from '../repositories/BlocksAuditsRepository';
+import blocks from '../api/blocks';
 
 const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
 
@@ -44,8 +49,6 @@ class Audit {
 
     displacedWeight += (4000 - transactions[0].weight);
 
-    logger.warn(`${fresh.length} fresh, ${Object.keys(isCensored).length} possibly censored, ${displacedWeight} displaced weight`);
-
     // we can expect an honest miner to include 'displaced' transactions in place of recent arrivals and censored txs
     // these displaced transactions should occupy the first N weight units of the next projected block
     let displacedWeightRemaining = displacedWeight;
@@ -73,6 +76,7 @@ class Audit {
 
     // mark unexpected transactions in the mined block as 'added'
     let overflowWeight = 0;
+    let totalWeight = 0;
     for (const tx of transactions) {
       if (inTemplate[tx.txid]) {
         matches.push(tx.txid);
@@ -82,11 +86,13 @@ class Audit {
         }
         overflowWeight += tx.weight;
       }
+      totalWeight += tx.weight;
     }
 
     // transactions missing from near the end of our template are probably not being censored
-    let overflowWeightRemaining = overflowWeight;
+    let overflowWeightRemaining = overflowWeight - (config.MEMPOOL.BLOCK_WEIGHT_UNITS - totalWeight);
-    let lastOverflowRate = 1.00;
+    let maxOverflowRate = 0;
+    let rateThreshold = 0;
     index = projectedBlocks[0].transactionIds.length - 1;
     while (index >= 0) {
       const txid = projectedBlocks[0].transactionIds[index];
@@ -94,8 +100,11 @@ class Audit {
         if (isCensored[txid]) {
           delete isCensored[txid];
         }
-        lastOverflowRate = mempool[txid].effectiveFeePerVsize;
+        if (mempool[txid].effectiveFeePerVsize > maxOverflowRate) {
-      } else if (Math.floor(mempool[txid].effectiveFeePerVsize * 100) <= Math.ceil(lastOverflowRate * 100)) { // tolerance of 0.01 sat/vb
+          maxOverflowRate = mempool[txid].effectiveFeePerVsize;
+          rateThreshold = (Math.ceil(maxOverflowRate * 100) / 100) + 0.005;
+        }
+      } else if (mempool[txid].effectiveFeePerVsize <= rateThreshold) { // tolerance of 0.01 sat/vb + rounding
         if (isCensored[txid]) {
           delete isCensored[txid];
         }
@@ -113,6 +122,45 @@ class Audit {
       score
     };
   }
+
+  public async $getBlockAuditScores(fromHeight?: number, limit: number = 15): Promise<AuditScore[]> {
+    let currentHeight = fromHeight !== undefined ? fromHeight : await blocksRepository.$mostRecentBlockHeight();
+    const returnScores: AuditScore[] = [];
+
+    if (currentHeight < 0) {
+      return returnScores;
+    }
+
+    for (let i = 0; i < limit && currentHeight >= 0; i++) {
+      const block = blocks.getBlocks().find((b) => b.height === currentHeight);
+      if (block?.extras?.matchRate != null) {
+        returnScores.push({
+          hash: block.id,
+          matchRate: block.extras.matchRate
+        });
+      } else {
+        let currentHash;
+        if (!currentHash && Common.indexingEnabled()) {
+          const dbBlock = await blocksRepository.$getBlockByHeight(currentHeight);
+          if (dbBlock && dbBlock['id']) {
+            currentHash = dbBlock['id'];
+          }
+        }
+        if (!currentHash) {
+          currentHash = await bitcoinApi.$getBlockHash(currentHeight);
+        }
+        if (currentHash) {
+          const auditScore = await blocksAuditsRepository.$getBlockAuditScore(currentHash);
+          returnScores.push({
+            hash: currentHash,
+            matchRate: auditScore?.matchRate
+          });
+        }
+      }
+      currentHeight--;
+    }
+    return returnScores;
+  }
 }
 
 export default new Audit();

backend/src/api/blocks.ts

@@ -34,6 +34,7 @@ class Blocks {
   private lastDifficultyAdjustmentTime = 0;
   private previousDifficultyRetarget = 0;
   private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = [];
+  private newAsyncBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => Promise<void>)[] = [];
 
   constructor() { }
 
@@ -57,6 +58,10 @@ class Blocks {
     this.newBlockCallbacks.push(fn);
   }
 
+  public setNewAsyncBlockCallback(fn: (block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => Promise<void>) {
+    this.newAsyncBlockCallbacks.push(fn);
+  }
+
   /**
    * Return the list of transaction for a block
    * @param blockHash
@@ -130,7 +135,7 @@ class Blocks {
     const stripped = block.tx.map((tx) => {
       return {
         txid: tx.txid,
-        vsize: tx.vsize,
+        vsize: tx.weight / 4,
         fee: tx.fee ? Math.round(tx.fee * 100000000) : 0,
         value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0) * 100000000)
       };
@@ -195,9 +200,9 @@ class Blocks {
         };
       }
 
-      const auditSummary = await BlocksAuditsRepository.$getShortBlockAudit(block.id);
+      const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
-      if (auditSummary) {
+      if (auditScore != null) {
-        blockExtended.extras.matchRate = auditSummary.matchRate;
+        blockExtended.extras.matchRate = auditScore.matchRate;
       }
     }
 
@@ -444,6 +449,9 @@ class Blocks {
       const blockExtended: BlockExtended = await this.$getBlockExtended(block, transactions);
       const blockSummary: BlockSummary = this.summarizeBlock(verboseBlock);
 
+      // start async callbacks
+      const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions));
+
       if (Common.indexingEnabled()) {
         if (!fastForwarded) {
           const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
@@ -514,6 +522,9 @@ class Blocks {
       if (!memPool.hasPriority()) {
         diskCache.$saveCacheToDisk();
       }
+
+      // wait for pending async callbacks to finish
+      await Promise.all(callbackPromises);
     }
   }
 

backend/src/api/database-migration.ts

@@ -4,8 +4,8 @@ import logger from '../logger';
 import { Common } from './common';
 
 class DatabaseMigration {
-  private static currentVersion = 41;
+  private static currentVersion = 44;
-  private queryTimeout = 120000;
+  private queryTimeout = 900_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
 
@@ -352,6 +352,19 @@ class DatabaseMigration {
     if (databaseSchemaVersion < 41 && isBitcoin === true) {
       await this.$executeQuery('UPDATE channels SET closing_reason = NULL WHERE closing_reason = 1');
     }
+
+    if (databaseSchemaVersion < 42 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `channels` ADD closing_resolved tinyint(1) DEFAULT 0');
+    }
+
+    if (databaseSchemaVersion < 43 && isBitcoin === true) {
+      await this.$executeQuery(this.getCreateLNNodeRecordsTableQuery(), await this.$checkIfTableExists('nodes_records'));
+    }
+
+    if (databaseSchemaVersion < 44 && isBitcoin === true) {
+      await this.$executeQuery('TRUNCATE TABLE `blocks_audits`');
+      await this.$executeQuery('UPDATE blocks_summaries SET template = NULL');
+    }
   }
 
   /**
@@ -787,6 +800,19 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getCreateLNNodeRecordsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS nodes_records (
+      public_key varchar(66) NOT NULL,
+      type int(10) unsigned NOT NULL,
+      payload blob NOT NULL,
+      UNIQUE KEY public_key_type (public_key, type),
+      INDEX (public_key),
+      FOREIGN KEY (public_key)
+        REFERENCES nodes (public_key)
+        ON DELETE CASCADE
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
   public async $truncateIndexedData(tables: string[]) {
     const allowedTables = ['blocks', 'hashrates', 'prices'];
 

backend/src/api/explorer/channels.api.ts

@@ -117,6 +117,17 @@ class ChannelsApi {
     }
   }
 
+  public async $getUnresolvedClosedChannels(): Promise<any[]> {
+    try {
+      const query = `SELECT * FROM channels WHERE status = 2 AND closing_reason = 2 AND closing_resolved = 0 AND closing_transaction_id != ''`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getUnresolvedClosedChannels error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   public async $getChannelsWithoutCreatedDate(): Promise<any[]> {
     try {
       const query = `SELECT * FROM channels WHERE created IS NULL`;

backend/src/api/explorer/nodes.api.ts

@@ -105,6 +105,18 @@ class NodesApi {
         node.closed_channel_count = rows[0].closed_channel_count;
       }
 
+      // Custom records
+      query = `
+        SELECT type, payload
+        FROM nodes_records
+        WHERE public_key = ?
+      `;
+      [rows] = await DB.query(query, [public_key]);
+      node.custom_records = {};
+      for (const record of rows) {
+        node.custom_records[record.type] = Buffer.from(record.payload, 'binary').toString('hex');
+      }
+
       return node;
     } catch (e) {
       logger.err(`Cannot get node information for ${public_key}. Reason: ${(e instanceof Error ? e.message : e)}`);
@@ -512,7 +524,37 @@ class NodesApi {
 
   public async $getNodesPerISP(ISPId: string) {
     try {
-      const query = `
+      let query = `
+        SELECT channels.node1_public_key AS node1PublicKey, isp1.id as isp1ID,
+          channels.node2_public_key AS node2PublicKey, isp2.id as isp2ID
+        FROM channels
+        JOIN nodes node1 ON node1.public_key = channels.node1_public_key
+        JOIN nodes node2 ON node2.public_key = channels.node2_public_key
+        JOIN geo_names isp1 ON isp1.id = node1.as_number
+        JOIN geo_names isp2 ON isp2.id = node2.as_number
+        WHERE channels.status = 1 AND (node1.as_number IN (?) OR node2.as_number IN (?))
+        ORDER BY short_id DESC
+      `;
+
+      const IPSIds = ISPId.split(',');
+      const [rows]: any = await DB.query(query, [IPSIds, IPSIds]);
+      const nodes = {};
+
+      const intISPIds: number[] = [];
+      for (const ispId of IPSIds) {
+        intISPIds.push(parseInt(ispId, 10));
+      }
+
+      for (const channel of rows) {
+        if (intISPIds.includes(channel.isp1ID)) {
+          nodes[channel.node1PublicKey] = true;
+        }
+        if (intISPIds.includes(channel.isp2ID)) {
+          nodes[channel.node2PublicKey] = true;
+        }
+      }
+
+      query = `
         SELECT nodes.public_key, CAST(COALESCE(nodes.capacity, 0) as INT) as capacity, CAST(COALESCE(nodes.channels, 0) as INT) as channels,
           nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
           geo_names_city.names as city, geo_names_country.names as country,
@@ -523,17 +565,18 @@ class NodesApi {
         LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
         LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
         LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = nodes.subdivision_id AND geo_names_subdivision.type = 'division'
-        WHERE nodes.as_number IN (?)
+        WHERE nodes.public_key IN (?)
         ORDER BY capacity DESC
       `;
 
-      const [rows]: any = await DB.query(query, [ISPId.split(',')]);
+      const [rows2]: any = await DB.query(query, [Object.keys(nodes)]);
-      for (let i = 0; i < rows.length; ++i) {
+      for (let i = 0; i < rows2.length; ++i) {
-        rows[i].country = JSON.parse(rows[i].country);
+        rows2[i].country = JSON.parse(rows2[i].country);
-        rows[i].city = JSON.parse(rows[i].city);
+        rows2[i].city = JSON.parse(rows2[i].city);
-        rows[i].subdivision = JSON.parse(rows[i].subdivision);
+        rows2[i].subdivision = JSON.parse(rows2[i].subdivision);
       }
-      return rows;
+      return rows2;
+
     } catch (e) {
       logger.err(`Cannot get nodes for ISP id ${ISPId}. Reason: ${e instanceof Error ? e.message : e}`);
       throw e;

backend/src/api/lightning/clightning/clightning-convert.ts

@@ -7,6 +7,15 @@ import { Common } from '../../common';
  * Convert a clightning "listnode" entry to a lnd node entry
  */
 export function convertNode(clNode: any): ILightningApi.Node {
+  let custom_records: { [type: number]: string } | undefined = undefined;
+  if (clNode.option_will_fund) {
+    try {
+      custom_records = { '1': Buffer.from(clNode.option_will_fund.compact_lease || '', 'hex').toString('base64') };
+    } catch (e) {
+      logger.err(`Cannot decode option_will_fund compact_lease for ${clNode.nodeid}). Reason: ` + (e instanceof Error ? e.message : e));
+      custom_records = undefined;
+    }
+  }
   return {
     alias: clNode.alias ?? '',
     color: `#${clNode.color ?? ''}`,
@@ -23,6 +32,7 @@ export function convertNode(clNode: any): ILightningApi.Node {
       };
     }) ?? [],
     last_update: clNode?.last_timestamp ?? 0,
+    custom_records
   };
 }
 

backend/src/api/lightning/lightning-api.interface.ts

@@ -49,6 +49,7 @@ export namespace ILightningApi {
     }[];
     color: string;
     features: { [key: number]: Feature };
+    custom_records?: { [type: number]: string };
   }
 
   export interface Info {

backend/src/api/mempool-blocks.ts

@@ -1,12 +1,17 @@
 import logger from '../logger';
-import { MempoolBlock, TransactionExtended, AuditTransaction, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor } from '../mempool.interfaces';
+import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta } from '../mempool.interfaces';
 import { Common } from './common';
 import config from '../config';
-import { PairingHeap } from '../utils/pairing-heap';
+import { StaticPool } from 'node-worker-threads-pool';
+import path from 'path';
 
 class MempoolBlocks {
   private mempoolBlocks: MempoolBlockWithTransactions[] = [];
   private mempoolBlockDeltas: MempoolBlockDelta[] = [];
+  private makeTemplatesPool = new StaticPool({
+    size: 1,
+    task: path.resolve(__dirname, './tx-selection-worker.js'),
+  });
 
   constructor() {}
 
@@ -72,16 +77,15 @@ class MempoolBlocks {
     const time = end - start;
     logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
 
-    const { blocks, deltas } = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks);
+    const blocks = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks);
+    const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks);
 
     this.mempoolBlocks = blocks;
     this.mempoolBlockDeltas = deltas;
   }
 
-  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[], prevBlocks: MempoolBlockWithTransactions[]):
+  private calculateMempoolBlocks(transactionsSorted: TransactionExtended[], prevBlocks: MempoolBlockWithTransactions[]): MempoolBlockWithTransactions[] {
-    { blocks: MempoolBlockWithTransactions[], deltas: MempoolBlockDelta[] } {
     const mempoolBlocks: MempoolBlockWithTransactions[] = [];
-    const mempoolBlockDeltas: MempoolBlockDelta[] = [];
     let blockWeight = 0;
     let blockSize = 0;
     let transactions: TransactionExtended[] = [];
@@ -102,7 +106,11 @@ class MempoolBlocks {
       mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
     }
 
-    // Calculate change from previous block states
+    return mempoolBlocks;
+  }
+
+  private calculateMempoolDeltas(prevBlocks: MempoolBlockWithTransactions[], mempoolBlocks: MempoolBlockWithTransactions[]): MempoolBlockDelta[] {
+    const mempoolBlockDeltas: MempoolBlockDelta[] = [];
     for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) {
       let added: TransactionStripped[] = [];
       let removed: string[] = [];
@@ -135,284 +143,25 @@ class MempoolBlocks {
         removed
       });
     }
-
+    return mempoolBlockDeltas;
-    return {
-      blocks: mempoolBlocks,
-      deltas: mempoolBlockDeltas
-    };
   }
 
-  /*
+  public async makeBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, blockLimit: number, weightLimit: number | null = null, condenseRest = false): Promise<void> {
-  * Build projected mempool blocks using an approximation of the transaction selection algorithm from Bitcoin Core
+    const { mempool, blocks } = await this.makeTemplatesPool.exec({ mempool: newMempool, blockLimit, weightLimit, condenseRest });
-  * (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp)
+    const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks);
-  *
-  * blockLimit: number of blocks to build in total.
-  * weightLimit: maximum weight of transactions to consider using the selection algorithm.
-  *              if weightLimit is significantly lower than the mempool size, results may start to diverge from getBlockTemplate 
-  * condenseRest: whether to ignore excess transactions or append them to the final block.
-  */
-  public makeBlockTemplates(mempool: { [txid: string]: TransactionExtended }, blockLimit: number, weightLimit: number | null = null, condenseRest = false): MempoolBlockWithTransactions[] {
-    const start = Date.now();
-    const auditPool: { [txid: string]: AuditTransaction } = {};
-    const mempoolArray: AuditTransaction[] = [];
-    const restOfArray: TransactionExtended[] = [];
-    
-    let weight = 0;
-    const maxWeight = weightLimit ? Math.max(4_000_000 * blockLimit, weightLimit) : Infinity;
-    // grab the top feerate txs up to maxWeight
-    Object.values(mempool).sort((a, b) => b.feePerVsize - a.feePerVsize).forEach(tx => {
-      weight += tx.weight;
-      if (weight >= maxWeight) {
-        restOfArray.push(tx);
-        return;
-      }
-      // initializing everything up front helps V8 optimize property access later
-      auditPool[tx.txid] = {
-        txid: tx.txid,
-        fee: tx.fee,
-        size: tx.size,
-        weight: tx.weight,
-        feePerVsize: tx.feePerVsize,
-        vin: tx.vin,
-        relativesSet: false,
-        ancestorMap: new Map<string, AuditTransaction>(),
-        children: new Set<AuditTransaction>(),
-        ancestorFee: 0,
-        ancestorWeight: 0,
-        score: 0,
-        used: false,
-        modified: false,
-        modifiedNode: null,
-      }
-      mempoolArray.push(auditPool[tx.txid]);
-    })
 
-    // Build relatives graph & calculate ancestor scores
+    // copy CPFP info across to main thread's mempool
-    for (const tx of mempoolArray) {
+    Object.keys(newMempool).forEach((txid) => {
-      if (!tx.relativesSet) {
+      if (newMempool[txid] && mempool[txid]) {
-        this.setRelatives(tx, auditPool);
+        newMempool[txid].effectiveFeePerVsize = mempool[txid].effectiveFeePerVsize;
-      }
+        newMempool[txid].ancestors = mempool[txid].ancestors;
-    }
+        newMempool[txid].bestDescendant = mempool[txid].bestDescendant;
-
+        newMempool[txid].cpfpChecked = mempool[txid].cpfpChecked;
-    // Sort by descending ancestor score
-    mempoolArray.sort((a, b) => (b.score || 0) - (a.score || 0));
-
-    // Build blocks by greedily choosing the highest feerate package
-    // (i.e. the package rooted in the transaction with the best ancestor score)
-    const blocks: MempoolBlockWithTransactions[] = [];
-    let blockWeight = 4000;
-    let blockSize = 0;
-    let transactions: AuditTransaction[] = [];
-    const modified: PairingHeap<AuditTransaction> = new PairingHeap((a, b): boolean => (a.score || 0) > (b.score || 0));
-    let overflow: AuditTransaction[] = [];
-    let failures = 0;
-    let top = 0;
-    while ((top < mempoolArray.length || !modified.isEmpty()) && (condenseRest || blocks.length < blockLimit)) {
-      // skip invalid transactions
-      while (top < mempoolArray.length && (mempoolArray[top].used || mempoolArray[top].modified)) {
-        top++;
-      }
-
-      // Select best next package
-      let nextTx;
-      const nextPoolTx = mempoolArray[top];
-      const nextModifiedTx = modified.peek();
-      if (nextPoolTx && (!nextModifiedTx || (nextPoolTx.score || 0) > (nextModifiedTx.score || 0))) {
-        nextTx = nextPoolTx;
-        top++;
-      } else {
-        modified.pop();
-        if (nextModifiedTx) {
-          nextTx = nextModifiedTx;
-          nextTx.modifiedNode = undefined;
-        }
-      }
-
-      if (nextTx && !nextTx?.used) {
-        // Check if the package fits into this block
-        if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
-          blockWeight += nextTx.ancestorWeight;
-          const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values());
-          // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count)
-          const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx];
-          const effectiveFeeRate = nextTx.ancestorFee / (nextTx.ancestorWeight / 4);
-          sortedTxSet.forEach((ancestor, i, arr) => {
-            const mempoolTx = mempool[ancestor.txid];
-            if (ancestor && !ancestor?.used) {
-              ancestor.used = true;
-              // update original copy of this tx with effective fee rate & relatives data
-              mempoolTx.effectiveFeePerVsize = effectiveFeeRate;
-              mempoolTx.ancestors = (Array.from(ancestor.ancestorMap?.values()) as AuditTransaction[]).map((a) => {
-                return {
-                  txid: a.txid,
-                  fee: a.fee,
-                  weight: a.weight,
-                }
-              })
-              if (i < arr.length - 1) {
-                mempoolTx.bestDescendant = {
-                  txid: arr[arr.length - 1].txid,
-                  fee: arr[arr.length - 1].fee,
-                  weight: arr[arr.length - 1].weight,
-                };
-              }
-              transactions.push(ancestor);
-              blockSize += ancestor.size;
-            }
-          });
-
-          // remove these as valid package ancestors for any descendants remaining in the mempool
-          if (sortedTxSet.length) {
-            sortedTxSet.forEach(tx => {
-              this.updateDescendants(tx, auditPool, modified);
-            });
-          }
-
-          failures = 0;
-        } else {
-          // hold this package in an overflow list while we check for smaller options
-          overflow.push(nextTx);
-          failures++;
-        }
-      }
-
-      // this block is full
-      const exceededPackageTries = failures > 1000 && blockWeight > (config.MEMPOOL.BLOCK_WEIGHT_UNITS - 4000);
-      if (exceededPackageTries && (!condenseRest || blocks.length < blockLimit - 1)) {
-        // construct this block
-        if (transactions.length) {
-          blocks.push(this.dataToMempoolBlocks(transactions.map(t => mempool[t.txid]), blockSize, blockWeight, blocks.length));
-        }
-        // reset for the next block
-        transactions = [];
-        blockSize = 0;
-        blockWeight = 4000;
-
-        // 'overflow' packages didn't fit in this block, but are valid candidates for the next
-        for (const overflowTx of overflow.reverse()) {
-          if (overflowTx.modified) {
-            overflowTx.modifiedNode = modified.add(overflowTx);
-          } else {
-            top--;
-            mempoolArray[top] = overflowTx;
-          }
-        }
-        overflow = [];
-      }
-    }
-    if (condenseRest) {
-      // pack any leftover transactions into the last block
-      for (const tx of overflow) {
-        if (!tx || tx?.used) {
-          continue;
-        }
-        blockWeight += tx.weight;
-        blockSize += tx.size;
-        transactions.push(tx);
-        tx.used = true;
-      }
-      const blockTransactions = transactions.map(t => mempool[t.txid])
-      restOfArray.forEach(tx => {
-        blockWeight += tx.weight;
-        blockSize += tx.size;
-        blockTransactions.push(tx);
-      });
-      if (blockTransactions.length) {
-        blocks.push(this.dataToMempoolBlocks(blockTransactions, blockSize, blockWeight, blocks.length));
-      }
-      transactions = [];
-    } else if (transactions.length) {
-      blocks.push(this.dataToMempoolBlocks(transactions.map(t => mempool[t.txid]), blockSize, blockWeight, blocks.length));
-    }
-
-    const end = Date.now();
-    const time = end - start;
-    logger.debug('Mempool templates calculated in ' + time / 1000 + ' seconds');
-
-    return blocks;
-  }
-
-  // traverse in-mempool ancestors
-  // recursion unavoidable, but should be limited to depth < 25 by mempool policy
-  public setRelatives(
-    tx: AuditTransaction,
-    mempool: { [txid: string]: AuditTransaction },
-  ): void {
-    for (const parent of tx.vin) {
-      const parentTx = mempool[parent.txid];
-      if (parentTx && !tx.ancestorMap!.has(parent.txid)) {
-        tx.ancestorMap.set(parent.txid, parentTx);
-        parentTx.children.add(tx);
-        // visit each node only once
-        if (!parentTx.relativesSet) {
-          this.setRelatives(parentTx, mempool);
-        }
-        parentTx.ancestorMap.forEach((ancestor) => {
-          tx.ancestorMap.set(ancestor.txid, ancestor);
-        });
-      }
-    };
-    tx.ancestorFee = tx.fee || 0;
-    tx.ancestorWeight = tx.weight || 0;
-    tx.ancestorMap.forEach((ancestor) => {
-      tx.ancestorFee += ancestor.fee;
-      tx.ancestorWeight += ancestor.weight;
-    });
-    tx.score = tx.ancestorFee / (tx.ancestorWeight || 1);
-    tx.relativesSet = true;
-  }
-
-  // iterate over remaining descendants, removing the root as a valid ancestor & updating the ancestor score
-  // avoids recursion to limit call stack depth
-  private updateDescendants(
-    rootTx: AuditTransaction,
-    mempool: { [txid: string]: AuditTransaction },
-    modified: PairingHeap<AuditTransaction>,
-  ): void {
-    const descendantSet: Set<AuditTransaction> = new Set();
-    // stack of nodes left to visit
-    const descendants: AuditTransaction[] = [];
-    let descendantTx;
-    let ancestorIndex;
-    let tmpScore;
-    rootTx.children.forEach(childTx => {
-      if (!descendantSet.has(childTx)) {
-        descendants.push(childTx);
-        descendantSet.add(childTx);
       }
     });
-    while (descendants.length) {
-      descendantTx = descendants.pop();
-      if (descendantTx && descendantTx.ancestorMap && descendantTx.ancestorMap.has(rootTx.txid)) {
-        // remove tx as ancestor
-        descendantTx.ancestorMap.delete(rootTx.txid);
-        descendantTx.ancestorFee -= rootTx.fee;
-        descendantTx.ancestorWeight -= rootTx.weight;
-        tmpScore = descendantTx.score;
-        descendantTx.score = descendantTx.ancestorFee / descendantTx.ancestorWeight;
 
-        if (!descendantTx.modifiedNode) {
+    this.mempoolBlocks = blocks;
-          descendantTx.modified = true;
+    this.mempoolBlockDeltas = deltas;
-          descendantTx.modifiedNode = modified.add(descendantTx);
-        } else {
-          // rebalance modified heap if score has changed
-          if (descendantTx.score < tmpScore) {
-            modified.decreasePriority(descendantTx.modifiedNode);
-          } else if (descendantTx.score > tmpScore) {
-            modified.increasePriority(descendantTx.modifiedNode);
-          }
-        }
-
-        // add this node's children to the stack
-        descendantTx.children.forEach(childTx => {
-          // visit each node only once
-          if (!descendantSet.has(childTx)) {
-            descendants.push(childTx);
-            descendantSet.add(childTx);
-          }
-        });
-      }
-    }
   }
 
   private dataToMempoolBlocks(transactions: TransactionExtended[],

backend/src/api/mempool.ts

@@ -20,6 +20,8 @@ class Mempool {
                                                     maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 };
   private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
     deletedTransactions: TransactionExtended[]) => void) | undefined;
+  private asyncMempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
+    deletedTransactions: TransactionExtended[]) => void) | undefined;
 
   private txPerSecondArray: number[] = [];
   private txPerSecond: number = 0;
@@ -63,6 +65,11 @@ class Mempool {
     this.mempoolChangedCallback = fn;
   }
 
+  public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: TransactionExtended; },
+    newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) => Promise<void>) {
+    this.asyncMempoolChangedCallback = fn;
+  }
+
   public getMempool(): { [txid: string]: TransactionExtended } {
     return this.mempoolCache;
   }
@@ -72,6 +79,9 @@ class Mempool {
     if (this.mempoolChangedCallback) {
       this.mempoolChangedCallback(this.mempoolCache, [], []);
     }
+    if (this.asyncMempoolChangedCallback) {
+      this.asyncMempoolChangedCallback(this.mempoolCache, [], []);
+    }
   }
 
   public async $updateMemPoolInfo() {
@@ -103,12 +113,11 @@ class Mempool {
     return txTimes;
   }
 
-  public async $updateMempool() {
+  public async $updateMempool(): Promise<void> {
-    logger.debug('Updating mempool');
+    logger.debug(`Updating mempool...`);
     const start = new Date().getTime();
     let hasChange: boolean = false;
     const currentMempoolSize = Object.keys(this.mempoolCache).length;
-    let txCount = 0;
     const transactions = await bitcoinApi.$getRawMempool();
     const diff = transactions.length - currentMempoolSize;
     const newTransactions: TransactionExtended[] = [];
@@ -124,7 +133,6 @@ class Mempool {
         try {
           const transaction = await transactionUtils.$getTransactionExtended(txid);
           this.mempoolCache[txid] = transaction;
-          txCount++;
           if (this.inSync) {
             this.txPerSecondArray.push(new Date().getTime());
             this.vBytesPerSecondArray.push({
@@ -133,14 +141,9 @@ class Mempool {
             });
           }
           hasChange = true;
-          if (diff > 0) {
-            logger.debug('Fetched transaction ' + txCount + ' / ' + diff);
-          } else {
-            logger.debug('Fetched transaction ' + txCount);
-          }
           newTransactions.push(transaction);
         } catch (e) {
-          logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
+          logger.debug(`Error finding transaction '${txid}' in the mempool: ` + (e instanceof Error ? e.message : e));
         }
       }
 
@@ -194,11 +197,13 @@ class Mempool {
     if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
       this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
     }
+    if (this.asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) {
+      await this.asyncMempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
+    }
 
     const end = new Date().getTime();
     const time = end - start;
-    logger.debug(`New mempool size: ${Object.keys(this.mempoolCache).length} Change: ${diff}`);
+    logger.debug(`Mempool updated in ${time / 1000} seconds. New size: ${Object.keys(this.mempoolCache).length} (${diff > 0 ? '+' + diff : diff})`);
-    logger.debug('Mempool updated in ' + time / 1000 + ' seconds');
   }
 
   public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended; }) {

backend/src/api/mining/mining-routes.ts

@@ -1,6 +1,7 @@
 import { Application, Request, Response } from 'express';
 import config from "../../config";
 import logger from '../../logger';
+import audits from '../audit';
 import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
 import BlocksRepository from '../../repositories/BlocksRepository';
 import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository';
@@ -26,7 +27,11 @@ class MiningRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', this.$getHistoricalBlockSizeAndWeight)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments/:interval', this.$getDifficultyAdjustments)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/predictions/:interval', this.$getHistoricalBlockPrediction)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/scores', this.$getBlockAuditScores)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/scores/:height', this.$getBlockAuditScores)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/score/:hash', this.$getBlockAuditScore)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/:hash', this.$getBlockAudit)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/timestamp/:timestamp', this.$getHeightFromTimestamp)
     ;
   }
 
@@ -252,6 +257,52 @@ class MiningRoutes {
       res.status(500).send(e instanceof Error ? e.message : e);
     }
   }
+
+  private async $getHeightFromTimestamp(req: Request, res: Response) {
+    try {
+      const timestamp = parseInt(req.params.timestamp, 10);
+      // This will prevent people from entering milliseconds etc.
+      // Block timestamps are allowed to be up to 2 hours off, so 24 hours
+      // will never put the maximum value before the most recent block
+      const nowPlus1day = Math.floor(Date.now() / 1000) + 60 * 60 * 24;
+      // Prevent non-integers that are not seconds
+      if (!/^[1-9][0-9]*$/.test(req.params.timestamp) || timestamp > nowPlus1day) {
+        throw new Error(`Invalid timestamp, value must be Unix seconds`);
+      }
+      const result = await BlocksRepository.$getBlockHeightFromTimestamp(
+        timestamp,
+      );
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getBlockAuditScores(req: Request, res: Response) {
+    try {
+      const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(await audits.$getBlockAuditScores(height, 15));
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  public async $getBlockAuditScore(req: Request, res: Response) {
+    try {
+      const audit = await BlocksAuditsRepository.$getBlockAuditScore(req.params.hash);
+
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());
+      res.json(audit || 'null');
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
 }
 
 export default new MiningRoutes();

backend/src/api/pools-parser.ts

@@ -14,10 +14,10 @@ interface Pool {
 class PoolsParser {
   miningPools: any[] = [];
   unknownPool: any = {
-    'name': "Unknown",
+    'name': 'Unknown',
-    'link': "https://learnmeabitcoin.com/technical/coinbase-transaction",
+    'link': 'https://learnmeabitcoin.com/technical/coinbase-transaction',
-    'regexes': "[]",
+    'regexes': '[]',
-    'addresses': "[]",
+    'addresses': '[]',
     'slug': 'unknown'
   };
   slugWarnFlag = false;
@@ -25,7 +25,7 @@ class PoolsParser {
   /**
    * Parse the pools.json file, consolidate the data and dump it into the database
    */
-  public async migratePoolsJson(poolsJson: object) {
+  public async migratePoolsJson(poolsJson: object): Promise<void> {
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
       return;
     }
@@ -81,6 +81,7 @@ class PoolsParser {
     // Finally, we generate the final consolidated pools data
     const finalPoolDataAdd: Pool[] = [];
     const finalPoolDataUpdate: Pool[] = [];
+    const finalPoolDataRename: Pool[] = [];
     for (let i = 0; i < poolNames.length; ++i) {
       let allAddresses: string[] = [];
       let allRegexes: string[] = [];
@@ -127,8 +128,26 @@ class PoolsParser {
           finalPoolDataUpdate.push(poolObj);
         }
       } else {
-        logger.debug(`Add '${finalPoolName}' mining pool`);
+        // Double check that if we're not just renaming a pool (same address same regex)
-        finalPoolDataAdd.push(poolObj);
+        const [poolToRename]: any[] = await DB.query(`
+          SELECT * FROM pools
+          WHERE addresses = ? OR regexes = ?`,
+          [JSON.stringify(poolObj.addresses), JSON.stringify(poolObj.regexes)]
+        );
+        if (poolToRename && poolToRename.length > 0) {
+          // We're actually renaming an existing pool
+          finalPoolDataRename.push({
+            'name': poolObj.name,
+            'link': poolObj.link,
+            'regexes': allRegexes,
+            'addresses': allAddresses,
+            'slug': slug
+          });
+          logger.debug(`Rename '${poolToRename[0].name}' mining pool to ${poolObj.name}`);
+        } else {
+          logger.debug(`Add '${finalPoolName}' mining pool`);
+          finalPoolDataAdd.push(poolObj);
+        }
       }
 
       this.miningPools.push({
@@ -145,7 +164,9 @@ class PoolsParser {
       return;
     }
 
-    if (finalPoolDataAdd.length > 0 || finalPoolDataUpdate.length > 0) {    
+    if (finalPoolDataAdd.length > 0 || finalPoolDataUpdate.length > 0 ||
+      finalPoolDataRename.length > 0
+    ) {    
       logger.debug(`Update pools table now`);
 
       // Add new mining pools into the database
@@ -169,8 +190,22 @@ class PoolsParser {
         ;`);
       }
 
+      // Rename mining pools
+      const renameQueries: string[] = [];
+      for (let i = 0; i < finalPoolDataRename.length; ++i) {
+        renameQueries.push(`
+          UPDATE pools
+          SET name='${finalPoolDataRename[i].name}', link='${finalPoolDataRename[i].link}',
+            slug='${finalPoolDataRename[i].slug}'
+          WHERE regexes='${JSON.stringify(finalPoolDataRename[i].regexes)}'
+            AND addresses='${JSON.stringify(finalPoolDataRename[i].addresses)}'
+        ;`);
+      }
+
       try {
-        await this.$deleteBlocskToReindex(finalPoolDataUpdate);
+        if (finalPoolDataAdd.length > 0 || updateQueries.length > 0) {
+          await this.$deleteBlocskToReindex(finalPoolDataUpdate);
+        }
 
         if (finalPoolDataAdd.length > 0) {
           await DB.query({ sql: queryAdd, timeout: 120000 });
@@ -178,6 +213,9 @@ class PoolsParser {
         for (const query of updateQueries) {
           await DB.query({ sql: query, timeout: 120000 });
         }
+        for (const query of renameQueries) {
+          await DB.query({ sql: query, timeout: 120000 });
+        }
         await this.insertUnknownPool();
         logger.info('Mining pools.json import completed');
       } catch (e) {

backend/src/api/tx-selection-worker.ts

@@ -0,0 +1,336 @@
+import config from '../config';
+import logger from '../logger';
+import { TransactionExtended, MempoolBlockWithTransactions, AuditTransaction } from '../mempool.interfaces';
+import { PairingHeap } from '../utils/pairing-heap';
+import { Common } from './common';
+import { parentPort } from 'worker_threads';
+
+if (parentPort) {
+  parentPort.on('message', (params: { mempool: { [txid: string]: TransactionExtended }, blockLimit: number, weightLimit: number | null, condenseRest: boolean}) => {
+    const { mempool, blocks } = makeBlockTemplates(params);
+
+    // return the result to main thread.
+    if (parentPort) {
+     parentPort.postMessage({ mempool, blocks });
+    }
+  });
+}
+
+/*
+* Build projected mempool blocks using an approximation of the transaction selection algorithm from Bitcoin Core
+* (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp)
+*
+* blockLimit: number of blocks to build in total.
+* weightLimit: maximum weight of transactions to consider using the selection algorithm.
+*              if weightLimit is significantly lower than the mempool size, results may start to diverge from getBlockTemplate 
+* condenseRest: whether to ignore excess transactions or append them to the final block.
+*/
+function makeBlockTemplates({ mempool, blockLimit, weightLimit, condenseRest }: { mempool: { [txid: string]: TransactionExtended }, blockLimit: number, weightLimit?: number | null, condenseRest?: boolean | null })
+  : { mempool: { [txid: string]: TransactionExtended }, blocks: MempoolBlockWithTransactions[] } {
+  const start = Date.now();
+  const auditPool: { [txid: string]: AuditTransaction } = {};
+  const mempoolArray: AuditTransaction[] = [];
+  const restOfArray: TransactionExtended[] = [];
+  
+  let weight = 0;
+  const maxWeight = weightLimit ? Math.max(4_000_000 * blockLimit, weightLimit) : Infinity;
+  // grab the top feerate txs up to maxWeight
+  Object.values(mempool).sort((a, b) => b.feePerVsize - a.feePerVsize).forEach(tx => {
+    weight += tx.weight;
+    if (weight >= maxWeight) {
+      restOfArray.push(tx);
+      return;
+    }
+    // initializing everything up front helps V8 optimize property access later
+    auditPool[tx.txid] = {
+      txid: tx.txid,
+      fee: tx.fee,
+      size: tx.size,
+      weight: tx.weight,
+      feePerVsize: tx.feePerVsize,
+      vin: tx.vin,
+      relativesSet: false,
+      ancestorMap: new Map<string, AuditTransaction>(),
+      children: new Set<AuditTransaction>(),
+      ancestorFee: 0,
+      ancestorWeight: 0,
+      score: 0,
+      used: false,
+      modified: false,
+      modifiedNode: null,
+    };
+    mempoolArray.push(auditPool[tx.txid]);
+  });
+
+  // Build relatives graph & calculate ancestor scores
+  for (const tx of mempoolArray) {
+    if (!tx.relativesSet) {
+      setRelatives(tx, auditPool);
+    }
+  }
+
+  // Sort by descending ancestor score
+  mempoolArray.sort((a, b) => (b.score || 0) - (a.score || 0));
+
+  // Build blocks by greedily choosing the highest feerate package
+  // (i.e. the package rooted in the transaction with the best ancestor score)
+  const blocks: MempoolBlockWithTransactions[] = [];
+  let blockWeight = 4000;
+  let blockSize = 0;
+  let transactions: AuditTransaction[] = [];
+  const modified: PairingHeap<AuditTransaction> = new PairingHeap((a, b): boolean => (a.score || 0) > (b.score || 0));
+  let overflow: AuditTransaction[] = [];
+  let failures = 0;
+  let top = 0;
+  while ((top < mempoolArray.length || !modified.isEmpty()) && (condenseRest || blocks.length < blockLimit)) {
+    // skip invalid transactions
+    while (top < mempoolArray.length && (mempoolArray[top].used || mempoolArray[top].modified)) {
+      top++;
+    }
+
+    // Select best next package
+    let nextTx;
+    const nextPoolTx = mempoolArray[top];
+    const nextModifiedTx = modified.peek();
+    if (nextPoolTx && (!nextModifiedTx || (nextPoolTx.score || 0) > (nextModifiedTx.score || 0))) {
+      nextTx = nextPoolTx;
+      top++;
+    } else {
+      modified.pop();
+      if (nextModifiedTx) {
+        nextTx = nextModifiedTx;
+        nextTx.modifiedNode = undefined;
+      }
+    }
+
+    if (nextTx && !nextTx?.used) {
+      // Check if the package fits into this block
+      if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
+        blockWeight += nextTx.ancestorWeight;
+        const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values());
+        // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count)
+        const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx];
+        const effectiveFeeRate = nextTx.ancestorFee / (nextTx.ancestorWeight / 4);
+        sortedTxSet.forEach((ancestor, i, arr) => {
+          const mempoolTx = mempool[ancestor.txid];
+          if (ancestor && !ancestor?.used) {
+            ancestor.used = true;
+            // update original copy of this tx with effective fee rate & relatives data
+            mempoolTx.effectiveFeePerVsize = effectiveFeeRate;
+            mempoolTx.ancestors = (Array.from(ancestor.ancestorMap?.values()) as AuditTransaction[]).map((a) => {
+              return {
+                txid: a.txid,
+                fee: a.fee,
+                weight: a.weight,
+              };
+            });
+            mempoolTx.cpfpChecked = true;
+            if (i < arr.length - 1) {
+              mempoolTx.bestDescendant = {
+                txid: arr[arr.length - 1].txid,
+                fee: arr[arr.length - 1].fee,
+                weight: arr[arr.length - 1].weight,
+              };
+            } else {
+              mempoolTx.bestDescendant = null;
+            }
+            transactions.push(ancestor);
+            blockSize += ancestor.size;
+          }
+        });
+
+        // remove these as valid package ancestors for any descendants remaining in the mempool
+        if (sortedTxSet.length) {
+          sortedTxSet.forEach(tx => {
+            updateDescendants(tx, auditPool, modified);
+          });
+        }
+
+        failures = 0;
+      } else {
+        // hold this package in an overflow list while we check for smaller options
+        overflow.push(nextTx);
+        failures++;
+      }
+    }
+
+    // this block is full
+    const exceededPackageTries = failures > 1000 && blockWeight > (config.MEMPOOL.BLOCK_WEIGHT_UNITS - 4000);
+    const queueEmpty = top >= mempoolArray.length && modified.isEmpty();
+    if ((exceededPackageTries || queueEmpty) && (!condenseRest || blocks.length < blockLimit - 1)) {
+      // construct this block
+      if (transactions.length) {
+        blocks.push(dataToMempoolBlocks(transactions.map(t => mempool[t.txid]), blockSize, blockWeight, blocks.length));
+      }
+      // reset for the next block
+      transactions = [];
+      blockSize = 0;
+      blockWeight = 4000;
+
+      // 'overflow' packages didn't fit in this block, but are valid candidates for the next
+      for (const overflowTx of overflow.reverse()) {
+        if (overflowTx.modified) {
+          overflowTx.modifiedNode = modified.add(overflowTx);
+        } else {
+          top--;
+          mempoolArray[top] = overflowTx;
+        }
+      }
+      overflow = [];
+    }
+  }
+  if (condenseRest) {
+    // pack any leftover transactions into the last block
+    for (const tx of overflow) {
+      if (!tx || tx?.used) {
+        continue;
+      }
+      blockWeight += tx.weight;
+      blockSize += tx.size;
+      const mempoolTx = mempool[tx.txid];
+      // update original copy of this tx with effective fee rate & relatives data
+      mempoolTx.effectiveFeePerVsize = tx.score;
+      mempoolTx.ancestors = (Array.from(tx.ancestorMap?.values()) as AuditTransaction[]).map((a) => {
+        return {
+          txid: a.txid,
+          fee: a.fee,
+          weight: a.weight,
+        };
+      });
+      mempoolTx.bestDescendant = null;
+      mempoolTx.cpfpChecked = true;
+      transactions.push(tx);
+      tx.used = true;
+    }
+    const blockTransactions = transactions.map(t => mempool[t.txid]);
+    restOfArray.forEach(tx => {
+      blockWeight += tx.weight;
+      blockSize += tx.size;
+      tx.effectiveFeePerVsize = tx.feePerVsize;
+      tx.cpfpChecked = false;
+      tx.ancestors = [];
+      tx.bestDescendant = null;
+      blockTransactions.push(tx);
+    });
+    if (blockTransactions.length) {
+      blocks.push(dataToMempoolBlocks(blockTransactions, blockSize, blockWeight, blocks.length));
+    }
+    transactions = [];
+  } else if (transactions.length) {
+    blocks.push(dataToMempoolBlocks(transactions.map(t => mempool[t.txid]), blockSize, blockWeight, blocks.length));
+  }
+
+  const end = Date.now();
+  const time = end - start;
+  logger.debug('Mempool templates calculated in ' + time / 1000 + ' seconds');
+
+  return {
+    mempool,
+    blocks
+  };
+}
+
+// traverse in-mempool ancestors
+// recursion unavoidable, but should be limited to depth < 25 by mempool policy
+function setRelatives(
+  tx: AuditTransaction,
+  mempool: { [txid: string]: AuditTransaction },
+): void {
+  for (const parent of tx.vin) {
+    const parentTx = mempool[parent.txid];
+    if (parentTx && !tx.ancestorMap?.has(parent.txid)) {
+      tx.ancestorMap.set(parent.txid, parentTx);
+      parentTx.children.add(tx);
+      // visit each node only once
+      if (!parentTx.relativesSet) {
+        setRelatives(parentTx, mempool);
+      }
+      parentTx.ancestorMap.forEach((ancestor) => {
+        tx.ancestorMap.set(ancestor.txid, ancestor);
+      });
+    }
+  };
+  tx.ancestorFee = tx.fee || 0;
+  tx.ancestorWeight = tx.weight || 0;
+  tx.ancestorMap.forEach((ancestor) => {
+    tx.ancestorFee += ancestor.fee;
+    tx.ancestorWeight += ancestor.weight;
+  });
+  tx.score = tx.ancestorFee / ((tx.ancestorWeight / 4) || 1);
+  tx.relativesSet = true;
+}
+
+// iterate over remaining descendants, removing the root as a valid ancestor & updating the ancestor score
+// avoids recursion to limit call stack depth
+function updateDescendants(
+  rootTx: AuditTransaction,
+  mempool: { [txid: string]: AuditTransaction },
+  modified: PairingHeap<AuditTransaction>,
+): void {
+  const descendantSet: Set<AuditTransaction> = new Set();
+  // stack of nodes left to visit
+  const descendants: AuditTransaction[] = [];
+  let descendantTx;
+  let tmpScore;
+  rootTx.children.forEach(childTx => {
+    if (!descendantSet.has(childTx)) {
+      descendants.push(childTx);
+      descendantSet.add(childTx);
+    }
+  });
+  while (descendants.length) {
+    descendantTx = descendants.pop();
+    if (descendantTx && descendantTx.ancestorMap && descendantTx.ancestorMap.has(rootTx.txid)) {
+      // remove tx as ancestor
+      descendantTx.ancestorMap.delete(rootTx.txid);
+      descendantTx.ancestorFee -= rootTx.fee;
+      descendantTx.ancestorWeight -= rootTx.weight;
+      tmpScore = descendantTx.score;
+      descendantTx.score = descendantTx.ancestorFee / (descendantTx.ancestorWeight / 4);
+
+      if (!descendantTx.modifiedNode) {
+        descendantTx.modified = true;
+        descendantTx.modifiedNode = modified.add(descendantTx);
+      } else {
+        // rebalance modified heap if score has changed
+        if (descendantTx.score < tmpScore) {
+          modified.decreasePriority(descendantTx.modifiedNode);
+        } else if (descendantTx.score > tmpScore) {
+          modified.increasePriority(descendantTx.modifiedNode);
+        }
+      }
+
+      // add this node's children to the stack
+      descendantTx.children.forEach(childTx => {
+        // visit each node only once
+        if (!descendantSet.has(childTx)) {
+          descendants.push(childTx);
+          descendantSet.add(childTx);
+        }
+      });
+    }
+  }
+}
+
+function dataToMempoolBlocks(transactions: TransactionExtended[],
+  blockSize: number, blockWeight: number, blocksIndex: number): MempoolBlockWithTransactions {
+  let rangeLength = 4;
+  if (blocksIndex === 0) {
+    rangeLength = 8;
+  }
+  if (transactions.length > 4000) {
+    rangeLength = 6;
+  } else if (transactions.length > 10000) {
+    rangeLength = 8;
+  }
+  return {
+    blockSize: blockSize,
+    blockVSize: blockWeight / 4,
+    nTx: transactions.length,
+    totalFees: transactions.reduce((acc, cur) => acc + cur.fee, 0),
+    medianFee: Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
+    feeRange: Common.getFeesInRange(transactions, rangeLength),
+    transactionIds: transactions.map((tx) => tx.txid),
+    transactions: transactions.map((tx) => Common.stripTransaction(tx)),
+  };
+}

backend/src/api/websocket-handler.ts

@@ -244,13 +244,18 @@ class WebsocketHandler {
     });
   }
 
-  handleMempoolChange(newMempool: { [txid: string]: TransactionExtended },
+  async handleMempoolChange(newMempool: { [txid: string]: TransactionExtended },
-    newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) {
+    newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]): Promise<void> {
     if (!this.wss) {
       throw new Error('WebSocket.Server is not set');
     }
 
-    mempoolBlocks.updateMempoolBlocks(newMempool);
+    if (config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) {
+      await mempoolBlocks.makeBlockTemplates(newMempool, 8, null, true);
+    }
+    else {
+      mempoolBlocks.updateMempoolBlocks(newMempool);
+    }
     const mBlocks = mempoolBlocks.getMempoolBlocks();
     const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
     const mempoolInfo = memPool.getMempoolInfo();
@@ -405,22 +410,25 @@ class WebsocketHandler {
       }
     });
   }
-
+ 
-  handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]): void {
+  async handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]): Promise<void> {
     if (!this.wss) {
       throw new Error('WebSocket.Server is not set');
     }
 
-    let mBlocks: undefined | MempoolBlock[];
-    let mBlockDeltas: undefined | MempoolBlockDelta[];
-    let matchRate;
     const _memPool = memPool.getMempool();
+    let matchRate;
 
-    if (Common.indexingEnabled()) {
+    if (config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) {
-      const mempoolCopy = cloneMempool(_memPool);
+      await mempoolBlocks.makeBlockTemplates(_memPool, 2);
-      const projectedBlocks = mempoolBlocks.makeBlockTemplates(mempoolCopy, 2);
+    } else {
+      mempoolBlocks.updateMempoolBlocks(_memPool);
+    }
 
-      const { censored, added, score } = Audit.auditBlock(transactions, projectedBlocks, mempoolCopy);
+    if (Common.indexingEnabled() && memPool.isInSync()) {
+      const projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
+
+      const { censored, added, score } = Audit.auditBlock(transactions, projectedBlocks, _memPool);
       matchRate = Math.round(score * 100 * 100) / 100;
 
       const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
@@ -459,9 +467,13 @@ class WebsocketHandler {
       delete _memPool[txId];
     }
 
-    mempoolBlocks.updateMempoolBlocks(_memPool);
+    if (config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) {
-    mBlocks = mempoolBlocks.getMempoolBlocks();
+      await mempoolBlocks.makeBlockTemplates(_memPool, 2);
-    mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
+    } else {
+      mempoolBlocks.updateMempoolBlocks(_memPool);
+    }
+    const mBlocks = mempoolBlocks.getMempoolBlocks();
+    const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
 
     const da = difficultyAdjustment.getDifficultyAdjustment();
     const fees = feeApi.getRecommendedFee();
@@ -569,14 +581,4 @@ class WebsocketHandler {
   }
 }
 
-function cloneMempool(mempool: { [txid: string]: TransactionExtended }): { [txid: string]: TransactionExtended } {
-  const cloned = {};
-  Object.keys(mempool).forEach(id => {
-    cloned[id] = {
-      ...mempool[id]
-    };
-  });
-  return cloned;
-}
-
 export default new WebsocketHandler();

backend/src/config.ts

@@ -4,6 +4,7 @@ const configFromFile = require(
 
 interface IConfig {
   MEMPOOL: {
+    ENABLED: boolean;
     NETWORK: 'mainnet' | 'testnet' | 'signet' | 'liquid' | 'liquidtestnet';
     BACKEND: 'esplora' | 'electrum' | 'none';
     HTTP_PORT: number;
@@ -28,6 +29,7 @@ interface IConfig {
     AUTOMATIC_BLOCK_REINDEXING: boolean;
     POOLS_JSON_URL: string,
     POOLS_JSON_TREE_URL: string,
+    ADVANCED_TRANSACTION_SELECTION: boolean;
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -119,6 +121,7 @@ interface IConfig {
 
 const defaults: IConfig = {
   'MEMPOOL': {
+    'ENABLED': true,
     'NETWORK': 'mainnet',
     'BACKEND': 'none',
     'HTTP_PORT': 8999,
@@ -143,6 +146,7 @@ const defaults: IConfig = {
     'AUTOMATIC_BLOCK_REINDEXING': false,
     'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
     'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
+    'ADVANCED_TRANSACTION_SELECTION': false,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',
@@ -224,11 +228,11 @@ const defaults: IConfig = {
     'BISQ_URL': 'https://bisq.markets/api',
     'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
   },
-  "MAXMIND": {
+  'MAXMIND': {
     'ENABLED': false,
-    "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb",
+    'GEOLITE2_CITY': '/usr/local/share/GeoIP/GeoLite2-City.mmdb',
-    "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb",
+    'GEOLITE2_ASN': '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb',
-    "GEOIP2_ISP": "/usr/local/share/GeoIP/GeoIP2-ISP.mmdb"
+    'GEOIP2_ISP': '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb'
   },
 };
 

backend/src/index.ts

@@ -1,4 +1,4 @@
-import express from "express";
+import express from 'express';
 import { Application, Request, Response, NextFunction } from 'express';
 import * as http from 'http';
 import * as WebSocket from 'ws';
@@ -34,7 +34,7 @@ import miningRoutes from './api/mining/mining-routes';
 import bisqRoutes from './api/bisq/bisq.routes';
 import liquidRoutes from './api/liquid/liquid.routes';
 import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
-import fundingTxFetcher from "./tasks/lightning/sync-tasks/funding-tx-fetcher";
+import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher';
 
 class Server {
   private wss: WebSocket.Server | undefined;
@@ -74,7 +74,7 @@ class Server {
     }
   }
 
-  async startServer(worker = false) {
+  async startServer(worker = false): Promise<void> {
     logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`);
 
     this.app
@@ -92,7 +92,9 @@ class Server {
     this.setUpWebsocketHandling();
 
     await syncAssets.syncAssets$();
-    diskCache.loadMempoolCache();
+    if (config.MEMPOOL.ENABLED) {
+      diskCache.loadMempoolCache();
+    }
 
     if (config.DATABASE.ENABLED) {
       await DB.checkDbConnection();
@@ -127,7 +129,10 @@ class Server {
     fiatConversion.startService();
 
     this.setUpHttpApiRoutes();
-    this.runMainUpdateLoop();
+
+    if (config.MEMPOOL.ENABLED) {
+      this.runMainUpdateLoop();
+    }
 
     if (config.BISQ.ENABLED) {
       bisq.startBisqService();
@@ -149,7 +154,7 @@ class Server {
     });
   }
 
-  async runMainUpdateLoop() {
+  async runMainUpdateLoop(): Promise<void> {
     try {
       try {
         await memPool.$updateMemPoolInfo();
@@ -183,7 +188,7 @@ class Server {
     }
   }
 
-  async $runLightningBackend() {
+  async $runLightningBackend(): Promise<void> {
     try {
       await fundingTxFetcher.$init();
       await networkSyncService.$startService();
@@ -195,7 +200,7 @@ class Server {
     };
 }
 
-  setUpWebsocketHandling() {
+  setUpWebsocketHandling(): void {
     if (this.wss) {
       websocketHandler.setWebsocketServer(this.wss);
     }
@@ -209,19 +214,21 @@ class Server {
       });
     }
     websocketHandler.setupConnectionHandling();
-    statistics.setNewStatisticsEntryCallback(websocketHandler.handleNewStatistic.bind(websocketHandler));
+    if (config.MEMPOOL.ENABLED) {
-    blocks.setNewBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler));
+      statistics.setNewStatisticsEntryCallback(websocketHandler.handleNewStatistic.bind(websocketHandler));
-    memPool.setMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler));
+      memPool.setAsyncMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler));
+      blocks.setNewAsyncBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler));
+    }
     fiatConversion.setProgressChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));
     loadingIndicators.setProgressChangedCallback(websocketHandler.handleLoadingChanged.bind(websocketHandler));
   }
-
+  
-  setUpHttpApiRoutes() {
+  setUpHttpApiRoutes(): void {
     bitcoinRoutes.initRoutes(this.app);
-    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED) {
+    if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && config.MEMPOOL.ENABLED) {
       statisticsRoutes.initRoutes(this.app);
     }
-    if (Common.indexingEnabled()) {
+    if (Common.indexingEnabled() && config.MEMPOOL.ENABLED) {
       miningRoutes.initRoutes(this.app);
     }
     if (config.BISQ.ENABLED) {
@@ -238,4 +245,4 @@ class Server {
   }
 }
 
-const server = new Server();
+((): Server => new Server())();

backend/src/mempool.interfaces.ts

@@ -32,6 +32,11 @@ export interface BlockAudit {
   matchRate: number,
 }
 
+export interface AuditScore {
+  hash: string,
+  matchRate?: number,
+}
+
 export interface MempoolBlock {
   blockSize: number;
   blockVSize: number;

backend/src/repositories/BlocksAuditsRepository.ts

@@ -1,6 +1,6 @@
 import DB from '../database';
 import logger from '../logger';
-import { BlockAudit } from '../mempool.interfaces';
+import { BlockAudit, AuditScore } from '../mempool.interfaces';
 
 class BlocksAuditRepositories {
   public async $saveAudit(audit: BlockAudit): Promise<void> {
@@ -72,10 +72,10 @@ class BlocksAuditRepositories {
     }
   }
 
-  public async $getShortBlockAudit(hash: string): Promise<any> {
+  public async $getBlockAuditScore(hash: string): Promise<AuditScore> {
     try {
       const [rows]: any[] = await DB.query(
-        `SELECT hash as id, match_rate as matchRate
+        `SELECT hash, match_rate as matchRate
         FROM blocks_audits
         WHERE blocks_audits.hash = "${hash}"
       `);

backend/src/repositories/BlocksRepository.ts

@@ -392,6 +392,36 @@ class BlocksRepository {
     }
   }
 
+  /**
+   * Get the first block at or directly after a given timestamp
+   * @param timestamp number unix time in seconds
+   * @returns The height and timestamp of a block (timestamp might vary from given timestamp)
+   */
+  public async $getBlockHeightFromTimestamp(
+    timestamp: number,
+  ): Promise<{ height: number; hash: string; timestamp: number }> {
+    try {
+      // Get first block at or after the given timestamp
+      const query = `SELECT height, hash, blockTimestamp as timestamp FROM blocks
+        WHERE blockTimestamp <= FROM_UNIXTIME(?)
+        ORDER BY blockTimestamp DESC
+        LIMIT 1`;
+      const params = [timestamp];
+      const [rows]: any[][] = await DB.query(query, params);
+      if (rows.length === 0) {
+        throw new Error(`No block was found before timestamp ${timestamp}`);
+      }
+
+      return rows[0];
+    } catch (e) {
+      logger.err(
+        'Cannot get block height from timestamp from the db. Reason: ' +
+          (e instanceof Error ? e.message : e),
+      );
+      throw e;
+    }
+  }
+
   /**
    * Return blocks height
    */

backend/src/repositories/NodeRecordsRepository.ts

@@ -0,0 +1,67 @@
+import { ResultSetHeader, RowDataPacket } from 'mysql2';
+import DB from '../database';
+import logger from '../logger';
+
+export interface NodeRecord {
+  publicKey: string; // node public key
+  type: number; // TLV extension record type
+  payload: string; // base64 record payload
+}
+
+class NodesRecordsRepository {
+  public async $saveRecord(record: NodeRecord): Promise<void> {
+    try {
+      const payloadBytes = Buffer.from(record.payload, 'base64');
+      await DB.query(`
+        INSERT INTO nodes_records(public_key, type, payload)
+        VALUE (?, ?, ?)
+        ON DUPLICATE KEY UPDATE
+          payload = ?
+      `, [record.publicKey, record.type, payloadBytes, payloadBytes]);
+    } catch (e: any) {
+      if (e.errno !== 1062) { // ER_DUP_ENTRY - Not an issue, just ignore this
+        logger.err(`Cannot save node record (${[record.publicKey, record.type, record.payload]}) into db. Reason: ` + (e instanceof Error ? e.message : e));
+        // We don't throw, not a critical issue if we miss some nodes records
+      }
+    }
+   }
+
+  public async $getRecordTypes(publicKey: string): Promise<any> {
+    try {
+      const query = `
+        SELECT type FROM nodes_records
+        WHERE public_key = ?
+      `;
+      const [rows] = await DB.query<RowDataPacket[][]>(query, [publicKey]);
+      return rows.map(row => row['type']);
+    } catch (e) {
+      logger.err(`Cannot retrieve custom records for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
+  public async $deleteUnusedRecords(publicKey: string, recordTypes: number[]): Promise<number> {
+    try {
+      let query;
+      if (recordTypes.length) {
+        query = `
+          DELETE FROM nodes_records
+          WHERE public_key = ?
+          AND type NOT IN (${recordTypes.map(type => `${type}`).join(',')})
+        `;
+      } else {
+        query = `
+          DELETE FROM nodes_records
+          WHERE public_key = ?
+        `;
+      }
+      const [result] = await DB.query<ResultSetHeader>(query, [publicKey]);
+      return result.affectedRows;
+    } catch (e) {
+      logger.err(`Cannot delete unused custom records for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e));
+      return 0;
+    }
+  }
+}
+
+export default new NodesRecordsRepository();

backend/src/tasks/lightning/network-sync.service.ts

@@ -13,6 +13,7 @@ import fundingTxFetcher from './sync-tasks/funding-tx-fetcher';
 import NodesSocketsRepository from '../../repositories/NodesSocketsRepository';
 import { Common } from '../../api/common';
 import blocks from '../../api/blocks';
+import NodeRecordsRepository from '../../repositories/NodeRecordsRepository';
 
 class NetworkSyncService {
   loggerTimer = 0;
@@ -63,6 +64,7 @@ class NetworkSyncService {
     let progress = 0;
 
     let deletedSockets = 0;
+    let deletedRecords = 0;
     const graphNodesPubkeys: string[] = [];
     for (const node of nodes) {
       const latestUpdated = await channelsApi.$getLatestChannelUpdateForNode(node.pub_key);
@@ -84,8 +86,23 @@ class NetworkSyncService {
         addresses.push(socket.addr);
       }
       deletedSockets += await NodesSocketsRepository.$deleteUnusedSockets(node.pub_key, addresses);
+
+      const oldRecordTypes = await NodeRecordsRepository.$getRecordTypes(node.pub_key);
+      const customRecordTypes: number[] = [];
+      for (const [type, payload] of Object.entries(node.custom_records || {})) {
+        const numericalType = parseInt(type);
+        await NodeRecordsRepository.$saveRecord({
+          publicKey: node.pub_key,
+          type: numericalType,
+          payload,
+        });
+        customRecordTypes.push(numericalType);
+      }
+      if (oldRecordTypes.reduce((changed, type) => changed || customRecordTypes.indexOf(type) === -1, false)) {
+        deletedRecords += await NodeRecordsRepository.$deleteUnusedRecords(node.pub_key, customRecordTypes);
+      }
     }
-    logger.info(`${progress} nodes updated. ${deletedSockets} sockets deleted`);
+    logger.info(`${progress} nodes updated. ${deletedSockets} sockets deleted. ${deletedRecords} custom records deleted.`);
 
     // If a channel if not present in the graph, mark it as inactive
     await nodesApi.$setNodesInactive(graphNodesPubkeys);
@@ -309,7 +326,7 @@ class NetworkSyncService {
          └──────────────────┘
   */
 
-  private async $runClosedChannelsForensics(): Promise<void> {
+  private async $runClosedChannelsForensics(skipUnresolved: boolean = false): Promise<void> {
     if (!config.ESPLORA.REST_API_URL) {
       return;
     }
@@ -318,9 +335,18 @@ class NetworkSyncService {
 
     try {
       logger.info(`Started running closed channel forensics...`);
-      const channels = await channelsApi.$getClosedChannelsWithoutReason();
+      let channels;
+      const closedChannels = await channelsApi.$getClosedChannelsWithoutReason();
+      if (skipUnresolved) {
+        channels = closedChannels;
+      } else {
+        const unresolvedChannels = await channelsApi.$getUnresolvedClosedChannels();
+        channels = [...closedChannels, ...unresolvedChannels];
+      }
+
       for (const channel of channels) {
         let reason = 0;
+        let resolvedForceClose = false;
         // Only Esplora backend can retrieve spent transaction outputs
         try {
           let outspends: IEsploraApi.Outspend[] | undefined;
@@ -350,6 +376,7 @@ class NetworkSyncService {
               reason = 3;
             } else {
               reason = 2;
+              resolvedForceClose = true;
             }
           } else {
             /*
@@ -374,6 +401,9 @@ class NetworkSyncService {
           if (reason) {
             logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.');
             await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]);
+            if (reason === 2 && resolvedForceClose) {
+              await DB.query(`UPDATE channels SET closing_resolved = ? WHERE id = ?`, [true, channel.id]);
+            }
           }
         } catch (e) {
           logger.err(`$runClosedChannelsForensics() failed for channel ${channel.short_id}. Reason: ${e instanceof Error ? e.message : e}`);

backend/src/tasks/lightning/sync-tasks/node-locations.ts

@@ -6,6 +6,7 @@ import DB from '../../../database';
 import logger from '../../../logger';
 import { ResultSetHeader } from 'mysql2';
 import * as IPCheck from '../../../utils/ipcheck.js';
+import { Reader } from 'mmdb-lib';
 
 export async function $lookupNodeLocation(): Promise<void> {
   let loggerTimer = new Date().getTime() / 1000;
@@ -18,7 +19,10 @@ export async function $lookupNodeLocation(): Promise<void> {
     const nodes = await nodesApi.$getAllNodes();
     const lookupCity = await maxmind.open<CityResponse>(config.MAXMIND.GEOLITE2_CITY);
     const lookupAsn = await maxmind.open<AsnResponse>(config.MAXMIND.GEOLITE2_ASN);
-    const lookupIsp = await maxmind.open<IspResponse>(config.MAXMIND.GEOIP2_ISP);
+    let lookupIsp: Reader<IspResponse> | null = null;
+    try {
+      lookupIsp = await maxmind.open<IspResponse>(config.MAXMIND.GEOIP2_ISP);
+    } catch (e) { }
 
     for (const node of nodes) {
       const sockets: string[] = node.sockets.split(',');
@@ -29,7 +33,10 @@ export async function $lookupNodeLocation(): Promise<void> {
         if (hasClearnet && ip !== '127.0.1.1' && ip !== '127.0.0.1') {
           const city = lookupCity.get(ip);
           const asn = lookupAsn.get(ip);
-          const isp = lookupIsp.get(ip);
+          let isp: IspResponse | null = null;
+          if (lookupIsp) {
+            isp = lookupIsp.get(ip);
+          }
 
           let asOverwrite: any | undefined;
           if (asn && (IPCheck.match(ip, '170.75.160.0/20') || IPCheck.match(ip, '172.81.176.0/21'))) {

docker/README.md

@@ -89,6 +89,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over
   "MEMPOOL": {
     "NETWORK": "mainnet",
     "BACKEND": "electrum",
+    "ENABLED": true,
     "HTTP_PORT": 8999,
     "SPAWN_CLUSTER_PROCS": 0,
     "API_URL_PREFIX": "/api/v1/",

docker/backend/mempool-config.json

@@ -2,6 +2,7 @@
   "MEMPOOL": {
     "NETWORK": "__MEMPOOL_NETWORK__",
     "BACKEND": "__MEMPOOL_BACKEND__",
+    "ENABLED": __MEMPOOL_ENABLED__,
     "HTTP_PORT": __MEMPOOL_HTTP_PORT__,
     "SPAWN_CLUSTER_PROCS": __MEMPOOL_SPAWN_CLUSTER_PROCS__,
     "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",

docker/backend/start.sh

@@ -3,6 +3,7 @@
 # MEMPOOL
 __MEMPOOL_NETWORK__=${MEMPOOL_NETWORK:=mainnet}
 __MEMPOOL_BACKEND__=${MEMPOOL_BACKEND:=electrum}
+__MEMPOOL_ENABLED__=${MEMPOOL_ENABLED:=true}
 __MEMPOOL_HTTP_PORT__=${BACKEND_HTTP_PORT:=8999}
 __MEMPOOL_SPAWN_CLUSTER_PROCS__=${MEMPOOL_SPAWN_CLUSTER_PROCS:=0}
 __MEMPOOL_API_URL_PREFIX__=${MEMPOOL_API_URL_PREFIX:=/api/v1/}
@@ -111,6 +112,7 @@ mkdir -p "${__MEMPOOL_CACHE_DIR__}"
 
 sed -i "s/__MEMPOOL_NETWORK__/${__MEMPOOL_NETWORK__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_BACKEND__/${__MEMPOOL_BACKEND__}/g" mempool-config.json
+sed -i "s/__MEMPOOL_ENABLED__/${__MEMPOOL_ENABLED__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_HTTP_PORT__/${__MEMPOOL_HTTP_PORT__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_SPAWN_CLUSTER_PROCS__/${__MEMPOOL_SPAWN_CLUSTER_PROCS__}/g" mempool-config.json
 sed -i "s!__MEMPOOL_API_URL_PREFIX__!${__MEMPOOL_API_URL_PREFIX__}!g" mempool-config.json

docker/frontend/Dockerfile

@@ -8,7 +8,9 @@ WORKDIR /build
 COPY . .
 RUN apt-get update
 RUN apt-get install -y build-essential rsync
+RUN cp mempool-frontend-config.sample.json mempool-frontend-config.json
 RUN npm install --omit=dev --omit=optional
+
 RUN npm run build
 
 FROM nginx:1.17.8-alpine
@@ -28,7 +30,9 @@ RUN chown -R 1000:1000 /patch && chmod -R 755 /patch && \
         chown -R 1000:1000 /var/cache/nginx && \
         chown -R 1000:1000 /var/log/nginx && \
         chown -R 1000:1000 /etc/nginx/nginx.conf && \
-        chown -R 1000:1000 /etc/nginx/conf.d
+        chown -R 1000:1000 /etc/nginx/conf.d && \
+        chown -R 1000:1000 /var/www/mempool
+
 RUN touch /var/run/nginx.pid && \
         chown -R 1000:1000 /var/run/nginx.pid
 

docker/frontend/entrypoint.sh

@@ -10,4 +10,51 @@ cp /etc/nginx/nginx.conf /patch/nginx.conf
 sed -i "s/__MEMPOOL_FRONTEND_HTTP_PORT__/${__MEMPOOL_FRONTEND_HTTP_PORT__}/g" /patch/nginx.conf
 cat /patch/nginx.conf > /etc/nginx/nginx.conf
 
+# Runtime overrides - read env vars defined in docker compose
+
+__TESTNET_ENABLED__=${TESTNET_ENABLED:=false}
+__SIGNET_ENABLED__=${SIGNET_ENABLED:=false}
+__LIQUID_ENABLED__=${LIQUID_EANBLED:=false}
+__LIQUID_TESTNET_ENABLED__=${LIQUID_TESTNET_ENABLED:=false}
+__BISQ_ENABLED__=${BISQ_ENABLED:=false}
+__BISQ_SEPARATE_BACKEND__=${BISQ_SEPARATE_BACKEND:=false}
+__ITEMS_PER_PAGE__=${ITEMS_PER_PAGE:=10}
+__KEEP_BLOCKS_AMOUNT__=${KEEP_BLOCKS_AMOUNT:=8}
+__NGINX_PROTOCOL__=${NGINX_PROTOCOL:=http}
+__NGINX_HOSTNAME__=${NGINX_HOSTNAME:=localhost}
+__NGINX_PORT__=${NGINX_PORT:=8999}
+__BLOCK_WEIGHT_UNITS__=${BLOCK_WEIGHT_UNITS:=4000000}
+__MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_BLOCKS_AMOUNT:=8}
+__BASE_MODULE__=${BASE_MODULE:=mempool}
+__MEMPOOL_WEBSITE_URL__=${MEMPOOL_WEBSITE_URL:=https://mempool.space}
+__LIQUID_WEBSITE_URL__=${LIQUID_WEBSITE_URL:=https://liquid.network}
+__BISQ_WEBSITE_URL__=${BISQ_WEBSITE_URL:=https://bisq.markets}
+__MINING_DASHBOARD__=${MINING_DASHBOARD:=true}
+__LIGHTNING__=${LIGHTNING:=false}
+
+# Export as environment variables to be used by envsubst
+export __TESTNET_ENABLED__
+export __SIGNET_ENABLED__
+export __LIQUID_ENABLED__
+export __LIQUID_TESTNET_ENABLED__
+export __BISQ_ENABLED__
+export __BISQ_SEPARATE_BACKEND__
+export __ITEMS_PER_PAGE__
+export __KEEP_BLOCKS_AMOUNT__
+export __NGINX_PROTOCOL__
+export __NGINX_HOSTNAME__
+export __NGINX_PORT__
+export __BLOCK_WEIGHT_UNITS__
+export __MEMPOOL_BLOCKS_AMOUNT__
+export __BASE_MODULE__
+export __MEMPOOL_WEBSITE_URL__
+export __LIQUID_WEBSITE_URL__
+export __BISQ_WEBSITE_URL__
+export __MINING_DASHBOARD__
+export __LIGHTNING__
+
+folder=$(find /var/www/mempool -name "config.js" | xargs dirname)
+echo ${folder}
+envsubst < ${folder}/config.template.js > ${folder}/config.js
+
 exec "$@"

frontend/angular.json

@@ -152,15 +152,14 @@
             "assets": [
               "src/favicon.ico",
               "src/resources",
-              "src/robots.txt"
+              "src/robots.txt",
+              "src/config.js",
+              "src/config.template.js"
             ],
             "styles": [
               "src/styles.scss",
               "node_modules/@fortawesome/fontawesome-svg-core/styles.css"
             ],
-            "scripts": [
-              "generated-config.js"
-            ],
             "vendorChunk": true,
             "extractLicenses": false,
             "buildOptimizer": false,

frontend/generate-config.js

@@ -2,7 +2,8 @@ var fs = require('fs');
 const { spawnSync } = require('child_process');
 
 const CONFIG_FILE_NAME = 'mempool-frontend-config.json';
-const GENERATED_CONFIG_FILE_NAME = 'generated-config.js';
+const GENERATED_CONFIG_FILE_NAME = 'src/resources/config.js';
+const GENERATED_TEMPLATE_CONFIG_FILE_NAME = 'src/resources/config.template.js';
 
 let settings = [];
 let configContent = {};
@@ -67,10 +68,17 @@ if (process.env.DOCKER_COMMIT_HASH) {
 
 const newConfig = `(function (window) {
   window.__env = window.__env || {};${settings.reduce((str, obj) => `${str}
-    window.__env.${obj.key} = ${ typeof obj.value === 'string' ? `'${obj.value}'` : obj.value };`, '')}
+    window.__env.${obj.key} = ${typeof obj.value === 'string' ? `'${obj.value}'` : obj.value};`, '')}
     window.__env.GIT_COMMIT_HASH = '${gitCommitHash}';
     window.__env.PACKAGE_JSON_VERSION = '${packetJsonVersion}';
-  }(global || this));`;
+  }(this));`;
+
+const newConfigTemplate = `(function (window) {
+  window.__env = window.__env || {};${settings.reduce((str, obj) => `${str}
+    window.__env.${obj.key} = ${typeof obj.value === 'string' ? `'\${__${obj.key}__}'` : `\${__${obj.key}__}`};`, '')}
+    window.__env.GIT_COMMIT_HASH = '${gitCommitHash}';
+    window.__env.PACKAGE_JSON_VERSION = '${packetJsonVersion}';
+  }(this));`;
 
 function readConfig(path) {
   try {
@@ -89,6 +97,16 @@ function writeConfig(path, config) {
   }
 }
 
+function writeConfigTemplate(path, config) {
+  try {
+    fs.writeFileSync(path, config, 'utf8');
+  } catch (e) {
+    throw new Error(e);
+  }
+}
+
+writeConfigTemplate(GENERATED_TEMPLATE_CONFIG_FILE_NAME, newConfigTemplate);
+
 const currentConfig = readConfig(GENERATED_CONFIG_FILE_NAME);
 
 if (currentConfig && currentConfig === newConfig) {
@@ -106,4 +124,4 @@ if (currentConfig && currentConfig === newConfig) {
   console.log('NEW CONFIG: ', newConfig);
   writeConfig(GENERATED_CONFIG_FILE_NAME, newConfig);
   console.log(`${GENERATED_CONFIG_FILE_NAME} file updated`);
-};
+}

frontend/src/app/app.constants.ts

@@ -79,7 +79,7 @@ export const poolsColor = {
    'binancepool': '#1E88E5',
    'viabtc': '#039BE5',
    'btccom': '#00897B',
-   'slushpool': '#00ACC1',
+   'braiinspool': '#00ACC1',
    'sbicrypto': '#43A047',
    'marapool': '#7CB342',
    'luxor': '#C0CA33',

frontend/src/app/components/about/about.component.html

@@ -129,7 +129,7 @@
         <span>Gemini</span>
       </a>
       <a href="https://exodus.com/" target="_blank" title="Exodus">
-        <svg width="80" height="80" viewBox="0 0 500 500" fill="none" xmlns="http://www.w3.org/2000/svg">
+        <svg width="81" height="81" viewBox="0 0 500 500" fill="none" xmlns="http://www.w3.org/2000/svg">
           <circle cx="250" cy="250" r="250" fill="#1F2033"/>
           <g clip-path="url(#clip0_2_14)">
             <path d="M411.042 178.303L271.79 87V138.048L361.121 196.097L350.612 229.351H271.79V271.648H350.612L361.121 304.903L271.79 362.952V414L411.042 322.989L388.271 250.646L411.042 178.303Z" fill="url(#paint0_linear_2_14)"/>
@@ -274,6 +274,10 @@
         <img class="image" src="/resources/profile/schildbach.svg" />
         <span>Schildbach</span>
       </a>
+      <a href="https://github.com/nunchuk-io" target="_blank" title="Nunchuck">
+        <img class="image" src="/resources/profile/nunchuk.svg" />
+        <span>Nunchuk</span>
+      </a>
     </div>
   </div>
 

frontend/src/app/components/about/about.component.scss

@@ -3,8 +3,8 @@
   text-align: center;
 
   .image {
-    width: 80px;
+    width: 81px;
-    height: 80px;
+    height: 81px;
     background-size: 100%, 100%;
     border-radius: 50%;
     margin: 25px;

frontend/src/app/components/block-audit/block-audit.component.html

@@ -41,10 +41,6 @@
                   </div>
                 </td>
               </tr>
-              <tr>
-                <td class="td-width" i18n="shared.transaction-count">Transactions</td>
-                <td>{{ blockAudit.tx_count }}</td>
-              </tr>
               <tr>
                 <td i18n="blockAudit.size">Size</td>
                 <td [innerHTML]="'&lrm;' + (blockAudit.size | bytes: 2)"></td>
@@ -61,6 +57,10 @@
         <div class="col-sm" *ngIf="blockAudit">
           <table class="table table-borderless table-striped">
             <tbody>
+              <tr>
+                <td class="td-width" i18n="shared.transaction-count">Transactions</td>
+                <td>{{ blockAudit.tx_count }}</td>
+              </tr>
               <tr>
                 <td i18n="block.health">Block health</td>
                 <td>{{ blockAudit.matchRate }}%</td>
@@ -69,18 +69,10 @@
                 <td i18n="block.missing-txs">Removed txs</td>
                 <td>{{ blockAudit.missingTxs.length }}</td>
               </tr>
-              <tr>
-                <td i18n="block.missing-txs">Omitted txs</td>
-                <td>{{ numMissing }}</td>
-              </tr>
               <tr>
                 <td i18n="block.added-txs">Added txs</td>
                 <td>{{ blockAudit.addedTxs.length }}</td>
               </tr>
-              <tr>
-                <td i18n="block.missing-txs">Included txs</td>
-                <td>{{ numUnexpected }}</td>
-              </tr>
             </tbody>
           </table>
         </div>
@@ -97,21 +89,6 @@
   </div>
 
   <ng-template [ngIf]="!error && isLoading">
-    <div class="title-block" id="block">
-      <h1>
-        <span class="next-previous-blocks">
-          <span i18n="shared.block-audit-title">Block Audit</span>
-          &nbsp;
-          <a *ngIf="blockAudit" [routerLink]="['/block/' | relativeUrl, blockHash]">{{ blockAudit.height }}</a>
-          &nbsp;
-        </span>
-      </h1>
-
-      <div class="grow"></div>
-
-      <button [routerLink]="['/' | relativeUrl]" class="btn btn-sm">&#10005;</button>
-    </div>
-
     <!-- OVERVIEW -->
     <div class="box mb-3">
       <div class="row">
@@ -123,7 +100,6 @@
               <tr><td class="td-width" colspan="2"><span class="skeleton-loader"></span></td></tr>
               <tr><td class="td-width" colspan="2"><span class="skeleton-loader"></span></td></tr>
               <tr><td class="td-width" colspan="2"><span class="skeleton-loader"></span></td></tr>
-              <tr><td class="td-width" colspan="2"><span class="skeleton-loader"></span></td></tr>
             </tbody>
           </table>
         </div>
@@ -136,7 +112,6 @@
               <tr><td class="td-width" colspan="2"><span class="skeleton-loader"></span></td></tr>
               <tr><td class="td-width" colspan="2"><span class="skeleton-loader"></span></td></tr>
               <tr><td class="td-width" colspan="2"><span class="skeleton-loader"></span></td></tr>
-              <tr><td class="td-width" colspan="2"><span class="skeleton-loader"></span></td></tr>
             </tbody>
           </table>
         </div>
@@ -180,16 +155,16 @@
       <div class="col-sm" *ngIf="webGlEnabled">
         <h3 class="block-subtitle" *ngIf="!isMobile" i18n="block.projected-block">Projected Block</h3>
         <app-block-overview-graph #blockGraphProjected [isLoading]="isLoading" [resolution]="75"
-          [blockLimit]="stateService.blockVSize" [orientation]="'top'" [flip]="false"
+          [blockLimit]="stateService.blockVSize" [orientation]="'top'" [flip]="false" [mirrorTxid]="hoverTx"
-          (txClickEvent)="onTxClick($event)"></app-block-overview-graph>
+          (txClickEvent)="onTxClick($event)" (txHoverEvent)="onTxHover($event)"></app-block-overview-graph>
       </div>
 
       <!-- ADDED TX RENDERING -->
       <div class="col-sm" *ngIf="webGlEnabled && !isMobile">
         <h3 class="block-subtitle" *ngIf="!isMobile" i18n="block.actual-block">Actual Block</h3>
         <app-block-overview-graph #blockGraphActual [isLoading]="isLoading" [resolution]="75"
-          [blockLimit]="stateService.blockVSize" [orientation]="'top'" [flip]="false"
+          [blockLimit]="stateService.blockVSize" [orientation]="'top'" [flip]="false" [mirrorTxid]="hoverTx"
-          (txClickEvent)="onTxClick($event)"></app-block-overview-graph>
+          (txClickEvent)="onTxClick($event)" (txHoverEvent)="onTxHover($event)"></app-block-overview-graph>
       </div>
     </div> <!-- row -->
   </div> <!-- box -->

frontend/src/app/components/block-audit/block-audit.component.ts

@@ -1,9 +1,10 @@
 import { Component, OnDestroy, OnInit, AfterViewInit, ViewChildren, QueryList } from '@angular/core';
 import { ActivatedRoute, ParamMap, Router } from '@angular/router';
-import { Subscription, combineLatest } from 'rxjs';
+import { Subscription, combineLatest, of } from 'rxjs';
-import { map, switchMap, startWith, catchError } from 'rxjs/operators';
+import { map, switchMap, startWith, catchError, filter } from 'rxjs/operators';
 import { BlockAudit, TransactionStripped } from '../../interfaces/node-api.interface';
 import { ApiService } from '../../services/api.service';
+import { ElectrsApiService } from '../../services/electrs-api.service';
 import { StateService } from '../../services/state.service';
 import { detectWebGL } from '../../shared/graphs.utils';
 import { RelativeUrlPipe } from '../../shared/pipes/relative-url/relative-url.pipe';
@@ -37,6 +38,7 @@ export class BlockAuditComponent implements OnInit, AfterViewInit, OnDestroy {
   isLoading = true;
   webGlEnabled = true;
   isMobile = window.innerWidth <= 767.98;
+  hoverTx: string;
 
   childChangeSubscription: Subscription;
 
@@ -51,7 +53,8 @@ export class BlockAuditComponent implements OnInit, AfterViewInit, OnDestroy {
     private route: ActivatedRoute,
     public stateService: StateService,
     private router: Router,
-    private apiService: ApiService
+    private apiService: ApiService,
+    private electrsApiService: ElectrsApiService,
   ) {
     this.webGlEnabled = detectWebGL();
   }
@@ -76,69 +79,95 @@ export class BlockAuditComponent implements OnInit, AfterViewInit, OnDestroy {
 
     this.auditSubscription = this.route.paramMap.pipe(
       switchMap((params: ParamMap) => {
-        this.blockHash = params.get('id') || null;
+        const blockHash = params.get('id') || null;
-        if (!this.blockHash) {
+        if (!blockHash) {
           return null;
         }
+
+        let isBlockHeight = false;
+        if (/^[0-9]+$/.test(blockHash)) {
+          isBlockHeight = true;
+        } else {
+          this.blockHash = blockHash;
+        }
+
+        if (isBlockHeight) {
+          return this.electrsApiService.getBlockHashFromHeight$(parseInt(blockHash, 10))
+            .pipe(
+              switchMap((hash: string) => {
+                if (hash) {
+                  this.blockHash = hash;
+                  return this.apiService.getBlockAudit$(this.blockHash)
+                } else {
+                  return null;
+                }
+              }),
+              catchError((err) => {
+                this.error = err;
+                return of(null);
+              }),
+            );
+        }
         return this.apiService.getBlockAudit$(this.blockHash)
-          .pipe(
+      }),
-            map((response) => {
+      filter((response) => response != null),
-              const blockAudit = response.body;
+      map((response) => {
-              const inTemplate = {};
+        const blockAudit = response.body;
-              const inBlock = {};
+        const inTemplate = {};
-              const isAdded = {};
+        const inBlock = {};
-              const isCensored = {};
+        const isAdded = {};
-              const isMissing = {};
+        const isCensored = {};
-              const isSelected = {};
+        const isMissing = {};
-              this.numMissing = 0;
+        const isSelected = {};
-              this.numUnexpected = 0;
+        this.numMissing = 0;
-              for (const tx of blockAudit.template) {
+        this.numUnexpected = 0;
-                inTemplate[tx.txid] = true;
+        for (const tx of blockAudit.template) {
-              }
+          inTemplate[tx.txid] = true;
-              for (const tx of blockAudit.transactions) {
+        }
-                inBlock[tx.txid] = true;
+        for (const tx of blockAudit.transactions) {
-              }
+          inBlock[tx.txid] = true;
-              for (const txid of blockAudit.addedTxs) {
+        }
-                isAdded[txid] = true;
+        for (const txid of blockAudit.addedTxs) {
-              }
+          isAdded[txid] = true;
-              for (const txid of blockAudit.missingTxs) {
+        }
-                isCensored[txid] = true;
+        for (const txid of blockAudit.missingTxs) {
-              }
+          isCensored[txid] = true;
-              // set transaction statuses
+        }
-              for (const tx of blockAudit.template) {
+        // set transaction statuses
-                if (isCensored[tx.txid]) {
+        for (const tx of blockAudit.template) {
-                  tx.status = 'censored';
+          if (isCensored[tx.txid]) {
-                } else if (inBlock[tx.txid]) {
+            tx.status = 'censored';
-                  tx.status = 'found';
+          } else if (inBlock[tx.txid]) {
-                } else {
+            tx.status = 'found';
-                  tx.status = 'missing';
+          } else {
-                  isMissing[tx.txid] = true;
+            tx.status = 'missing';
-                  this.numMissing++;
+            isMissing[tx.txid] = true;
-                }
+            this.numMissing++;
-              }
+          }
-              for (const [index, tx] of blockAudit.transactions.entries()) {
+        }
-                if (isAdded[tx.txid]) {
+        for (const [index, tx] of blockAudit.transactions.entries()) {
-                  tx.status = 'added';
+          if (index === 0) {
-                } else if (index === 0 || inTemplate[tx.txid]) {
+            tx.status = null;
-                  tx.status = 'found';
+          } else if (isAdded[tx.txid]) {
-                } else {
+            tx.status = 'added';
-                  tx.status = 'selected';
+          } else if (inTemplate[tx.txid]) {
-                  isSelected[tx.txid] = true;
+            tx.status = 'found';
-                  this.numUnexpected++;
+          } else {
-                }
+            tx.status = 'selected';
-              }
+            isSelected[tx.txid] = true;
-              for (const tx of blockAudit.transactions) {
+            this.numUnexpected++;
-                inBlock[tx.txid] = true;
+          }
-              }
+        }
-              return blockAudit;
+        for (const tx of blockAudit.transactions) {
-            })
+          inBlock[tx.txid] = true;
-          );
+        }
+        return blockAudit;
       }),
       catchError((err) => {
         console.log(err);
         this.error = err;
         this.isLoading = false;
-        return null;
+        return of(null);
       }),
     ).subscribe((blockAudit) => {
       this.blockAudit = blockAudit;
@@ -189,4 +218,12 @@ export class BlockAuditComponent implements OnInit, AfterViewInit, OnDestroy {
     const url = new RelativeUrlPipe(this.stateService).transform(`/tx/${event.txid}`);
     this.router.navigate([url]);
   }
+
+  onTxHover(txid: string): void {
+    if (txid && txid.length) {
+      this.hoverTx = txid;
+    } else {
+      this.hoverTx = null;
+    }
+  }
 }

frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts

@@ -18,7 +18,9 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
   @Input() orientation = 'left';
   @Input() flip = true;
   @Input() disableSpinner = false;
+  @Input() mirrorTxid: string | void;
   @Output() txClickEvent = new EventEmitter<TransactionStripped>();
+  @Output() txHoverEvent = new EventEmitter<string>();
   @Output() readyEvent = new EventEmitter();
 
   @ViewChild('blockCanvas')
@@ -37,6 +39,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
   scene: BlockScene;
   hoverTx: TxView | void;
   selectedTx: TxView | void;
+  mirrorTx: TxView | void;
   tooltipPosition: Position;
 
   readyNextFrame = false;
@@ -63,6 +66,9 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
         this.scene.setOrientation(this.orientation, this.flip);
       }
     }
+    if (changes.mirrorTxid) {
+      this.setMirror(this.mirrorTxid);
+    }
   }
 
   ngOnDestroy(): void {
@@ -76,6 +82,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
     this.exit(direction);
     this.hoverTx = null;
     this.selectedTx = null;
+    this.onTxHover(null);
     this.start();
   }
 
@@ -181,7 +188,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
       this.gl.viewport(0, 0, this.displayWidth, this.displayHeight);
     }
     if (this.scene) {
-      this.scene.resize({ width: this.displayWidth, height: this.displayHeight });
+      this.scene.resize({ width: this.displayWidth, height: this.displayHeight, animate: false });
       this.start();
     } else {
       this.scene = new BlockScene({ width: this.displayWidth, height: this.displayHeight, resolution: this.resolution,
@@ -301,6 +308,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
       }
       this.hoverTx = null;
       this.selectedTx = null;
+      this.onTxHover(null);
     }
   }
 
@@ -352,17 +360,20 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
             this.selectedTx = selected;
           } else {
             this.hoverTx = selected;
+            this.onTxHover(this.hoverTx ? this.hoverTx.txid : null);
           }
         } else {
           if (clicked) {
             this.selectedTx = null;
           }
           this.hoverTx = null;
+          this.onTxHover(null);
         }
       } else if (clicked) {
         if (selected === this.selectedTx) {
           this.hoverTx = this.selectedTx;
           this.selectedTx = null;
+          this.onTxHover(this.hoverTx ? this.hoverTx.txid : null);
         } else {
           this.selectedTx = selected;
         }
@@ -370,6 +381,18 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
     }
   }
 
+  setMirror(txid: string | void) {
+    if (this.mirrorTx) {
+      this.scene.setHover(this.mirrorTx, false);
+      this.start();
+    }
+    if (txid && this.scene.txs[txid]) {
+      this.mirrorTx = this.scene.txs[txid];
+      this.scene.setHover(this.mirrorTx, true);
+      this.start();
+    }
+  }
+
   onTxClick(cssX: number, cssY: number) {
     const x = cssX * window.devicePixelRatio;
     const y = cssY * window.devicePixelRatio;
@@ -378,6 +401,10 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
       this.txClickEvent.emit(selected);
     }
   }
+
+  onTxHover(hoverId: string) {
+    this.txHoverEvent.emit(hoverId);
+  }
 }
 
 // WebGL shader attributes

frontend/src/app/components/block-overview-graph/block-scene.ts

@@ -29,7 +29,7 @@ export default class BlockScene {
     this.init({ width, height, resolution, blockLimit, orientation, flip, vertexArray });
   }
 
-  resize({ width = this.width, height = this.height }: { width?: number, height?: number}): void {
+  resize({ width = this.width, height = this.height, animate = true }: { width?: number, height?: number, animate: boolean }): void {
     this.width = width;
     this.height = height;
     this.gridSize = this.width / this.gridWidth;
@@ -38,7 +38,7 @@ export default class BlockScene {
 
     this.dirty = true;
     if (this.initialised && this.scene) {
-      this.updateAll(performance.now(), 50);
+      this.updateAll(performance.now(), 50, 'left', animate);
     }
   }
 
@@ -212,7 +212,7 @@ export default class BlockScene {
     this.vbytesPerUnit = blockLimit / Math.pow(resolution / 1.02, 2);
     this.gridWidth = resolution;
     this.gridHeight = resolution;
-    this.resize({ width, height });
+    this.resize({ width, height, animate: true });
     this.layout = new BlockLayout({ width: this.gridWidth, height: this.gridHeight });
 
     this.txs = {};
@@ -225,14 +225,14 @@ export default class BlockScene {
     this.animateUntil = Math.max(this.animateUntil, tx.update(update));
   }
 
-  private updateTx(tx: TxView, startTime: number, delay: number, direction: string = 'left'): void {
+  private updateTx(tx: TxView, startTime: number, delay: number, direction: string = 'left', animate: boolean = true): void {
     if (tx.dirty || this.dirty) {
       this.saveGridToScreenPosition(tx);
-      this.setTxOnScreen(tx, startTime, delay, direction);
+      this.setTxOnScreen(tx, startTime, delay, direction, animate);
     }
   }
 
-  private setTxOnScreen(tx: TxView, startTime: number, delay: number = 50, direction: string = 'left'): void {
+  private setTxOnScreen(tx: TxView, startTime: number, delay: number = 50, direction: string = 'left', animate: boolean = true): void {
     if (!tx.initialised) {
       const txColor = tx.getColor();
       this.applyTxUpdate(tx, {
@@ -252,30 +252,42 @@ export default class BlockScene {
           position: tx.screenPosition,
           color: txColor
         },
-        duration: 1000,
+        duration: animate ? 1000 : 1,
         start: startTime,
-        delay,
+        delay: animate ? delay : 0,
       });
     } else {
       this.applyTxUpdate(tx, {
         display: {
           position: tx.screenPosition
         },
-        duration: 1000,
+        duration: animate ? 1000 : 0,
-        minDuration: 500,
+        minDuration: animate ? 500 : 0,
         start: startTime,
-        delay,
+        delay: animate ? delay : 0,
-        adjust: true
+        adjust: animate
       });
+      if (!animate) {
+        this.applyTxUpdate(tx, {
+          display: {
+            position: tx.screenPosition
+          },
+          duration: 0,
+          minDuration: 0,
+          start: startTime,
+          delay: 0,
+          adjust: false
+        });
+      }
     }
   }
 
-  private updateAll(startTime: number, delay: number = 50, direction: string = 'left'): void {
+  private updateAll(startTime: number, delay: number = 50, direction: string = 'left', animate: boolean = true): void {
     this.scene.count = 0;
     const ids = this.getTxList();
     startTime = startTime || performance.now();
     for (const id of ids) {
-      this.updateTx(this.txs[id], startTime, delay, direction);
+      this.updateTx(this.txs[id], startTime, delay, direction, animate);
     }
     this.dirty = false;
   }

frontend/src/app/components/block-overview-graph/tx-view.ts

@@ -12,8 +12,8 @@ const auditFeeColors = feeColors.map((color) => desaturate(color, 0.3));
 const auditColors = {
   censored: hexToColor('f344df'),
   missing: darken(desaturate(hexToColor('f344df'), 0.3), 0.7),
-  added: hexToColor('03E1E5'),
+  added: hexToColor('0099ff'),
-  selected: darken(desaturate(hexToColor('039BE5'), 0.3), 0.7),
+  selected: darken(desaturate(hexToColor('0099ff'), 0.3), 0.7),
 }
 
 // convert from this class's update format to TxSprite's update format

frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html

@@ -37,9 +37,9 @@
         <ng-container [ngSwitch]="tx?.status">
           <td *ngSwitchCase="'found'" i18n="transaction.audit.match">match</td>
           <td *ngSwitchCase="'censored'" i18n="transaction.audit.removed">removed</td>
-          <td *ngSwitchCase="'missing'" i18n="transaction.audit.missing">missing</td>
+          <td *ngSwitchCase="'missing'" i18n="transaction.audit.omitted">omitted</td>
           <td *ngSwitchCase="'added'" i18n="transaction.audit.added">added</td>
-          <td *ngSwitchCase="'selected'" i18n="transaction.audit.included">included</td>
+          <td *ngSwitchCase="'selected'" i18n="transaction.audit.extra">extra</td>
         </ng-container>
       </tr>
     </tbody>

frontend/src/app/components/block/block.component.html

@@ -114,7 +114,7 @@
                   <td i18n="block.health">Block health</td>
                   <td>
                     <a *ngIf="block.extras?.matchRate != null" [routerLink]="['/block-audit/' | relativeUrl, blockHash]">{{ block.extras.matchRate }}%</a>
-                    <span *ngIf="block.extras?.matchRate == null" i18n="unknown">Unknown</span>
+                    <span *ngIf="block.extras?.matchRate === null" i18n="unknown">Unknown</span>
                   </td>
                 </tr>
               </ng-template>

frontend/src/app/components/block/block.component.ts

@@ -4,7 +4,7 @@ import { ActivatedRoute, ParamMap, Router } from '@angular/router';
 import { ElectrsApiService } from '../../services/electrs-api.service';
 import { switchMap, tap, throttleTime, catchError, map, shareReplay, startWith, pairwise } from 'rxjs/operators';
 import { Transaction, Vout } from '../../interfaces/electrs.interface';
-import { Observable, of, Subscription, asyncScheduler, EMPTY } from 'rxjs';
+import { Observable, of, Subscription, asyncScheduler, EMPTY, Subject } from 'rxjs';
 import { StateService } from '../../services/state.service';
 import { SeoService } from '../../services/seo.service';
 import { WebsocketService } from '../../services/websocket.service';
@@ -60,6 +60,8 @@ export class BlockComponent implements OnInit, OnDestroy {
   nextBlockTxListSubscription: Subscription = undefined;
   timeLtrSubscription: Subscription;
   timeLtr: boolean;
+  fetchAuditScore$ = new Subject<string>();
+  fetchAuditScoreSubscription: Subscription;
 
   @ViewChild('blockGraph') blockGraph: BlockOverviewGraphComponent;
 
@@ -105,12 +107,30 @@ export class BlockComponent implements OnInit, OnDestroy {
 
         if (block.id === this.blockHash) {
           this.block = block;
+          if (this.block.id && this.block?.extras?.matchRate == null) {
+            this.fetchAuditScore$.next(this.block.id);
+          }
           if (block?.extras?.reward != undefined) {
             this.fees = block.extras.reward / 100000000 - this.blockSubsidy;
           }
         }
       });
 
+    if (this.indexingAvailable) {
+      this.fetchAuditScoreSubscription = this.fetchAuditScore$
+        .pipe(
+          switchMap((hash) => this.apiService.getBlockAuditScore$(hash)),
+          catchError(() => EMPTY),
+        )
+        .subscribe((score) => {
+          if (score && score.hash === this.block.id) {
+            this.block.extras.matchRate = score.matchRate || null;
+          } else {
+            this.block.extras.matchRate = null;
+          }
+        });
+    }
+
     const block$ = this.route.paramMap.pipe(
       switchMap((params: ParamMap) => {
         const blockHash: string = params.get('id') || '';
@@ -209,6 +229,9 @@ export class BlockComponent implements OnInit, OnDestroy {
           this.fees = block.extras.reward / 100000000 - this.blockSubsidy;
         }
         this.stateService.markBlock$.next({ blockHeight: this.blockHeight });
+        if (this.block.id && this.block?.extras?.matchRate == null) {
+          this.fetchAuditScore$.next(this.block.id);
+        }
         this.isLoadingTransactions = true;
         this.transactions = null;
         this.transactionsError = null;
@@ -311,6 +334,7 @@ export class BlockComponent implements OnInit, OnDestroy {
     this.networkChangedSubscription.unsubscribe();
     this.queryParamsSubscription.unsubscribe();
     this.timeLtrSubscription.unsubscribe();
+    this.fetchAuditScoreSubscription?.unsubscribe();
     this.unsubscribeNextBlockSubscriptions();
   }
 

frontend/src/app/components/blocks-list/blocks-list.component.html

@@ -46,22 +46,17 @@
             <app-time-since [time]="block.timestamp" [fastRender]="true"></app-time-since>
           </td>
           <td *ngIf="indexingAvailable" class="health text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
-            <a *ngIf="block.extras?.matchRate != null" class="clear-link" [routerLink]="['/block-audit/' | relativeUrl, block.id]">
+            <a class="clear-link" [routerLink]="auditScores[block.id] != null ? ['/block-audit/' | relativeUrl, block.id] : null">
               <div class="progress progress-health">
                 <div class="progress-bar progress-bar-health" role="progressbar"
-                  [ngStyle]="{'width': (100 - (block.extras?.matchRate || 0)) + '%' }"></div>
+                  [ngStyle]="{'width': (100 - (auditScores[block.id] || 0)) + '%' }"></div>
                 <div class="progress-text">
-                  <span>{{ block.extras.matchRate }}%</span>
+                  <span *ngIf="auditScores[block.id] != null;">{{ auditScores[block.id] }}%</span>
+                  <span *ngIf="auditScores[block.id] === undefined" class="skeleton-loader"></span>
+                  <span *ngIf="auditScores[block.id] === null">~</span>
                 </div>
               </div>
             </a>
-            <div *ngIf="block.extras?.matchRate == null" class="progress progress-health">
-              <div class="progress-bar progress-bar-health" role="progressbar"
-                [ngStyle]="{'width': '100%' }"></div>
-              <div class="progress-text">
-                <span>~</span>
-              </div>
-            </div>
           </td>
           <td *ngIf="indexingAvailable" class="reward text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
             <app-amount [satoshis]="block.extras.reward" [noFiat]="true" digitsInfo="1.2-2"></app-amount>

frontend/src/app/components/blocks-list/blocks-list.component.scss

@@ -196,6 +196,10 @@ tr, td, th {
   @media (max-width: 950px) {
     display: none;
   }
+
+  .progress-text .skeleton-loader {
+    top: -8.5px;
+  }
 }
 .health.widget {
   width: 25%;

frontend/src/app/components/blocks-list/blocks-list.component.ts

@@ -1,6 +1,6 @@
-import { Component, OnInit, ChangeDetectionStrategy, Input } from '@angular/core';
+import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, Input } from '@angular/core';
-import { BehaviorSubject, combineLatest, concat, Observable, timer } from 'rxjs';
+import { BehaviorSubject, combineLatest, concat, Observable, timer, EMPTY, Subscription, of } from 'rxjs';
-import { delayWhen, map, retryWhen, scan, skip, switchMap, tap } from 'rxjs/operators';
+import { catchError, delayWhen, map, retryWhen, scan, skip, switchMap, tap } from 'rxjs/operators';
 import { BlockExtended } from '../../interfaces/node-api.interface';
 import { ApiService } from '../../services/api.service';
 import { StateService } from '../../services/state.service';
@@ -12,10 +12,14 @@ import { WebsocketService } from '../../services/websocket.service';
   styleUrls: ['./blocks-list.component.scss'],
   changeDetection: ChangeDetectionStrategy.OnPush,
 })
-export class BlocksList implements OnInit {
+export class BlocksList implements OnInit, OnDestroy {
   @Input() widget: boolean = false;
 
   blocks$: Observable<BlockExtended[]> = undefined;
+  auditScores: { [hash: string]: number | void } = {};
+
+  auditScoreSubscription: Subscription;
+  latestScoreSubscription: Subscription;
 
   indexingAvailable = false;
   isLoading = true;
@@ -105,6 +109,53 @@ export class BlocksList implements OnInit {
           return acc;
         }, [])
       );
+
+    if (this.indexingAvailable) {
+      this.auditScoreSubscription = this.fromHeightSubject.pipe(
+        switchMap((fromBlockHeight) => {
+          return this.apiService.getBlockAuditScores$(this.page === 1 ? undefined : fromBlockHeight)
+            .pipe(
+              catchError(() => {
+                return EMPTY;
+              })
+            );
+        })
+      ).subscribe((scores) => {
+        Object.values(scores).forEach(score => {
+          this.auditScores[score.hash] = score?.matchRate != null ? score.matchRate : null;
+        });
+      });
+
+      this.latestScoreSubscription = this.stateService.blocks$.pipe(
+        switchMap((block) => {
+          if (block[0]?.extras?.matchRate != null) {
+            return of({
+              hash: block[0].id,
+              matchRate: block[0]?.extras?.matchRate,
+            });
+          }
+          else if (block[0]?.id && this.auditScores[block[0].id] === undefined) {
+            return this.apiService.getBlockAuditScore$(block[0].id)
+              .pipe(
+                catchError(() => {
+                  return EMPTY;
+                })
+              );
+          } else {
+            return EMPTY;
+          }
+        }),
+      ).subscribe((score) => {
+        if (score && score.hash) {
+          this.auditScores[score.hash] = score?.matchRate != null ? score.matchRate : null;
+        }
+      });
+    }
+  }
+
+  ngOnDestroy(): void {
+    this.auditScoreSubscription?.unsubscribe();
+    this.latestScoreSubscription?.unsubscribe();
   }
 
   pageChange(page: number) {

frontend/src/app/components/search-form/search-form.component.html

@@ -2,9 +2,7 @@
   <div class="d-flex">
     <div class="search-box-container mr-2">
       <input (focus)="focus$.next($any($event).target.value)" (click)="click$.next($any($event).target.value)" formControlName="searchText" type="text" class="form-control" i18n-placeholder="search-form.searchbar-placeholder" placeholder="Explore the full Bitcoin ecosystem">
-      
+      <app-search-results #searchResults [hidden]="dropdownHidden" [results]="typeAhead$ | async" (selectedResult)="selectedResult($event)"></app-search-results>
-      <app-search-results #searchResults [results]="typeAhead$ | async" (selectedResult)="selectedResult($event)"></app-search-results>
-    
     </div>
     <div>
       <button [disabled]="isSearching" type="submit" class="btn btn-block btn-primary">

frontend/src/app/components/search-form/search-form.component.ts

@@ -1,4 +1,4 @@
-import { Component, OnInit, ChangeDetectionStrategy, EventEmitter, Output, ViewChild, HostListener } from '@angular/core';
+import { Component, OnInit, ChangeDetectionStrategy, EventEmitter, Output, ViewChild, HostListener, ElementRef } from '@angular/core';
 import { FormBuilder, FormGroup, Validators } from '@angular/forms';
 import { Router } from '@angular/router';
 import { AssetsService } from '../../services/assets.service';
@@ -23,6 +23,16 @@ export class SearchFormComponent implements OnInit {
   isTypeaheading$ = new BehaviorSubject<boolean>(false);
   typeAhead$: Observable<any>;
   searchForm: FormGroup;
+  dropdownHidden = false;
+
+  @HostListener('document:click', ['$event'])
+  onDocumentClick(event) {
+    if (this.elementRef.nativeElement.contains(event.target)) {
+      this.dropdownHidden = false;
+    } else {
+      this.dropdownHidden = true;
+    }
+  }
 
   regexAddress = /^([a-km-zA-HJ-NP-Z1-9]{26,35}|[a-km-zA-HJ-NP-Z1-9]{80}|[A-z]{2,5}1[a-zA-HJ-NP-Z0-9]{39,59})$/;
   regexBlockhash = /^[0]{8}[a-fA-F0-9]{56}$/;
@@ -45,6 +55,7 @@ export class SearchFormComponent implements OnInit {
     private electrsApiService: ElectrsApiService,
     private apiService: ApiService,
     private relativeUrlPipe: RelativeUrlPipe,
+    private elementRef: ElementRef,
   ) { }
 
   ngOnInit(): void {

frontend/src/app/components/transaction/liquid-ublinding.ts

@@ -126,9 +126,13 @@ export class LiquidUnblinding {
   }
 
   async checkUnblindedTx(tx: Transaction) {
-    const windowLocationHash = window.location.hash.substring('#blinded='.length);
+    if (!window.location.hash?.length) {
-    if (windowLocationHash.length > 0) {
+      return tx;
-      const blinders = this.parseBlinders(windowLocationHash);
+    }
+    const fragmentParams = new URLSearchParams(window.location.hash.slice(1) || '');
+    const blinderStr = fragmentParams.get('blinded');
+    if (blinderStr && blinderStr.length) {
+      const blinders = this.parseBlinders(blinderStr);
       if (blinders) {
         this.commitments = await this.makeCommitmentMap(blinders);
         return this.tryUnblindTx(tx);

frontend/src/app/components/transaction/transaction-preview.component.html

@@ -29,7 +29,7 @@
 
 
   <div class="row graph-wrapper">
-    <tx-bowtie-graph [tx]="tx" [width]="1112" [height]="346" [network]="network"></tx-bowtie-graph>
+    <tx-bowtie-graph [tx]="tx" [width]="1132" [height]="346" [network]="network"></tx-bowtie-graph>
     <div class="above-bow">
       <p class="field pair">
         <span [innerHTML]="'&lrm;' + (tx.size | bytes: 2)"></span>
@@ -41,24 +41,20 @@
     </div>
     <div class="overlaid">
       <ng-container [ngSwitch]="extraData">
-        <table class="opreturns" *ngSwitchCase="'coinbase'">
+        <div class="opreturns" *ngSwitchCase="'coinbase'">
-          <tbody>
+            <div class="opreturn-row">
-            <tr>
+              <span class="label">Coinbase</span>
-              <td class="label">Coinbase</td>
+              <span class="message">{{ tx.vin[0].scriptsig | hex2ascii }}</span>
-              <td class="message">{{ tx.vin[0].scriptsig | hex2ascii }}</td>
+            </div>
-            </tr>
+        </div>
-          </tbody>
+        <div class="opreturns" *ngSwitchCase="'opreturn'">
-        </table>
-        <table class="opreturns" *ngSwitchCase="'opreturn'">
-          <tbody>
             <ng-container *ngFor="let vout of opReturns.slice(0,3)">
-              <tr>
+              <div class="opreturn-row">
-                <td class="label">OP_RETURN</td>
+                <span class="label">OP_RETURN</span>
-                <td *ngIf="vout.scriptpubkey_asm !== 'OP_RETURN'" class="message">{{ vout.scriptpubkey_asm | hex2ascii }}</td>
+                <span *ngIf="vout.scriptpubkey_asm !== 'OP_RETURN'" class="message">{{ vout.scriptpubkey_asm | hex2ascii }}</span>
-              </tr>
+              </div>
             </ng-container>
-          </tbody>
+        </div>
-        </table>
       </ng-container>
     </div>
   </div>

frontend/src/app/components/transaction/transaction-preview.component.scss

@@ -29,6 +29,8 @@
 .features {
   font-size: 24px;
   margin-left: 1em;
+  margin-top: 0.5em;
+  margin-right: -4px;
 }
 
 .top-data {
@@ -60,6 +62,15 @@
   }
 }
 
+.top-data .field {
+  &:first-child {
+    padding-left: 0;
+  }
+  &:last-child {
+    padding-right: 0;
+  }
+}
+
 .tx-link {
   display: inline;
   font-size: 28px;
@@ -69,7 +80,7 @@
 .graph-wrapper {
   position: relative;
   background: #181b2d;
-  padding: 10px;
+  padding: 10px 0;
   padding-bottom: 0;
 
   .above-bow {
@@ -92,26 +103,37 @@
     max-width: 90%;
     margin: auto;
     overflow: hidden;
+    display: flex;
+    flex-direction: row;
+    justify-content: center;
 
     .opreturns {
+      display: inline-block;
       width: auto;
+      max-width: 100%;
       margin: auto;
       table-layout: auto;
       background: #2d3348af;
       border-top-left-radius: 5px;
       border-top-right-radius: 5px;
 
-      td {
+      .opreturn-row {
-        padding: 10px 10px;
+        width: 100%;
+        display: flex;
+        flex-direction: row;
+        justify-content: flex-start;
+        padding: 0 10px;
+      }
 
-        &.message {
+      .label {
-          overflow: hidden;
+        margin-right: 1em;
-          display: inline-block;
+      }
-          vertical-align: bottom;
+
-          text-overflow: ellipsis;
+      .message {
-          white-space: nowrap;
+        flex-shrink: 1;
-          text-align: left;
+        white-space: nowrap;
-        }
+        overflow: hidden;
+        text-overflow: ellipsis;
       }
     }
   }

frontend/src/app/components/transaction/transaction-preview.component.ts

@@ -117,8 +117,9 @@ export class TransactionPreviewComponent implements OnInit, OnDestroy {
         }),
         switchMap(() => {
           let transactionObservable$: Observable<Transaction>;
-          if (history.state.data && history.state.data.fee !== -1) {
+          const cached = this.stateService.getTxFromCache(this.txId);
-            transactionObservable$ = of(history.state.data);
+          if (cached && cached.fee !== -1) {
+            transactionObservable$ = of(cached);
           } else {
             transactionObservable$ = this.electrsApiService
               .getTransaction$(this.txId)

frontend/src/app/components/transaction/transaction.component.html

@@ -3,7 +3,7 @@
   <div class="title-block">
     <div *ngIf="rbfTransaction" class="alert alert-mempool" role="alert">
       <span i18n="transaction.rbf.replacement|RBF replacement">This transaction has been replaced by:</span>
-      <a class="alert-link" [routerLink]="['/tx/' | relativeUrl, rbfTransaction.txid]" [state]="{ data: rbfTransaction.size ? rbfTransaction : null }">
+      <a class="alert-link" [routerLink]="['/tx/' | relativeUrl, rbfTransaction.txid]">
         <span class="d-inline d-lg-none">{{ rbfTransaction.txid | shortenString : 24 }}</span>
         <span class="d-none d-lg-inline">{{ rbfTransaction.txid }}</span>
       </a>
@@ -209,6 +209,7 @@
             [maxStrands]="graphExpanded ? maxInOut : 24"
             [network]="network"
             [tooltip]="true"
+            [connectors]="true"
             [inputIndex]="inputIndex" [outputIndex]="outputIndex"
           >
           </tx-bowtie-graph>

frontend/src/app/components/transaction/transaction.component.scss

@@ -86,7 +86,7 @@
   position: relative;
   width: 100%;
   background: #181b2d;
-  padding: 10px;
+  padding: 10px 0;
   padding-bottom: 0;
 }
 

frontend/src/app/components/transaction/transaction.component.ts

@@ -183,8 +183,9 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
         }),
         switchMap(() => {
           let transactionObservable$: Observable<Transaction>;
-          if (history.state.data && history.state.data.fee !== -1) {
+          const cached = this.stateService.getTxFromCache(this.txId);
-            transactionObservable$ = of(history.state.data);
+          if (cached && cached.fee !== -1) {
+            transactionObservable$ = of(cached);
           } else {
             transactionObservable$ = this.electrsApiService
               .getTransaction$(this.txId)
@@ -279,6 +280,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
         this.waitingForTransaction = false;
       }
       this.rbfTransaction = rbfTransaction;
+      this.stateService.setTxCache([this.rbfTransaction]);
     });
 
     this.queryParamsSubscription = this.route.queryParams.subscribe((params) => {
@@ -402,7 +404,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
   @HostListener('window:resize', ['$event'])
   setGraphSize(): void {
     if (this.graphContainer) {
-      this.graphWidth = this.graphContainer.nativeElement.clientWidth - 24;
+      this.graphWidth = this.graphContainer.nativeElement.clientWidth;
     }
   }
 

frontend/src/app/components/transactions-list/transactions-list.component.html

@@ -1,6 +1,6 @@
 <ng-container *ngFor="let tx of transactions; let i = index; trackBy: trackByFn">
   <div *ngIf="!transactionPage" class="header-bg box tx-page-container">
-    <a class="float-left" [routerLink]="['/tx/' | relativeUrl, tx.txid]" [state]="{ data: tx }">
+    <a class="float-left" [routerLink]="['/tx/' | relativeUrl, tx.txid]">
       <span style="float: left;" class="d-block d-md-none">{{ tx.txid | shortenString : 16 }}</span>
       <span style="float: left;" class="d-none d-md-block">{{ tx.txid }}</span>
     </a>

frontend/src/app/components/transactions-list/transactions-list.component.ts

@@ -119,7 +119,7 @@ export class TransactionsListComponent implements OnInit, OnChanges {
       }
 
       this.transactionsLength = this.transactions.length;
-
+      this.stateService.setTxCache(this.transactions);
 
       this.transactions.forEach((tx) => {
         tx['@voutLimit'] = true;

frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.html

@@ -22,13 +22,13 @@
 
   <ng-template #pegin>
     <ng-container *ngIf="line.pegin; else pegout">
-      <p>Peg In</p>
+      <p *ngIf="!isConnector">Peg In</p>
     </ng-container>
   </ng-template>
 
   <ng-template #pegout>
     <ng-container *ngIf="line.pegout; else normal">
-      <p>Peg Out</p>
+      <p *ngIf="!isConnector">Peg Out</p>
       <p *ngIf="line.value != null"><app-amount [satoshis]="line.value"></app-amount></p>
       <p class="address">
         <span class="first">{{ line.pegout.slice(0, -4) }}</span>
@@ -38,7 +38,7 @@
   </ng-template>
 
   <ng-template #normal>
-      <p>
+      <p *ngIf="!isConnector">
         <ng-container [ngSwitch]="line.type">
           <span *ngSwitchCase="'input'" i18n="transaction.input">Input</span>
           <span *ngSwitchCase="'output'" i18n="transaction.output">Output</span>
@@ -46,6 +46,17 @@
         </ng-container>
         <span *ngIf="line.type !== 'fee'"> #{{ line.index + 1 }}</span>
       </p>
+      <ng-container *ngIf="isConnector && line.txid">
+        <p>
+          <span i18n="transaction">Transaction</span>&nbsp;
+          <span class="first">{{ line.txid.slice(0, 8) }}</span>...
+          <span class="last-four">{{ line.txid.slice(-4) }}</span>
+        </p>
+          <ng-container [ngSwitch]="line.type">
+            <p *ngSwitchCase="'input'"><span i18n="transaction.output">Output</span>&nbsp; #{{ line.vout + 1 }}</p>
+            <p *ngSwitchCase="'output'"><span i18n="transaction.input">Input</span>&nbsp; #{{ line.vin + 1 }}</p>
+          </ng-container>
+      </ng-container>
       <p *ngIf="line.value == null && line.confidential" i18n="shared.confidential">Confidential</p>
       <p *ngIf="line.value != null"><app-amount [satoshis]="line.value"></app-amount></p>
       <p *ngIf="line.type !== 'fee' && line.address" class="address">

frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts

@@ -5,6 +5,9 @@ interface Xput {
   type: 'input' | 'output' | 'fee';
   value?: number;
   index?: number;
+  txid?: string;
+  vin?: number;
+  vout?: number;
   address?: string;
   rest?: number;
   coinbase?: boolean;
@@ -21,6 +24,7 @@ interface Xput {
 export class TxBowtieGraphTooltipComponent implements OnChanges {
   @Input() line: Xput | void;
   @Input() cursorPosition: { x: number, y: number };
+  @Input() isConnector: boolean = false;
 
   tooltipPosition = { x: 0, y: 0 };
 

frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html

@@ -1,5 +1,5 @@
 <div class="bowtie-graph">
-  <svg *ngIf="inputs && outputs" class="bowtie" [attr.height]="(height + 10) + 'px'" [attr.width]="width + 'px'">
+  <svg *ngIf="inputs && outputs" class="bowtie" [class.rtl]="dir === 'rtl'" [attr.height]="(height + 10) + 'px'" [attr.width]="width + 'px'">
     <defs>
       <marker id="input-arrow" viewBox="-5 -5 10 10"
           refX="0" refY="0"
@@ -21,6 +21,15 @@
           markerWidth="1.5" markerHeight="1"
           orient="auto">
       </marker>
+      <radialGradient id="gradient0" x1="0%" y1="0%" x2="100%" y2="100%" gradientUnits="userSpaceOnUse">
+        <stop [attr.stop-color]="gradient[0]" />
+      </radialGradient>
+      <radialGradient id="gradient1" x1="0%" y1="0%" x2="100%" y2="100%" gradientUnits="userSpaceOnUse">
+        <stop [attr.stop-color]="gradient[1]" />
+      </radialGradient>
+      <radialGradient id="gradient2" x1="0%" y1="0%" x2="100%" y2="100%" gradientUnits="userSpaceOnUse">
+        <stop [attr.stop-color]="gradient[2]" />
+      </radialGradient>
       <linearGradient id="input-gradient" x1="0%" y1="0%" x2="100%" y2="0%">
         <stop offset="0%" [attr.stop-color]="gradient[0]" />
         <stop offset="100%" [attr.stop-color]="gradient[1]" />
@@ -29,6 +38,14 @@
         <stop offset="0%" [attr.stop-color]="gradient[1]" />
         <stop offset="100%" [attr.stop-color]="gradient[0]" />
       </linearGradient>
+      <linearGradient id="input-connector-gradient" x1="0%" y1="0%" x2="100%" y2="0%">
+        <stop offset="0%" [attr.stop-color]="gradient[2]" />
+        <stop offset="80%" [attr.stop-color]="gradient[0]" />
+      </linearGradient>
+      <linearGradient id="output-connector-gradient" x1="0%" y1="0%" x2="100%" y2="0%">
+        <stop offset="20%" [attr.stop-color]="gradient[0]" />
+        <stop offset="100%" [attr.stop-color]="gradient[2]" />
+      </linearGradient>
       <linearGradient id="input-hover-gradient" x1="0%" y1="0%" x2="100%" y2="0%">
       <stop offset="0%" [attr.stop-color]="gradient[0]" />
       <stop offset="2%" [attr.stop-color]="gradient[0]" />
@@ -41,6 +58,14 @@
         <stop offset="98%" [attr.stop-color]="gradient[0]" />
         <stop offset="100%" [attr.stop-color]="gradient[0]" />
       </linearGradient>
+      <linearGradient id="input-hover-connector-gradient" x1="0%" y1="0%" x2="100%" y2="0%">
+        <stop offset="0%" stop-color="white" />
+        <stop offset="80%" [attr.stop-color]="gradient[0]" />
+      </linearGradient>
+      <linearGradient id="output-hover-connector-gradient" x1="0%" y1="0%" x2="100%" y2="0%">
+        <stop offset="20%" [attr.stop-color]="gradient[0]" />
+        <stop offset="100%" stop-color="white" />
+      </linearGradient>
       <linearGradient id="input-highlight-gradient" x1="0%" y1="0%" x2="100%" y2="0%">
       <stop offset="0%" [attr.stop-color]="gradient[0]" />
       <stop offset="2%" [attr.stop-color]="gradient[0]" />
@@ -65,6 +90,22 @@
     </defs>
     <path [attr.d]="middle.path" class="line middle" [style]="middle.style"/>
     <ng-container *ngFor="let input of inputs; let i = index">
+      <path *ngIf="connectors && !inputData[i].coinbase && !inputData[i].pegin"
+        [attr.d]="input.connectorPath"
+        class="input connector {{input.class}}"
+        [class.highlight]="inputData[i].index === inputIndex"
+        (pointerover)="onHover($event, 'input-connector', i);"
+        (pointerout)="onBlur($event, 'input-connector', i);"
+        (click)="onClick($event, 'input-connector', inputData[i].index);"
+      />
+      <path
+        [attr.d]="input.markerPath"
+        class="input marker-target {{input.class}}"
+        [class.highlight]="inputData[i].index === inputIndex"
+        (pointerover)="onHover($event, 'input', i);"
+        (pointerout)="onBlur($event, 'input', i);"
+        (click)="onClick($event, 'input', inputData[i].index);"
+      />
       <path
         [attr.d]="input.path"
         class="line {{input.class}}"
@@ -77,7 +118,23 @@
       />
     </ng-container>
     <ng-container *ngFor="let output of outputs; let i = index">
-      <path
+      <path *ngIf="connectors && outspends[outputData[i].index]?.spent"
+        [attr.d]="output.connectorPath"
+        class="output connector {{output.class}}"
+        [class.highlight]="outputData[i].index === outputIndex"
+        (pointerover)="onHover($event, 'output-connector', i);"
+        (pointerout)="onBlur($event, 'output-connector', i);"
+        (click)="onClick($event, 'output-connector', outputData[i].index);"
+      />
+      <path *ngIf="!output.zeroValue"
+        [attr.d]="output.markerPath"
+        class="output marker-target {{output.class}}"
+        [class.highlight]="outputData[i].index === outputIndex"
+        (pointerover)="onHover($event, 'output', i);"
+        (pointerout)="onBlur($event, 'output', i);"
+        (click)="onClick($event, 'output', outputData[i].index);"
+      />
+      <path *ngIf="!output.zeroValue"
         [attr.d]="output.path"
         class="line {{output.class}}"
         [class.highlight]="outputIndex != null && outputData[i].index === outputIndex"
@@ -87,6 +144,16 @@
         (pointerout)="onBlur($event, 'output', i);"
         (click)="onClick($event, 'output', outputData[i].index);"
       />
+      <path *ngIf="output.zeroValue"
+        [attr.d]="output.path"
+        class="line {{output.class}} zerovalue"
+        [class.highlight]="outputIndex != null && outputData[i].index === outputIndex"
+        [class.zerovalue]="output.zeroValue"
+        [style]="output.style"
+        (pointerover)="onHover($event, 'output', i);"
+        (pointerout)="onBlur($event, 'output', i);"
+        (click)="onClick($event, 'output', outputData[i].index);"
+      />
     </ng-container>
   </svg>
 
@@ -94,5 +161,6 @@
     *ngIf=[tooltip]
     [line]="hoverLine"
     [cursorPosition]="tooltipPosition"
+    [isConnector]="hoverConnector"
   ></app-tx-bowtie-graph-tooltip>
 </div>

frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.scss

@@ -1,4 +1,8 @@
 .bowtie {
+  &.rtl {
+    transform: scale(-1, 1);
+  }
+
   .line {
     fill: none;
 
@@ -11,6 +15,10 @@
     &.fee {
       stroke: url(#fee-gradient);
     }
+    &.zerovalue {
+      stroke: url(#gradient0);
+      stroke-linecap: round;
+    }
 
     &.highlight {
       z-index: 8;
@@ -21,20 +29,53 @@
       &.output {
         stroke: url(#output-highlight-gradient);
       }
-    }
+      &.zerovalue {
-
+        stroke: #1bd8f4;
-    &:hover {
-      z-index: 10;
-      cursor: pointer;
-      &.input {
-        stroke: url(#input-hover-gradient);
-      }
-      &.output {
-        stroke: url(#output-hover-gradient);
-      }
-      &.fee {
-        stroke: url(#fee-hover-gradient);
       }
     }
   }
-}
+
+  .line:hover, .marker-target:hover + .line {
+    z-index: 10;
+    cursor: pointer;
+    &.input {
+      stroke: url(#input-hover-gradient);
+    }
+    &.output {
+      stroke: url(#output-hover-gradient);
+    }
+    &.fee {
+      stroke: url(#fee-hover-gradient);
+    }
+    &.zerovalue {
+      stroke: white;
+    }
+  }
+
+  .connector {
+    stroke: none;
+    opacity: 0.75;
+    cursor: pointer;
+    &.input {
+      fill: url(#input-connector-gradient);
+    }
+    &.output {
+      fill: url(#output-connector-gradient);
+    }
+  }
+
+  .connector:hover {
+    &.input {
+      fill: url(#input-hover-connector-gradient);
+    }
+    &.output {
+      fill: url(#output-hover-connector-gradient);
+    }
+  }
+
+  .marker-target {
+    stroke: none;
+    fill: transparent;
+    cursor: pointer;
+  }
+}

frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts

@@ -1,4 +1,4 @@
-import { Component, OnInit, Input, OnChanges, HostListener } from '@angular/core';
+import { Component, OnInit, Input, OnChanges, HostListener, Inject, LOCALE_ID } from '@angular/core';
 import { StateService } from '../../services/state.service';
 import { Outspend, Transaction } from '../../interfaces/electrs.interface';
 import { Router } from '@angular/router';
@@ -11,12 +11,18 @@ interface SvgLine {
   path: string;
   style: string;
   class?: string;
+  connectorPath?: string;
+  markerPath?: string;
+  zeroValue?: boolean;
 }
 
 interface Xput {
   type: 'input' | 'output' | 'fee';
   value?: number;
   index?: number;
+  txid?: string;
+  vin?: number;
+  vout?: number;
   address?: string;
   rest?: number;
   coinbase?: boolean;
@@ -40,35 +46,43 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
   @Input() minWeight = 2; //
   @Input() maxStrands = 24; // number of inputs/outputs to keep fully on-screen.
   @Input() tooltip = false;
+  @Input() connectors = false;
   @Input() inputIndex: number;
   @Input() outputIndex: number;
 
+  dir: 'rtl' | 'ltr' = 'ltr';
+
   inputData: Xput[];
   outputData: Xput[];
   inputs: SvgLine[];
   outputs: SvgLine[];
   middle: SvgLine;
   midWidth: number;
+  txWidth: number;
+  connectorWidth: number;
   combinedWeight: number;
   isLiquid: boolean = false;
   hoverLine: Xput | void = null;
+  hoverConnector: boolean = false;
   tooltipPosition = { x: 0, y: 0 };
   outspends: Outspend[] = [];
+  zeroValueWidth = 60;
+  zeroValueThickness = 20;
 
   outspendsSubscription: Subscription;
   refreshOutspends$: ReplaySubject<string> = new ReplaySubject();
 
   gradientColors = {
-    '': ['#9339f4', '#105fb0'],
+    '': ['#9339f4', '#105fb0', '#9339f400'],
-    bisq: ['#9339f4', '#105fb0'],
+    bisq: ['#9339f4', '#105fb0', '#9339f400'],
     // liquid: ['#116761', '#183550'],
-    liquid: ['#09a197', '#0f62af'],
+    liquid: ['#09a197', '#0f62af', '#09a19700'],
     // 'liquidtestnet': ['#494a4a', '#272e46'],
-    'liquidtestnet': ['#d2d2d2', '#979797'],
+    'liquidtestnet': ['#d2d2d2', '#979797', '#d2d2d200'],
     // testnet: ['#1d486f', '#183550'],
-    testnet: ['#4edf77', '#10a0af'],
+    testnet: ['#4edf77', '#10a0af', '#4edf7700'],
     // signet: ['#6f1d5d', '#471850'],
-    signet: ['#d24fc8', '#a84fd2'],
+    signet: ['#d24fc8', '#a84fd2', '#d24fc800'],
   };
 
   gradient: string[] = ['#105fb0', '#105fb0'];
@@ -78,7 +92,12 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
     private relativeUrlPipe: RelativeUrlPipe,
     private stateService: StateService,
     private apiService: ApiService,
-  ) { }
+    @Inject(LOCALE_ID) private locale: string,
+  ) {
+    if (this.locale.startsWith('ar') || this.locale.startsWith('fa') || this.locale.startsWith('he')) {
+      this.dir = 'rtl';
+    }
+  }
 
   ngOnInit(): void {
     this.initGraph();
@@ -118,7 +137,10 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
     this.isLiquid = (this.network === 'liquid' || this.network === 'liquidtestnet');
     this.gradient = this.gradientColors[this.network];
     this.midWidth = Math.min(10, Math.ceil(this.width / 100));
-    this.combinedWeight = Math.min(this.maxCombinedWeight, Math.floor((this.width - (2 * this.midWidth)) / 6));
+    this.txWidth = this.connectors ? Math.max(this.width - 200, this.width * 0.8) : this.width - 20;
+    this.combinedWeight = Math.min(this.maxCombinedWeight, Math.floor((this.txWidth - (2 * this.midWidth)) / 6));
+    this.connectorWidth = (this.width - this.txWidth) / 2;
+    this.zeroValueWidth = Math.max(20, Math.min((this.txWidth / 2) - this.midWidth - 110, 60));
 
     const totalValue = this.calcTotalValue(this.tx);
     let voutWithFee = this.tx.vout.map((v, i) => {
@@ -141,6 +163,8 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
       return {
         type: 'input',
         value: v?.prevout?.value,
+        txid: v.txid,
+        vout: v.vout,
         address: v?.prevout?.scriptpubkey_address || v?.prevout?.scriptpubkey_type?.toUpperCase(),
         index: i,
         coinbase: v?.is_coinbase,
@@ -223,10 +247,10 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
   }
 
   linesFromWeights(side: 'in' | 'out', xputs: Xput[], weights: number[], maxVisibleStrands: number): SvgLine[] {
-    const lineParams = weights.map((w) => {
+    const lineParams = weights.map((w, i) => {
       return {
         weight: w,
-        thickness: Math.max(this.minWeight - 1, w) + 1,
+        thickness: xputs[i].value === 0 ? this.zeroValueThickness : Math.max(this.minWeight - 1, w) + 1,
         offset: 0,
         innerY: 0,
         outerY: 0,
@@ -243,7 +267,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
     let lastOuter = 0;
     let lastInner = innerTop;
     // gap between strands
-    const spacing = (this.height - visibleWeight) / gaps;
+    const spacing = Math.max(4, (this.height - visibleWeight) / gaps);
 
     // curve adjustments to prevent overlaps
     let offset = 0;
@@ -252,6 +276,12 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
     let lastWeight = 0;
     let pad = 0;
     lineParams.forEach((line, i) => {
+      if (xputs[i].value === 0) {
+        line.outerY = lastOuter + (this.zeroValueThickness / 2);
+        lastOuter += this.zeroValueThickness + spacing;
+        return;
+      }
+
       // set the vertical position of the (center of the) outer side of the line
       line.outerY = lastOuter + (line.thickness / 2);
       line.innerY = Math.min(innerBottom + (line.thickness / 2), Math.max(innerTop + (line.thickness / 2), lastInner + (line.weight / 2)));
@@ -268,7 +298,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
       // required to prevent this line overlapping its neighbor
 
       if (this.tooltip || !xputs[i].rest) {
-        const w = (this.width - Math.max(lastWeight, line.weight)) / 2; // approximate horizontal width of the curved section of the line
+        const w = (this.width - Math.max(lastWeight, line.weight) - (2 * this.connectorWidth)) / 2; // approximate horizontal width of the curved section of the line
         const y1 = line.outerY;
         const y2 = line.innerY;
         const t = (lastWeight + line.weight) / 2; // distance between center of this line and center of previous line
@@ -305,17 +335,28 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
     maxOffset -= minOffset;
 
     return lineParams.map((line, i) => {
-      return {
+      if (xputs[i].value === 0) {
-        path: this.makePath(side, line.outerY, line.innerY, line.thickness, line.offset, pad + maxOffset),
+        return {
-        style: this.makeStyle(line.thickness, xputs[i].type),
+          path: this.makeZeroValuePath(side, line.outerY),
-        class: xputs[i].type
+          style: this.makeStyle(this.zeroValueThickness, xputs[i].type),
-      };
+          class: xputs[i].type,
+          zeroValue: true,
+        };
+      } else {
+        return {
+          path: this.makePath(side, line.outerY, line.innerY, line.thickness, line.offset, pad + maxOffset),
+          style: this.makeStyle(line.thickness, xputs[i].type),
+          class: xputs[i].type,
+          connectorPath: this.connectors ? this.makeConnectorPath(side, line.outerY, line.innerY, line.thickness): null,
+          markerPath: this.makeMarkerPath(side, line.outerY, line.innerY, line.thickness),
+        };
+      }
     });
   }
 
   makePath(side: 'in' | 'out', outer: number, inner: number, weight: number, offset: number, pad: number): string {
-    const start = (weight * 0.5);
+    const start = (weight * 0.5) + this.connectorWidth;
-    const curveStart = Math.max(start + 1, pad - offset);
+    const curveStart = Math.max(start + 5, pad - offset);
     const end =  this.width / 2 - (this.midWidth * 0.9) + 1;
     const curveEnd = end - offset - 10;
     const midpoint = (curveStart + curveEnd) / 2;
@@ -332,6 +373,50 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
     }
   }
 
+  makeZeroValuePath(side: 'in' | 'out', y: number): string {
+    const offset = this.zeroValueThickness / 2;
+    const start = (this.connectorWidth / 2) + 10;
+    if (side === 'in') {
+      return `M ${start + offset} ${y} L ${start + this.zeroValueWidth + offset} ${y}`;
+    } else { // mirrored in y-axis for the right hand side
+      return `M ${this.width - start - offset} ${y} L ${this.width - start - this.zeroValueWidth - offset} ${y}`;
+    }
+  }
+
+  makeConnectorPath(side: 'in' | 'out', y: number, inner, weight: number): string {
+    const halfWidth = weight * 0.5;
+    const offset = 10; //Math.max(2, halfWidth * 0.2);
+    const lineEnd = this.connectorWidth;
+
+    // align with for svg horizontal gradient bug correction
+    if (Math.round(y) === Math.round(inner)) {
+      y -= 1;
+    }
+
+    if (side === 'in') {
+      return `M ${lineEnd - offset} ${y - halfWidth} L ${halfWidth + lineEnd - offset} ${y} L ${lineEnd - offset} ${y + halfWidth} L -${10} ${ y + halfWidth} L -${10} ${y - halfWidth}`;
+    } else {
+      return `M ${this.width - halfWidth - lineEnd + offset} ${y - halfWidth} L ${this.width - lineEnd + offset} ${y} L ${this.width - halfWidth - lineEnd + offset} ${y + halfWidth} L ${this.width + 10} ${ y + halfWidth} L ${this.width + 10} ${y - halfWidth}`;
+    }
+  }
+
+  makeMarkerPath(side: 'in' | 'out', y: number, inner, weight: number): string {
+    const halfWidth = weight * 0.5;
+    const offset = 10; //Math.max(2, halfWidth * 0.2);
+    const lineEnd = this.connectorWidth;
+
+    // align with for svg horizontal gradient bug correction
+    if (Math.round(y) === Math.round(inner)) {
+      y -= 1;
+    }
+
+    if (side === 'in') {
+      return `M ${lineEnd - offset} ${y - halfWidth} L ${halfWidth + lineEnd - offset} ${y} L ${lineEnd - offset} ${y + halfWidth} L ${weight + lineEnd} ${ y + halfWidth} L ${weight + lineEnd} ${y - halfWidth}`;
+    } else {
+      return `M ${this.width - halfWidth - lineEnd + offset} ${y - halfWidth} L ${this.width - lineEnd + offset} ${y} L ${this.width - halfWidth - lineEnd + offset} ${y + halfWidth} L ${this.width - halfWidth - lineEnd} ${ y + halfWidth} L ${this.width - halfWidth - lineEnd} ${y - halfWidth}`;
+    }
+  }
+
   makeStyle(minWeight, type): string {
     if (type === 'fee') {
       return `stroke-width: ${minWeight}`;
@@ -342,30 +427,39 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
 
   @HostListener('pointermove', ['$event'])
   onPointerMove(event) {
-    this.tooltipPosition = { x: event.offsetX, y: event.offsetY };
+    if (this.dir === 'rtl') {
+      this.tooltipPosition = { x: this.width - event.offsetX, y: event.offsetY };
+    } else {
+     this.tooltipPosition = { x: event.offsetX, y: event.offsetY };
+    }
   }
 
   onHover(event, side, index): void {
-    if (side === 'input') {
+    if (side.startsWith('input')) {
       this.hoverLine = {
         ...this.inputData[index],
         index
       };
+      this.hoverConnector = (side === 'input-connector');
+
     } else {
       this.hoverLine = {
-        ...this.outputData[index]
+        ...this.outputData[index],
+        ...this.outspends[this.outputData[index].index]
       };
+      this.hoverConnector = (side === 'output-connector');
     }
   }
 
   onBlur(event, side, index): void {
     this.hoverLine = null;
+    this.hoverConnector = false;
   }
 
   onClick(event, side, index): void {
-    if (side === 'input') {
+    if (side.startsWith('input')) {
       const input = this.tx.vin[index];
-      if (input && !input.is_coinbase && !input.is_pegin && input.txid && input.vout != null) {
+      if (side === 'input-connector' && input && !input.is_coinbase && !input.is_pegin && input.txid && input.vout != null) {
         this.router.navigate([this.relativeUrlPipe.transform('/tx'), input.txid], {
           queryParamsHandling: 'merge',
           fragment: (new URLSearchParams({
@@ -385,7 +479,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
     } else {
       const output = this.tx.vout[index];
       const outspend = this.outspends[index];
-      if (output && outspend && outspend.spent && outspend.txid) {
+      if (side === 'output-connector' && output && outspend && outspend.spent && outspend.txid) {
         this.router.navigate([this.relativeUrlPipe.transform('/tx'), outspend.txid], {
           queryParamsHandling: 'merge',
           fragment: (new URLSearchParams({

frontend/src/app/docs/api-docs/api-docs-data.ts

@@ -8562,20 +8562,6 @@ export const faqData = [
     fragment: "what-is-svb",
     title: "What is sat/vB?",
   },
-  {
-    type: "endpoint",
-    category: "basics",
-    showConditions: bitcoinNetworks,
-    fragment: "what-is-full-mempool",
-    title: "What does it mean for the mempool to be \"full\"?",
-  },
-  {
-    type: "endpoint",
-    category: "basics",
-    showConditions: bitcoinNetworks,
-    fragment: "why-empty-blocks",
-    title: "Why are there empty blocks?",
-  },
   {
     type: "category",
     category: "help",
@@ -8657,33 +8643,68 @@ export const faqData = [
     type: "endpoint",
     category: "advanced",
     showConditions: bitcoinNetworks,
+    fragment: "what-is-full-mempool",
+    title: "What does it mean for the mempool to be \"full\"?",
+  },
+  {
+    type: "endpoint",
+    category: "advanced",
+    showConditions: bitcoinNetworks,
+    fragment: "why-empty-blocks",
+    title: "Why are there empty blocks?",
+  },
+  {
+    type: "endpoint",
+    category: "advanced",
+    showConditions: bitcoinNetworks,
+    fragment: "why-block-timestamps-dont-always-increase",
+    title: "Why don't block timestamps always increase?",
+  },
+  {
+    type: "endpoint",
+    category: "advanced",
+    showConditions: bitcoinNetworks,
+    fragment: "why-dont-fee-ranges-match",
+    title: "Why doesn't the fee range shown for a block match the feerates of transactions within the block?",
+  },
+  {
+    type: "category",
+    category: "self-hosting",
+    fragment: "self-hosting",
+    title: "Self-Hosting",
+    showConditions: bitcoinNetworks
+  },
+  {
+    type: "endpoint",
+    category: "self-hosting",
+    showConditions: bitcoinNetworks,
     fragment: "who-runs-this-website",
     title: "Who runs this website?",
   },
   {
     type: "endpoint",
-    category: "advanced",
+    category: "self-hosting",
     showConditions: bitcoinNetworks,
     fragment: "host-my-own-instance-raspberry-pi",
     title: "How can I host my own instance on a Raspberry Pi?",
   },
   {
     type: "endpoint",
-    category: "advanced",
+    category: "self-hosting",
     showConditions: bitcoinNetworks,
     fragment: "host-my-own-instance-linux-server",
     title: "How can I host my own instance on a Linux server?",
   },
   {
     type: "endpoint",
-    category: "advanced",
+    category: "self-hosting",
     showConditions: bitcoinNetworks,
     fragment: "install-mempool-with-docker",
     title: "Can I install Mempool using Docker?",
   },
   {
     type: "endpoint",
-    category: "advanced",
+    category: "self-hosting",
     showConditions: bitcoinNetworks,
     fragment: "address-lookup-issues",
     title: "Why do I get an error for certain address lookups on my Mempool instance?",

frontend/src/app/docs/api-docs/api-docs.component.html

@@ -9,6 +9,11 @@
 
       <div class="doc-content">
 
+        <div id="disclaimer">
+          <table><tr><td><svg viewBox="0 0 304 304" xmlns="http://www.w3.org/2000/svg"><g fill-rule="evenodd" style="fill:#ffc107;fill-opacity:1"><path d="M135.3 34.474c-15.62 27.306-54.206 95.63-85.21 150.534L9.075 257.583C5.382 264.08 6.76 269.217 7.908 271.7c2.326 5.028 7.29 7.537 11.155 8.215l.78.133 264.698.006-.554-.02c4.152.255 9.664-1.24 12.677-6.194 1.926-3.18 3.31-8.589-1.073-16.278L213.637 114.37l-45.351-79.205c-5.681-9.932-12.272-12.022-16.8-12.022-4.42 0-10.818 1.964-16.181 11.331h-.006zm-69.072 159.94c30.997-54.885 69.563-123.184 85.16-150.446l.186-.297c.2.303.393.582.618.981l45.363 79.22s72.377 126.47 78.569 137.283l-247.618-.007 37.719-66.734" style="fill:#ffc107;fill-opacity:1"/><path d="M152.597 247.445c8.02 0 14.518-6.728 14.518-15.025 0-8.29-6.499-15.018-14.518-15.018-8.031 0-14.529 6.728-14.529 15.018 0 8.297 6.498 15.025 14.53 15.025m-.001-147.18c11.586 0 22.23 10.958 20.977 21.7l-9.922 75.564c-.966 6.601-4.95 11.433-11.055 11.433s-10.102-4.832-11.056-11.433l-9.927-75.564c-1.26-10.742 9.39-21.7 20.983-21.7" style="fill:#ffc107;fill-opacity:1"/></g></svg></td><td><p><b>mempool.space merely provides data about the Bitcoin network.</b> It cannot help you with retrieving funds, confirming your transaction quicker, etc.</p><p>For any such requests, you need to get in touch with the entity that helped make the transaction (wallet software, exchange company, etc).</p></td></tr></table>
+
+        </div>
+
         <div class="doc-item-container" *ngFor="let item of faq">
           <h3 *ngIf="item.type === 'category'">{{ item.title }}</h3>
           <div *ngIf="item.type !== 'category'" class="endpoint-container" id="{{ item.fragment }}">
@@ -163,14 +168,6 @@
   <p>There are feerate estimates on the top of <a [routerLink]="['/' | relativeUrl]">the main dashboard</a> you can use as a guide. See <a [routerLink]="['/docs/faq' | relativeUrl]" fragment="looking-up-fee-estimates">this FAQ</a> for more on picking the right feerate.</p>
 </ng-template>
 
-<ng-template type="what-is-full-mempool">
-  <p>When a Bitcoin transaction is made, it is stored in a Bitcoin node's mempool before it is confirmed into a block. When the rate of incoming transactions exceeds the rate transactions are confirmed, the mempool grows in size.</p><p>The default maximum size of a Bitcoin node's mempool is 300MB, so when there are 300MB of transactions in the mempool, we say it's "full".</p>
-</ng-template>
-
-<ng-template type="why-empty-blocks">
-  <p>When a new block is found, mining pools send miners a block template with no transactions so they can start searching for the next block as soon as possible. They send a block template full of transactions right afterward, but a full block template is a bigger data transfer and takes slightly longer to reach miners.</p><p>In this intervening time, which is usually no more than 1-2 seconds, miners sometimes get lucky and find a new block using the empty block template.</p>
-</ng-template>
-
 <ng-template type="why-is-transaction-stuck-in-mempool">
   <p>If it's been a while and your transaction hasn't confirmed, your transaction is probably using a lower feerate relative to other transactions currently in the mempool. Depending on how you made your transaction, there may be <a [routerLink]="['/docs/faq' | relativeUrl]" fragment="how-to-get-transaction-confirmed-quickly">ways to accelerate the process</a>.</p><p>There's no need to panic—a Bitcoin transaction will always either confirm completely (or not at all) at some point. As long as you have your transaction's ID, you can always see where your funds are.</p><p style='font-weight:700'>This site only provides data about the Bitcoin network—it cannot help you get your transaction confirmed quicker.</p>
 </ng-template>
@@ -203,6 +200,24 @@
   See the <a [routerLink]="['/graphs' | relativeUrl]">graphs page</a> for aggregate trends over time: mempool size over time and incoming transaction velocity over time.
 </ng-template>
 
+<ng-template type="what-is-full-mempool">
+  <p>When a Bitcoin transaction is made, it is stored in a Bitcoin node's mempool before it is confirmed into a block. When the rate of incoming transactions exceeds the rate transactions are confirmed, the mempool grows in size.</p><p>By default, Bitcoin Core allocates 300MB of memory for its mempool, so when a node's mempool grows big enough to use all 300MB of allocated memory, we say it's "full".</p><p>Once a node's mempool is using all of its allocated memory, it will start rejecting new transactions below a certain feerate threshold—so when this is the case, be extra sure to set a feerate that (at a minimum) exceeds that threshold. The current threshold feerate (and memory usage) are displayed right on Mempool's front page.</p>
+</ng-template>
+
+<ng-template type="why-empty-blocks">
+  <p>When a new block is found, mining pools send miners a block template with no transactions so they can start searching for the next block as soon as possible. They send a block template full of transactions right afterward, but a full block template is a bigger data transfer and takes slightly longer to reach miners.</p><p>In this intervening time, which is usually no more than 1-2 seconds, miners sometimes get lucky and find a new block using the empty block template.</p>
+</ng-template>
+
+<ng-template type="why-block-timestamps-dont-always-increase">
+  <p>Block validation rules do not strictly require that a block's timestamp be more recent than the timestamp of the block preceding it. Without a central authority, it's impossible to know what the exact correct time is. Instead, the Bitcoin protocol requires that a block's timestamp meet certain requirements. One of those requirements is that a block's timestamp cannot be older than the median timestamp of the 12 blocks that came before it. See more details <a href="https://en.bitcoin.it/wiki/Block_timestamp" target="_blank">here</a>.</p><p>As a result, timestamps are only accurate to within an hour or so, which sometimes results in blocks with timestamps that appear out of order.</p>
+</ng-template>
+
+<ng-template type="why-dont-fee-ranges-match">
+  <p>Mempool aims to show you the <i>effective feerate</i> range for blocks—how much would you actually need to pay to get a transaction included in a block.</p>
+  <p>A transaction's effective feerate is not always the same as the feerate explicitly set for it. For example, if you see a 1 s/vb transaction in a block with a displayed feerate range of 5 s/vb to 72 s/vb, chances are that 1 s/vb transaction had a high-feerate child transaction that boosted its effective feerate to 5 s/vb or higher (this is how CPFP fee-bumping works). In such a case, it would be misleading to use 1 s/vb as the lower bound of the block's feerate range because it actually required more than 1 s/vb to confirm that transaction in that block.</p>
+  <p>For unconfirmed CPFP transactions, Mempool will show the effective feerate (along with descendent & ancestor transaction information) on the transaction page. For confirmed transactions, CPFP relationships are not stored, so this additional information is not shown.</p>
+</ng-template>
+
 <ng-template type="who-runs-this-website">
   The official mempool.space website is operated by The Mempool Open Source Project. See more information on our <a [routerLink]="['/about']">About page</a>. There are also many unofficial instances of this website operated by individual members of the Bitcoin community.
 </ng-template>

frontend/src/app/docs/api-docs/api-docs.component.scss

@@ -172,6 +172,7 @@ h3 {
   border-radius: 0.25rem;
   font-family: monospace;
   float: right;
+  white-space: nowrap;
 }
 
 .endpoint-container .section-header table {
@@ -219,6 +220,22 @@ h3 {
   display: none;
 }
 
+#disclaimer {
+  background-color: #1d1f31;
+  padding: 24px;
+  margin: 24px 0;
+}
+
+#disclaimer svg {
+  width: 50px;
+  height: auto;
+  margin-right: 32px;
+}
+
+#disclaimer p:last-child {
+  margin-bottom: 0;
+}
+
 @media (max-width: 992px) {
 
   h3 {

frontend/src/app/docs/code-template/code-template.component.html

@@ -30,7 +30,7 @@
         <pre><code [innerText]="wrapEsModule(code)"></code></pre>
       </ng-template>
     </li>
-    <li ngbNavItem *ngIf="showCodeExample[network] && network !== 'liquid' && network !== 'liquidtestnet'" role="presentation">
+    <li ngbNavItem *ngIf="code.codeTemplate.python && network !== 'liquid' && network !== 'liquidtestnet'" role="presentation">
       <a ngbNavLink (click)="adjustContainerHeight( $event )" role="tab">Python</a>
       <ng-template ngbNavContent>
         <div class="subtitle"><ng-container i18n="API Docs code example">Code Example</ng-container> <app-clipboard [text]="wrapEsModule(code)"></app-clipboard></div>

frontend/src/app/interfaces/node-api.interface.ts

@@ -152,6 +152,11 @@ export interface RewardStats {
   totalTx: number;
 }
 
+export interface AuditScore {
+  hash: string;
+  matchRate?: number;
+}
+
 export interface ITopNodesPerChannels {
   publicKey: string,
   alias: string,

frontend/src/app/lightning/node/liquidity-ad.ts

@@ -0,0 +1,31 @@
+export interface ILiquidityAd {
+  funding_weight: number;
+  lease_fee_basis: number; // lease fee rate in parts-per-thousandth
+  lease_fee_base_sat: number; // fixed lease fee in sats
+  channel_fee_max_rate: number; // max routing fee rate in parts-per-thousandth
+  channel_fee_max_base: number; // max routing base fee in milli-sats
+  compact_lease?: string;
+}
+
+export function parseLiquidityAdHex(compact_lease: string): ILiquidityAd | false {
+  if (!compact_lease || compact_lease.length < 20 || compact_lease.length > 28) {
+    return false;
+  }
+  try {
+    const liquidityAd: ILiquidityAd = {
+      funding_weight: parseInt(compact_lease.slice(0, 4), 16),
+      lease_fee_basis: parseInt(compact_lease.slice(4, 8), 16),
+      channel_fee_max_rate: parseInt(compact_lease.slice(8, 12), 16),
+      lease_fee_base_sat: parseInt(compact_lease.slice(12, 20), 16),
+      channel_fee_max_base: compact_lease.length > 20 ? parseInt(compact_lease.slice(20), 16) : 0,
+    }
+    if (Object.values(liquidityAd).reduce((valid: boolean, value: number): boolean => (valid && !isNaN(value) && value >= 0), true)) {
+      liquidityAd.compact_lease = compact_lease;
+      return liquidityAd;
+    } else {
+      return false;
+    }
+  } catch (err) {
+    return false;
+  }
+}

frontend/src/app/lightning/node/node.component.html

@@ -52,6 +52,10 @@
                 <span i18n="unknown">Unknown</span>
               </td>
             </tr>
+            <tr *ngIf="(avgChannelDistance$ | async) as avgDistance;">
+              <td i18n="lightning.avg-distance" class="text-truncate">Avg channel distance</td>
+              <td>{{ avgDistance | number : '1.0-0' }} <span class="symbol">km</span> <span class="separator">/</span> {{ kmToMiles(avgDistance) | number : '1.0-0' }} <span class="symbol">mi</span></td>
+            </tr>
           </tbody>
         </table>
       </div>
@@ -125,6 +129,93 @@
     <app-clipboard [button]="true" [text]="node.socketsObject[selectedSocketIndex].socket" [leftPadding]="false"></app-clipboard>
   </div>
 
+  <div *ngIf="hasDetails" [hidden]="!showDetails" id="details" class="details mt-3">
+    <div class="box">
+      <ng-template [ngIf]="liquidityAd">
+        <div class="detail-section">
+          <h5 class="mb-3" i18n="node.liquidity-ad">Liquidity ad</h5>
+          <div class="row">
+            <div class="col-md">
+              <table class="table table-borderless table-striped">
+                <tbody>
+                  <tr>
+                    <td class="label" i18n="liquidity-ad.lease-fee-rate|Liquidity ad lease fee rate">Lease fee rate</td>
+                    <td>
+                      <span class="d-inline-block">
+                        {{ liquidityAd.lease_fee_basis !== null ? ((liquidityAd.lease_fee_basis * 1000) | amountShortener : 2 : undefined : true) : '-' }} <span class="symbol">ppm {{ liquidityAd.lease_fee_basis !== null ? '(' + (liquidityAd.lease_fee_basis / 10 | amountShortener : 2 : undefined : true) + '%)' : '' }}</span>
+                      </span>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td class="label" i18n="liquidity-ad.lease-base-fee">Lease base fee</td>
+                    <td>
+                      <app-sats [valueOverride]="liquidityAd.lease_fee_base_sat === null ? '- ' : undefined" [satoshis]="liquidityAd.lease_fee_base_sat"></app-sats>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td class="label" i18n="liquidity-ad.funding-weight">Funding weight</td>
+                    <td [innerHTML]="'&lrm;' + (liquidityAd.funding_weight | wuBytes: 2)"></td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+            <div class="col-md">
+              <table class="table table-borderless table-striped">
+                <tbody>
+                  <tr>
+                    <td class="label" i18n="liquidity-ad.channel-fee-rate|Liquidity ad channel fee rate">Channel fee rate</td>
+                    <td>
+                      <span class="d-inline-block">
+                        {{ liquidityAd.channel_fee_max_rate !== null ? ((liquidityAd.channel_fee_max_rate * 1000) | amountShortener : 2 : undefined : true) : '-' }} <span class="symbol">ppm {{ liquidityAd.channel_fee_max_rate !== null ? '(' + (liquidityAd.channel_fee_max_rate / 10 | amountShortener : 2 : undefined : true) + '%)' : '' }}</span>
+                      </span>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td class="label" i18n="liquidity-ad.channel-base-fee">Channel base fee</td>
+                    <td>
+                      <span *ngIf="liquidityAd.channel_fee_max_base !== null">
+                        {{ liquidityAd.channel_fee_max_base | amountShortener : 0 }}
+                        <span class="symbol" i18n="shared.m-sats">mSats</span>
+                      </span>
+                      <span *ngIf="liquidityAd.channel_fee_max_base === null">
+                        -
+                      </span>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td class="label" i18n="liquidity-ad.compact-lease">Compact lease</td>
+                    <td class="compact-lease">{{ liquidityAd.compact_lease }}</td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+          </div>
+        </div>
+      </ng-template>
+      <ng-template [ngIf]="tlvRecords?.length">
+        <div class="detail-section">
+          <h5 class="mb-3" i18n="node.tlv.records">TLV extension records</h5>
+          <div class="row">
+            <div class="col">
+              <table class="table table-borderless table-striped">
+                <tbody>
+                  <tr *ngFor="let recordItem of tlvRecords">
+                    <td class="tlv-type">{{ recordItem.type }}</td>
+                    <td class="tlv-payload">{{ recordItem.payload }}</td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+          </div>
+        </div>
+      </ng-template>
+    </div>
+  </div>
+
+  <div *ngIf="hasDetails" class="text-right mt-3">
+    <button type="button" class="btn btn-outline-info btn-sm btn-details" (click)="toggleShowDetails()" i18n="node.details|Node Details">Details</button>
+  </div>
+
   <div *ngIf="!error">
     <div class="row" *ngIf="node.as_number && node.active_channel_count">
       <div class="col-sm">

frontend/src/app/lightning/node/node.component.scss

@@ -72,3 +72,36 @@ app-fiat {
     height: 28px !important;
   };
 }
+
+.details {
+
+  .detail-section {
+    margin-bottom: 1.5rem;
+    &:last-child {
+      margin-bottom: 0;
+    }
+  }
+
+  .tlv-type {
+    font-size: 12px;
+    color: #ffffff66;
+  }
+
+  .tlv-payload {
+    font-size: 12px;
+    width: 100%;
+    word-break: break-all;
+    white-space: normal;
+    font-family: "Courier New", Courier, monospace;
+  }
+
+  .compact-lease {
+    word-break: break-all;
+    white-space: normal;
+    font-family: "Courier New", Courier, monospace;
+  }
+}
+
+.separator {
+  margin: 0 1em;
+}

frontend/src/app/lightning/node/node.component.ts

@@ -1,10 +1,18 @@
 import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core';
 import { ActivatedRoute, ParamMap } from '@angular/router';
 import { Observable } from 'rxjs';
-import { catchError, map, switchMap } from 'rxjs/operators';
+import { catchError, map, switchMap, tap } from 'rxjs/operators';
 import { SeoService } from '../../services/seo.service';
+import { ApiService } from '../../services/api.service';
 import { LightningApiService } from '../lightning-api.service';
 import { GeolocationData } from '../../shared/components/geolocation/geolocation.component';
+import { ILiquidityAd, parseLiquidityAdHex } from './liquidity-ad';
+import { haversineDistance, kmToMiles } from 'src/app/shared/common.utils';
+
+interface CustomRecord {
+  type: string;
+  payload: string;
+}
 
 @Component({
   selector: 'app-node',
@@ -24,8 +32,16 @@ export class NodeComponent implements OnInit {
   channelListLoading = false;
   clearnetSocketCount = 0;
   torSocketCount = 0;
+  hasDetails = false;
+  showDetails = false;
+  liquidityAd: ILiquidityAd;
+  tlvRecords: CustomRecord[];
+  avgChannelDistance$: Observable<number | null>;
+
+  kmToMiles = kmToMiles;
 
   constructor(
+    private apiService: ApiService,
     private lightningApiService: LightningApiService,
     private activatedRoute: ActivatedRoute,
     private seoService: SeoService,
@@ -36,6 +52,8 @@ export class NodeComponent implements OnInit {
       .pipe(
         switchMap((params: ParamMap) => {
           this.publicKey = params.get('public_key');
+          this.tlvRecords = [];
+          this.liquidityAd = null;
           return this.lightningApiService.getNode$(params.get('public_key'));
         }),
         map((node) => {
@@ -79,6 +97,26 @@ export class NodeComponent implements OnInit {
 
           return node;
         }),
+        tap((node) => {
+          this.hasDetails = Object.keys(node.custom_records).length > 0;
+          for (const [type, payload] of Object.entries(node.custom_records)) {
+            if (typeof payload !== 'string') {
+              break;
+            }
+
+            let parsed = false;
+            if (type === '1') {
+              const ad = parseLiquidityAdHex(payload);
+              if (ad) {
+                parsed = true;
+                this.liquidityAd = ad;
+              }
+            }
+            if (!parsed) {
+              this.tlvRecords.push({ type, payload });
+            }
+          }
+        }),
         catchError(err => {
           this.error = err;
           return [{
@@ -87,6 +125,30 @@ export class NodeComponent implements OnInit {
           }];
         })
       );
+
+    this.avgChannelDistance$ = this.activatedRoute.paramMap
+    .pipe(
+      switchMap((params: ParamMap) => {
+        return this.apiService.getChannelsGeo$(params.get('public_key'), 'nodepage');
+      }),
+      map((channelsGeo) => {
+        if (channelsGeo?.length) {
+          const totalDistance = channelsGeo.reduce((sum, chan) => {
+            return sum + haversineDistance(chan[3], chan[2], chan[7], chan[6]);
+          }, 0);
+          return totalDistance / channelsGeo.length;
+        } else {
+          return null;
+        }
+      }),
+      catchError(() => {
+        return null;
+      })
+    ) as Observable<number | null>;
+  }
+
+  toggleShowDetails(): void {
+    this.showDetails = !this.showDetails;
   }
 
   changeSocket(index: number) {

frontend/src/app/lightning/nodes-per-isp/nodes-per-isp.component.html

@@ -11,7 +11,7 @@
               <td>{{ isp?.id }}</td>
             </tr>
             <tr>
-              <td i18n="lightning.node-count">Nodes</td>
+              <td i18n="lightning.active-node-count">Active nodes</td>
               <td>{{ ispNodes.nodes.length }}</td>
             </tr>
             <tr>

frontend/src/app/services/api.service.ts

@@ -1,7 +1,7 @@
 import { Injectable } from '@angular/core';
 import { HttpClient, HttpParams } from '@angular/common/http';
 import { CpfpInfo, OptimizedMempoolStats, AddressInformation, LiquidPegs, ITranslators,
-  PoolStat, BlockExtended, TransactionStripped, RewardStats } from '../interfaces/node-api.interface';
+  PoolStat, BlockExtended, TransactionStripped, RewardStats, AuditScore } from '../interfaces/node-api.interface';
 import { Observable } from 'rxjs';
 import { StateService } from './state.service';
 import { WebsocketResponse } from '../interfaces/websocket.interface';
@@ -234,6 +234,19 @@ export class ApiService {
     );
   }
 
+  getBlockAuditScores$(from: number): Observable<AuditScore[]> {
+    return this.httpClient.get<AuditScore[]>(
+      this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/blocks/audit/scores` +
+      (from !== undefined ? `/${from}` : ``)
+    );
+  }
+
+  getBlockAuditScore$(hash: string) : Observable<any> {
+    return this.httpClient.get<any>(
+      this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/blocks/audit/score/` + hash
+    );
+  }
+
   getRewardStats$(blockCount: number = 144): Observable<RewardStats> {
     return this.httpClient.get<RewardStats>(this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/reward-stats/${blockCount}`);
   }

frontend/src/app/services/state.service.ts

@@ -112,6 +112,8 @@ export class StateService {
   timeLtr: BehaviorSubject<boolean>;
   hideFlow: BehaviorSubject<boolean>;
 
+  txCache: { [txid: string]: Transaction } = {};
+
   constructor(
     @Inject(PLATFORM_ID) private platformId: any,
     @Inject(LOCALE_ID) private locale: string,
@@ -265,4 +267,19 @@ export class StateService {
   isLiquid() {
     return this.network === 'liquid' || this.network === 'liquidtestnet';
   }
+
+  setTxCache(transactions) {
+    this.txCache = {};
+    transactions.forEach(tx => {
+      this.txCache[tx.txid] = tx;
+    });
+  }
+ 
+  getTxFromCache(txid) {
+    if (this.txCache && this.txCache[txid]) {
+      return this.txCache[txid];
+    } else {
+      return null;
+    }
+  }
 }

frontend/src/app/shared/common.utils.ts

@@ -118,3 +118,21 @@ export function convertRegion(input, to: 'name' | 'abbreviated'): string {
     }
   }
 }
+
+export function haversineDistance(lat1: number, lon1: number, lat2: number, lon2: number): number {
+  const rlat1 = lat1 * Math.PI / 180;
+  const rlon1 = lon1 * Math.PI / 180;
+  const rlat2 = lat2 * Math.PI / 180;
+  const rlon2 = lon2 * Math.PI / 180;
+
+  const dlat = Math.sin((rlat2 - rlat1) / 2);
+  const dlon = Math.sin((rlon2 - rlon1) / 2);
+  const a = Math.min(1, Math.max(0, (dlat * dlat) + (Math.cos(rlat1) * Math.cos(rlat2) * dlon * dlon)));
+  const d = 2 * 6371 * Math.asin(Math.sqrt(a));
+
+  return d;
+}
+
+export function kmToMiles(km: number): number {
+  return km * 0.62137119;
+}

frontend/src/index.bisq.html

@@ -1,8 +1,10 @@
 <!doctype html>
 <html lang="en">
+
 <head>
   <meta charset="utf-8">
   <title>mempool - Bisq Markets</title>
+  <script src="/resources/config.js"></script>
   <base href="/">
 
   <meta name="description" content="The Mempool Open Source Project™ - Explore the full Bitcoin ecosystem.">
@@ -31,11 +33,13 @@
   <link rel="manifest" href="/resources/bisq/favicons/site.webmanifest">
   <link rel="mask-icon" href="/resources/bisq/favicons/safari-pinned-tab.svg" color="#5bbad5">
   <link rel="shortcut icon" href="/resources/bisq/favicons/favicon.ico">
- 
+
   <link id="canonical" rel="canonical" href="https://bisq.markets">
 
 </head>
+
 <body>
   <app-root></app-root>
 </body>
+
 </html>

frontend/src/index.liquid.html

@@ -1,8 +1,10 @@
 <!doctype html>
 <html lang="en">
+
 <head>
   <meta charset="utf-8">
   <title>mempool - Liquid Network</title>
+  <script src="/resources/config.js"></script>
   <base href="/">
 
   <meta name="description" content="The Mempool Open Source Project™ - Explore the full Bitcoin ecosystem.">
@@ -17,7 +19,7 @@
   <meta property="twitter:description" content="Explore the full Bitcoin ecosystem with mempool.space™" />
   <meta property="twitter:image:src" content="https://liquid.network/resources/liquid/liquid-network-preview.png" />
   <meta property="twitter:domain" content="liquid.network">
-  
+
   <link rel="apple-touch-icon" sizes="180x180" href="/resources/liquid/favicons/apple-touch-icon.png">
   <link rel="icon" type="image/png" sizes="48x48" href="/resources/liquid/favicons/favicon-48x48.png">
   <link rel="icon" type="image/png" sizes="32x32" href="/resources/liquid/favicons/favicon-32x32.png">
@@ -33,7 +35,9 @@
 
   <meta name="viewport" content="width=device-width, initial-scale=1">
 </head>
+
 <body>
   <app-root></app-root>
 </body>
+
 </html>

frontend/src/index.mempool.html

@@ -1,8 +1,10 @@
 <!doctype html>
 <html lang="en">
+
 <head>
   <meta charset="utf-8">
   <title>mempool - Bitcoin Explorer</title>
+  <script src="/resources/config.js"></script>
   <base href="/">
 
   <meta name="description" content="The Mempool Open Source Project™ - Explore the full Bitcoin ecosystem." />
@@ -17,7 +19,7 @@
   <meta property="twitter:description" content="Explore the full Bitcoin ecosystem with mempool.space™" />
   <meta property="twitter:image:src" content="https://mempool.space/resources/mempool-space-preview.png" />
   <meta property="twitter:domain" content="mempool.space">
-  
+
   <link rel="apple-touch-icon" sizes="180x180" href="/resources/favicons/apple-touch-icon.png">
   <link rel="icon" type="image/png" sizes="32x32" href="/resources/favicons/favicon-32x32.png">
   <link rel="icon" type="image/png" sizes="16x16" href="/resources/favicons/favicon-16x16.png">
@@ -32,7 +34,9 @@
 
   <meta name="viewport" content="width=device-width, initial-scale=1">
 </head>
+
 <body>
   <app-root></app-root>
 </body>
+
 </html>

frontend/src/resources/profile/nunchuk.svg

@@ -0,0 +1 @@
+<svg width="120" height="120" fill="none" xmlns="http://www.w3.org/2000/svg"><g clip-path="url(#a)"><path d="M93.28 40.282 69.81 63.75a1.107 1.107 0 0 1-1.889-.784v-10.81h-14.24a1.107 1.107 0 0 1-1.106-1.108V40.3H38.939L13.156 66.15 26.72 79.718l23.265-23.27a1.108 1.108 0 0 1 1.89.783v10.976h14.442a1.108 1.108 0 0 1 1.107 1.108V79.69H81.06l25.783-25.85-13.564-13.559Z" fill="#FFCB2E"/></g><defs><clipPath id="a"><path fill="#fff" d="M0 0h120v120H0z"/></clipPath></defs></svg>

nginx-mempool.conf

@@ -21,6 +21,13 @@
 		try_files $uri @index-redirect;
 		expires 1h;
 	}
+
+	# only cache /resources/config.* for 5 minutes since it changes often
+	location /resources/config. {
+		try_files $uri =404;
+		expires 5m;
+	}
+
 	location @index-redirect {
 		rewrite (.*) /$lang/index.html;
 	}

production/mempool-config.mainnet-lightning.json

@@ -1,5 +1,6 @@
 {
   "MEMPOOL": {
+    "ENABLED": false,
     "NETWORK": "mainnet",
     "BACKEND": "esplora",
     "HTTP_PORT": 8993,

production/mempool-config.signet-lightning.json

@@ -1,5 +1,6 @@
 {
   "MEMPOOL": {
+    "ENABLED": false,
     "NETWORK": "signet",
     "BACKEND": "esplora",
     "HTTP_PORT": 8991,

production/mempool-config.testnet-lightning.json

@@ -1,5 +1,6 @@
 {
   "MEMPOOL": {
+    "ENABLED": false,
     "NETWORK": "testnet",
     "BACKEND": "esplora",
     "HTTP_PORT": 8992,

production/nginx/server-common.conf

@@ -81,6 +81,13 @@ location /resources {
 	try_files $uri /en-US/index.html;
 	expires 1w;
 }
+
+# only cache /resources/config.* for 5 minutes since it changes often
+location /resources/config. {
+	try_files $uri =404;
+	expires 5m;
+}
+
 # cache /main.f40e91d908a068a2.js forever since they never change
 location ~* ^/.+\..+\.(js|css) {
 	try_files /$lang/$uri /en-US/$uri =404;