Merge branch 'master' into simon/disable-mempool-config

backend/src/api/audit.ts

@@ -0,0 +1,118 @@
+import logger from '../logger';
+import { BlockExtended, TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
+
+const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
+
+class Audit {
+  auditBlock(transactions: TransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended })
+   : { censored: string[], added: string[], score: number } {
+    if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
+      return { censored: [], added: [], score: 0 };
+    }
+
+    const matches: string[] = []; // present in both mined block and template
+    const added: string[] = []; // present in mined block, not in template
+    const fresh: string[] = []; // missing, but firstSeen within PROPAGATION_MARGIN
+    const isCensored = {}; // missing, without excuse
+    const isDisplaced = {};
+    let displacedWeight = 0;
+
+    const inBlock = {};
+    const inTemplate = {};
+
+    const now = Math.round((Date.now() / 1000));
+    for (const tx of transactions) {
+      inBlock[tx.txid] = tx;
+    }
+    // coinbase is always expected
+    if (transactions[0]) {
+      inTemplate[transactions[0].txid] = true;
+    }
+    // look for transactions that were expected in the template, but missing from the mined block
+    for (const txid of projectedBlocks[0].transactionIds) {
+      if (!inBlock[txid]) {
+        // tx is recent, may have reached the miner too late for inclusion
+        if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) {
+          fresh.push(txid);
+        } else {
+          isCensored[txid] = true;
+        }
+        displacedWeight += mempool[txid].weight;
+      }
+      inTemplate[txid] = true;
+    }
+
+    displacedWeight += (4000 - transactions[0].weight);
+
+    logger.warn(`${fresh.length} fresh, ${Object.keys(isCensored).length} possibly censored, ${displacedWeight} displaced weight`);
+
+    // we can expect an honest miner to include 'displaced' transactions in place of recent arrivals and censored txs
+    // these displaced transactions should occupy the first N weight units of the next projected block
+    let displacedWeightRemaining = displacedWeight;
+    let index = 0;
+    let lastFeeRate = Infinity;
+    let failures = 0;
+    while (projectedBlocks[1] && index < projectedBlocks[1].transactionIds.length && failures < 500) {
+      const txid = projectedBlocks[1].transactionIds[index];
+      const fits = (mempool[txid].weight - displacedWeightRemaining) < 4000;
+      const feeMatches = mempool[txid].effectiveFeePerVsize >= lastFeeRate;
+      if (fits || feeMatches) {
+        isDisplaced[txid] = true;
+        if (fits) {
+          lastFeeRate = Math.min(lastFeeRate, mempool[txid].effectiveFeePerVsize);
+        }
+        if (mempool[txid].firstSeen == null || (now - (mempool[txid]?.firstSeen || 0)) > PROPAGATION_MARGIN) {
+          displacedWeightRemaining -= mempool[txid].weight;
+        }
+        failures = 0;
+      } else {
+        failures++;
+      }
+      index++;
+    }
+
+    // mark unexpected transactions in the mined block as 'added'
+    let overflowWeight = 0;
+    for (const tx of transactions) {
+      if (inTemplate[tx.txid]) {
+        matches.push(tx.txid);
+      } else {
+        if (!isDisplaced[tx.txid]) {
+          added.push(tx.txid);
+        }
+        overflowWeight += tx.weight;
+      }
+    }
+
+    // transactions missing from near the end of our template are probably not being censored
+    let overflowWeightRemaining = overflowWeight;
+    let lastOverflowRate = 1.00;
+    index = projectedBlocks[0].transactionIds.length - 1;
+    while (index >= 0) {
+      const txid = projectedBlocks[0].transactionIds[index];
+      if (overflowWeightRemaining > 0) {
+        if (isCensored[txid]) {
+          delete isCensored[txid];
+        }
+        lastOverflowRate = mempool[txid].effectiveFeePerVsize;
+      } else if (Math.floor(mempool[txid].effectiveFeePerVsize * 100) <= Math.ceil(lastOverflowRate * 100)) { // tolerance of 0.01 sat/vb
+        if (isCensored[txid]) {
+          delete isCensored[txid];
+        }
+      }
+      overflowWeightRemaining -= (mempool[txid]?.weight || 0);
+      index--;
+    }
+
+    const numCensored = Object.keys(isCensored).length;
+    const score = matches.length > 0 ? (matches.length / (matches.length + numCensored)) : 0;
+
+    return {
+      censored: Object.keys(isCensored),
+      added,
+      score
+    };
+  }
+}
+
+export default new Audit();

backend/src/api/blocks.ts

@@ -20,6 +20,7 @@ import indexer from '../indexer';
 import fiatConversion from './fiat-conversion';
 import poolsParser from './pools-parser';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
+import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
 import mining from './mining/mining';
 import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
 import PricesRepository from '../repositories/PricesRepository';
@@ -186,9 +187,7 @@ class Blocks {
       if (!pool) { // We should never have this situation in practise
         logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` +
           `Check your "pools" table entries`);
-        return blockExtended;
+      } else {
-      }
-
         blockExtended.extras.pool = {
           id: pool.id,
           name: pool.name,
@@ -196,6 +195,12 @@ class Blocks {
         };
       }
 
+      const auditSummary = await BlocksAuditsRepository.$getShortBlockAudit(block.id);
+      if (auditSummary) {
+        blockExtended.extras.matchRate = auditSummary.matchRate;
+      }
+    }
+
     return blockExtended;
   }
 

backend/src/api/database-migration.ts

@@ -4,7 +4,7 @@ import logger from '../logger';
 import { Common } from './common';
 
 class DatabaseMigration {
-  private static currentVersion = 40;
+  private static currentVersion = 41;
   private queryTimeout = 120000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -348,6 +348,10 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `nodes` ADD channels int(11) unsigned DEFAULT NULL');
       await this.$executeQuery('ALTER TABLE `nodes` ADD INDEX `capacity` (`capacity`);');
     }
+
+    if (databaseSchemaVersion < 41 && isBitcoin === true) {
+      await this.$executeQuery('UPDATE channels SET closing_reason = NULL WHERE closing_reason = 1');
+    }
   }
 
   /**

backend/src/api/mempool-blocks.ts

@@ -1,7 +1,8 @@
 import logger from '../logger';
-import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta } from '../mempool.interfaces';
+import { MempoolBlock, TransactionExtended, AuditTransaction, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor } from '../mempool.interfaces';
 import { Common } from './common';
 import config from '../config';
+import { PairingHeap } from '../utils/pairing-heap';
 
 class MempoolBlocks {
   private mempoolBlocks: MempoolBlockWithTransactions[] = [];
@@ -72,6 +73,7 @@ class MempoolBlocks {
     logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
 
     const { blocks, deltas } = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks);
+
     this.mempoolBlocks = blocks;
     this.mempoolBlockDeltas = deltas;
   }
@@ -99,6 +101,7 @@ class MempoolBlocks {
     if (transactions.length) {
       mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
     }
+
     // Calculate change from previous block states
     for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) {
       let added: TransactionStripped[] = [];
@@ -132,12 +135,286 @@ class MempoolBlocks {
         removed
       });
     }
+
     return {
       blocks: mempoolBlocks,
       deltas: mempoolBlockDeltas
     };
   }
 
+  /*
+  * Build projected mempool blocks using an approximation of the transaction selection algorithm from Bitcoin Core
+  * (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp)
+  *
+  * blockLimit: number of blocks to build in total.
+  * weightLimit: maximum weight of transactions to consider using the selection algorithm.
+  *              if weightLimit is significantly lower than the mempool size, results may start to diverge from getBlockTemplate 
+  * condenseRest: whether to ignore excess transactions or append them to the final block.
+  */
+  public makeBlockTemplates(mempool: { [txid: string]: TransactionExtended }, blockLimit: number, weightLimit: number | null = null, condenseRest = false): MempoolBlockWithTransactions[] {
+    const start = Date.now();
+    const auditPool: { [txid: string]: AuditTransaction } = {};
+    const mempoolArray: AuditTransaction[] = [];
+    const restOfArray: TransactionExtended[] = [];
+    
+    let weight = 0;
+    const maxWeight = weightLimit ? Math.max(4_000_000 * blockLimit, weightLimit) : Infinity;
+    // grab the top feerate txs up to maxWeight
+    Object.values(mempool).sort((a, b) => b.feePerVsize - a.feePerVsize).forEach(tx => {
+      weight += tx.weight;
+      if (weight >= maxWeight) {
+        restOfArray.push(tx);
+        return;
+      }
+      // initializing everything up front helps V8 optimize property access later
+      auditPool[tx.txid] = {
+        txid: tx.txid,
+        fee: tx.fee,
+        size: tx.size,
+        weight: tx.weight,
+        feePerVsize: tx.feePerVsize,
+        vin: tx.vin,
+        relativesSet: false,
+        ancestorMap: new Map<string, AuditTransaction>(),
+        children: new Set<AuditTransaction>(),
+        ancestorFee: 0,
+        ancestorWeight: 0,
+        score: 0,
+        used: false,
+        modified: false,
+        modifiedNode: null,
+      }
+      mempoolArray.push(auditPool[tx.txid]);
+    })
+
+    // Build relatives graph & calculate ancestor scores
+    for (const tx of mempoolArray) {
+      if (!tx.relativesSet) {
+        this.setRelatives(tx, auditPool);
+      }
+    }
+
+    // Sort by descending ancestor score
+    mempoolArray.sort((a, b) => (b.score || 0) - (a.score || 0));
+
+    // Build blocks by greedily choosing the highest feerate package
+    // (i.e. the package rooted in the transaction with the best ancestor score)
+    const blocks: MempoolBlockWithTransactions[] = [];
+    let blockWeight = 4000;
+    let blockSize = 0;
+    let transactions: AuditTransaction[] = [];
+    const modified: PairingHeap<AuditTransaction> = new PairingHeap((a, b): boolean => (a.score || 0) > (b.score || 0));
+    let overflow: AuditTransaction[] = [];
+    let failures = 0;
+    let top = 0;
+    while ((top < mempoolArray.length || !modified.isEmpty()) && (condenseRest || blocks.length < blockLimit)) {
+      // skip invalid transactions
+      while (top < mempoolArray.length && (mempoolArray[top].used || mempoolArray[top].modified)) {
+        top++;
+      }
+
+      // Select best next package
+      let nextTx;
+      const nextPoolTx = mempoolArray[top];
+      const nextModifiedTx = modified.peek();
+      if (nextPoolTx && (!nextModifiedTx || (nextPoolTx.score || 0) > (nextModifiedTx.score || 0))) {
+        nextTx = nextPoolTx;
+        top++;
+      } else {
+        modified.pop();
+        if (nextModifiedTx) {
+          nextTx = nextModifiedTx;
+          nextTx.modifiedNode = undefined;
+        }
+      }
+
+      if (nextTx && !nextTx?.used) {
+        // Check if the package fits into this block
+        if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
+          blockWeight += nextTx.ancestorWeight;
+          const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values());
+          // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count)
+          const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx];
+          const effectiveFeeRate = nextTx.ancestorFee / (nextTx.ancestorWeight / 4);
+          sortedTxSet.forEach((ancestor, i, arr) => {
+            const mempoolTx = mempool[ancestor.txid];
+            if (ancestor && !ancestor?.used) {
+              ancestor.used = true;
+              // update original copy of this tx with effective fee rate & relatives data
+              mempoolTx.effectiveFeePerVsize = effectiveFeeRate;
+              mempoolTx.ancestors = (Array.from(ancestor.ancestorMap?.values()) as AuditTransaction[]).map((a) => {
+                return {
+                  txid: a.txid,
+                  fee: a.fee,
+                  weight: a.weight,
+                }
+              })
+              if (i < arr.length - 1) {
+                mempoolTx.bestDescendant = {
+                  txid: arr[arr.length - 1].txid,
+                  fee: arr[arr.length - 1].fee,
+                  weight: arr[arr.length - 1].weight,
+                };
+              }
+              transactions.push(ancestor);
+              blockSize += ancestor.size;
+            }
+          });
+
+          // remove these as valid package ancestors for any descendants remaining in the mempool
+          if (sortedTxSet.length) {
+            sortedTxSet.forEach(tx => {
+              this.updateDescendants(tx, auditPool, modified);
+            });
+          }
+
+          failures = 0;
+        } else {
+          // hold this package in an overflow list while we check for smaller options
+          overflow.push(nextTx);
+          failures++;
+        }
+      }
+
+      // this block is full
+      const exceededPackageTries = failures > 1000 && blockWeight > (config.MEMPOOL.BLOCK_WEIGHT_UNITS - 4000);
+      if (exceededPackageTries && (!condenseRest || blocks.length < blockLimit - 1)) {
+        // construct this block
+        if (transactions.length) {
+          blocks.push(this.dataToMempoolBlocks(transactions.map(t => mempool[t.txid]), blockSize, blockWeight, blocks.length));
+        }
+        // reset for the next block
+        transactions = [];
+        blockSize = 0;
+        blockWeight = 4000;
+
+        // 'overflow' packages didn't fit in this block, but are valid candidates for the next
+        for (const overflowTx of overflow.reverse()) {
+          if (overflowTx.modified) {
+            overflowTx.modifiedNode = modified.add(overflowTx);
+          } else {
+            top--;
+            mempoolArray[top] = overflowTx;
+          }
+        }
+        overflow = [];
+      }
+    }
+    if (condenseRest) {
+      // pack any leftover transactions into the last block
+      for (const tx of overflow) {
+        if (!tx || tx?.used) {
+          continue;
+        }
+        blockWeight += tx.weight;
+        blockSize += tx.size;
+        transactions.push(tx);
+        tx.used = true;
+      }
+      const blockTransactions = transactions.map(t => mempool[t.txid])
+      restOfArray.forEach(tx => {
+        blockWeight += tx.weight;
+        blockSize += tx.size;
+        blockTransactions.push(tx);
+      });
+      if (blockTransactions.length) {
+        blocks.push(this.dataToMempoolBlocks(blockTransactions, blockSize, blockWeight, blocks.length));
+      }
+      transactions = [];
+    } else if (transactions.length) {
+      blocks.push(this.dataToMempoolBlocks(transactions.map(t => mempool[t.txid]), blockSize, blockWeight, blocks.length));
+    }
+
+    const end = Date.now();
+    const time = end - start;
+    logger.debug('Mempool templates calculated in ' + time / 1000 + ' seconds');
+
+    return blocks;
+  }
+
+  // traverse in-mempool ancestors
+  // recursion unavoidable, but should be limited to depth < 25 by mempool policy
+  public setRelatives(
+    tx: AuditTransaction,
+    mempool: { [txid: string]: AuditTransaction },
+  ): void {
+    for (const parent of tx.vin) {
+      const parentTx = mempool[parent.txid];
+      if (parentTx && !tx.ancestorMap!.has(parent.txid)) {
+        tx.ancestorMap.set(parent.txid, parentTx);
+        parentTx.children.add(tx);
+        // visit each node only once
+        if (!parentTx.relativesSet) {
+          this.setRelatives(parentTx, mempool);
+        }
+        parentTx.ancestorMap.forEach((ancestor) => {
+          tx.ancestorMap.set(ancestor.txid, ancestor);
+        });
+      }
+    };
+    tx.ancestorFee = tx.fee || 0;
+    tx.ancestorWeight = tx.weight || 0;
+    tx.ancestorMap.forEach((ancestor) => {
+      tx.ancestorFee += ancestor.fee;
+      tx.ancestorWeight += ancestor.weight;
+    });
+    tx.score = tx.ancestorFee / (tx.ancestorWeight || 1);
+    tx.relativesSet = true;
+  }
+
+  // iterate over remaining descendants, removing the root as a valid ancestor & updating the ancestor score
+  // avoids recursion to limit call stack depth
+  private updateDescendants(
+    rootTx: AuditTransaction,
+    mempool: { [txid: string]: AuditTransaction },
+    modified: PairingHeap<AuditTransaction>,
+  ): void {
+    const descendantSet: Set<AuditTransaction> = new Set();
+    // stack of nodes left to visit
+    const descendants: AuditTransaction[] = [];
+    let descendantTx;
+    let ancestorIndex;
+    let tmpScore;
+    rootTx.children.forEach(childTx => {
+      if (!descendantSet.has(childTx)) {
+        descendants.push(childTx);
+        descendantSet.add(childTx);
+      }
+    });
+    while (descendants.length) {
+      descendantTx = descendants.pop();
+      if (descendantTx && descendantTx.ancestorMap && descendantTx.ancestorMap.has(rootTx.txid)) {
+        // remove tx as ancestor
+        descendantTx.ancestorMap.delete(rootTx.txid);
+        descendantTx.ancestorFee -= rootTx.fee;
+        descendantTx.ancestorWeight -= rootTx.weight;
+        tmpScore = descendantTx.score;
+        descendantTx.score = descendantTx.ancestorFee / descendantTx.ancestorWeight;
+
+        if (!descendantTx.modifiedNode) {
+          descendantTx.modified = true;
+          descendantTx.modifiedNode = modified.add(descendantTx);
+        } else {
+          // rebalance modified heap if score has changed
+          if (descendantTx.score < tmpScore) {
+            modified.decreasePriority(descendantTx.modifiedNode);
+          } else if (descendantTx.score > tmpScore) {
+            modified.increasePriority(descendantTx.modifiedNode);
+          }
+        }
+
+        // add this node's children to the stack
+        descendantTx.children.forEach(childTx => {
+          // visit each node only once
+          if (!descendantSet.has(childTx)) {
+            descendants.push(childTx);
+            descendantSet.add(childTx);
+          }
+        });
+      }
+    }
+  }
+
   private dataToMempoolBlocks(transactions: TransactionExtended[],
     blockSize: number, blockWeight: number, blocksIndex: number): MempoolBlockWithTransactions {
     let rangeLength = 4;

backend/src/api/mempool.ts

@@ -103,12 +103,11 @@ class Mempool {
     return txTimes;
   }
 
-  public async $updateMempool() {
+  public async $updateMempool(): Promise<void> {
-    logger.debug('Updating mempool');
+    logger.debug(`Updating mempool...`);
     const start = new Date().getTime();
     let hasChange: boolean = false;
     const currentMempoolSize = Object.keys(this.mempoolCache).length;
-    let txCount = 0;
     const transactions = await bitcoinApi.$getRawMempool();
     const diff = transactions.length - currentMempoolSize;
     const newTransactions: TransactionExtended[] = [];
@@ -124,7 +123,6 @@ class Mempool {
         try {
           const transaction = await transactionUtils.$getTransactionExtended(txid);
           this.mempoolCache[txid] = transaction;
-          txCount++;
           if (this.inSync) {
             this.txPerSecondArray.push(new Date().getTime());
             this.vBytesPerSecondArray.push({
@@ -133,14 +131,9 @@ class Mempool {
             });
           }
           hasChange = true;
-          if (diff > 0) {
-            logger.debug('Fetched transaction ' + txCount + ' / ' + diff);
-          } else {
-            logger.debug('Fetched transaction ' + txCount);
-          }
           newTransactions.push(transaction);
         } catch (e) {
-          logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
+          logger.debug(`Error finding transaction '${txid}' in the mempool: ` + (e instanceof Error ? e.message : e));
         }
       }
 
@@ -197,8 +190,7 @@ class Mempool {
 
     const end = new Date().getTime();
     const time = end - start;
-    logger.debug(`New mempool size: ${Object.keys(this.mempoolCache).length} Change: ${diff}`);
+    logger.debug(`Mempool updated in ${time / 1000} seconds. New size: ${Object.keys(this.mempoolCache).length} (${diff > 0 ? '+' + diff : diff})`);
-    logger.debug('Mempool updated in ' + time / 1000 + ' seconds');
   }
 
   public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended; }) {

backend/src/api/mining/mining-routes.ts

@@ -238,6 +238,12 @@ class MiningRoutes {
   public async $getBlockAudit(req: Request, res: Response) {
     try {
       const audit = await BlocksAuditsRepository.$getBlockAudit(req.params.hash);
+
+      if (!audit) {
+        res.status(404).send(`This block has not been audited.`);
+        return;
+      }
+
       res.header('Pragma', 'public');
       res.header('Cache-control', 'public');
       res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());

backend/src/api/websocket-handler.ts

@@ -18,6 +18,7 @@ import difficultyAdjustment from './difficulty-adjustment';
 import feeApi from './fee-api';
 import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
+import Audit from './audit';
 
 class WebsocketHandler {
   private wss: WebSocket.Server | undefined;
@@ -405,52 +406,32 @@ class WebsocketHandler {
     });
   }
 
-  handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) {
+  handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]): void {
     if (!this.wss) {
       throw new Error('WebSocket.Server is not set');
     }
 
     let mBlocks: undefined | MempoolBlock[];
     let mBlockDeltas: undefined | MempoolBlockDelta[];
-    let matchRate = 0;
+    let matchRate;
     const _memPool = memPool.getMempool();
-    const _mempoolBlocks = mempoolBlocks.getMempoolBlocksWithTransactions();
-
-    if (_mempoolBlocks[0]) {
-      const matches: string[] = [];
-      const added: string[] = [];
-      const missing: string[] = [];
-
-      for (const txId of txIds) {
-        if (_mempoolBlocks[0].transactionIds.indexOf(txId) > -1) {
-          matches.push(txId);
-        } else {
-          added.push(txId);
-        }
-        delete _memPool[txId];
-      }
-
-      for (const txId of _mempoolBlocks[0].transactionIds) {
-        if (matches.includes(txId) || added.includes(txId)) {
-          continue;
-        }
-        missing.push(txId);
-      }
-
-      matchRate = Math.round((Math.max(0, matches.length - missing.length - added.length) / txIds.length * 100) * 100) / 100;
-      mempoolBlocks.updateMempoolBlocks(_memPool);
-      mBlocks = mempoolBlocks.getMempoolBlocks();
-      mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
 
     if (Common.indexingEnabled()) {
-        const stripped = _mempoolBlocks[0].transactions.map((tx) => {
+      const mempoolCopy = cloneMempool(_memPool);
+      const projectedBlocks = mempoolBlocks.makeBlockTemplates(mempoolCopy, 2);
+
+      const { censored, added, score } = Audit.auditBlock(transactions, projectedBlocks, mempoolCopy);
+      matchRate = Math.round(score * 100 * 100) / 100;
+
+      const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
         return {
           txid: tx.txid,
           vsize: tx.vsize,
           fee: tx.fee ? Math.round(tx.fee) : 0,
           value: tx.value,
         };
-        });  
+      }) : [];
+
       BlocksSummariesRepository.$saveSummary({
         height: block.height,
         template: {
@@ -464,15 +445,23 @@ class WebsocketHandler {
         height: block.height,
         hash: block.id,
         addedTxs: added,
-          missingTxs: missing,
+        missingTxs: censored,
         matchRate: matchRate,
       });
-      }
-    }
 
       if (block.extras) {
         block.extras.matchRate = matchRate;
       }
+    }
+
+    // Update mempool to remove transactions included in the new block
+    for (const txId of txIds) {
+      delete _memPool[txId];
+    }
+
+    mempoolBlocks.updateMempoolBlocks(_memPool);
+    mBlocks = mempoolBlocks.getMempoolBlocks();
+    mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
 
     const da = difficultyAdjustment.getDifficultyAdjustment();
     const fees = feeApi.getRecommendedFee();
@@ -580,4 +569,14 @@ class WebsocketHandler {
   }
 }
 
+function cloneMempool(mempool: { [txid: string]: TransactionExtended }): { [txid: string]: TransactionExtended } {
+  const cloned = {};
+  Object.keys(mempool).forEach(id => {
+    cloned[id] = {
+      ...mempool[id]
+    };
+  });
+  return cloned;
+}
+
 export default new WebsocketHandler();

backend/src/mempool.interfaces.ts

@@ -1,4 +1,5 @@
 import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
+import { HeapNode } from "./utils/pairing-heap";
 
 export interface PoolTag {
   id: number; // mysql row id
@@ -70,12 +71,40 @@ export interface TransactionExtended extends IEsploraApi.Transaction {
   deleteAfter?: number;
 }
 
-interface Ancestor {
+export interface AuditTransaction {
+  txid: string;
+  fee: number;
+  size: number;
+  weight: number;
+  feePerVsize: number;
+  vin: IEsploraApi.Vin[];
+  relativesSet: boolean;
+  ancestorMap: Map<string, AuditTransaction>;
+  children: Set<AuditTransaction>;
+  ancestorFee: number;
+  ancestorWeight: number;
+  score: number;
+  used: boolean;
+  modified: boolean;
+  modifiedNode: HeapNode<AuditTransaction>;
+}
+
+export interface Ancestor {
   txid: string;
   weight: number;
   fee: number;
 }
 
+export interface TransactionSet {
+  fee: number;
+  weight: number;
+  score: number;
+  children?: Set<string>;
+  available?: boolean;
+  modified?: boolean;
+  modifiedNode?: HeapNode<string>;
+}
+
 interface BestDescendant {
   txid: string;
   weight: number;

backend/src/repositories/BlocksAuditsRepository.ts

@@ -58,10 +58,12 @@ class BlocksAuditRepositories {
         WHERE blocks_audits.hash = "${hash}"
       `);
       
+      if (rows.length) {
         rows[0].missingTxs = JSON.parse(rows[0].missingTxs);
         rows[0].addedTxs = JSON.parse(rows[0].addedTxs);
         rows[0].transactions = JSON.parse(rows[0].transactions);
         rows[0].template = JSON.parse(rows[0].template);
+      }
             
       return rows[0];
     } catch (e: any) {
@@ -69,6 +71,20 @@ class BlocksAuditRepositories {
       throw e;
     }
   }
+
+  public async $getShortBlockAudit(hash: string): Promise<any> {
+    try {
+      const [rows]: any[] = await DB.query(
+        `SELECT hash as id, match_rate as matchRate
+        FROM blocks_audits
+        WHERE blocks_audits.hash = "${hash}"
+      `);
+      return rows[0];
+    } catch (e: any) {
+      logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
 }
 
 export default new BlocksAuditRepositories();

backend/src/tasks/lightning/network-sync.service.ts

@@ -289,6 +289,24 @@ class NetworkSyncService {
     1. Mutually closed
     2. Forced closed
     3. Forced closed with penalty
+
+    ┌────────────────────────────────────┐       ┌────────────────────────────┐
+    │ outputs contain revocation script? ├──yes──► force close w/ penalty = 3 │
+    └──────────────┬─────────────────────┘       └────────────────────────────┘
+                   no
+    ┌──────────────▼──────────────────────────┐
+    │ outputs contain other lightning script? ├──┐
+    └──────────────┬──────────────────────────┘  │
+                   no                           yes
+    ┌──────────────▼─────────────┐               │
+    │ sequence starts with 0x80  │      ┌────────▼────────┐
+    │           and              ├──────► force close = 2 │
+    │ locktime starts with 0x20? │      └─────────────────┘
+    └──────────────┬─────────────┘
+                   no
+         ┌─────────▼────────┐
+         │ mutual close = 1 │
+         └──────────────────┘
   */
 
   private async $runClosedChannelsForensics(): Promise<void> {
@@ -326,10 +344,6 @@ class NetworkSyncService {
               lightningScriptReasons.push(lightningScript);
             }
           }
-          if (lightningScriptReasons.length === outspends.length
-            && lightningScriptReasons.filter((r) => r === 1).length === outspends.length) {
-            reason = 1;
-          } else {
           const filteredReasons = lightningScriptReasons.filter((r) => r !== 1);
           if (filteredReasons.length) {
             if (filteredReasons.some((r) => r === 2 || r === 4)) {
@@ -357,7 +371,6 @@ class NetworkSyncService {
               reason = 1;
             }
           }
-          }
           if (reason) {
             logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.');
             await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]);

backend/src/utils/pairing-heap.ts

@@ -0,0 +1,174 @@
+export type HeapNode<T> = {
+  element: T
+  child?: HeapNode<T>
+  next?: HeapNode<T>
+  prev?: HeapNode<T>
+} | null | undefined;
+
+// minimal pairing heap priority queue implementation
+export class PairingHeap<T> {
+  private root: HeapNode<T> = null;
+  private comparator: (a: T, b: T) => boolean;
+
+  // comparator function should return 'true' if a is higher priority than b
+  constructor(comparator: (a: T, b: T) => boolean) {
+    this.comparator = comparator;
+  }
+
+  isEmpty(): boolean {
+    return !this.root;
+  }
+
+  add(element: T): HeapNode<T> {
+    const node: HeapNode<T> = {
+      element
+    };
+
+    this.root = this.meld(this.root, node);
+
+    return node;
+  }
+
+  // returns the top priority element without modifying the queue
+  peek(): T | void {
+    return this.root?.element;
+  }
+
+  // removes and returns the top priority element
+  pop(): T | void {
+    let element;
+    if (this.root) {
+      const node = this.root;
+      element = node.element;
+      this.root = this.mergePairs(node.child);
+    }
+    return element;
+  }
+
+  deleteNode(node: HeapNode<T>): void {
+    if (!node) {
+      return;
+    }
+
+    if (node === this.root) {
+      this.root = this.mergePairs(node.child);
+    }
+    else {
+      if (node.prev) {
+        if (node.prev.child === node) {
+          node.prev.child = node.next;
+        }
+        else {
+          node.prev.next = node.next;
+        }
+      }
+      if (node.next) {
+        node.next.prev = node.prev;
+      }
+      this.root = this.meld(this.root, this.mergePairs(node.child));
+    }
+
+    node.child = null;
+    node.prev = null;
+    node.next = null;
+  }
+
+  // fix the heap after increasing the priority of a given node
+  increasePriority(node: HeapNode<T>): void {
+    // already the top priority element
+    if (!node || node === this.root) {
+      return;
+    }
+    // extract from siblings
+    if (node.prev) {
+      if (node.prev?.child === node) {
+        if (this.comparator(node.prev.element, node.element)) {
+          // already in a valid position
+          return;
+        }
+        node.prev.child = node.next;
+      }
+      else {
+        node.prev.next = node.next;
+      }
+    }
+    if (node.next) {
+      node.next.prev = node.prev;
+    }
+
+    this.root = this.meld(this.root, node);
+  }
+
+  decreasePriority(node: HeapNode<T>): void {
+    this.deleteNode(node);
+    this.root = this.meld(this.root, node);
+  }
+
+  meld(a: HeapNode<T>, b: HeapNode<T>): HeapNode<T> {
+    if (!a) {
+      return b;
+    }
+    if (!b || a === b) {
+      return a;
+    }
+
+    let parent: HeapNode<T> = b;
+    let child: HeapNode<T> = a;
+    if (this.comparator(a.element, b.element)) {
+      parent = a;
+      child = b;
+    }
+
+    child.next = parent.child;
+    if (parent.child) {
+      parent.child.prev = child;
+    }
+    child.prev = parent;
+    parent.child = child;
+
+    parent.next = null;
+    parent.prev = null;
+
+    return parent;
+  }
+
+  mergePairs(node: HeapNode<T>): HeapNode<T> {
+    if (!node) {
+      return null;
+    }
+
+    let current: HeapNode<T> = node;
+    let next: HeapNode<T>;
+    let nextCurrent: HeapNode<T>;
+    let pairs: HeapNode<T>;
+    let melded: HeapNode<T>;
+    while (current) {
+      next = current.next;
+      if (next) {
+        nextCurrent = next.next;
+        melded = this.meld(current, next);
+        if (melded) {
+          melded.prev = pairs;
+        }
+        pairs = melded;
+      }
+      else {
+        nextCurrent = null;
+        current.prev = pairs;
+        pairs = current;
+        break;
+      }
+      current = nextCurrent;
+    }
+
+    melded = null;
+    let prev: HeapNode<T>;
+    while (pairs) {
+      prev = pairs.prev;
+      melded = this.meld(melded, pairs);
+      pairs = prev;
+    }
+
+    return melded;
+  }
+}

frontend/angular.json

@@ -222,6 +222,10 @@
               "proxyConfig": "proxy.conf.local.js",
               "verbose": true
             },
+            "local-esplora": {
+              "proxyConfig": "proxy.conf.local-esplora.js",
+              "verbose": true
+            },
             "mixed": {
               "proxyConfig": "proxy.conf.mixed.js",
               "verbose": true

frontend/package.json

@@ -29,6 +29,7 @@
     "serve:local-prod": "npm run generate-config && npm run ng -- serve -c local-prod",
     "serve:local-staging": "npm run generate-config && npm run ng -- serve -c local-staging",
     "start": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local",
+    "start:local-esplora": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local-esplora",
     "start:stg": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c staging",
     "start:local-prod": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local-prod",
     "start:local-staging": "npm run generate-config && npm run sync-assets-dev && npm run ng -- serve -c local-staging",

frontend/proxy.conf.local-esplora.js

@@ -0,0 +1,137 @@
+const fs = require('fs');
+
+const FRONTEND_CONFIG_FILE_NAME = 'mempool-frontend-config.json';
+
+let configContent;
+
+// Read frontend config 
+try {
+    const rawConfig = fs.readFileSync(FRONTEND_CONFIG_FILE_NAME);
+    configContent = JSON.parse(rawConfig);
+    console.log(`${FRONTEND_CONFIG_FILE_NAME} file found, using provided config`);
+} catch (e) {
+    console.log(e);
+    if (e.code !== 'ENOENT') {
+      throw new Error(e);
+  } else {
+      console.log(`${FRONTEND_CONFIG_FILE_NAME} file not found, using default config`);
+  }
+}
+
+let PROXY_CONFIG = [];
+
+if (configContent && configContent.BASE_MODULE === 'liquid') {
+  PROXY_CONFIG.push(...[
+    {
+      context: ['/liquid/api/v1/**'],
+      target: `http://127.0.0.1:8999`,
+      secure: false,
+      ws: true,
+      changeOrigin: true,
+      proxyTimeout: 30000,
+      pathRewrite: {
+          "^/liquid": ""
+      },
+    },
+    {
+      context: ['/liquid/api/**'],
+      target: `http://127.0.0.1:3000`,
+      secure: false,
+      changeOrigin: true,
+      proxyTimeout: 30000,
+      pathRewrite: {
+          "^/liquid/api/": ""
+      },
+    },
+    {
+      context: ['/liquidtestnet/api/v1/**'],
+      target: `http://127.0.0.1:8999`,
+      secure: false,
+      ws: true,
+      changeOrigin: true,
+      proxyTimeout: 30000,
+      pathRewrite: {
+          "^/liquidtestnet": ""
+      },
+    },
+    {
+      context: ['/liquidtestnet/api/**'],
+      target: `http://127.0.0.1:3000`,
+      secure: false,
+      changeOrigin: true,
+      proxyTimeout: 30000,
+      pathRewrite: {
+          "^/liquidtestnet/api/": "/"
+      },
+    },
+  ]);
+}
+
+
+if (configContent && configContent.BASE_MODULE === 'bisq') {
+  PROXY_CONFIG.push(...[
+    {
+      context: ['/bisq/api/v1/ws'],
+      target: `http://127.0.0.1:8999`,
+      secure: false,
+      ws: true,
+      changeOrigin: true,
+      proxyTimeout: 30000,
+      pathRewrite: {
+          "^/bisq": ""
+      },
+    },
+    {
+      context: ['/bisq/api/v1/**'],
+      target: `http://127.0.0.1:8999`,
+      secure: false,
+      changeOrigin: true,
+      proxyTimeout: 30000,
+    },
+    {
+      context: ['/bisq/api/**'],
+      target: `http://127.0.0.1:8999`,
+      secure: false,
+      changeOrigin: true,
+      proxyTimeout: 30000,
+      pathRewrite: {
+          "^/bisq/api/": "/api/v1/bisq/"
+      },
+    }
+  ]);
+}
+
+PROXY_CONFIG.push(...[
+  {
+    context: ['/testnet/api/v1/lightning/**'],
+    target: `http://127.0.0.1:8999`,
+    secure: false,
+    changeOrigin: true,
+    proxyTimeout: 30000,
+    pathRewrite: {
+        "^/testnet": ""
+    },
+  },
+  {
+    context: ['/api/v1/**'],
+    target: `http://127.0.0.1:8999`,
+    secure: false,
+    ws: true,
+    changeOrigin: true,
+    proxyTimeout: 30000,
+  },
+  {
+    context: ['/api/**'],
+    target: `http://127.0.0.1:3000`,
+    secure: false,
+    changeOrigin: true,
+    proxyTimeout: 30000,
+    pathRewrite: {
+        "^/api": ""
+    },
+  }
+]);
+
+console.log(PROXY_CONFIG);
+
+module.exports = PROXY_CONFIG;

frontend/proxy.conf.staging.js

@@ -3,9 +3,9 @@ const fs = require('fs');
 let PROXY_CONFIG = require('./proxy.conf');
 
 PROXY_CONFIG.forEach(entry => {
-  entry.target = entry.target.replace("mempool.space", "mempool.ninja");
+  entry.target = entry.target.replace("mempool.space", "mempool-staging.fra.mempool.space");
-  entry.target = entry.target.replace("liquid.network", "liquid.place");
+  entry.target = entry.target.replace("liquid.network", "liquid-staging.fra.mempool.space");
-  entry.target = entry.target.replace("bisq.markets", "bisq.ninja");
+  entry.target = entry.target.replace("bisq.markets", "bisq-staging.fra.mempool.space");
 });
 
 module.exports = PROXY_CONFIG;

frontend/src/app/components/block-audit/block-audit.component.html

@@ -1,22 +1,23 @@
 <div class="container-xl" (window:resize)="onResize($event)">
 
-  <div *ngIf="(auditObservable$ | async) as blockAudit; else skeleton">
   <div class="title-block" id="block">
     <h1>
       <span class="next-previous-blocks">
-          <span i18n="shared.block-title">Block </span>
+        <span i18n="shared.block-audit-title">Block Audit</span>
         &nbsp;
-          <a [routerLink]="['/block/' | relativeUrl, blockAudit.id]">{{ blockAudit.height }}</a>
+        <a *ngIf="blockAudit" [routerLink]="['/block/' | relativeUrl, blockHash]">{{ blockAudit.height }}</a>
         &nbsp;
-          <span i18n="shared.template-vs-mined">Template vs Mined</span>
       </span>
     </h1>
 
     <div class="grow"></div>
 
-      <button [routerLink]="['/' | relativeUrl]" class="btn btn-sm">&#10005;</button>
+    <button [routerLink]="['/block/' | relativeUrl, blockHash]" class="btn btn-sm">&#10005;</button>
   </div>
 
+  <div *ngIf="!error && !isLoading">
+    
+
     <!-- OVERVIEW -->
     <div class="box mb-3">
       <div class="row">
@@ -26,8 +27,8 @@
             <tbody>
               <tr>
                 <td class="td-width" i18n="block.hash">Hash</td>
-                <td><a [routerLink]="['/block/' | relativeUrl, blockAudit.id]" title="{{ blockAudit.id }}">{{ blockAudit.id | shortenString : 13 }}</a>
+                <td><a [routerLink]="['/block/' | relativeUrl, blockHash]" title="{{ blockHash }}">{{ blockHash | shortenString : 13 }}</a>
-                  <app-clipboard class="d-none d-sm-inline-block" [text]="blockAudit.id"></app-clipboard>
+                  <app-clipboard class="d-none d-sm-inline-block" [text]="blockHash"></app-clipboard>
                 </td>
               </tr>
               <tr>
@@ -40,6 +41,10 @@
                   </div>
                 </td>
               </tr>
+              <tr>
+                <td class="td-width" i18n="shared.transaction-count">Transactions</td>
+                <td>{{ blockAudit.tx_count }}</td>
+              </tr>
               <tr>
                 <td i18n="blockAudit.size">Size</td>
                 <td [innerHTML]="'&lrm;' + (blockAudit.size | bytes: 2)"></td>
@@ -57,21 +62,25 @@
           <table class="table table-borderless table-striped">
             <tbody>
               <tr>
-                <td class="td-width" i18n="shared.transaction-count">Transactions</td>
+                <td i18n="block.health">Block health</td>
-                <td>{{ blockAudit.tx_count }}</td>
-              </tr>
-              <tr>
-                <td i18n="block.match-rate">Match rate</td>
                 <td>{{ blockAudit.matchRate }}%</td>
               </tr>
               <tr>
-                <td i18n="block.missing-txs">Missing txs</td>
+                <td i18n="block.missing-txs">Removed txs</td>
                 <td>{{ blockAudit.missingTxs.length }}</td>
               </tr>
+              <tr>
+                <td i18n="block.missing-txs">Omitted txs</td>
+                <td>{{ numMissing }}</td>
+              </tr>
               <tr>
                 <td i18n="block.added-txs">Added txs</td>
                 <td>{{ blockAudit.addedTxs.length }}</td>
               </tr>
+              <tr>
+                <td i18n="block.missing-txs">Included txs</td>
+                <td>{{ numUnexpected }}</td>
+              </tr>
             </tbody>
           </table>
         </div>
@@ -79,33 +88,110 @@
     </div> <!-- box -->
 
     <!-- ADDED vs MISSING button -->
-    <div class="d-flex justify-content-center menu mt-3" *ngIf="isMobile">
+    <div class="d-flex justify-content-center menu mt-3 mb-3" *ngIf="isMobile">
-      <a routerLinkActive="active" class="btn btn-primary w-50 mr-1 ml-1 menu-button" i18n="block.missing-txs"
+      <a class="btn btn-primary w-50 mr-1 ml-1 menu-button" [class.active]="mode === 'projected'" i18n="block.projected"
-        fragment="missing" (click)="changeMode('missing')">Missing</a>
+        fragment="projected" (click)="changeMode('projected')">Projected</a>
-      <a routerLinkActive="active" class="btn btn-primary w-50 mr-1 ml-1 menu-button" i18n="block.added-txs"
+      <a class="btn btn-primary w-50 mr-1 ml-1 menu-button" [class.active]="mode === 'actual'" i18n="block.actual"
-        fragment="added" (click)="changeMode('added')">Added</a>
+        fragment="actual" (click)="changeMode('actual')">Actual</a>
     </div>
   </div>
 
+  <ng-template [ngIf]="!error && isLoading">
+    <div class="title-block" id="block">
+      <h1>
+        <span class="next-previous-blocks">
+          <span i18n="shared.block-audit-title">Block Audit</span>
+          &nbsp;
+          <a *ngIf="blockAudit" [routerLink]="['/block/' | relativeUrl, blockHash]">{{ blockAudit.height }}</a>
+          &nbsp;
+        </span>
+      </h1>
+
+      <div class="grow"></div>
+
+      <button [routerLink]="['/' | relativeUrl]" class="btn btn-sm">&#10005;</button>
+    </div>
+
+    <!-- OVERVIEW -->
+    <div class="box mb-3">
+      <div class="row">
+        <!-- LEFT COLUMN -->
+        <div class="col-sm">
+          <table class="table table-borderless table-striped">
+            <tbody>
+              <tr><td class="td-width" colspan="2"><span class="skeleton-loader"></span></td></tr>
+              <tr><td class="td-width" colspan="2"><span class="skeleton-loader"></span></td></tr>
+              <tr><td class="td-width" colspan="2"><span class="skeleton-loader"></span></td></tr>
+              <tr><td class="td-width" colspan="2"><span class="skeleton-loader"></span></td></tr>
+              <tr><td class="td-width" colspan="2"><span class="skeleton-loader"></span></td></tr>
+            </tbody>
+          </table>
+        </div>
+
+        <!-- RIGHT COLUMN -->
+        <div class="col-sm">
+          <table class="table table-borderless table-striped">
+            <tbody>
+              <tr><td class="td-width" colspan="2"><span class="skeleton-loader"></span></td></tr>
+              <tr><td class="td-width" colspan="2"><span class="skeleton-loader"></span></td></tr>
+              <tr><td class="td-width" colspan="2"><span class="skeleton-loader"></span></td></tr>
+              <tr><td class="td-width" colspan="2"><span class="skeleton-loader"></span></td></tr>
+              <tr><td class="td-width" colspan="2"><span class="skeleton-loader"></span></td></tr>
+            </tbody>
+          </table>
+        </div>
+      </div> <!-- row -->
+    </div> <!-- box -->
+
+    <!-- ADDED vs MISSING button -->
+    <div class="d-flex justify-content-center menu mt-3 mb-3" *ngIf="isMobile">
+      <a class="btn btn-primary w-50 mr-1 ml-1 menu-button" [class.active]="mode === 'projected'" i18n="block.projected"
+        fragment="projected" (click)="changeMode('projected')">Projected</a>
+      <a class="btn btn-primary w-50 mr-1 ml-1 menu-button" [class.active]="mode === 'actual'" i18n="block.actual"
+        fragment="actual" (click)="changeMode('actual')">Actual</a>
+    </div>
+  </ng-template>
+
+  <ng-template [ngIf]="error">
+    <div *ngIf="error && error.status === 404; else generalError" class="text-center">
+      <br>
+      <b i18n="error.audit-unavailable">audit unavailable</b>
+      <br><br>
+      <i>{{ error.error }}</i>
+      <br>
+      <br>
+    </div>
+    <ng-template #generalError>
+      <div class="text-center">
+        <br>
+        <span i18n="error.general-loading-data">Error loading data.</span>
+        <br><br>
+        <i>{{ error }}</i>
+        <br>
+        <br>
+      </div>
+    </ng-template>
+  </ng-template>
+
   <!-- VISUALIZATIONS -->
-  <div class="box">
+  <div class="box" *ngIf="!error">
     <div class="row">
       <!-- MISSING TX RENDERING -->
       <div class="col-sm" *ngIf="webGlEnabled">
-        <app-block-overview-graph #blockGraphTemplate [isLoading]="isLoading" [resolution]="75"
+        <h3 class="block-subtitle" *ngIf="!isMobile" i18n="block.projected-block">Projected Block</h3>
+        <app-block-overview-graph #blockGraphProjected [isLoading]="isLoading" [resolution]="75"
           [blockLimit]="stateService.blockVSize" [orientation]="'top'" [flip]="false"
           (txClickEvent)="onTxClick($event)"></app-block-overview-graph>
       </div>
 
       <!-- ADDED TX RENDERING -->
       <div class="col-sm" *ngIf="webGlEnabled && !isMobile">
-        <app-block-overview-graph #blockGraphMined [isLoading]="isLoading" [resolution]="75"
+        <h3 class="block-subtitle" *ngIf="!isMobile" i18n="block.actual-block">Actual Block</h3>
+        <app-block-overview-graph #blockGraphActual [isLoading]="isLoading" [resolution]="75"
           [blockLimit]="stateService.blockVSize" [orientation]="'top'" [flip]="false"
           (txClickEvent)="onTxClick($event)"></app-block-overview-graph>
       </div>
     </div> <!-- row -->
   </div> <!-- box -->
 
-  <ng-template #skeleton></ng-template>
-
 </div>

frontend/src/app/components/block-audit/block-audit.component.scss

@@ -38,3 +38,7 @@
     max-width: 150px;
   }
 }
+
+.block-subtitle {
+  text-align: center;
+}

frontend/src/app/components/block-audit/block-audit.component.ts

@@ -1,7 +1,7 @@
-import { Component, OnDestroy, OnInit, ViewChild } from '@angular/core';
+import { Component, OnDestroy, OnInit, AfterViewInit, ViewChildren, QueryList } from '@angular/core';
 import { ActivatedRoute, ParamMap, Router } from '@angular/router';
-import { Observable } from 'rxjs';
+import { Subscription, combineLatest } from 'rxjs';
-import { map, share, switchMap, tap } from 'rxjs/operators';
+import { map, switchMap, startWith, catchError } from 'rxjs/operators';
 import { BlockAudit, TransactionStripped } from '../../interfaces/node-api.interface';
 import { ApiService } from '../../services/api.service';
 import { StateService } from '../../services/state.service';
@@ -22,22 +22,30 @@ import { BlockOverviewGraphComponent } from '../block-overview-graph/block-overv
     }
   `],
 })
-export class BlockAuditComponent implements OnInit, OnDestroy {
+export class BlockAuditComponent implements OnInit, AfterViewInit, OnDestroy {
   blockAudit: BlockAudit = undefined;
   transactions: string[];
-  auditObservable$: Observable<BlockAudit>;
+  auditSubscription: Subscription;
+  urlFragmentSubscription: Subscription;
 
   paginationMaxSize: number;
   page = 1;
   itemsPerPage: number;
 
-  mode: 'missing' | 'added' = 'missing';
+  mode: 'projected' | 'actual' = 'projected';
+  error: any;
   isLoading = true;
   webGlEnabled = true;
   isMobile = window.innerWidth <= 767.98;
 
-  @ViewChild('blockGraphTemplate') blockGraphTemplate: BlockOverviewGraphComponent;
+  childChangeSubscription: Subscription;
-  @ViewChild('blockGraphMined') blockGraphMined: BlockOverviewGraphComponent;
+
+  blockHash: string;
+  numMissing: number = 0;
+  numUnexpected: number = 0;
+
+  @ViewChildren('blockGraphProjected') blockGraphProjected: QueryList<BlockOverviewGraphComponent>;
+  @ViewChildren('blockGraphActual') blockGraphActual: QueryList<BlockOverviewGraphComponent>;
 
   constructor(
     private route: ActivatedRoute,
@@ -48,73 +56,137 @@ export class BlockAuditComponent implements OnInit, OnDestroy {
     this.webGlEnabled = detectWebGL();
   }
 
-  ngOnDestroy(): void {
+  ngOnDestroy() {
+    this.childChangeSubscription.unsubscribe();
+    this.urlFragmentSubscription.unsubscribe();
   }
 
   ngOnInit(): void {
     this.paginationMaxSize = window.matchMedia('(max-width: 670px)').matches ? 3 : 5;
     this.itemsPerPage = this.stateService.env.ITEMS_PER_PAGE;
 
-    this.auditObservable$ = this.route.paramMap.pipe(
+    this.urlFragmentSubscription = this.route.fragment.subscribe((fragment) => {
+      if (fragment === 'actual') {
+        this.mode = 'actual';
+      } else {
+        this.mode = 'projected'
+      }
+      this.setupBlockGraphs();
+    });
+
+    this.auditSubscription = this.route.paramMap.pipe(
       switchMap((params: ParamMap) => {
-        const blockHash: string = params.get('id') || '';
+        this.blockHash = params.get('id') || null;
-        return this.apiService.getBlockAudit$(blockHash)
+        if (!this.blockHash) {
+          return null;
+        }
+        return this.apiService.getBlockAudit$(this.blockHash)
           .pipe(
             map((response) => {
               const blockAudit = response.body;
-              for (let i = 0; i < blockAudit.template.length; ++i) {
+              const inTemplate = {};
-                if (blockAudit.missingTxs.includes(blockAudit.template[i].txid)) {
+              const inBlock = {};
-                  blockAudit.template[i].status = 'missing';
+              const isAdded = {};
-                } else if (blockAudit.addedTxs.includes(blockAudit.template[i].txid)) {
+              const isCensored = {};
-                  blockAudit.template[i].status = 'added';
+              const isMissing = {};
+              const isSelected = {};
+              this.numMissing = 0;
+              this.numUnexpected = 0;
+              for (const tx of blockAudit.template) {
+                inTemplate[tx.txid] = true;
+              }
+              for (const tx of blockAudit.transactions) {
+                inBlock[tx.txid] = true;
+              }
+              for (const txid of blockAudit.addedTxs) {
+                isAdded[txid] = true;
+              }
+              for (const txid of blockAudit.missingTxs) {
+                isCensored[txid] = true;
+              }
+              // set transaction statuses
+              for (const tx of blockAudit.template) {
+                if (isCensored[tx.txid]) {
+                  tx.status = 'censored';
+                } else if (inBlock[tx.txid]) {
+                  tx.status = 'found';
                 } else {
-                  blockAudit.template[i].status = 'found';
+                  tx.status = 'missing';
+                  isMissing[tx.txid] = true;
+                  this.numMissing++;
                 }
               }
-              for (let i = 0; i < blockAudit.transactions.length; ++i) {
+              for (const [index, tx] of blockAudit.transactions.entries()) {
-                if (blockAudit.missingTxs.includes(blockAudit.transactions[i].txid)) {
+                if (isAdded[tx.txid]) {
-                  blockAudit.transactions[i].status = 'missing';
+                  tx.status = 'added';
-                } else if (blockAudit.addedTxs.includes(blockAudit.transactions[i].txid)) {
+                } else if (index === 0 || inTemplate[tx.txid]) {
-                  blockAudit.transactions[i].status = 'added';
+                  tx.status = 'found';
                 } else {
-                  blockAudit.transactions[i].status = 'found';
+                  tx.status = 'selected';
+                  isSelected[tx.txid] = true;
+                  this.numUnexpected++;
                 }
               }
+              for (const tx of blockAudit.transactions) {
+                inBlock[tx.txid] = true;
+              }
               return blockAudit;
+            })
+          );
       }),
-            tap((blockAudit) => {
+      catchError((err) => {
-              this.changeMode(this.mode);
+        console.log(err);
-              if (this.blockGraphTemplate) {
+        this.error = err;
-                this.blockGraphTemplate.destroy();
-                this.blockGraphTemplate.setup(blockAudit.template);
-              }
-              if (this.blockGraphMined) {
-                this.blockGraphMined.destroy();
-                this.blockGraphMined.setup(blockAudit.transactions);
-              }
         this.isLoading = false;
+        return null;
       }),
-          );
+    ).subscribe((blockAudit) => {
-      }),
+      this.blockAudit = blockAudit;
-      share()
+      this.setupBlockGraphs();
-    );
+      this.isLoading = false;
+    });
+  }
+
+  ngAfterViewInit() {
+    this.childChangeSubscription = combineLatest([this.blockGraphProjected.changes.pipe(startWith(null)), this.blockGraphActual.changes.pipe(startWith(null))]).subscribe(() => {
+      this.setupBlockGraphs();
+    })
+  }
+
+  setupBlockGraphs() {
+    if (this.blockAudit) {
+      this.blockGraphProjected.forEach(graph => {
+        graph.destroy();
+        if (this.isMobile && this.mode === 'actual') {
+          graph.setup(this.blockAudit.transactions);
+        } else {
+          graph.setup(this.blockAudit.template);
+        }
+      })
+      this.blockGraphActual.forEach(graph => {
+        graph.destroy();
+        graph.setup(this.blockAudit.transactions);
+      })
+    }
   }
 
   onResize(event: any) {
-    this.isMobile = event.target.innerWidth <= 767.98;
+    const isMobile = event.target.innerWidth <= 767.98;
+    const changed = isMobile !== this.isMobile;
+    this.isMobile = isMobile;
     this.paginationMaxSize = event.target.innerWidth < 670 ? 3 : 5;
+
+    if (changed) {
+      this.changeMode(this.mode);
+    }
   }
 
-  changeMode(mode: 'missing' | 'added') {
+  changeMode(mode: 'projected' | 'actual') {
     this.router.navigate([], { fragment: mode });
-    this.mode = mode;
   }
 
   onTxClick(event: TransactionStripped): void {
     const url = new RelativeUrlPipe(this.stateService).transform(`/tx/${event.txid}`);
     this.router.navigate([url]);
   }
-
-  pageChange(page: number, target: HTMLElement) {
-  }
 }

frontend/src/app/components/block-overview-graph/tx-view.ts

@@ -7,6 +7,15 @@ import { feeLevels, mempoolFeeColors } from '../../app.constants';
 const hoverTransitionTime = 300;
 const defaultHoverColor = hexToColor('1bd8f4');
 
+const feeColors = mempoolFeeColors.map(hexToColor);
+const auditFeeColors = feeColors.map((color) => desaturate(color, 0.3));
+const auditColors = {
+  censored: hexToColor('f344df'),
+  missing: darken(desaturate(hexToColor('f344df'), 0.3), 0.7),
+  added: hexToColor('03E1E5'),
+  selected: darken(desaturate(hexToColor('039BE5'), 0.3), 0.7),
+}
+
 // convert from this class's update format to TxSprite's update format
 function toSpriteUpdate(params: ViewUpdateParams): SpriteUpdateParams {
   return {
@@ -25,7 +34,7 @@ export default class TxView implements TransactionStripped {
   vsize: number;
   value: number;
   feerate: number;
-  status?: 'found' | 'missing' | 'added';
+  status?: 'found' | 'missing' | 'added' | 'censored' | 'selected';
 
   initialised: boolean;
   vertexArray: FastVertexArray;
@@ -142,16 +151,23 @@ export default class TxView implements TransactionStripped {
   }
 
   getColor(): Color {
-    // Block audit
-    if (this.status === 'missing') {
-      return hexToColor('039BE5');
-    } else if (this.status === 'added') {
-      return hexToColor('D81B60');
-    }
-
-    // Block component
     const feeLevelIndex = feeLevels.findIndex((feeLvl) => Math.max(1, this.feerate) < feeLvl) - 1;
-    return hexToColor(mempoolFeeColors[feeLevelIndex] || mempoolFeeColors[mempoolFeeColors.length - 1]);
+    const feeLevelColor = feeColors[feeLevelIndex] || feeColors[mempoolFeeColors.length - 1];
+    // Block audit
+    switch(this.status) {
+      case 'censored':
+        return auditColors.censored;
+      case 'missing':
+        return auditColors.missing;
+      case 'added':
+        return auditColors.added;
+      case 'selected':
+        return auditColors.selected;
+      case 'found':
+        return auditFeeColors[feeLevelIndex] || auditFeeColors[mempoolFeeColors.length - 1];
+      default:
+        return feeLevelColor;
+    }
   }
 }
 
@@ -163,3 +179,22 @@ function hexToColor(hex: string): Color {
     a: 1
   };
 }
+
+function desaturate(color: Color, amount: number): Color {
+  const gray = (color.r + color.g + color.b) / 6;
+  return {
+    r: color.r + ((gray - color.r) * amount),
+    g: color.g + ((gray - color.g) * amount),
+    b: color.b + ((gray - color.b) * amount),
+    a: color.a,
+  };
+}
+
+function darken(color: Color, amount: number): Color {
+  return {
+    r: color.r * amount,
+    g: color.g * amount,
+    b: color.b * amount,
+    a: color.a,
+  }
+}

frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html

@@ -32,6 +32,16 @@
         <td class="td-width" i18n="transaction.vsize|Transaction Virtual Size">Virtual size</td>
         <td [innerHTML]="'&lrm;' + (vsize | vbytes: 2)"></td>
       </tr>
+      <tr *ngIf="tx && tx.status && tx.status.length">
+        <td class="td-width" i18n="transaction.audit-status">Audit status</td>
+        <ng-container [ngSwitch]="tx?.status">
+          <td *ngSwitchCase="'found'" i18n="transaction.audit.match">match</td>
+          <td *ngSwitchCase="'censored'" i18n="transaction.audit.removed">removed</td>
+          <td *ngSwitchCase="'missing'" i18n="transaction.audit.missing">missing</td>
+          <td *ngSwitchCase="'added'" i18n="transaction.audit.added">added</td>
+          <td *ngSwitchCase="'selected'" i18n="transaction.audit.included">included</td>
+        </ng-container>
+      </tr>
     </tbody>
   </table>
 </div>

frontend/src/app/components/block/block.component.html

@@ -110,6 +110,13 @@
                   </span>
                   </td>
                 </tr>
+                <tr *ngIf="indexingAvailable">
+                  <td i18n="block.health">Block health</td>
+                  <td>
+                    <a *ngIf="block.extras?.matchRate != null" [routerLink]="['/block-audit/' | relativeUrl, blockHash]">{{ block.extras.matchRate }}%</a>
+                    <span *ngIf="block.extras?.matchRate == null" i18n="unknown">Unknown</span>
+                  </td>
+                </tr>
               </ng-template>
             </tbody>
           </table>

frontend/src/app/components/block/block.component.ts

@@ -47,6 +47,7 @@ export class BlockComponent implements OnInit, OnDestroy {
   transactionsError: any = null;
   overviewError: any = null;
   webGlEnabled = true;
+  indexingAvailable = false;
 
   transactionSubscription: Subscription;
   overviewSubscription: Subscription;
@@ -86,6 +87,9 @@ export class BlockComponent implements OnInit, OnDestroy {
       this.timeLtr = !!ltr;
     });
 
+    this.indexingAvailable = (this.stateService.env.BASE_MODULE === 'mempool' &&
+      this.stateService.env.MINING_DASHBOARD === true);
+
     this.txsLoadingStatus$ = this.route.paramMap
       .pipe(
         switchMap(() => this.stateService.loadingIndicators$),

frontend/src/app/components/blocks-list/blocks-list.component.html

@@ -14,6 +14,8 @@
           i18n-ngbTooltip="mining.pool-name" ngbTooltip="Pool" placement="bottom" #miningpool [disableTooltip]="!isEllipsisActive(miningpool)">Pool</th>
         <th class="timestamp" i18n="latest-blocks.timestamp" *ngIf="!widget" [class]="indexingAvailable ? '' : 'legacy'">Timestamp</th>
         <th class="mined" i18n="latest-blocks.mined" *ngIf="!widget" [class]="indexingAvailable ? '' : 'legacy'">Mined</th>
+        <th *ngIf="indexingAvailable" class="health text-left" i18n="latest-blocks.health" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}"
+          i18n-ngbTooltip="latest-blocks.health" ngbTooltip="Health" placement="bottom" #health [disableTooltip]="!isEllipsisActive(health)">Health</th>
         <th *ngIf="indexingAvailable" class="reward text-right" i18n="latest-blocks.reward" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}"
           i18n-ngbTooltip="latest-blocks.reward" ngbTooltip="Reward" placement="bottom" #reward [disableTooltip]="!isEllipsisActive(reward)">Reward</th>
         <th *ngIf="indexingAvailable && !widget" class="fees text-right" i18n="latest-blocks.fees" [class]="indexingAvailable ? '' : 'legacy'">Fees</th>
@@ -37,12 +39,30 @@
               <span *ngIf="!widget" class="tooltiptext badge badge-secondary scriptmessage">{{ block.extras.coinbaseRaw | hex2ascii }}</span>
             </div>
           </td>
-          <td class="timestamp" *ngIf="!widget">
+          <td class="timestamp" *ngIf="!widget" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
             &lrm;{{ block.timestamp * 1000 | date:'yyyy-MM-dd HH:mm' }}
           </td>
           <td class="mined" *ngIf="!widget" [class]="indexingAvailable ? '' : 'legacy'">
             <app-time-since [time]="block.timestamp" [fastRender]="true"></app-time-since>
           </td>
+          <td *ngIf="indexingAvailable" class="health text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
+            <a *ngIf="block.extras?.matchRate != null" class="clear-link" [routerLink]="['/block-audit/' | relativeUrl, block.id]">
+              <div class="progress progress-health">
+                <div class="progress-bar progress-bar-health" role="progressbar"
+                  [ngStyle]="{'width': (100 - (block.extras?.matchRate || 0)) + '%' }"></div>
+                <div class="progress-text">
+                  <span>{{ block.extras.matchRate }}%</span>
+                </div>
+              </div>
+            </a>
+            <div *ngIf="block.extras?.matchRate == null" class="progress progress-health">
+              <div class="progress-bar progress-bar-health" role="progressbar"
+                [ngStyle]="{'width': '100%' }"></div>
+              <div class="progress-text">
+                <span>~</span>
+              </div>
+            </div>
+          </td>
           <td *ngIf="indexingAvailable" class="reward text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
             <app-amount [satoshis]="block.extras.reward" [noFiat]="true" digitsInfo="1.2-2"></app-amount>
           </td>
@@ -77,6 +97,9 @@
             <td class="mined" *ngIf="!widget" [class]="indexingAvailable ? '' : 'legacy'">
               <span class="skeleton-loader" style="max-width: 125px"></span>
             </td>
+            <td *ngIf="indexingAvailable" class="health text-left" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
+              <span class="skeleton-loader" style="max-width: 75px"></span>
+            </td>
             <td *ngIf="indexingAvailable" class="reward text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
               <span class="skeleton-loader" style="max-width: 75px"></span>
             </td>

frontend/src/app/components/blocks-list/blocks-list.component.scss

@@ -63,7 +63,7 @@ tr, td, th {
 }
 
 .height {
-  width: 10%;
+  width: 8%;
 }
 .height.widget {
   width: 15%;
@@ -77,12 +77,18 @@ tr, td, th {
 
 .timestamp {
   width: 18%;
-  @media (max-width: 900px) {
+  @media (max-width: 1100px) {
     display: none;
   }
 }
 .timestamp.legacy {
   width: 20%;
+  @media (max-width: 1100px) {
+    display: table-cell;
+  }
+  @media (max-width: 850px) {
+    display: none;
+  }
 }
 
 .mined {
@@ -93,6 +99,10 @@ tr, td, th {
 }
 .mined.legacy {
   width: 15%;
+  @media (max-width: 1000px) {
+    padding-right: 20px;
+    width: 20%;
+  }
   @media (max-width: 576px) {
     display: table-cell;
   }
@@ -100,6 +110,7 @@ tr, td, th {
 
 .txs {
   padding-right: 40px;
+  width: 8%;
   @media (max-width: 1100px) {
     padding-right: 10px;
   }
@@ -113,17 +124,21 @@ tr, td, th {
 }
 .txs.widget {
   padding-right: 0;
+  display: none;
   @media (max-width: 650px) {
     display: none;
   }
 }
 .txs.legacy {
-  padding-right: 80px;
+  width: 18%;
-  width: 10%;
+  display: table-cell;
+  @media (max-width: 1000px) {
+    padding-right: 20px;
+  }
 }
 
 .fees {
-  width: 10%;
+  width: 8%;
   @media (max-width: 650px) {
     display: none;
   }
@@ -133,7 +148,7 @@ tr, td, th {
 }
 
 .reward {
-  width: 10%;
+  width: 8%;
   @media (max-width: 576px) {
     width: 7%;
     padding-right: 30px;
@@ -152,8 +167,11 @@ tr, td, th {
 }
 
 .size {
-  width: 12%;
+  width: 10%;
   @media (max-width: 1000px) {
+    width: 13%;
+  }
+  @media (max-width: 950px) {
     width: 15%;
   }
   @media (max-width: 650px) {
@@ -164,12 +182,34 @@ tr, td, th {
   }
 }
 .size.legacy {
-  width: 20%;
+  width: 30%;
   @media (max-width: 576px) {
     display: table-cell;
   }
 }
 
+.health {
+  width: 10%;
+  @media (max-width: 1000px) {
+    width: 13%;
+  }
+  @media (max-width: 950px) {
+    display: none;
+  }
+}
+.health.widget {
+  width: 25%;
+  @media (max-width: 1000px) {
+    display: none;
+  }
+  @media (max-width: 767px) {
+    display: table-cell;
+  }
+  @media (max-width: 500px) {
+    display: none;
+  }
+}
+
 /* Tooltip text */
 .tooltip-custom {
   position: relative; 

frontend/src/app/components/search-form/search-form.component.html

@@ -2,9 +2,7 @@
   <div class="d-flex">
     <div class="search-box-container mr-2">
       <input (focus)="focus$.next($any($event).target.value)" (click)="click$.next($any($event).target.value)" formControlName="searchText" type="text" class="form-control" i18n-placeholder="search-form.searchbar-placeholder" placeholder="Explore the full Bitcoin ecosystem">
-      
+      <app-search-results #searchResults [hidden]="dropdownHidden" [results]="typeAhead$ | async" (selectedResult)="selectedResult($event)"></app-search-results>
-      <app-search-results #searchResults [results]="typeAhead$ | async" (selectedResult)="selectedResult($event)"></app-search-results>
-    
     </div>
     <div>
       <button [disabled]="isSearching" type="submit" class="btn btn-block btn-primary">

frontend/src/app/components/search-form/search-form.component.ts

@@ -1,4 +1,4 @@
-import { Component, OnInit, ChangeDetectionStrategy, EventEmitter, Output, ViewChild, HostListener } from '@angular/core';
+import { Component, OnInit, ChangeDetectionStrategy, EventEmitter, Output, ViewChild, HostListener, ElementRef } from '@angular/core';
 import { FormBuilder, FormGroup, Validators } from '@angular/forms';
 import { Router } from '@angular/router';
 import { AssetsService } from '../../services/assets.service';
@@ -23,6 +23,16 @@ export class SearchFormComponent implements OnInit {
   isTypeaheading$ = new BehaviorSubject<boolean>(false);
   typeAhead$: Observable<any>;
   searchForm: FormGroup;
+  dropdownHidden = false;
+
+  @HostListener('document:click', ['$event'])
+  onDocumentClick(event) {
+    if (this.elementRef.nativeElement.contains(event.target)) {
+      this.dropdownHidden = false;
+    } else {
+      this.dropdownHidden = true;
+    }
+  }
 
   regexAddress = /^([a-km-zA-HJ-NP-Z1-9]{26,35}|[a-km-zA-HJ-NP-Z1-9]{80}|[A-z]{2,5}1[a-zA-HJ-NP-Z0-9]{39,59})$/;
   regexBlockhash = /^[0]{8}[a-fA-F0-9]{56}$/;
@@ -45,6 +55,7 @@ export class SearchFormComponent implements OnInit {
     private electrsApiService: ElectrsApiService,
     private apiService: ApiService,
     private relativeUrlPipe: RelativeUrlPipe,
+    private elementRef: ElementRef,
   ) { }
 
   ngOnInit(): void {

frontend/src/app/components/television/television.component.html

@@ -11,11 +11,15 @@
         [showZoom]="false"
       ></app-mempool-graph>
     </div>
-    <div class="blockchain-wrapper">
+    <div class="blockchain-wrapper" [dir]="timeLtr ? 'rtl' : 'ltr'" [class.time-ltr]="timeLtr">
       <div class="position-container">
+        <span>
+          <div class="blocks-wrapper">
             <app-mempool-blocks></app-mempool-blocks>
             <app-blockchain-blocks></app-blockchain-blocks>
+          </div>
           <div id="divider"></div>
+        </span>
       </div>
     </div>
   </div>

frontend/src/app/components/television/television.component.scss

@@ -31,8 +31,9 @@
 
   .position-container {
     position: absolute;
-    left: 50%;
+    left: 0;
     bottom: 170px;
+    transform: translateX(50vw);
   }
 
   #divider {
@@ -47,9 +48,33 @@
       top: -28px;
     }
   }
+
+  &.time-ltr {
+    .blocks-wrapper {
+      transform: scaleX(-1);
     }
+  }
+}
+
+:host-context(.ltr-layout) {
+  .blockchain-wrapper.time-ltr .blocks-wrapper,
+  .blockchain-wrapper .blocks-wrapper {
+    direction: ltr;
+  }
+}
+
+:host-context(.rtl-layout) {
+  .blockchain-wrapper.time-ltr .blocks-wrapper,
+  .blockchain-wrapper .blocks-wrapper {
+    direction: rtl;
+  }
+}
+
 .tv-container {
   display: flex;
   margin-top: 0px;
   flex-direction: column;
 }
+
+
+

frontend/src/app/components/television/television.component.ts

@@ -1,4 +1,4 @@
-import { Component, OnInit } from '@angular/core';
+import { Component, OnInit, OnDestroy } from '@angular/core';
 import { WebsocketService } from '../../services/websocket.service';
 import { OptimizedMempoolStats } from '../../interfaces/node-api.interface';
 import { StateService } from '../../services/state.service';
@@ -6,7 +6,7 @@ import { ApiService } from '../../services/api.service';
 import { SeoService } from '../../services/seo.service';
 import { ActivatedRoute } from '@angular/router';
 import { map, scan, startWith, switchMap, tap } from 'rxjs/operators';
-import { interval, merge, Observable } from 'rxjs';
+import { interval, merge, Observable, Subscription } from 'rxjs';
 import { ChangeDetectionStrategy } from '@angular/core';
 
 @Component({
@@ -15,11 +15,13 @@ import { ChangeDetectionStrategy } from '@angular/core';
   styleUrls: ['./television.component.scss'],
   changeDetection: ChangeDetectionStrategy.OnPush
 })
-export class TelevisionComponent implements OnInit {
+export class TelevisionComponent implements OnInit, OnDestroy {
 
   mempoolStats: OptimizedMempoolStats[] = [];
   statsSubscription$: Observable<OptimizedMempoolStats[]>;
   fragment: string;
+  timeLtrSubscription: Subscription;
+  timeLtr: boolean = this.stateService.timeLtr.value;
 
   constructor(
     private websocketService: WebsocketService,
@@ -37,6 +39,10 @@ export class TelevisionComponent implements OnInit {
     this.seoService.setTitle($localize`:@@46ce8155c9ab953edeec97e8950b5a21e67d7c4e:TV view`);
     this.websocketService.want(['blocks', 'live-2h-chart', 'mempool-blocks']);
 
+    this.timeLtrSubscription = this.stateService.timeLtr.subscribe((ltr) => {
+      this.timeLtr = !!ltr;
+    });
+
     this.statsSubscription$ = merge(
       this.stateService.live2Chart$.pipe(map(stats => [stats])),
       this.route.fragment
@@ -70,4 +76,8 @@ export class TelevisionComponent implements OnInit {
       })
     );
   }
+
+  ngOnDestroy() {
+    this.timeLtrSubscription.unsubscribe();
+  }
 }

frontend/src/app/docs/api-docs/api-docs.component.html

@@ -106,6 +106,20 @@
       </div>
     </div>
 
+    <div id="electrs" *ngIf="whichTab === 'electrs'">
+      <div class="doc-content no-sidebar">
+        <div class="doc-item-container">
+          <p class='subtitle'>Hostname</p>
+          <p>{{plainHostname}}</p>
+          <p class="subtitle">Port</p>
+          <p>{{electrsPort}}</p>
+          <p class="subtitle">SSL</p>
+          <p>Enabled</p>
+          <p class="note" *ngIf="network.val !== 'signet'">Electrum RPC interface for Bitcoin Signet is <a href="/signet/docs/api/electrs">publicly available</a>. Electrum RPC interface for all other networks is available to <a href='/enterprise'>sponsors</a> only—whitelisting is required.</p>
+        </div>
+      </div>
+    </div>
+
   </div>
 </ng-container>
 

frontend/src/app/docs/api-docs/api-docs.component.scss

@@ -1,7 +1,21 @@
+.center {
+  text-align: center;
+}
+
+.note {
+  font-style: italic;
+}
+
 .text-small {
   font-size: 12px;
 }
 
+.container-xl {
+  display: flex;
+  min-height: 75vh;
+  flex-direction: column;
+}
+
 code {
   background-color: #1d1f31;
   font-family: Consolas,Monaco,Lucida Console,Liberation Mono,DejaVu Sans Mono,Bitstream Vera Sans Mono,Courier New;
@@ -116,6 +130,10 @@ li.nav-item {
   float: right;
 }
 
+.doc-content.no-sidebar {
+  width: 100%
+}
+
 h3 {
   margin: 2rem 0 0 0;
 }

frontend/src/app/docs/api-docs/api-docs.component.ts

@@ -12,6 +12,8 @@ import { FaqTemplateDirective } from '../faq-template/faq-template.component';
   styleUrls: ['./api-docs.component.scss']
 })
 export class ApiDocsComponent implements OnInit, AfterViewInit {
+  plainHostname = document.location.hostname;
+  electrsPort = 0;
   hostname = document.location.hostname;
   network$: Observable<string>;
   active = 0;
@@ -82,6 +84,20 @@ export class ApiDocsComponent implements OnInit, AfterViewInit {
 
     this.network$.subscribe((network) => {
       this.active = (network === 'liquid' || network === 'liquidtestnet') ? 2 : 0;
+      switch( network ) {
+        case "":
+          this.electrsPort = 50002; break;
+        case "mainnet":
+          this.electrsPort = 50002; break;
+        case "testnet":
+          this.electrsPort = 60002; break;
+        case "signet":
+          this.electrsPort = 60602; break;
+        case "liquid":
+          this.electrsPort = 51002; break;
+        case "liquidtestnet":
+          this.electrsPort = 51302; break;
+      }
     });
   }
 

frontend/src/app/docs/docs/docs.component.html

@@ -32,6 +32,15 @@
         </ng-template>
       </li>
 
+      <li [ngbNavItem]="3" *ngIf="showElectrsTab" role="presentation">
+        <a ngbNavLink [routerLink]="['/docs/api/electrs' | relativeUrl]" role="tab">API - Electrum RPC</a>
+        <ng-template ngbNavContent>
+
+          <app-api-docs [whichTab]="'electrs'"></app-api-docs>
+
+        </ng-template>
+      </li>
+
     </ul>
 
     <div id="main-tab-content" [ngbNavOutlet]="nav"></div>

frontend/src/app/docs/docs/docs.component.ts

@@ -15,6 +15,7 @@ export class DocsComponent implements OnInit {
   env: Env;
   showWebSocketTab = true;
   showFaqTab = true;
+  showElectrsTab = true;
 
   @HostBinding('attr.dir') dir = 'ltr';
 
@@ -34,14 +35,18 @@ export class DocsComponent implements OnInit {
     } else if( url[1].path === "rest" ) {
         this.activeTab = 1;
         this.seoService.setTitle($localize`:@@e351b40b3869a5c7d19c3d4918cb1ac7aaab95c4:API`);
-    } else {
+    } else if( url[1].path === "websocket" ) {
         this.activeTab = 2;
         this.seoService.setTitle($localize`:@@e351b40b3869a5c7d19c3d4918cb1ac7aaab95c4:API`);
+    } else {
+        this.activeTab = 3;
+        this.seoService.setTitle($localize`:@@e351b40b3869a5c7d19c3d4918cb1ac7aaab95c4:API`);
     }
 
     this.env = this.stateService.env;
     this.showWebSocketTab = ( ! ( ( this.stateService.network === "bisq" ) || ( this.stateService.network === "liquidtestnet" ) ) );
     this.showFaqTab = ( this.env.BASE_MODULE === 'mempool' ) ? true : false;
+    this.showElectrsTab = this.stateService.env.OFFICIAL_MEMPOOL_SPACE && ( this.stateService.network !== "bisq" );
 
     document.querySelector<HTMLElement>( "html" ).style.scrollBehavior = "smooth";
   }

frontend/src/app/interfaces/node-api.interface.ts

@@ -141,7 +141,7 @@ export interface TransactionStripped {
   fee: number;
   vsize: number;
   value: number;
-  status?: 'found' | 'missing' | 'added';
+  status?: 'found' | 'missing' | 'added' | 'censored' | 'selected';
 }
 
 export interface RewardStats {

frontend/src/app/interfaces/websocket.interface.ts

@@ -70,7 +70,7 @@ export interface TransactionStripped {
   fee: number;
   vsize: number;
   value: number;
-  status?: 'found' | 'missing' | 'added';
+  status?: 'found' | 'missing' | 'added' | 'censored' | 'selected';
 }
 
 export interface IBackendInfo {

frontend/src/styles.scss

@@ -668,6 +668,15 @@ h1, h2, h3 {
   background-color: #2e324e;
 }
 
+.progress.progress-health {
+  background: repeating-linear-gradient(to right, #2d3348, #2d3348 0%, #105fb0 0%, #1a9436 100%);
+  justify-content: flex-end;
+}
+
+.progress-bar.progress-bar-health {
+  background: #2d3348;
+}
+
 .mt-2-5, .my-2-5 {
   margin-top: 0.75rem !important;
 }

production/install

@@ -401,7 +401,7 @@ FREEBSD_PKG+=(nginx rsync py39-certbot-nginx mariadb105-server keybase)
 FREEBSD_PKG+=(geoipupdate)
 
 FREEBSD_UNFURL_PKG=()
-FREEBSD_UNFURL_PKG+=(nvidia-driver-470-470.129.06 chromium xinit xterm twm ja-sourcehansans-otf)
+FREEBSD_UNFURL_PKG+=(nvidia-driver-470 chromium xinit xterm twm ja-sourcehansans-otf)
 FREEBSD_UNFURL_PKG+=(zh-sourcehansans-sc-otf ko-aleefonts-ttf lohit tlwg-ttf)
 
 #############################
@@ -1324,9 +1324,9 @@ case $OS in
         osPackageInstall ${CLN_PKG}
 
         echo "[*] Installing Core Lightning mainnet Cronjob"
-        crontab_cln+='@reboot sleep 60 ; screen -dmS main lightningd --alias `hostname` --bitcoin-datadir /bitcoin\n'
+        crontab_cln+='@reboot sleep 60 ; screen -dmS main lightningd --alias `hostname` --fee-base 0 --bitcoin-datadir /bitcoin\n'
-        crontab_cln+='@reboot sleep 90 ; screen -dmS tes lightningd --alias `hostname` --bitcoin-datadir /bitcoin --network testnet\n'
+        crontab_cln+='@reboot sleep 90 ; screen -dmS tes lightningd --alias `hostname` --fee-base 0 --bitcoin-datadir /bitcoin --network testnet\n'
-        crontab_cln+='@reboot sleep 120 ; screen -dmS sig lightningd --alias `hostname` --bitcoin-datadir /bitcoin --network signet\n'
+        crontab_cln+='@reboot sleep 120 ; screen -dmS sig lightningd --alias `hostname` --fee-base 0 --bitcoin-datadir /bitcoin --network signet\n'
         echo "${crontab_cln}" | crontab -u "${CLN_USER}" -
     ;;
     Debian)

unfurler/src/language/lang.ts

@@ -62,15 +62,15 @@ export const languages = languageDict;
 
 // expects path to start with a leading '/'
 export function parseLanguageUrl(path) {
-  const parts = path.split('/');
+  const parts = path.split('/').filter(part => part.length);
   let lang;
   let rest;
-  if (languages[parts[1]]) {
+  if (languages[parts[0]]) {
-    lang = parts[1];
+    lang = parts[0];
-    rest = '/' + parts.slice(2).join('/');
+    rest = '/' + parts.slice(1).join('/');
   } else {
     lang = null;
-    rest = path;
+    rest = '/' + parts.join('/');
   }
   if (lang === 'en') {
     lang = null;