Merge branch 'master' into nymkappa/network-switch-align

backend/mempool-config.sample.json

@@ -125,5 +125,16 @@
     "LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1",
     "BISQ_URL": "https://bisq.markets/api",
     "BISQ_ONION": "http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api"
+  },
+  "REPLICATION": {
+    "ENABLED": false,
+    "AUDIT": false,
+    "AUDIT_START_HEIGHT": 774000,
+    "SERVERS": [
+      "list",
+      "of",
+      "trusted",
+      "servers"
+    ]
   }
 }

backend/src/__fixtures__/mempool-config.template.json

@@ -121,5 +121,11 @@
   },
   "CLIGHTNING": {
     "SOCKET": "__CLIGHTNING_SOCKET__"
+  },
+  "REPLICATION": {
+    "ENABLED": false,
+    "AUDIT": false,
+    "AUDIT_START_HEIGHT": 774000,
+    "SERVERS": []
   }
 }

backend/src/__tests__/config.test.ts

@@ -120,6 +120,13 @@ describe('Mempool Backend Config', () => {
         GEOLITE2_ASN: '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb',
         GEOIP2_ISP: '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb'
       });
+
+      expect(config.REPLICATION).toStrictEqual({
+        ENABLED: false,
+        AUDIT: false,
+        AUDIT_START_HEIGHT: 774000,
+        SERVERS: []
+      });
     });
   });
 

backend/src/api/audit.ts

@@ -1,12 +1,12 @@
 import config from '../config';
 import logger from '../logger';
-import { TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
+import { MempoolTransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
 import rbfCache from './rbf-cache';
 
 const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
 
 class Audit {
-  auditBlock(transactions: TransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended })
+  auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended })
    : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], score: number, similarity: number } {
     if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
       return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], score: 0, similarity: 1 };
@@ -14,7 +14,7 @@ class Audit {
 
     const matches: string[] = []; // present in both mined block and template
     const added: string[] = []; // present in mined block, not in template
-    const fresh: string[] = []; // missing, but firstSeen within PROPAGATION_MARGIN
+    const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN
     const fullrbf: string[] = []; // either missing or present, and part of a fullrbf replacement
     const isCensored = {}; // missing, without excuse
     const isDisplaced = {};
@@ -36,10 +36,13 @@ class Audit {
     // look for transactions that were expected in the template, but missing from the mined block
     for (const txid of projectedBlocks[0].transactionIds) {
       if (!inBlock[txid]) {
-        // tx is recent, may have reached the miner too late for inclusion
         if (rbfCache.isFullRbf(txid)) {
           fullrbf.push(txid);
         } else if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) {
+          // tx is recent, may have reached the miner too late for inclusion
+          fresh.push(txid);
+        } else if (mempool[txid]?.lastBoosted != null && (now - (mempool[txid]?.lastBoosted || 0)) <= PROPAGATION_MARGIN) {
+          // tx was recently cpfp'd, miner may not have the latest effective rate
           fresh.push(txid);
         } else {
           isCensored[txid] = true;

backend/src/api/bitcoin/bitcoin-api.ts

@@ -65,17 +65,11 @@ class BitcoinApi implements AbstractBitcoinApi {
   }
 
   $getBlockHeightTip(): Promise<number> {
-    return this.bitcoindClient.getChainTips()
+    return this.bitcoindClient.getBlockCount();
-      .then((result: IBitcoinApi.ChainTips[]) => {
-        return result.find(tip => tip.status === 'active')!.height;
-      });
   }
 
   $getBlockHashTip(): Promise<string> {
-    return this.bitcoindClient.getChainTips()
+    return this.bitcoindClient.getBestBlockHash();
-      .then((result: IBitcoinApi.ChainTips[]) => {
-        return result.find(tip => tip.status === 'active')!.hash;
-      });
   }
 
   $getTxIdsForBlock(hash: string): Promise<string[]> {

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -121,7 +121,6 @@ class BitcoinRoutes {
           .get(config.MEMPOOL.API_URL_PREFIX + 'block-height/:height', this.getBlockHeight)
           .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address', this.getAddress)
           .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', this.getAddressTransactions)
-          .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs/chain/:txId', this.getAddressTransactions)
           .get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', this.getAddressPrefix)
           ;
       }
@@ -546,27 +545,28 @@ class BitcoinRoutes {
     }
   }
 
-  private async getAddressTransactions(req: Request, res: Response) {
+  private async getAddressTransactions(req: Request, res: Response): Promise<void> {
     if (config.MEMPOOL.BACKEND === 'none') {
       res.status(405).send('Address lookups cannot be used with bitcoind as backend.');
       return;
     }
 
     try {
-      const transactions = await bitcoinApi.$getAddressTransactions(req.params.address, req.params.txId);
+      let lastTxId: string = '';
+      if (req.query.after_txid && typeof req.query.after_txid === 'string') {
+        lastTxId = req.query.after_txid;
+      }
+      const transactions = await bitcoinApi.$getAddressTransactions(req.params.address, lastTxId);
       res.json(transactions);
     } catch (e) {
       if (e instanceof Error && e.message && (e.message.indexOf('too long') > 0 || e.message.indexOf('confirmed status') > 0)) {
-        return res.status(413).send(e instanceof Error ? e.message : e);
+        res.status(413).send(e instanceof Error ? e.message : e);
+        return;
       }
       res.status(500).send(e instanceof Error ? e.message : e);
     }
   }
 
-  private async getAdressTxChain(req: Request, res: Response) {
-    res.status(501).send('Not implemented');
-  }
-
   private async getAddressPrefix(req: Request, res: Response) {
     try {
       const blockHash = await bitcoinApi.$getAddressPrefix(req.params.prefix);

backend/src/api/blocks.ts

@@ -76,11 +76,14 @@ class Blocks {
     blockHash: string,
     blockHeight: number,
     onlyCoinbase: boolean,
+    txIds: string[] | null = null,
     quiet: boolean = false,
     addMempoolData: boolean = false,
   ): Promise<TransactionExtended[]> {
     const transactions: TransactionExtended[] = [];
-    const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
+    if (!txIds) {
+      txIds = await bitcoinApi.$getTxIdsForBlock(blockHash);
+    }
 
     const mempool = memPool.getMempool();
     let transactionsFound = 0;
@@ -554,7 +557,7 @@ class Blocks {
           }
           const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
           const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
-          const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
+          const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, null, true);
           const blockExtended = await this.$getBlockExtended(block, transactions);
 
           newlyIndexed++;
@@ -586,7 +589,7 @@ class Blocks {
 
     let fastForwarded = false;
     let handledBlocks = 0;
-    const blockHeightTip = await bitcoinApi.$getBlockHeightTip();
+    const blockHeightTip = await bitcoinCoreApi.$getBlockHeightTip();
     this.updateTimerProgress(timer, 'got block height tip');
 
     if (this.blocks.length === 0) {
@@ -639,11 +642,11 @@ class Blocks {
       }
 
       this.updateTimerProgress(timer, `getting block data for ${this.currentBlockHeight}`);
-      const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
+      const blockHash = await bitcoinCoreApi.$getBlockHash(this.currentBlockHeight);
       const verboseBlock = await bitcoinClient.getBlock(blockHash, 2);
       const block = BitcoinApi.convertBlock(verboseBlock);
-      const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
+      const txIds: string[] = verboseBlock.tx.map(tx => tx.txid);
-      const transactions = await this.$getTransactionsExtended(blockHash, block.height, false, false, true) as MempoolTransactionExtended[];
+      const transactions = await this.$getTransactionsExtended(blockHash, block.height, false, txIds, false, true) as MempoolTransactionExtended[];
       if (config.MEMPOOL.BACKEND !== 'esplora') {
         // fill in missing transaction fee data from verboseBlock
         for (let i = 0; i < transactions.length; i++) {

backend/src/api/database-migration.ts

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 63;
+  private static currentVersion = 64;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -543,6 +543,11 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fullrbf_txs JSON DEFAULT "[]"');
       await this.updateToSchemaVersion(63);
     }
+
+    if (databaseSchemaVersion < 64 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `nodes` ADD features text NULL');
+      await this.updateToSchemaVersion(64);
+    }
   }
 
   /**

backend/src/api/explorer/nodes.api.ts

@@ -3,6 +3,7 @@ import DB from '../../database';
 import { ResultSetHeader } from 'mysql2';
 import { ILightningApi } from '../lightning/lightning-api.interface';
 import { ITopNodesPerCapacity, ITopNodesPerChannels } from '../../mempool.interfaces';
+import { bin2hex } from '../../utils/format';
 
 class NodesApi {
   public async $getWorldNodes(): Promise<any> {
@@ -56,7 +57,8 @@ class NodesApi {
           UNIX_TIMESTAMP(updated_at) AS updated_at, color, sockets as sockets,
           as_number, city_id, country_id, subdivision_id, longitude, latitude,
           geo_names_iso.names as iso_code, geo_names_as.names as as_organization, geo_names_city.names as city,
-          geo_names_country.names as country, geo_names_subdivision.names as subdivision
+          geo_names_country.names as country, geo_names_subdivision.names as subdivision,
+          features
         FROM nodes
         LEFT JOIN geo_names geo_names_as on geo_names_as.id = as_number
         LEFT JOIN geo_names geo_names_city on geo_names_city.id = city_id
@@ -76,6 +78,23 @@ class NodesApi {
       node.city = JSON.parse(node.city);
       node.country = JSON.parse(node.country);
 
+      // Features      
+      node.features = JSON.parse(node.features);
+      node.featuresBits = null;
+      if (node.features) {
+        let maxBit = 0;
+        for (const feature of node.features) {
+          maxBit = Math.max(maxBit, feature.bit);
+        }
+        maxBit = Math.ceil(maxBit / 4) * 4 - 1;
+        
+        node.featuresBits = new Array(maxBit + 1).fill(0);
+        for (const feature of node.features) {
+          node.featuresBits[feature.bit] = 1;
+        }
+        node.featuresBits = bin2hex(node.featuresBits.reverse().join(''));
+      }
+
       // Active channels and capacity
       const activeChannelsStats: any = await this.$getActiveChannelsStats(public_key);
       node.active_channel_count = activeChannelsStats.active_channel_count ?? 0;
@@ -656,10 +675,19 @@ class NodesApi {
           alias_search,
           color,
           sockets,
-          status
+          status,
+          features
         )
-        VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, ?, 1)
+        VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, ?, 1, ?)
-        ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, alias_search = ?, color = ?, sockets = ?, status = 1`;
+        ON DUPLICATE KEY UPDATE
+          updated_at = FROM_UNIXTIME(?),
+          alias = ?,
+          alias_search = ?,
+          color = ?,
+          sockets = ?,
+          status = 1,
+          features = ?
+      `;
 
       await DB.query(query, [
         node.pub_key,
@@ -668,11 +696,13 @@ class NodesApi {
         this.aliasToSearchText(node.alias),
         node.color,
         sockets,
+        JSON.stringify(node.features),
         node.last_update,
         node.alias,
         this.aliasToSearchText(node.alias),
         node.color,
         sockets,
+        JSON.stringify(node.features),
       ]);
     } catch (e) {
       logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));

backend/src/api/lightning/clightning/clightning-convert.ts

@@ -2,8 +2,91 @@ import { ILightningApi } from '../lightning-api.interface';
 import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher';
 import logger from '../../../logger';
 import { Common } from '../../common';
+import { hex2bin } from '../../../utils/format';
 import config from '../../../config';
 
+// https://github.com/lightningnetwork/lnd/blob/master/lnwire/features.go
+export enum FeatureBits {
+	DataLossProtectRequired = 0,
+	DataLossProtectOptional = 1,
+	InitialRoutingSync = 3,
+	UpfrontShutdownScriptRequired = 4,
+	UpfrontShutdownScriptOptional = 5,
+	GossipQueriesRequired = 6,
+	GossipQueriesOptional = 7,
+	TLVOnionPayloadRequired = 8,
+	TLVOnionPayloadOptional = 9,
+	StaticRemoteKeyRequired = 12,
+	StaticRemoteKeyOptional = 13,
+	PaymentAddrRequired = 14,
+	PaymentAddrOptional = 15,
+	MPPRequired = 16,
+	MPPOptional = 17,
+	WumboChannelsRequired = 18,
+	WumboChannelsOptional = 19,
+	AnchorsRequired = 20,
+	AnchorsOptional = 21,
+	AnchorsZeroFeeHtlcTxRequired = 22,
+	AnchorsZeroFeeHtlcTxOptional = 23,
+	ShutdownAnySegwitRequired = 26,
+	ShutdownAnySegwitOptional = 27,
+	AMPRequired = 30,
+	AMPOptional = 31,
+	ExplicitChannelTypeRequired = 44,
+	ExplicitChannelTypeOptional = 45,
+	ScidAliasRequired = 46,
+	ScidAliasOptional = 47,
+	PaymentMetadataRequired = 48,
+	PaymentMetadataOptional = 49,
+	ZeroConfRequired = 50,
+	ZeroConfOptional = 51,
+	KeysendRequired = 54,
+	KeysendOptional = 55,
+	ScriptEnforcedLeaseRequired = 2022,
+	ScriptEnforcedLeaseOptional = 2023,
+	MaxBolt11Feature = 5114,
+};
+  
+export const FeaturesMap = new Map<FeatureBits, string>([
+	[FeatureBits.DataLossProtectRequired, 'data-loss-protect'],
+	[FeatureBits.DataLossProtectOptional, 'data-loss-protect'],
+	[FeatureBits.InitialRoutingSync, 'initial-routing-sync'],
+	[FeatureBits.UpfrontShutdownScriptRequired, 'upfront-shutdown-script'],
+	[FeatureBits.UpfrontShutdownScriptOptional, 'upfront-shutdown-script'],
+	[FeatureBits.GossipQueriesRequired, 'gossip-queries'],
+	[FeatureBits.GossipQueriesOptional, 'gossip-queries'],
+	[FeatureBits.TLVOnionPayloadRequired, 'tlv-onion'],
+	[FeatureBits.TLVOnionPayloadOptional, 'tlv-onion'],
+	[FeatureBits.StaticRemoteKeyOptional, 'static-remote-key'],
+	[FeatureBits.StaticRemoteKeyRequired, 'static-remote-key'],
+	[FeatureBits.PaymentAddrOptional, 'payment-addr'],
+	[FeatureBits.PaymentAddrRequired, 'payment-addr'],
+	[FeatureBits.MPPOptional, 'multi-path-payments'],
+	[FeatureBits.MPPRequired, 'multi-path-payments'],
+	[FeatureBits.AnchorsRequired, 'anchor-commitments'],
+	[FeatureBits.AnchorsOptional, 'anchor-commitments'],
+	[FeatureBits.AnchorsZeroFeeHtlcTxRequired, 'anchors-zero-fee-htlc-tx'],
+	[FeatureBits.AnchorsZeroFeeHtlcTxOptional, 'anchors-zero-fee-htlc-tx'],
+	[FeatureBits.WumboChannelsRequired, 'wumbo-channels'],
+	[FeatureBits.WumboChannelsOptional, 'wumbo-channels'],
+	[FeatureBits.AMPRequired, 'amp'],
+	[FeatureBits.AMPOptional, 'amp'],
+	[FeatureBits.PaymentMetadataOptional, 'payment-metadata'],
+	[FeatureBits.PaymentMetadataRequired, 'payment-metadata'],
+	[FeatureBits.ExplicitChannelTypeOptional, 'explicit-commitment-type'],
+	[FeatureBits.ExplicitChannelTypeRequired, 'explicit-commitment-type'],
+	[FeatureBits.KeysendOptional, 'keysend'],
+	[FeatureBits.KeysendRequired, 'keysend'],
+	[FeatureBits.ScriptEnforcedLeaseRequired, 'script-enforced-lease'],
+	[FeatureBits.ScriptEnforcedLeaseOptional, 'script-enforced-lease'],
+	[FeatureBits.ScidAliasRequired, 'scid-alias'],
+	[FeatureBits.ScidAliasOptional, 'scid-alias'],
+	[FeatureBits.ZeroConfRequired, 'zero-conf'],
+	[FeatureBits.ZeroConfOptional, 'zero-conf'],
+	[FeatureBits.ShutdownAnySegwitRequired, 'shutdown-any-segwit'],
+	[FeatureBits.ShutdownAnySegwitOptional, 'shutdown-any-segwit'],
+]);
+
 /**
  * Convert a clightning "listnode" entry to a lnd node entry
  */
@@ -17,10 +100,36 @@ export function convertNode(clNode: any): ILightningApi.Node {
       custom_records = undefined;
     }
   }
+
+  const nodeFeatures: ILightningApi.Feature[] = [];
+  const nodeFeaturesBinary = hex2bin(clNode.features).split('').reverse().join('');
+
+  for (let i = 0; i < nodeFeaturesBinary.length; i++) {
+    if (nodeFeaturesBinary[i] === '0') {
+      continue;
+    }
+    const feature = FeaturesMap.get(i);
+    if (!feature) {
+      nodeFeatures.push({
+        bit: i,
+        name: 'unknown',
+        is_required: i % 2 === 0,
+        is_known: false
+      });
+    } else {
+      nodeFeatures.push({
+        bit: i,
+        name: feature,
+        is_required: i % 2 === 0,
+        is_known: true
+      });
+    }
+  }
+
   return {
     alias: clNode.alias ?? '',
     color: `#${clNode.color ?? ''}`,
-    features: [], // TODO parse and return clNode.feature
+    features: nodeFeatures,
     pub_key: clNode.nodeid,
     addresses: clNode.addresses?.map((addr) => {
       let address = addr.address;

backend/src/api/lightning/lightning-api.interface.ts

@@ -79,6 +79,7 @@ export namespace ILightningApi {
   }
 
   export interface Feature {
+    bit: number;
     name: string;
     is_required: boolean;
     is_known: boolean;

backend/src/api/lightning/lnd/lnd-api.ts

@@ -41,8 +41,23 @@ class LndApi implements AbstractLightningApi {
   }
 
   async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
-    return axios.get<ILightningApi.NetworkGraph>(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig)
+    const graph = await axios.get<ILightningApi.NetworkGraph>(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig)
       .then((response) => response.data);
+
+    for (const node of graph.nodes) {
+      const nodeFeatures: ILightningApi.Feature[] = [];
+      for (const bit in node.features) {        
+        nodeFeatures.push({
+          bit: parseInt(bit, 10),
+          name: node.features[bit].name,  
+          is_required: node.features[bit].is_required,
+          is_known: node.features[bit].is_known,
+        });
+      }
+      node.features = nodeFeatures;
+    }
+
+    return graph;
   }
 }
 

backend/src/api/mempool-blocks.ts

@@ -457,6 +457,7 @@ class MempoolBlocks {
               };
               if (matched) {
                 descendants.push(relative);
+                mempoolTx.lastBoosted = Math.max(mempoolTx.lastBoosted || 0, mempool[txid].firstSeen || 0);
               } else {
                 ancestors.push(relative);
               }

backend/src/api/websocket-handler.ts

@@ -229,14 +229,16 @@ class WebsocketHandler {
           if (parsedMessage && parsedMessage['track-rbf-summary'] != null) {
             if (parsedMessage['track-rbf-summary']) {
               client['track-rbf-summary'] = true;
+              if (this.socketData['rbfSummary'] != null) {
                 response['rbfLatestSummary'] = this.socketData['rbfSummary'];
+              }
             } else {
               client['track-rbf-summary'] = false;
             }
           }
 
           if (parsedMessage.action === 'init') {
-            if (!this.socketData['blocks']?.length || !this.socketData['da']) {
+            if (!this.socketData['blocks']?.length || !this.socketData['da'] || !this.socketData['backendInfo'] || !this.socketData['conversions']) {
               this.updateSocketData();
             }
             if (!this.socketData['blocks']?.length) {
@@ -419,7 +421,7 @@ class WebsocketHandler {
     memPool.addToSpendMap(newTransactions);
     const recommendedFees = feeApi.getRecommendedFee();
 
-    const latestTransactions = newTransactions.slice(0, 6).map((tx) => Common.stripTransaction(tx));
+    const latestTransactions = memPool.getLatestTransactions();
 
     // update init data
     const socketDataFields = {

backend/src/config.ts

@@ -132,6 +132,12 @@ interface IConfig {
     GEOLITE2_ASN: string;
     GEOIP2_ISP: string;
   },
+  REPLICATION: {
+    ENABLED: boolean;
+    AUDIT: boolean;
+    AUDIT_START_HEIGHT: number;
+    SERVERS: string[];
+  }
 }
 
 const defaults: IConfig = {
@@ -264,6 +270,12 @@ const defaults: IConfig = {
     'GEOLITE2_ASN': '/usr/local/share/GeoIP/GeoLite2-ASN.mmdb',
     'GEOIP2_ISP': '/usr/local/share/GeoIP/GeoIP2-ISP.mmdb'
   },
+  'REPLICATION': {
+    'ENABLED': false,
+    'AUDIT': false,
+    'AUDIT_START_HEIGHT': 774000,
+    'SERVERS': [],
+  }
 };
 
 class Config implements IConfig {
@@ -283,6 +295,7 @@ class Config implements IConfig {
   PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
   EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
   MAXMIND: IConfig['MAXMIND'];
+  REPLICATION: IConfig['REPLICATION'];
 
   constructor() {
     const configs = this.merge(configFromFile, defaults);
@@ -302,6 +315,7 @@ class Config implements IConfig {
     this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
     this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;
     this.MAXMIND = configs.MAXMIND;
+    this.REPLICATION = configs.REPLICATION;
   }
 
   merge = (...objects: object[]): IConfig => {

backend/src/index.ts

@@ -169,6 +169,7 @@ class Server {
   }
 
   async runMainUpdateLoop(): Promise<void> {
+    const start = Date.now();
     try {
       try {
         await memPool.$updateMemPoolInfo();
@@ -188,7 +189,9 @@ class Server {
       indexer.$run();
 
       // rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS
-      setTimeout(this.runMainUpdateLoop.bind(this), numHandledBlocks > 0 ? 1 : config.MEMPOOL.POLL_RATE_MS);
+      const elapsed = Date.now() - start;
+      const remainingTime = Math.max(0, config.MEMPOOL.POLL_RATE_MS - elapsed)
+      setTimeout(this.runMainUpdateLoop.bind(this), numHandledBlocks > 0 ? 0 : remainingTime);
       this.backendRetryCount = 0;
     } catch (e: any) {
       this.backendRetryCount++;

backend/src/indexer.ts

@@ -7,6 +7,7 @@ import bitcoinClient from './api/bitcoin/bitcoin-client';
 import priceUpdater from './tasks/price-updater';
 import PricesRepository from './repositories/PricesRepository';
 import config from './config';
+import auditReplicator from './replication/AuditReplication';
 
 export interface CoreIndex {
   name: string;
@@ -136,6 +137,7 @@ class Indexer {
       await blocks.$generateBlocksSummariesDatabase();
       await blocks.$generateCPFPDatabase();
       await blocks.$generateAuditStats();
+      await auditReplicator.$sync();
     } catch (e) {
       this.indexerRunning = false;
       logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e));

backend/src/mempool.interfaces.ts

@@ -100,6 +100,7 @@ export interface MempoolTransactionExtended extends TransactionExtended {
   adjustedVsize: number;
   adjustedFeePerVsize: number;
   inputs?: number[];
+  lastBoosted?: number;
 }
 
 export interface AuditTransaction {
@@ -236,6 +237,15 @@ export interface BlockSummary {
   transactions: TransactionStripped[];
 }
 
+export interface AuditSummary extends BlockAudit {
+  timestamp?: number,
+  size?: number,
+  weight?: number,
+  tx_count?: number,
+  transactions: TransactionStripped[];
+  template?: TransactionStripped[];
+}
+
 export interface BlockPrice {
   height: number;
   priceId: number;

backend/src/replication/AuditReplication.ts

@@ -0,0 +1,134 @@
+import DB from '../database';
+import logger from '../logger';
+import { AuditSummary } from '../mempool.interfaces';
+import blocksAuditsRepository from '../repositories/BlocksAuditsRepository';
+import blocksSummariesRepository from '../repositories/BlocksSummariesRepository';
+import { $sync } from './replicator';
+import config from '../config';
+import { Common } from '../api/common';
+import blocks from '../api/blocks';
+
+const BATCH_SIZE = 16;
+
+/**
+ * Syncs missing block template and audit data from trusted servers
+ */
+class AuditReplication {
+  inProgress: boolean = false;
+  skip: Set<string> = new Set();
+
+  public async $sync(): Promise<void> {
+    if (!config.REPLICATION.ENABLED || !config.REPLICATION.AUDIT) {
+      // replication not enabled
+      return;
+    }
+    if (this.inProgress) {
+      logger.info(`AuditReplication sync already in progress`, 'Replication');
+      return;
+    }
+    this.inProgress = true;
+
+    const missingAudits = await this.$getMissingAuditBlocks();
+
+    logger.debug(`Fetching missing audit data for ${missingAudits.length} blocks from trusted servers`, 'Replication');
+    
+    let totalSynced = 0;
+    let totalMissed = 0;
+    let loggerTimer = Date.now();
+    // process missing audits in batches of 
+    for (let i = 0; i < missingAudits.length; i += BATCH_SIZE) {
+      const slice = missingAudits.slice(i, i + BATCH_SIZE);
+      const results = await Promise.all(slice.map(hash => this.$syncAudit(hash)));
+      const synced = results.reduce((total, status) => status ? total + 1 : total, 0);
+      totalSynced += synced;
+      totalMissed += (slice.length - synced);
+      if (Date.now() - loggerTimer > 10000) {
+        loggerTimer = Date.now();
+        logger.info(`Found ${totalSynced} / ${totalSynced + totalMissed} of ${missingAudits.length} missing audits`, 'Replication');
+      }
+      await Common.sleep$(1000);
+    }
+
+    logger.debug(`Fetched ${totalSynced} audits, ${totalMissed} still missing`, 'Replication');
+
+    this.inProgress = false;
+  }
+
+  private async $syncAudit(hash: string): Promise<boolean> {
+    if (this.skip.has(hash)) {
+      // we already know none of our trusted servers have this audit
+      return false;
+    }
+
+    let success = false;
+    // start with a random server so load is uniformly spread
+    const syncResult = await $sync(`/api/v1/block/${hash}/audit-summary`);
+    if (syncResult) {
+      if (syncResult.data?.template?.length) {
+        await this.$saveAuditData(hash, syncResult.data);
+        logger.info(`Imported audit data from ${syncResult.server} for block ${syncResult.data.height} (${hash})`);
+        success = true;
+      }
+      if (!syncResult.data && !syncResult.exists) {
+        this.skip.add(hash);
+      }
+    }
+
+    return success;
+  }
+
+  private async $getMissingAuditBlocks(): Promise<string[]> {
+    try {
+      const startHeight = config.REPLICATION.AUDIT_START_HEIGHT || 0;
+      const [rows]: any[] = await DB.query(`
+        SELECT auditable.hash, auditable.height
+        FROM (
+          SELECT hash, height
+          FROM blocks
+          WHERE height >= ?
+        ) AS auditable
+        LEFT JOIN blocks_audits ON auditable.hash = blocks_audits.hash
+        WHERE blocks_audits.hash IS NULL
+        ORDER BY auditable.height DESC
+      `, [startHeight]);
+      return rows.map(row => row.hash);
+    } catch (e: any) {
+      logger.err(`Cannot fetch missing audit blocks from db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  private async $saveAuditData(blockHash: string, auditSummary: AuditSummary): Promise<void> {
+    // save audit & template to DB
+    await blocksSummariesRepository.$saveTemplate({
+      height: auditSummary.height,
+      template: {
+        id: blockHash,
+        transactions: auditSummary.template || []
+      }
+    });
+    await blocksAuditsRepository.$saveAudit({
+      hash: blockHash,
+      height: auditSummary.height,
+      time: auditSummary.timestamp || auditSummary.time,
+      missingTxs: auditSummary.missingTxs || [],
+      addedTxs: auditSummary.addedTxs || [],
+      freshTxs: auditSummary.freshTxs || [],
+      sigopTxs: auditSummary.sigopTxs || [],
+      fullrbfTxs: auditSummary.fullrbfTxs || [],
+      matchRate: auditSummary.matchRate,
+      expectedFees: auditSummary.expectedFees,
+      expectedWeight: auditSummary.expectedWeight,
+    });
+    // add missing data to cached blocks
+    const cachedBlock = blocks.getBlocks().find(block => block.id === blockHash);
+    if (cachedBlock) {
+      cachedBlock.extras.matchRate = auditSummary.matchRate;
+      cachedBlock.extras.expectedFees = auditSummary.expectedFees || null;
+      cachedBlock.extras.expectedWeight = auditSummary.expectedWeight || null;
+    }
+  }
+}
+
+export default new AuditReplication();
+

backend/src/replication/replicator.ts

@@ -0,0 +1,70 @@
+import config from '../config';
+import backendInfo from '../api/backend-info';
+import axios, { AxiosResponse } from 'axios';
+import { SocksProxyAgent } from 'socks-proxy-agent';
+import * as https from 'https';
+
+export async function $sync(path): Promise<{ data?: any, exists: boolean, server?: string }> {
+  // start with a random server so load is uniformly spread
+  let allMissing = true;
+  const offset = Math.floor(Math.random() * config.REPLICATION.SERVERS.length);
+  for (let i = 0; i < config.REPLICATION.SERVERS.length; i++) {
+    const server = config.REPLICATION.SERVERS[(i + offset) % config.REPLICATION.SERVERS.length];
+    // don't query ourself
+    if (server === backendInfo.getBackendInfo().hostname) {
+      continue;
+    }
+    
+    try {
+      const result = await query(`https://${server}${path}`);
+      if (result) {
+        return { data: result, exists: true, server };
+      }
+    } catch (e: any) {
+      if (e?.response?.status === 404) {
+        // this server is also missing this data
+      } else {
+        // something else went wrong
+        allMissing = false;
+      }
+    }
+  }
+
+  return { exists: !allMissing };
+}
+
+export async function query(path): Promise<object> {
+  type axiosOptions = {
+    headers: {
+      'User-Agent': string
+    };
+    timeout: number;
+    httpsAgent?: https.Agent;
+  };
+  const axiosOptions: axiosOptions = {
+    headers: {
+      'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+    },
+    timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
+  };
+
+  if (config.SOCKS5PROXY.ENABLED) {
+    const socksOptions = {
+      agentOptions: {
+        keepAlive: true,
+      },
+      hostname: config.SOCKS5PROXY.HOST,
+      port: config.SOCKS5PROXY.PORT,
+      username: config.SOCKS5PROXY.USERNAME || 'circuit0',
+      password: config.SOCKS5PROXY.PASSWORD,
+    };
+
+    axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
+  }
+
+  const data: AxiosResponse = await axios.get(path, axiosOptions);
+  if (data.statusText === 'error' || !data.data) {
+    throw new Error(`${data.status}`);
+  }
+  return data.data;
+}

backend/src/tasks/lightning/network-sync.service.ts

@@ -3,7 +3,6 @@ import logger from '../../logger';
 import channelsApi from '../../api/explorer/channels.api';
 import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory';
 import config from '../../config';
-import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
 import { ILightningApi } from '../../api/lightning/lightning-api.interface';
 import { $lookupNodeLocation } from './sync-tasks/node-locations';
 import lightningApi from '../../api/lightning/lightning-api-factory';

backend/src/tasks/price-updater.ts

@@ -153,6 +153,7 @@ class PriceUpdater {
       try {
         const p = 60 * 60 * 1000; // milliseconds in an hour
         const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042
+        this.latestPrices.time = nowRounded.getTime() / 1000;
         await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices);
       } catch (e) {
         this.lastRun = previousRun + 5 * 60;

backend/src/utils/format.ts

@@ -27,3 +27,69 @@ export function formatBytes(bytes: number, toUnit: string, skipUnit = false): st
 
   return `${bytes.toFixed(2)}${skipUnit ? '' : ' ' + byteUnits[unitIndex]}`;
 }
+
+// https://stackoverflow.com/a/64235212
+export function hex2bin(hex: string): string {
+  if (!hex) {
+    return '';
+  }
+
+  hex = hex.replace('0x', '').toLowerCase();
+  let out = '';
+
+  for (const c of hex) {
+    switch (c) {
+      case '0': out += '0000'; break;
+      case '1': out += '0001'; break;
+      case '2': out += '0010'; break;
+      case '3': out += '0011'; break;
+      case '4': out += '0100'; break;
+      case '5': out += '0101'; break;
+      case '6': out += '0110'; break;
+      case '7': out += '0111'; break;
+      case '8': out += '1000'; break;
+      case '9': out += '1001'; break;
+      case 'a': out += '1010'; break;
+      case 'b': out += '1011'; break;
+      case 'c': out += '1100'; break;
+      case 'd': out += '1101'; break;
+      case 'e': out += '1110'; break;
+      case 'f': out += '1111'; break;
+      default: return '';
+    }
+  }
+  return out;
+}
+
+export function bin2hex(bin: string): string {
+  if (!bin) {
+    return '';
+  }
+
+  let out = '';
+
+  for (let i = 0; i < bin.length; i += 4) {
+    const c = bin.substring(i, i + 4);
+    switch (c) {
+      case '0000': out += '0'; break;
+      case '0001': out += '1'; break;
+      case '0010': out += '2'; break;
+      case '0011': out += '3'; break;
+      case '0100': out += '4'; break;
+      case '0101': out += '5'; break;
+      case '0110': out += '6'; break;
+      case '0111': out += '7'; break;
+      case '1000': out += '8'; break;
+      case '1001': out += '9'; break;
+      case '1010': out += 'a'; break;
+      case '1011': out += 'b'; break;
+      case '1100': out += 'c'; break;
+      case '1101': out += 'd'; break;
+      case '1110': out += 'e'; break;
+      case '1111': out += 'f'; break;
+      default: return '';
+    }
+  }
+
+  return out;
+}

docker/backend/mempool-config.json

@@ -127,5 +127,11 @@
     "GEOLITE2_CITY": "__MAXMIND_GEOLITE2_CITY__",
     "GEOLITE2_ASN": "__MAXMIND_GEOLITE2_ASN__",
     "GEOIP2_ISP": "__MAXMIND_GEOIP2_ISP__"
+  },
+  "REPLICATION": {
+    "ENABLED": __REPLICATION_ENABLED__,
+    "AUDIT": __REPLICATION_AUDIT__,
+    "AUDIT_START_HEIGHT": __REPLICATION_AUDIT_START_HEIGHT__,
+    "SERVERS": __REPLICATION_SERVERS__
   }
 }

docker/backend/start.sh

@@ -130,6 +130,12 @@ __MAXMIND_GEOLITE2_CITY__=${MAXMIND_GEOLITE2_CITY:="/backend/GeoIP/GeoLite2-City
 __MAXMIND_GEOLITE2_ASN__=${MAXMIND_GEOLITE2_ASN:="/backend/GeoIP/GeoLite2-ASN.mmdb"}
 __MAXMIND_GEOIP2_ISP__=${MAXMIND_GEOIP2_ISP:=""}
 
+# REPLICATION
+__REPLICATION_ENABLED__=${REPLICATION_ENABLED:=true}
+__REPLICATION_AUDIT__=${REPLICATION_AUDIT:=true}
+__REPLICATION_AUDIT_START_HEIGHT__=${REPLICATION_AUDIT_START_HEIGHT:=774000}
+__REPLICATION_SERVERS__=${REPLICATION_SERVERS:=[]}
+
 
 mkdir -p "${__MEMPOOL_CACHE_DIR__}"
 
@@ -250,5 +256,10 @@ sed -i "s!__MAXMIND_GEOLITE2_CITY__!${__MAXMIND_GEOLITE2_CITY__}!g" mempool-conf
 sed -i "s!__MAXMIND_GEOLITE2_ASN__!${__MAXMIND_GEOLITE2_ASN__}!g" mempool-config.json
 sed -i "s!__MAXMIND_GEOIP2_ISP__!${__MAXMIND_GEOIP2_ISP__}!g" mempool-config.json
 
+# REPLICATION
+sed -i "s!__REPLICATION_ENABLED__!${__REPLICATION_ENABLED__}!g" mempool-config.json
+sed -i "s!__REPLICATION_AUDIT__!${__REPLICATION_AUDIT__}!g" mempool-config.json
+sed -i "s!__REPLICATION_AUDIT_START_HEIGHT__!${__REPLICATION_AUDIT_START_HEIGHT__}!g" mempool-config.json
+sed -i "s!__REPLICATION_SERVERS__!${__REPLICATION_SERVERS__}!g" mempool-config.json
 
 node /backend/package/index.js

docker/frontend/entrypoint.sh

@@ -39,7 +39,6 @@ __AUDIT__=${AUDIT:=false}
 __MAINNET_BLOCK_AUDIT_START_HEIGHT__=${MAINNET_BLOCK_AUDIT_START_HEIGHT:=0}
 __TESTNET_BLOCK_AUDIT_START_HEIGHT__=${TESTNET_BLOCK_AUDIT_START_HEIGHT:=0}
 __SIGNET_BLOCK_AUDIT_START_HEIGHT__=${SIGNET_BLOCK_AUDIT_START_HEIGHT:=0}
-__FULL_RBF_ENABLED__=${FULL_RBF_ENABLED:=false}
 __HISTORICAL_PRICE__=${HISTORICAL_PRICE:=true}
 
 # Export as environment variables to be used by envsubst
@@ -66,7 +65,6 @@ export __AUDIT__
 export __MAINNET_BLOCK_AUDIT_START_HEIGHT__
 export __TESTNET_BLOCK_AUDIT_START_HEIGHT__
 export __SIGNET_BLOCK_AUDIT_START_HEIGHT__
-export __FULL_RBF_ENABLED__
 export __HISTORICAL_PRICE__
 
 folder=$(find /var/www/mempool -name "config.js" | xargs dirname)

frontend/mempool-frontend-config.sample.json

@@ -22,6 +22,5 @@
   "TESTNET_BLOCK_AUDIT_START_HEIGHT": 0,
   "SIGNET_BLOCK_AUDIT_START_HEIGHT": 0,
   "LIGHTNING": false,
-  "FULL_RBF_ENABLED": false,
   "HISTORICAL_PRICE": true
 }

frontend/package.json

@@ -61,60 +61,60 @@
     "cypress:run:ci:staging": "node update-config.js TESTNET_ENABLED=true SIGNET_ENABLED=true LIQUID_ENABLED=true BISQ_ENABLED=true ITEMS_PER_PAGE=25 && npm run generate-config && start-server-and-test serve:local-staging 4200 cypress:run:record"
   },
   "dependencies": {
-    "@angular-devkit/build-angular": "^14.2.10",
+    "@angular-devkit/build-angular": "^16.1.4",
-    "@angular/animations": "^14.2.12",
+    "@angular/animations": "^16.1.5",
-    "@angular/cli": "^14.2.10",
+    "@angular/cli": "^16.1.4",
-    "@angular/common": "^14.2.12",
+    "@angular/common": "^16.1.5",
-    "@angular/compiler": "^14.2.12",
+    "@angular/compiler": "^16.1.5",
-    "@angular/core": "^14.2.12",
+    "@angular/core": "^16.1.5",
-    "@angular/forms": "^14.2.12",
+    "@angular/forms": "^16.1.5",
-    "@angular/localize": "^14.2.12",
+    "@angular/localize": "^16.1.5",
-    "@angular/platform-browser": "^14.2.12",
+    "@angular/platform-browser": "^16.1.5",
-    "@angular/platform-browser-dynamic": "^14.2.12",
+    "@angular/platform-browser-dynamic": "^16.1.5",
-    "@angular/platform-server": "^14.2.12",
+    "@angular/platform-server": "^16.1.5",
-    "@angular/router": "^14.2.12",
+    "@angular/router": "^16.1.5",
-    "@fortawesome/angular-fontawesome": "~0.11.1",
+    "@fortawesome/angular-fontawesome": "~0.13.0",
-    "@fortawesome/fontawesome-common-types": "~6.2.1",
+    "@fortawesome/fontawesome-common-types": "~6.4.0",
-    "@fortawesome/fontawesome-svg-core": "~6.2.1",
+    "@fortawesome/fontawesome-svg-core": "~6.4.0",
-    "@fortawesome/free-solid-svg-icons": "~6.2.1",
+    "@fortawesome/free-solid-svg-icons": "~6.4.0",
     "@mempool/mempool.js": "2.3.0",
-    "@ng-bootstrap/ng-bootstrap": "^13.1.1",
+    "@ng-bootstrap/ng-bootstrap": "^15.1.0",
     "@types/qrcode": "~1.5.0",
-    "bootstrap": "~4.6.1",
+    "bootstrap": "~4.6.2",
     "browserify": "^17.0.0",
     "clipboard": "^2.0.11",
     "domino": "^2.1.6",
-    "echarts": "~5.4.1",
+    "echarts": "~5.4.3",
     "echarts-gl": "^2.0.9",
     "lightweight-charts": "~3.8.0",
-    "ngx-echarts": "~14.0.0",
+    "ngx-echarts": "~16.0.0",
-    "ngx-infinite-scroll": "^14.0.1",
+    "ngx-infinite-scroll": "^16.0.0",
     "qrcode": "1.5.1",
-    "rxjs": "~7.8.0",
+    "rxjs": "~7.8.1",
-    "tinyify": "^3.1.0",
+    "tinyify": "^4.0.0",
     "tlite": "^0.1.9",
-    "tslib": "~2.4.1",
+    "tslib": "~2.6.0",
-    "zone.js": "~0.12.0"
+    "zone.js": "~0.13.1"
   },
   "devDependencies": {
-    "@angular/compiler-cli": "^14.2.12",
+    "@angular/compiler-cli": "^16.1.5",
-    "@angular/language-service": "^14.2.12",
+    "@angular/language-service": "^16.1.5",
     "@types/node": "^18.11.9",
     "@typescript-eslint/eslint-plugin": "^5.48.1",
     "@typescript-eslint/parser": "^5.48.1",
     "eslint": "^8.31.0",
     "http-proxy-middleware": "~2.0.6",
-    "prettier": "^2.8.2",
+    "prettier": "^3.0.0",
     "ts-node": "~10.9.1",
-    "typescript": "~4.6.4"
+    "typescript": "~4.9.3"
   },
   "optionalDependencies": {
-    "@cypress/schematic": "^2.4.0",
+    "@cypress/schematic": "^2.5.0",
-    "cypress": "^12.7.0",
+    "cypress": "^12.17.1",
-    "cypress-fail-on-console-error": "~4.0.2",
+    "cypress-fail-on-console-error": "~4.0.3",
     "cypress-wait-until": "^1.7.2",
-    "mock-socket": "~9.1.5",
+    "mock-socket": "~9.2.1",
-    "start-server-and-test": "~1.14.0"
+    "start-server-and-test": "~2.0.0"
   },
   "scarfSettings": {
     "enabled": false

frontend/src/app/app.module.ts

@@ -1,4 +1,4 @@
-import { BrowserModule, BrowserTransferStateModule } from '@angular/platform-browser';
+import { BrowserModule } from '@angular/platform-browser';
 import { ModuleWithProviders, NgModule } from '@angular/core';
 import { HttpClientModule, HTTP_INTERCEPTORS } from '@angular/common/http';
 import { BrowserAnimationsModule } from '@angular/platform-browser/animations';
@@ -48,8 +48,7 @@ const providers = [
     AppComponent,
   ],
   imports: [
-    BrowserModule.withServerTransition({ appId: 'serverApp' }),
+    BrowserModule,
-    BrowserTransferStateModule,
     AppRoutingModule,
     HttpClientModule,
     BrowserAnimationsModule,

frontend/src/app/components/address/address.component.ts

@@ -207,7 +207,7 @@ export class AddressComponent implements OnInit, OnDestroy {
     }
     this.isLoadingTransactions = true;
     this.retryLoadMore = false;
-    this.electrsApiService.getAddressTransactionsFromHash$(this.address.address, this.lastTransactionTxId)
+    this.electrsApiService.getAddressTransactions$(this.address.address, this.lastTransactionTxId)
       .subscribe((transactions: Transaction[]) => {
         this.lastTransactionTxId = transactions[transactions.length - 1].txid;
         this.loadedConfirmedTxCount += transactions.length;
@@ -217,6 +217,10 @@ export class AddressComponent implements OnInit, OnDestroy {
       (error) => {
         this.isLoadingTransactions = false;
         this.retryLoadMore = true;
+        // In the unlikely event of the txid wasn't found in the mempool anymore and we must reload the page.
+        if (error.status === 422) {
+          window.location.reload();
+        }
       });
   }
 

frontend/src/app/components/block-overview-graph/tx-view.ts

@@ -38,7 +38,7 @@ export default class TxView implements TransactionStripped {
   value: number;
   feerate: number;
   rate?: number;
-  status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf';
+  status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'fullrbf';
   context?: 'projected' | 'actual';
   scene?: BlockScene;
 
@@ -210,6 +210,7 @@ export default class TxView implements TransactionStripped {
       case 'fullrbf':
         return marginalFeeColors[feeLevelIndex] || marginalFeeColors[mempoolFeeColors.length - 1];
       case 'fresh':
+      case 'freshcpfp':
         return auditColors.missing;
       case 'added':
         return auditColors.added;

frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html

@@ -50,6 +50,7 @@
           <td *ngSwitchCase="'missing'"><span class="badge badge-warning" i18n="transaction.audit.marginal">Marginal fee rate</span></td>
           <td *ngSwitchCase="'sigop'"><span class="badge badge-warning" i18n="transaction.audit.sigop">High sigop count</span></td>
           <td *ngSwitchCase="'fresh'"><span class="badge badge-warning" i18n="transaction.audit.recently-broadcasted">Recently broadcasted</span></td>
+          <td *ngSwitchCase="'freshcpfp'"><span class="badge badge-warning" i18n="transaction.audit.recently-cpfped">Recently CPFP'd</span></td>
           <td *ngSwitchCase="'added'"><span class="badge badge-warning" i18n="transaction.audit.added">Added</span></td>
           <td *ngSwitchCase="'selected'"><span class="badge badge-warning" i18n="transaction.audit.marginal">Marginal fee rate</span></td>
           <td *ngSwitchCase="'fullrbf'"><span class="badge badge-warning" i18n="transaction.audit.fullrbf">Full RBF</span></td>

frontend/src/app/components/block/block.component.ts

@@ -370,7 +370,11 @@ export class BlockComponent implements OnInit, OnDestroy {
               tx.status = 'found';
             } else {
               if (isFresh[tx.txid]) {
+                if (tx.rate - (tx.fee / tx.vsize) >= 0.1) {
+                  tx.status = 'freshcpfp';
+                } else {
                   tx.status = 'fresh';
+                }
               } else if (isSigop[tx.txid]) {
                 tx.status = 'sigop';
               } else if (isFullRbf[tx.txid]) {

frontend/src/app/components/calculator/calculator.component.html

@@ -12,7 +12,7 @@
           <div class="input-group-prepend">
             <span class="input-group-text">{{ currency$ | async }}</span>
           </div>
-          <input type="text" class="form-control" formControlName="fiat" (input)="transformInput('fiat')">
+          <input type="text" class="form-control" formControlName="fiat" (input)="transformInput('fiat')" (click)="selectAll($event)">
           <app-clipboard [button]="true" [text]="form.get('fiat').value" [class]="'btn btn-lg btn-secondary ml-1'"></app-clipboard>
         </div>
 
@@ -20,7 +20,7 @@
           <div class="input-group-prepend">
             <span class="input-group-text">BTC</span>
           </div>
-          <input type="text" class="form-control" formControlName="bitcoin" (input)="transformInput('bitcoin')">
+          <input type="text" class="form-control" formControlName="bitcoin" (input)="transformInput('bitcoin')" (click)="selectAll($event)">
           <app-clipboard [button]="true" [text]="form.get('bitcoin').value" [class]="'btn btn-lg btn-secondary ml-1'"></app-clipboard>
         </div>
 
@@ -28,7 +28,7 @@
           <div class="input-group-prepend">
             <span class="input-group-text">sats</span>
           </div>
-          <input type="text" class="form-control" formControlName="satoshis" (input)="transformInput('satoshis')">
+          <input type="text" class="form-control" formControlName="satoshis" (input)="transformInput('satoshis')" (click)="selectAll($event)">
           <app-clipboard [button]="true" [text]="form.get('satoshis').value" [class]="'btn btn-lg btn-secondary ml-1'"></app-clipboard>
         </div>
       </form>

frontend/src/app/components/calculator/calculator.component.scss

@@ -24,3 +24,7 @@
   font-size: 20px;
   margin-left: 5px;
 }
+
+.row {
+  margin: auto;
+}

frontend/src/app/components/calculator/calculator.component.ts

@@ -54,6 +54,9 @@ export class CalculatorComponent implements OnInit {
     ]).subscribe(([price, value]) => {
       const rate = (value / price).toFixed(8);
       const satsRate = Math.round(value / price * 100_000_000);
+      if (isNaN(value)) {
+        return;
+      }
       this.form.get('bitcoin').setValue(rate, { emitEvent: false });
       this.form.get('satoshis').setValue(satsRate, { emitEvent: false } );
     });
@@ -63,6 +66,9 @@ export class CalculatorComponent implements OnInit {
       this.form.get('bitcoin').valueChanges
     ]).subscribe(([price, value]) => {
       const rate = parseFloat((value * price).toFixed(8));
+      if (isNaN(value)) {
+        return;
+      }
       this.form.get('fiat').setValue(rate, { emitEvent: false } );
       this.form.get('satoshis').setValue(Math.round(value * 100_000_000), { emitEvent: false } );
     });
@@ -73,6 +79,9 @@ export class CalculatorComponent implements OnInit {
     ]).subscribe(([price, value]) => {
       const rate = parseFloat((value / 100_000_000 * price).toFixed(8));
       const bitcoinRate = (value / 100_000_000).toFixed(8);
+      if (isNaN(value)) {
+        return;
+      }
       this.form.get('fiat').setValue(rate, { emitEvent: false } );
       this.form.get('bitcoin').setValue(bitcoinRate, { emitEvent: false });
     });
@@ -88,7 +97,16 @@ export class CalculatorComponent implements OnInit {
     if (value === '.') {
       value = '0';
     }
-    const sanitizedValue = this.removeExtraDots(value);
+    let sanitizedValue = this.removeExtraDots(value);
+    if (name === 'bitcoin' && this.countDecimals(sanitizedValue) > 8) {
+      sanitizedValue = this.toFixedWithoutRounding(sanitizedValue, 8);
+    }
+    if (sanitizedValue === '') {
+      sanitizedValue = '0';
+    }
+    if (name === 'satoshis') {
+      sanitizedValue = parseFloat(sanitizedValue).toFixed(0);
+    }
     formControl.setValue(sanitizedValue, {emitEvent: true});
   }
 
@@ -100,4 +118,20 @@ export class CalculatorComponent implements OnInit {
     const afterDotReplaced = afterDot.replace(/\./g, '');
     return `${beforeDot}.${afterDotReplaced}`;
   }
+
+  countDecimals(numberString: string): number {
+    const decimalPos = numberString.indexOf('.');
+    if (decimalPos === -1) return 0;
+    return numberString.length - decimalPos - 1;
+  }
+
+  toFixedWithoutRounding(numStr: string, fixed: number): string {
+    const re = new RegExp(`^-?\\d+(?:.\\d{0,${(fixed || -1)}})?`);
+    const result = numStr.match(re);
+    return result ? result[0] : numStr;
+  }
+
+  selectAll(event): void {
+    event.target.select();
+  }
 }

frontend/src/app/components/clock/clock.component.scss

@@ -9,6 +9,7 @@
   display: flex;
   flex-direction: column;
   justify-content: flex-start;
+  overflow: hidden;
 
   --chain-height: 60px;
   --clock-width: 300px;

frontend/src/app/components/pool/pool.component.ts

@@ -37,7 +37,7 @@ export class PoolComponent implements OnInit {
 
   auditAvailable = false;
 
-  loadMoreSubject: BehaviorSubject<number> = new BehaviorSubject(this.blocks[0]?.height);
+  loadMoreSubject: BehaviorSubject<number> = new BehaviorSubject(this.blocks[this.blocks.length - 1]?.height);
 
   constructor(
     @Inject(LOCALE_ID) public locale: string,
@@ -91,7 +91,7 @@ export class PoolComponent implements OnInit {
           if (this.slug === undefined) {
             return [];
           }
-          return this.apiService.getPoolBlocks$(this.slug, this.blocks[0]?.height);
+          return this.apiService.getPoolBlocks$(this.slug, this.blocks[this.blocks.length - 1]?.height);
         }),
         tap((newBlocks) => {
           this.blocks = this.blocks.concat(newBlocks);
@@ -237,7 +237,7 @@ export class PoolComponent implements OnInit {
   }
 
   loadMore() {
-    this.loadMoreSubject.next(this.blocks[0]?.height);
+    this.loadMoreSubject.next(this.blocks[this.blocks.length - 1]?.height);
   }
 
   trackByBlock(index: number, block: BlockExtended) {

frontend/src/app/components/rbf-list/rbf-list.component.html

@@ -2,7 +2,7 @@
   <h1 class="float-left" i18n="page.rbf-replacements">RBF Replacements</h1>
   <div *ngIf="isLoading" class="spinner-border ml-3" role="status"></div>
 
-  <div class="mode-toggle float-right" *ngIf="fullRbfEnabled">
+  <div class="mode-toggle float-right">
     <form class="formRadioGroup">
       <div class="btn-group btn-group-toggle" name="radioBasic">
         <label class="btn btn-primary btn-sm" [class.active]="!fullRbf">

frontend/src/app/components/rbf-list/rbf-list.component.ts

@@ -17,7 +17,6 @@ export class RbfList implements OnInit, OnDestroy {
   rbfTrees$: Observable<RbfTree[]>;
   nextRbfSubject = new BehaviorSubject(null);
   urlFragmentSubscription: Subscription;
-  fullRbfEnabled: boolean;
   fullRbf: boolean;
   isLoading = true;
 
@@ -27,9 +26,7 @@ export class RbfList implements OnInit, OnDestroy {
     private apiService: ApiService,
     public stateService: StateService,
     private websocketService: WebsocketService,
-  ) {
+  ) { }
-    this.fullRbfEnabled = stateService.env.FULL_RBF_ENABLED;
-  }
 
   ngOnInit(): void {
     this.urlFragmentSubscription = this.route.fragment.subscribe((fragment) => {

frontend/src/app/components/transaction/transaction.component.html

@@ -19,6 +19,7 @@
 
       <div class="container-buttons">
         <app-confirmations
+          *ngIf="tx"
           [chainTip]="latestBlock?.height"
           [height]="tx?.status?.block_height"
           [replaced]="replaced"

frontend/src/app/components/transaction/transaction.component.ts

@@ -379,7 +379,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
                 ancestors: tx.ancestors,
                 bestDescendant: tx.bestDescendant,
               };
-              const hasRelatives = !!(tx.ancestors.length || tx.bestDescendant);
+              const hasRelatives = !!(tx.ancestors?.length || tx.bestDescendant);
               this.hasEffectiveFeeRate = hasRelatives || (tx.effectiveFeePerVsize && (Math.abs(tx.effectiveFeePerVsize - tx.feePerVsize) > 0.01));
             } else {
               this.fetchCpfp$.next(this.tx.txid);

frontend/src/app/interfaces/node-api.interface.ts

@@ -173,7 +173,8 @@ export interface TransactionStripped {
   fee: number;
   vsize: number;
   value: number;
-  status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf';
+  rate?: number; // effective fee rate
+  status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'fullrbf';
   context?: 'projected' | 'actual';
 }
 

frontend/src/app/interfaces/websocket.interface.ts

@@ -89,7 +89,7 @@ export interface TransactionStripped {
   vsize: number;
   value: number;
   rate?: number; // effective fee rate
-  status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf';
+  status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'fullrbf';
   context?: 'projected' | 'actual';
 }
 

frontend/src/app/lightning/node/node.component.html

@@ -21,7 +21,6 @@
   </div>
 
   <div class="box" *ngIf="!error">
-
     <div class="row">
       <div class="col-md">
         <table class="table table-borderless table-striped table-fixed">
@@ -59,6 +58,9 @@
               <td i18n="lightning.avg-distance" class="text-truncate">Avg channel distance</td>
               <td class="direction-ltr">{{ avgDistance | amountShortener: 1 }} <span class="symbol">km</span> <span class="separator">·</span>{{ kmToMiles(avgDistance) | amountShortener: 1 }} <span class="symbol">mi</span></td>
             </tr>
+            <tr *ngIf="!node.geolocation" class="d-none d-md-table-row">
+              <ng-container *ngTemplateOutlet="featurebits;context:{bits: node.featuresBits}"></ng-container>
+            </tr>
           </tbody>
         </table>
       </div>
@@ -100,11 +102,50 @@
                 </td>
               </ng-template>
             </tr>
+            <tr *ngIf="node.geolocation && node.featuresBits">
+              <ng-container *ngTemplateOutlet="featurebits;context:{bits: node.featuresBits}"></ng-container>
+            </tr>
+            <tr *ngIf="!node.geolocation && node.featuresBits" class="d-table-row d-md-none">
+              <ng-container *ngTemplateOutlet="featurebits;context:{bits: node.featuresBits}"></ng-container>
+            </tr>
           </tbody>
         </table>
       </div>
     </div>
+  </div>
 
+  <ng-template #featurebits let-bits="bits">
+    <td i18n="lightning.features" class="text-truncate label">Features</td>
+    <td class="d-flex justify-content-between">
+      <span class="text-truncate w-90">{{ bits }}</span>
+      <button type="button" class="btn btn-outline-info btn-xs" (click)="toggleFeatures()" i18n="transaction.details|Transaction Details">Details</button>
+    </td>
+  </ng-template>
+
+  <div class="box mt-2" *ngIf="!error && showFeatures">
+    <div class="row">
+      <div class="col-md">
+        <div class="mb-3">
+          <h5>Raw bits</h5>
+          <span class="text-wrap w-100"><small>{{ node.featuresBits }}</small></span>
+        </div>
+        <h5>Decoded</h5>
+        <table class="table table-borderless table-striped table-fixed">
+          <thead>
+            <th style="width: 13%">Bit</th>
+            <th>Name</th>
+            <th style="width: 25%; text-align: right">Required</th>
+          </thead>
+          <tbody>
+            <tr *ngFor="let feature of node.features">
+              <td style="width: 13%">{{ feature.bit }}</td>
+              <td>{{ feature.name }}</td>
+              <td style="width: 25%; text-align: right">{{ feature.is_required }}</td>
+            </tr>
+          </tbody>
+        </table>
+      </div>
+    </div>
   </div>
 
   <div class="input-group mt-3" *ngIf="!error && node.socketsObject.length">

frontend/src/app/lightning/node/node.component.ts

@@ -37,7 +37,7 @@ export class NodeComponent implements OnInit {
   liquidityAd: ILiquidityAd;
   tlvRecords: CustomRecord[];
   avgChannelDistance$: Observable<number | null>;
-
+  showFeatures = false;
   kmToMiles = kmToMiles;
 
   constructor(
@@ -164,4 +164,9 @@ export class NodeComponent implements OnInit {
   onLoadingEvent(e) {
     this.channelListLoading = e;
   }
+
+  toggleFeatures() {
+    this.showFeatures = !this.showFeatures;
+    return false;
+  }
 }

frontend/src/app/services/electrs-api.service.ts

@@ -1,5 +1,5 @@
 import { Injectable } from '@angular/core';
-import { HttpClient } from '@angular/common/http';
+import { HttpClient, HttpParams } from '@angular/common/http';
 import { Observable } from 'rxjs';
 import { Transaction, Address, Outspend, Recent, Asset } from '../interfaces/electrs.interface';
 import { StateService } from './state.service';
@@ -65,12 +65,12 @@ export class ElectrsApiService {
     return this.httpClient.get<Address>(this.apiBaseUrl + this.apiBasePath + '/api/address/' + address);
   }
 
-  getAddressTransactions$(address: string): Observable<Transaction[]> {
+  getAddressTransactions$(address: string,  txid?: string): Observable<Transaction[]> {
-    return this.httpClient.get<Transaction[]>(this.apiBaseUrl + this.apiBasePath + '/api/address/' + address + '/txs');
+    let params = new HttpParams();
+    if (txid) {
+      params = params.append('after_txid', txid);
     }
-
+    return this.httpClient.get<Transaction[]>(this.apiBaseUrl + this.apiBasePath + '/api/address/' + address + '/txs', { params });
-  getAddressTransactionsFromHash$(address: string, txid: string): Observable<Transaction[]> {
-    return this.httpClient.get<Transaction[]>(this.apiBaseUrl + this.apiBasePath + '/api/address/' + address + '/txs/chain/' + txid);
   }
 
   getAsset$(assetId: string): Observable<Asset> {

frontend/src/app/services/state.service.ts

@@ -45,7 +45,6 @@ export interface Env {
   MAINNET_BLOCK_AUDIT_START_HEIGHT: number;
   TESTNET_BLOCK_AUDIT_START_HEIGHT: number;
   SIGNET_BLOCK_AUDIT_START_HEIGHT: number;
-  FULL_RBF_ENABLED: boolean;
   HISTORICAL_PRICE: boolean;
 }
 
@@ -76,7 +75,6 @@ const defaultEnv: Env = {
   'MAINNET_BLOCK_AUDIT_START_HEIGHT': 0,
   'TESTNET_BLOCK_AUDIT_START_HEIGHT': 0,
   'SIGNET_BLOCK_AUDIT_START_HEIGHT': 0,
-  'FULL_RBF_ENABLED': false,
   'HISTORICAL_PRICE': true,
 };
 

frontend/src/app/shared/components/confirmations/confirmations.component.html

@@ -5,12 +5,15 @@
     <ng-template #confirmationPlural let-i i18n="shared.confirmation-count.plural|Transaction plural confirmation count">{{ i }} confirmations</ng-template>
   </button>
 </ng-template>
+<ng-template [ngIf]="!confirmations && height != null">
+  <button type="button" class="btn btn-sm btn-success {{buttonClass}}" i18n="transaction.confirmed|Transaction confirmed state">Confirmed</button>
+</ng-template>
 <ng-template [ngIf]="!hideUnconfirmed && !confirmations && replaced">
   <button type="button" class="btn btn-sm btn-warning {{buttonClass}}" i18n="transaction.replaced|Transaction replaced state">Replaced</button>
 </ng-template>
 <ng-template [ngIf]="!hideUnconfirmed && !confirmations && !replaced && removed">
   <button type="button" class="btn btn-sm btn-warning {{buttonClass}}" i18n="transaction.audit.removed|Transaction removed state">Removed</button>
 </ng-template>
-<ng-template [ngIf]="!hideUnconfirmed && !confirmations && !replaced && !removed">
+<ng-template [ngIf]="!hideUnconfirmed && chainTip != null && !confirmations && !replaced && !removed">
   <button type="button" class="btn btn-sm btn-danger {{buttonClass}}" i18n="transaction.unconfirmed|Transaction unconfirmed state">Unconfirmed</button>
 </ng-template>

frontend/src/styles.scss

@@ -45,8 +45,8 @@ $dropdown-link-hover-bg: #11131f;
 $dropdown-link-active-color: #fff;
 $dropdown-link-active-bg: #11131f;
 
-@import "~bootstrap/scss/bootstrap";
+@import "bootstrap/scss/bootstrap";
-@import '~tlite/tlite.css';
+@import 'tlite/tlite.css';
 
 html, body {
   height: 100%;
@@ -1164,3 +1164,10 @@ app-master-page, app-liquid-master-page, app-bisq-master-page {
 app-global-footer {
   margin-top: auto;
 }
+
+.btn-xs {
+  padding: 0.25rem 0.5rem;
+  font-size: 0.875rem;
+  line-height: 0.5;
+  border-radius: 0.2rem;
+}

production/bitcoin.conf

@@ -8,7 +8,8 @@ par=16
 dbcache=8192
 maxmempool=4096
 mempoolexpiry=999999
-maxconnections=42
+mempoolfullrbf=1
+maxconnections=100
 onion=127.0.0.1:9050
 rpcallowip=127.0.0.1
 rpcuser=__BITCOIN_RPC_USER__

production/bitcoin.minfee.conf

@@ -4,6 +4,7 @@ txindex=0
 listen=1
 daemon=1
 prune=1337
+mempoolfullrbf=1
 rpcallowip=127.0.0.1
 rpcuser=__BITCOIN_RPC_USER__
 rpcpassword=__BITCOIN_RPC_PASS__

production/install

@@ -353,7 +353,7 @@ ELEMENTS_REPO_URL=https://github.com/ElementsProject/elements
 ELEMENTS_REPO_NAME=elements
 ELEMENTS_REPO_BRANCH=master
 #ELEMENTS_LATEST_RELEASE=$(curl -s https://api.github.com/repos/ElementsProject/elements/releases/latest|grep tag_name|head -1|cut -d '"' -f4)
-ELEMENTS_LATEST_RELEASE=elements-22.1
+ELEMENTS_LATEST_RELEASE=elements-22.1.1
 echo -n '.'
 
 BITCOIN_ELECTRS_REPO_URL=https://github.com/mempool/electrs
@@ -1044,8 +1044,11 @@ osSudo "${ROOT_USER}" crontab -u "${MEMPOOL_USER}" "${MEMPOOL_HOME}/${MEMPOOL_RE
 echo "[*] Installing nvm.sh from GitHub"
 osSudo "${MEMPOOL_USER}" sh -c 'curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | zsh'
 
-echo "[*] Building NodeJS via nvm.sh"
+echo "[*] Building NodeJS v20.4.0 via nvm.sh"
-osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm install v16.16.0 --shared-zlib'
+osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm install v20.4.0 --shared-zlib'
+echo "[*] Building NodeJS v18.16.1 via nvm.sh"
+osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm install v18.16.1 --shared-zlib'
+osSudo "${MEMPOOL_USER}" zsh -c 'source ~/.zshrc ; nvm alias default 18.16.1'
 
 ####################
 # Tor installation #

production/mempool-config.mainnet.json

@@ -48,5 +48,30 @@
   "STATISTICS": {
     "ENABLED": true,
     "TX_PER_SECOND_SAMPLE_PERIOD": 150
+  },
+  "REPLICATION": {
+    "ENABLED": true,
+    "AUDIT": true,
+    "AUDIT_START_HEIGHT": 774000,
+    "SERVERS": [
+      "node201.fmt.mempool.space",
+      "node202.fmt.mempool.space",
+      "node203.fmt.mempool.space",
+      "node204.fmt.mempool.space",
+      "node205.fmt.mempool.space",
+      "node206.fmt.mempool.space",
+      "node201.fra.mempool.space",
+      "node202.fra.mempool.space",
+      "node203.fra.mempool.space",
+      "node204.fra.mempool.space",
+      "node205.fra.mempool.space",
+      "node206.fra.mempool.space",
+      "node201.tk7.mempool.space",
+      "node202.tk7.mempool.space",
+      "node203.tk7.mempool.space",
+      "node204.tk7.mempool.space",
+      "node205.tk7.mempool.space",
+      "node206.tk7.mempool.space"
+    ]
   }
 }

production/mempool-start-all

@@ -1,6 +1,7 @@
 #!/usr/bin/env zsh
 export NVM_DIR="$HOME/.nvm"
 source "$NVM_DIR/nvm.sh"
+nvm use v20.4.0
 
 # start all mempool backends that exist
 for site in mainnet mainnet-lightning testnet testnet-lightning signet signet-lightning bisq liquid liquidtestnet;do