Add note about enterprise support to readmes

README.md

@@ -30,6 +30,8 @@ Mempool can be conveniently installed on the following full-node distros:
 
 Mempool can be installed in other ways too, but we only recommend doing so if you're a developer, have experience managing servers, or otherwise know what you're doing.
 
+**We only provide support for advanced installation methods to <a href="https://mempool.space/enterprise">Enterprise sponsors</a>.**
+
 - See the [`docker/`](./docker/) directory for instructions on deploying Mempool with Docker.
 - See the [`backend/`](./backend/) and [`frontend/`](./frontend/) directories for manual install instructions oriented for developers.
 - See the [`production/`](./production/) directory for guidance on setting up a more serious Mempool instance designed for high performance at scale.

backend/README.md

@@ -2,7 +2,9 @@
 
 These instructions are mostly intended for developers. 
 
-If you choose to use these instructions for a production setup, be aware that you will still probably need to do additional configuration for your specific OS, environment, use-case, etc. We do our best here to provide a good starting point, but only proceed if you know what you're doing. Mempool does not provide support for custom setups.
+If you choose to use these instructions for a production setup, be aware that you will still probably need to do additional configuration for your specific OS, environment, use-case, etc. We do our best here to provide a good starting point, but only proceed if you know what you're doing.
+
+**We only provide support for this installation method to <a href="https://mempool.space/enterprise">Enterprise sponsors</a>.**
 
 See other ways to set up Mempool on [the main README](/../../#installation-methods).
 

backend/mempool-config.sample.json

@@ -44,8 +44,7 @@
   },
   "ESPLORA": {
     "REST_API_URL": "http://127.0.0.1:3000",
-    "UNIX_SOCKET_PATH": "/tmp/esplora-bitcoin-mainnet",
-    "RETRY_UNIX_SOCKET_AFTER": 30000
+    "UNIX_SOCKET_PATH": "/tmp/esplora-bitcoin-mainnet"
   },
   "SECOND_CORE_RPC": {
     "HOST": "127.0.0.1",

backend/src/__fixtures__/mempool-config.template.json

@@ -45,8 +45,7 @@
   },
   "ESPLORA": {
     "REST_API_URL": "__ESPLORA_REST_API_URL__",
-    "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__",
-    "RETRY_UNIX_SOCKET_AFTER": "__ESPLORA_RETRY_UNIX_SOCKET_AFTER__"
+    "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__"
   },
   "SECOND_CORE_RPC": {
     "HOST": "__SECOND_CORE_RPC_HOST__",

backend/src/__tests__/config.test.ts

@@ -47,7 +47,7 @@ describe('Mempool Backend Config', () => {
 
       expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
 
-      expect(config.ESPLORA).toStrictEqual({ REST_API_URL: 'http://127.0.0.1:3000', UNIX_SOCKET_PATH: null, RETRY_UNIX_SOCKET_AFTER: 30000 });
+      expect(config.ESPLORA).toStrictEqual({ REST_API_URL: 'http://127.0.0.1:3000', UNIX_SOCKET_PATH: null });
 
       expect(config.CORE_RPC).toStrictEqual({
         HOST: '127.0.0.1',

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -94,7 +94,6 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash', this.getBlock)
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/summary', this.getStrippedBlockTransactions)
       .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/audit-summary', this.getBlockAuditSummary)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', this.getBlockTipHeight)
       .post(config.MEMPOOL.API_URL_PREFIX + 'psbt/addparents', this.postPsbtCompletion)
       .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from', this.getBlocksByBulk.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-bulk/:from/:to', this.getBlocksByBulk.bind(this))
@@ -111,6 +110,7 @@ class BitcoinRoutes {
           .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/status', this.getTransactionStatus)
           .get(config.MEMPOOL.API_URL_PREFIX + 'tx/:txId/outspends', this.getTransactionOutspends)
           .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', this.getBlockHeader)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', this.getBlockTipHeight)
           .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/hash', this.getBlockTipHash)
           .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/raw', this.getRawBlock)
           .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txids', this.getTxIdsForBlock)
@@ -589,14 +589,10 @@ class BitcoinRoutes {
     }
   }
 
-  private getBlockTipHeight(req: Request, res: Response) {
+  private async getBlockTipHeight(req: Request, res: Response) {
     try {
-      const result = blocks.getCurrentBlockHeight();
-      if (!result) {
-        return res.status(503).send(`Service Temporarily Unavailable`);
-      }
-      res.setHeader('content-type', 'text/plain');
-      res.send(result.toString());
+      const result = await bitcoinApi.$getBlockHeightTip();
+      res.json(result);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
     }

backend/src/api/blocks.ts

@@ -36,8 +36,6 @@ class Blocks {
   private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = [];
   private newAsyncBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => Promise<void>)[] = [];
 
-  private mainLoopTimeout: number = 120000;
-
   constructor() { }
 
   public getBlocks(): BlockExtended[] {
@@ -530,12 +528,8 @@ class Blocks {
   }
 
   public async $updateBlocks() {
-    // throw an error if this stalls the main loop for more than 2 minutes
-    const timer = this.startTimer();
-
     let fastForwarded = false;
     const blockHeightTip = await bitcoinApi.$getBlockHeightTip();
-    this.updateTimerProgress(timer, 'got block height tip');
 
     if (this.blocks.length === 0) {
       this.currentBlockHeight = Math.max(blockHeightTip - config.MEMPOOL.INITIAL_BLOCKS_AMOUNT, -1);
@@ -553,21 +547,16 @@ class Blocks {
 
     if (!this.lastDifficultyAdjustmentTime) {
       const blockchainInfo = await bitcoinClient.getBlockchainInfo();
-      this.updateTimerProgress(timer, 'got blockchain info for initial difficulty adjustment');
       if (blockchainInfo.blocks === blockchainInfo.headers) {
         const heightDiff = blockHeightTip % 2016;
         const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff);
-        this.updateTimerProgress(timer, 'got block hash for initial difficulty adjustment');
         const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
-        this.updateTimerProgress(timer, 'got block for initial difficulty adjustment');
         this.lastDifficultyAdjustmentTime = block.timestamp;
         this.currentDifficulty = block.difficulty;
 
         if (blockHeightTip >= 2016) {
           const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
-          this.updateTimerProgress(timer, 'got previous block hash for initial difficulty adjustment');
           const previousPeriodBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(previousPeriodBlockHash);
-          this.updateTimerProgress(timer, 'got previous block for initial difficulty adjustment');
           this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
           logger.debug(`Initial difficulty adjustment data set.`);
         }
@@ -582,11 +571,9 @@ class Blocks {
       } else {
         this.currentBlockHeight++;
         logger.debug(`New block found (#${this.currentBlockHeight})!`);
-        this.updateTimerProgress(timer, `getting orphaned blocks for ${this.currentBlockHeight}`);
         await chainTips.updateOrphanedBlocks();
       }
 
-      this.updateTimerProgress(timer, `getting block data for ${this.currentBlockHeight}`);
       const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
       const verboseBlock = await bitcoinClient.getBlock(blockHash, 2);
       const block = BitcoinApi.convertBlock(verboseBlock);
@@ -595,51 +582,39 @@ class Blocks {
       const cpfpSummary: CpfpSummary = Common.calculateCpfp(block.height, transactions);
       const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions);
       const blockSummary: BlockSummary = this.summarizeBlock(verboseBlock);
-      this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`);
 
       // start async callbacks
-      this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`);
       const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions));
 
       if (Common.indexingEnabled()) {
         if (!fastForwarded) {
           const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
-          this.updateTimerProgress(timer, `got block by height for ${this.currentBlockHeight}`);
           if (lastBlock !== null && blockExtended.previousblockhash !== lastBlock.id) {
             logger.warn(`Chain divergence detected at block ${lastBlock.height}, re-indexing most recent data`, logger.tags.mining);
             // We assume there won't be a reorg with more than 10 block depth
-            this.updateTimerProgress(timer, `rolling back diverged chain from ${this.currentBlockHeight}`);
             await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10);
             await HashratesRepository.$deleteLastEntries();
             await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10);
-            this.updateTimerProgress(timer, `rolled back chain divergence from ${this.currentBlockHeight}`);
             for (let i = 10; i >= 0; --i) {
               const newBlock = await this.$indexBlock(lastBlock.height - i);
-              this.updateTimerProgress(timer, `reindexed block`);
               await this.$getStrippedBlockTransactions(newBlock.id, true, true);
-              this.updateTimerProgress(timer, `reindexed block summary`);
               if (config.MEMPOOL.CPFP_INDEXING) {
                 await this.$indexCPFP(newBlock.id, lastBlock.height - i);
-                this.updateTimerProgress(timer, `reindexed block cpfp`);
               }
             }
             await mining.$indexDifficultyAdjustments();
             await DifficultyAdjustmentsRepository.$deleteLastAdjustment();
-            this.updateTimerProgress(timer, `reindexed difficulty adjustments`);
             logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`, logger.tags.mining);
             indexer.reindex();
           }
           await blocksRepository.$saveBlockInDatabase(blockExtended);
-          this.updateTimerProgress(timer, `saved ${this.currentBlockHeight} to database`);
 
           const lastestPriceId = await PricesRepository.$getLatestPriceId();
-          this.updateTimerProgress(timer, `got latest price id ${this.currentBlockHeight}`);
           if (priceUpdater.historyInserted === true && lastestPriceId !== null) {
             await blocksRepository.$saveBlockPrices([{
               height: blockExtended.height,
               priceId: lastestPriceId,
             }]);
-            this.updateTimerProgress(timer, `saved prices for ${this.currentBlockHeight}`);
           } else {
             logger.debug(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`, logger.tags.mining);
             setTimeout(() => {
@@ -650,11 +625,9 @@ class Blocks {
           // Save blocks summary for visualization if it's enabled
           if (Common.blocksSummariesIndexingEnabled() === true) {
             await this.$getStrippedBlockTransactions(blockExtended.id, true);
-            this.updateTimerProgress(timer, `saved block summary for ${this.currentBlockHeight}`);
           }
           if (config.MEMPOOL.CPFP_INDEXING) {
             this.$saveCpfp(blockExtended.id, this.currentBlockHeight, cpfpSummary);
-            this.updateTimerProgress(timer, `saved cpfp for ${this.currentBlockHeight}`);
           }
         }
       }
@@ -667,7 +640,6 @@ class Blocks {
             difficulty: block.difficulty,
             adjustment: Math.round((block.difficulty / this.currentDifficulty) * 1000000) / 1000000, // Remove float point noise
           });
-          this.updateTimerProgress(timer, `saved difficulty adjustment for ${this.currentBlockHeight}`);
         }
 
         this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100;
@@ -692,38 +664,7 @@ class Blocks {
       }
 
       // wait for pending async callbacks to finish
-      this.updateTimerProgress(timer, `waiting for async callbacks to complete for ${this.currentBlockHeight}`);
       await Promise.all(callbackPromises);
-      this.updateTimerProgress(timer, `async callbacks completed for ${this.currentBlockHeight}`);
-    }
-
-    this.clearTimer(timer);
-  }
-
-  private startTimer() {
-    const state: any = {
-      start: Date.now(),
-      progress: 'begin $updateBlocks',
-      timer: null,
-    };
-    state.timer = setTimeout(() => {
-      logger.err(`$updateBlocks stalled at "${state.progress}`);
-    }, this.mainLoopTimeout);
-    return state;
-  }
-
-  private updateTimerProgress(state, msg) {
-    state.progress = msg;
-    if (Date.now() - state.start > this.mainLoopTimeout) {
-      // abort the function if it already timed out
-      logger.err(`$updateBlocks attempted to resume after "${state.progress}"`);
-      throw new Error(`$updateBlocks attempted to resume after "${state.progress}"`);
-    }
-  }
-
-  private clearTimer(state) {
-    if (state.timer) {
-      clearTimeout(state.timer);
     }
   }
 
@@ -916,7 +857,6 @@ class Blocks {
         }
         if (cleanBlock.fee_amt_percentiles !== null) {
           cleanBlock.median_fee_amt = cleanBlock.fee_amt_percentiles[3];
-          await blocksRepository.$updateFeeAmounts(cleanBlock.hash, cleanBlock.fee_amt_percentiles, cleanBlock.median_fee_amt);
         }
       }
 

backend/src/api/liquid/icons.ts

@@ -2,7 +2,7 @@ import * as fs from 'fs';
 import logger from '../../logger';
 
 class Icons {
-  private static FILE_NAME = '/elements/asset_registry_db/icons.json';
+  private static FILE_NAME = './icons.json';
   private iconIds: string[] = [];
   private icons: { [assetId: string]: string; } = {};
 

backend/src/api/mempool.ts

@@ -36,8 +36,6 @@ class Mempool {
   private timer = new Date().getTime();
   private missingTxCount = 0;
 
-  private mainLoopTimeout: number = 120000;
-
   constructor() {
     setInterval(this.updateTxPerSecond.bind(this), 1000);
   }
@@ -121,15 +119,10 @@ class Mempool {
 
   public async $updateMempool(): Promise<void> {
     logger.debug(`Updating mempool...`);
-
-    // throw an error if this stalls the main loop for more than 2 minutes
-    const timer = this.startTimer();
-
     const start = new Date().getTime();
     let hasChange: boolean = false;
     const currentMempoolSize = Object.keys(this.mempoolCache).length;
     const transactions = await bitcoinApi.$getRawMempool();
-    this.updateTimerProgress(timer, 'got raw mempool');
     const diff = transactions.length - currentMempoolSize;
     const newTransactions: TransactionExtended[] = [];
 
@@ -153,7 +146,6 @@ class Mempool {
       if (!this.mempoolCache[txid]) {
         try {
           const transaction = await transactionUtils.$getTransactionExtended(txid);
-          this.updateTimerProgress(timer, 'fetched new transaction');
           this.mempoolCache[txid] = transaction;
           if (this.inSync) {
             this.txPerSecondArray.push(new Date().getTime());
@@ -231,43 +223,12 @@ class Mempool {
       this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
     }
     if (this.asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) {
-      this.updateTimerProgress(timer, 'running async mempool callback');
       await this.asyncMempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
-      this.updateTimerProgress(timer, 'completed async mempool callback');
     }
 
     const end = new Date().getTime();
     const time = end - start;
     logger.debug(`Mempool updated in ${time / 1000} seconds. New size: ${Object.keys(this.mempoolCache).length} (${diff > 0 ? '+' + diff : diff})`);
-
-    this.clearTimer(timer);
-  }
-
-  private startTimer() {
-    const state: any = {
-      start: Date.now(),
-      progress: 'begin $updateMempool',
-      timer: null,
-    };
-    state.timer = setTimeout(() => {
-      logger.err(`$updateMempool stalled at "${state.progress}`);
-    }, this.mainLoopTimeout);
-    return state;
-  }
-
-  private updateTimerProgress(state, msg) {
-    state.progress = msg;
-    if (Date.now() - state.start > this.mainLoopTimeout) {
-      // abort the function if it already timed out
-      logger.err(`$updateMempool attempted to resume after "${state.progress}"`);
-      throw new Error(`$updateMempool attempted to resume after "${state.progress}"`);
-    }
-  }
-
-  private clearTimer(state) {
-    if (state.timer) {
-      clearTimeout(state.timer);
-    }
   }
 
   public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended; }) {

backend/src/api/tx-selection-worker.ts

@@ -107,7 +107,7 @@ function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction })
 
     if (nextTx && !nextTx?.used) {
       // Check if the package fits into this block
-      if (blocks.length >= 7 || (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS)) {
+      if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
         const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values());
         // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count)
         const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx];
@@ -175,14 +175,34 @@ function makeBlockTemplates(mempool: { [txid: string]: ThreadTransaction })
       overflow = [];
     }
   }
-
-  if (overflow.length > 0) {
-    logger.warn('GBT overflow list unexpectedly non-empty after final block constructed');
+  // pack any leftover transactions into the last block
+  for (const tx of overflow) {
+    if (!tx || tx?.used) {
+      continue;
+    }
+    blockWeight += tx.weight;
+    const mempoolTx = mempool[tx.txid];
+    // update original copy of this tx with effective fee rate & relatives data
+    mempoolTx.effectiveFeePerVsize = tx.score;
+    if (tx.ancestorMap.size > 0) {
+      cpfpClusters[tx.txid] = Array.from(tx.ancestorMap?.values()).map(a => a.txid);
+      mempoolTx.cpfpRoot = tx.txid;
+    }
+    mempoolTx.cpfpChecked = true;
+    transactions.push(tx);
+    tx.used = true;
   }
-  // add the final unbounded block if it contains any transactions
-  if (transactions.length > 0) {
-    blocks.push(transactions.map(t => mempool[t.txid]));
+  const blockTransactions = transactions.map(t => mempool[t.txid]);
+  restOfArray.forEach(tx => {
+    blockWeight += tx.weight;
+    tx.effectiveFeePerVsize = tx.feePerVsize;
+    tx.cpfpChecked = false;
+    blockTransactions.push(tx);
+  });
+  if (blockTransactions.length) {
+    blocks.push(blockTransactions);
   }
+  transactions = [];
 
   const end = Date.now();
   const time = end - start;

backend/src/config.ts

@@ -38,7 +38,6 @@ interface IConfig {
   ESPLORA: {
     REST_API_URL: string;
     UNIX_SOCKET_PATH: string | void | null;
-    RETRY_UNIX_SOCKET_AFTER: number;
   };
   LIGHTNING: {
     ENABLED: boolean;
@@ -166,7 +165,6 @@ const defaults: IConfig = {
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',
     'UNIX_SOCKET_PATH': null,
-    'RETRY_UNIX_SOCKET_AFTER': 30000,
   },
   'ELECTRUM': {
     'HOST': '127.0.0.1',

backend/src/index.ts

@@ -179,14 +179,8 @@ class Server {
         }
       }
       memPool.deleteExpiredTransactions();
-      await Promise.race([
-        blocks.$updateBlocks(),
-        new Promise((_, reject) => setTimeout(() => reject(new Error('$updateBlocks timed out')), 180000))
-      ]);
-      await Promise.race([
-        memPool.$updateMempool(),
-        new Promise((_, reject) => setTimeout(() => reject(new Error('$updateMempool timed out')), 180000))
-      ]);
+      await blocks.$updateBlocks();
+      await memPool.$updateMempool();
       indexer.$run();
 
       setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);

backend/src/repositories/BlocksRepository.ts

@@ -13,48 +13,6 @@ import chainTips from '../api/chain-tips';
 import blocks from '../api/blocks';
 import BlocksAuditsRepository from './BlocksAuditsRepository';
 
-interface DatabaseBlock {
-  id: string;
-  height: number;
-  version: number;
-  timestamp: number;
-  bits: number;
-  nonce: number;
-  difficulty: number;
-  merkle_root: string;
-  tx_count: number;
-  size: number;
-  weight: number;
-  previousblockhash: string;
-  mediantime: number;
-  totalFees: number;
-  medianFee: number;
-  feeRange: string;
-  reward: number;
-  poolId: number;
-  poolName: string;
-  poolSlug: string;
-  avgFee: number;
-  avgFeeRate: number;
-  coinbaseRaw: string;
-  coinbaseAddress: string;
-  coinbaseSignature: string;
-  coinbaseSignatureAscii: string;
-  avgTxSize: number;
-  totalInputs: number;
-  totalOutputs: number;
-  totalOutputAmt: number;
-  medianFeeAmt: number;
-  feePercentiles: string;
-  segwitTotalTxs: number;
-  segwitTotalSize: number;
-  segwitTotalWeight: number;
-  header: string;
-  utxoSetChange: number;
-  utxoSetSize: number;
-  totalInputAmt: number;
-}
-
 const BLOCK_DB_FIELDS = `
   blocks.hash AS id,
   blocks.height,
@@ -94,7 +52,7 @@ const BLOCK_DB_FIELDS = `
   blocks.header,
   blocks.utxoset_change AS utxoSetChange,
   blocks.utxoset_size AS utxoSetSize,
-  blocks.total_input_amt AS totalInputAmt
+  blocks.total_input_amt AS totalInputAmts
 `;
 
 class BlocksRepository {
@@ -213,32 +171,6 @@ class BlocksRepository {
     }
   }
 
-  /**
-   * Update missing fee amounts fields
-   *
-   * @param blockHash 
-   * @param feeAmtPercentiles 
-   * @param medianFeeAmt 
-   */
-  public async $updateFeeAmounts(blockHash: string, feeAmtPercentiles, medianFeeAmt) : Promise<void> {
-    try {
-      const query = `
-        UPDATE blocks
-        SET fee_percentiles = ?, median_fee_amt = ?
-        WHERE hash = ?
-      `;
-      const params: any[] = [
-        JSON.stringify(feeAmtPercentiles),
-        medianFeeAmt,
-        blockHash
-      ];
-      await DB.query(query, params);
-    } catch (e: any) {
-      logger.err(`Cannot update fee amounts for block ${blockHash}. Reason: ' + ${e instanceof Error ? e.message : e}`);
-      throw e;
-    }
-  }
-
   /**
    * Get all block height that have not been indexed between [startHeight, endHeight]
    */
@@ -500,7 +432,7 @@ class BlocksRepository {
 
       const blocks: BlockExtended[] = [];
       for (const block of rows) {
-        blocks.push(await this.formatDbBlockIntoExtendedBlock(block as DatabaseBlock));
+        blocks.push(await this.formatDbBlockIntoExtendedBlock(block));
       }
 
       return blocks;
@@ -527,7 +459,7 @@ class BlocksRepository {
         return null;
       }
 
-      return await this.formatDbBlockIntoExtendedBlock(rows[0] as DatabaseBlock);  
+      return await this.formatDbBlockIntoExtendedBlock(rows[0]);  
     } catch (e) {
       logger.err(`Cannot get indexed block ${height}. Reason: ` + (e instanceof Error ? e.message : e));
       throw e;
@@ -976,7 +908,7 @@ class BlocksRepository {
    * 
    * @param dbBlk 
    */
-  private async formatDbBlockIntoExtendedBlock(dbBlk: DatabaseBlock): Promise<BlockExtended> {
+  private async formatDbBlockIntoExtendedBlock(dbBlk: any): Promise<BlockExtended> {
     const blk: Partial<BlockExtended> = {};
     const extras: Partial<BlockExtension> = {};
 
@@ -1048,12 +980,11 @@ class BlocksRepository {
       if (extras.feePercentiles === null) {
         const block = await bitcoinClient.getBlock(dbBlk.id, 2);
         const summary = blocks.summarizeBlock(block);
-        await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.id, summary.transactions);
+        await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.hash, summary.transactions);
         extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
       }
       if (extras.feePercentiles !== null) {
         extras.medianFeeAmt = extras.feePercentiles[3];
-        await this.$updateFeeAmounts(dbBlk.id, extras.feePercentiles, extras.medianFeeAmt);
       }
     }
 

backend/src/repositories/BlocksSummariesRepository.ts

@@ -17,6 +17,26 @@ class BlocksSummariesRepository {
     return undefined;
   }
 
+  public async $saveSummary(params: { height: number, mined?: BlockSummary}): Promise<void> {
+    const blockId = params.mined?.id;
+    try {
+      const transactions = JSON.stringify(params.mined?.transactions || []);
+      await DB.query(`
+        INSERT INTO blocks_summaries (height, id, transactions, template)
+        VALUE (?, ?, ?, ?)
+        ON DUPLICATE KEY UPDATE
+          transactions = ?
+      `, [params.height, blockId, transactions, '[]', transactions]);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`Cannot save block summary for ${blockId} because it has already been indexed, ignoring`);
+      } else {
+        logger.debug(`Cannot save block summary for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`);
+        throw e;
+      }
+    }
+  }
+
   public async $saveTransactions(blockHeight: number, blockId: string, transactions: TransactionStripped[]): Promise<void> {
     try {
       const transactionsStr = JSON.stringify(transactions);

docker/README.md

@@ -6,6 +6,8 @@ If you are looking to use these Docker images to deploy your own instance of Mem
 
 See a video guide of this installation method by k3tan [on BitcoinTV.com](https://bitcointv.com/w/8fpAx6rf5CQ16mMhospwjg).
 
+**We only provide support for this installation method to <a href="https://mempool.space/enterprise">Enterprise sponsors</a>.**
+
 Jump to a section in this doc:
 - [Configure with Bitcoin Core Only](#configure-with-bitcoin-core-only)
 - [Configure with Bitcoin Core + Electrum Server](#configure-with-bitcoin-core--electrum-server)
@@ -205,8 +207,7 @@ Corresponding `docker-compose.yml` overrides:
 ```json
   "ESPLORA": {
     "REST_API_URL": "http://127.0.0.1:3000",
-    "UNIX_SOCKET_PATH": "/tmp/esplora-socket",
-    "RETRY_UNIX_SOCKET_AFTER": 30000
+    "UNIX_SOCKET_PATH": "/tmp/esplora-socket"
   },
 ```
 
@@ -216,7 +217,6 @@ Corresponding `docker-compose.yml` overrides:
     environment:
       ESPLORA_REST_API_URL: ""
       ESPLORA_UNIX_SOCKET_PATH: ""
-      ESPLORA_RETRY_UNIX_SOCKET_AFTER: ""
       ...
 ```
 

docker/backend/mempool-config.json

@@ -43,8 +43,7 @@
   },
   "ESPLORA": {
     "REST_API_URL": "__ESPLORA_REST_API_URL__",
-    "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__",
-    "RETRY_UNIX_SOCKET_AFTER": __ESPLORA_RETRY_UNIX_SOCKET_AFTER__
+    "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__"
   },
   "SECOND_CORE_RPC": {
     "HOST": "__SECOND_CORE_RPC_HOST__",

docker/backend/start.sh

@@ -47,7 +47,6 @@ __ELECTRUM_TLS_ENABLED__=${ELECTRUM_TLS_ENABLED:=false}
 # ESPLORA
 __ESPLORA_REST_API_URL__=${ESPLORA_REST_API_URL:=http://127.0.0.1:3000}
 __ESPLORA_UNIX_SOCKET_PATH__=${ESPLORA_UNIX_SOCKET_PATH:=null}
-__ESPLORA_RETRY_UNIX_SOCKET_AFTER__=${ESPLORA_RETRY_UNIX_SOCKET_AFTER:=30000}
 
 # SECOND_CORE_RPC
 __SECOND_CORE_RPC_HOST__=${SECOND_CORE_RPC_HOST:=127.0.0.1}
@@ -169,7 +168,6 @@ sed -i "s/__ELECTRUM_TLS_ENABLED__/${__ELECTRUM_TLS_ENABLED__}/g" mempool-config
 
 sed -i "s!__ESPLORA_REST_API_URL__!${__ESPLORA_REST_API_URL__}!g" mempool-config.json
 sed -i "s!__ESPLORA_UNIX_SOCKET_PATH__!${__ESPLORA_UNIX_SOCKET_PATH__}!g" mempool-config.json
-sed -i "s!__ESPLORA_RETRY_UNIX_SOCKET_AFTER__!${__ESPLORA_RETRY_UNIX_SOCKET_AFTER__}!g" mempool-config.json
 
 sed -i "s/__SECOND_CORE_RPC_HOST__/${__SECOND_CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__SECOND_CORE_RPC_PORT__/${__SECOND_CORE_RPC_PORT__}/g" mempool-config.json

frontend/src/app/components/mempool-blocks/mempool-blocks.component.html

@@ -2,7 +2,7 @@
   <div class="mempool-blocks-container" [class.time-ltr]="timeLtr" *ngIf="(difficultyAdjustments$ | async) as da;">
     <div class="flashing">
       <ng-template ngFor let-projectedBlock [ngForOf]="mempoolBlocks$ | async" let-i="index" [ngForTrackBy]="trackByFn">
-        <div @blockEntryTrigger [@.disabled]="i > 0 || !animateEntry" [attr.data-cy]="'mempool-block-' + i" class="bitcoin-block text-center mempool-block" id="mempool-block-{{ i }}" [ngStyle]="mempoolBlockStyles[i]" [class.blink-bg]="projectedBlock.blink">
+        <div @blockEntryTrigger [@.disabled]="!animateEntry" [attr.data-cy]="'mempool-block-' + i" class="bitcoin-block text-center mempool-block" id="mempool-block-{{ i }}" [ngStyle]="mempoolBlockStyles[i]" [class.blink-bg]="projectedBlock.blink">
           <a draggable="false" [routerLink]="['/mempool-block/' | relativeUrl, i]"
             class="blockLink" [ngClass]="{'disabled': (this.stateService.blockScrolling$ | async)}">&nbsp;</a>
           <div class="block-body">

frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts

@@ -1,4 +1,4 @@
-import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, ChangeDetectorRef, HostListener } from '@angular/core';
+import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, ChangeDetectorRef } from '@angular/core';
 import { Subscription, Observable, fromEvent, merge, of, combineLatest } from 'rxjs';
 import { MempoolBlock } from '../../interfaces/websocket.interface';
 import { StateService } from '../../services/state.service';
@@ -222,13 +222,8 @@ export class MempoolBlocksComponent implements OnInit, OnDestroy {
     clearTimeout(this.resetTransitionTimeout);
   }
 
-  @HostListener('window:resize', ['$event'])
-  onResize(): void {
-    this.animateEntry = false;
-  }
-
   trackByFn(index: number, block: MempoolBlock) {
-    return (block.isStack) ? `stack-${block.index}` : block.index;
+    return (block.isStack) ? 'stack' : block.index;
   }
 
   reduceMempoolBlocksToFitScreen(blocks: MempoolBlock[]): MempoolBlock[] {

frontend/src/app/components/start/start.component.ts

@@ -137,11 +137,9 @@ export class StartComponent implements OnInit, OnDestroy {
   }
 
   onMouseDown(event: MouseEvent) {
-    if (!(event.which > 1 || event.button > 0)) {
-      this.mouseDragStartX = event.clientX;
-      this.resetMomentum(event.clientX);
-      this.blockchainScrollLeftInit = this.blockchainContainer.nativeElement.scrollLeft;
-    }
+    this.mouseDragStartX = event.clientX;
+    this.resetMomentum(event.clientX);
+    this.blockchainScrollLeftInit = this.blockchainContainer.nativeElement.scrollLeft;
   }
   onPointerDown(event: PointerEvent) {
     if (this.isiOS) {

production/README.md

@@ -2,7 +2,7 @@
 
 These instructions are for setting up a serious production Mempool website for Bitcoin (mainnet, testnet, signet), Liquid (mainnet, testnet), and Bisq.
 
-Again, this setup is no joke—home users should use [one of the other installation methods](../#installation-methods).
+This setup is no joke—home users should use [one of the other installation methods](../#installation-methods). We only provide support for this installation method to <a href="https://mempool.space/enterprise">Enterprise sponsors</a>.
 
 ### Server Hardware
 

production/bitcoin.crontab

@@ -1,5 +1,5 @@
-@reboot screen -dmS mainnet /bitcoin/electrs/electrs-start-mainnet
-@reboot /usr/local/bin/bitcoind -testnet >/dev/null 2>&1
-@reboot screen -dmS testnet /bitcoin/electrs/electrs-start-testnet
-@reboot /usr/local/bin/bitcoind -signet >/dev/null 2>&1
-@reboot screen -dmS signet /bitcoin/electrs/electrs-start-signet
+@reboot sleep 30 ; screen -dmS mainnet /bitcoin/electrs/electrs-start-mainnet
+@reboot sleep 60 ; /usr/local/bin/bitcoind -testnet >/dev/null 2>&1
+@reboot sleep 70 ; screen -dmS testnet /bitcoin/electrs/electrs-start-testnet
+@reboot sleep 80 ; /usr/local/bin/bitcoind -signet >/dev/null 2>&1
+@reboot sleep 90 ; screen -dmS signet /bitcoin/electrs/electrs-start-signet

production/elements.crontab

@@ -1,10 +1,10 @@
 # start elements on reboot
-@reboot /usr/local/bin/elementsd -chain=liquidv1 >/dev/null 2>&1
-@reboot /usr/local/bin/elementsd -chain=liquidtestnet >/dev/null 2>&1
+@reboot sleep 60 ; /usr/local/bin/elementsd -chain=liquidv1 >/dev/null 2>&1
+@reboot sleep 60 ; /usr/local/bin/elementsd -chain=liquidtestnet >/dev/null 2>&1
 
 # start electrs on reboot
-@reboot screen -dmS liquidv1 /elements/electrs/electrs-start-liquid
-@reboot screen -dmS liquidtestnet /elements/electrs/electrs-start-liquidtestnet
+@reboot sleep 90 ; screen -dmS liquidv1 /elements/electrs/electrs-start-liquid
+@reboot sleep 90 ; screen -dmS liquidtestnet /elements/electrs/electrs-start-liquidtestnet
 
 # hourly asset update and electrs restart
 6 * * * * cd $HOME/asset_registry_db && git pull --quiet origin master && cd $HOME/asset_registry_testnet_db && git pull --quiet origin master && killall electrs

production/install

@@ -1325,10 +1325,10 @@ case $OS in
         public_ipv4=$( ifconfig | grep 'inet ' | awk -F ' ' '{ print $2 }' | grep -v '^103\.165\.192\.' | grep -v '^127\.0\.0\.1' )
         public_ipv6=$( ifconfig | grep 'inet6' | awk -F ' ' '{ print $2 }' | grep -v '^2001:df6:7280::' | grep -v '^fe80::' | grep -v '^::1' )
 
-        crontab_cln+="@reboot sleep 10 ; screen -dmS main lightningd --rpc-file-mode 0660 --alias `hostname` --disable-ip-discovery --autolisten false --bind-addr $public_ipv4 --announce-addr $public_ipv4 --bind-addr $public_ipv6 --announce-addr $public_ipv6\n"
-        crontab_cln+="@reboot sleep 10 ; screen -dmS tes lightningd --rpc-file-mode 0660 --alias `hostname` --network testnet --disable-ip-discovery --autolisten false --bind-addr $public_ipv4 --announce-addr $public_ipv4 --bind-addr $public_ipv6 --announce-addr $public_ipv6\n"
-        crontab_cln+="@reboot sleep 10 ; screen -dmS sig lightningd --rpc-file-mode 0660 --alias `hostname` --network signet --disable-ip-discovery --autolisten false --bind-addr $public_ipv4 --announce-addr $public_ipv4 --bind-addr $public_ipv6 --announce-addr $public_ipv6 \n"
-        crontab_cln+="@reboot sleep 20 ; /mempool/mempool.space/lightning-seeder >/dev/null 2>&1\n"
+        crontab_cln+="@reboot sleep 60 ; screen -dmS main lightningd --rpc-file-mode 0660 --alias `hostname` --disable-ip-discovery --autolisten false --bind-addr $public_ipv4 --announce-addr $public_ipv4 --bind-addr $public_ipv6 --announce-addr $public_ipv6\n"
+        crontab_cln+="@reboot sleep 90 ; screen -dmS tes lightningd --rpc-file-mode 0660 --alias `hostname` --network testnet --disable-ip-discovery --autolisten false --bind-addr $public_ipv4 --announce-addr $public_ipv4 --bind-addr $public_ipv6 --announce-addr $public_ipv6\n"
+        crontab_cln+="@reboot sleep 120 ; screen -dmS sig lightningd --rpc-file-mode 0660 --alias `hostname` --network signet --disable-ip-discovery --autolisten false --bind-addr $public_ipv4 --announce-addr $public_ipv4 --bind-addr $public_ipv6 --announce-addr $public_ipv6 \n"
+        crontab_cln+="@reboot sleep 180 ; /mempool/mempool.space/lightning-seeder >/dev/null 2>&1\n"
         crontab_cln+="1 * * * * /mempool/mempool.space/lightning-seeder >/dev/null 2>&1\n"
         echo "${crontab_cln}" | crontab -u "${CLN_USER}" -
     ;;

production/mempool-config.bisq.json

@@ -15,13 +15,12 @@
     "PASSWORD": "__BITCOIN_RPC_PASS__"
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:5000",
-    "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-mainnet"
+    "REST_API_URL": "http://127.0.0.1:4000"
   },
   "DATABASE": {
     "ENABLED": false,
     "HOST": "127.0.0.1",
-    "SOCKET": "/var/run/mysql/mysql.sock",
+    "PORT": 3306,
     "USERNAME": "__MEMPOOL_BISQ_USER__",
     "PASSWORD": "__MEMPOOL_BISQ_PASS__",
     "DATABASE": "mempool_bisq"

production/mempool-config.liquid.json

@@ -23,13 +23,12 @@
     "PASSWORD": "__BITCOIN_RPC_PASS__"
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:5001",
     "UNIX_SOCKET_PATH": "/elements/socket/esplora-liquid-mainnet"
   },
   "DATABASE": {
     "ENABLED": true,
     "HOST": "127.0.0.1",
-    "SOCKET": "/var/run/mysql/mysql.sock",
+    "PORT": 3306,
     "USERNAME": "__MEMPOOL_LIQUID_USER__",
     "PASSWORD": "__MEMPOOL_LIQUID_PASS__",
     "DATABASE": "mempool_liquid"

production/mempool-config.liquidtestnet.json

@@ -23,13 +23,12 @@
     "PASSWORD": "__BITCOIN_RPC_PASS__"
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:5004",
     "UNIX_SOCKET_PATH": "/elements/socket/esplora-liquid-testnet"
   },
   "DATABASE": {
     "ENABLED": true,
     "HOST": "127.0.0.1",
-    "SOCKET": "/var/run/mysql/mysql.sock",
+    "PORT": 3306,
     "USERNAME": "__MEMPOOL_LIQUIDTESTNET_USER__",
     "PASSWORD": "__MEMPOOL_LIQUIDTESTNET_PASS__",
     "DATABASE": "mempool_liquidtestnet"

production/mempool-config.mainnet-lightning.json

@@ -15,8 +15,7 @@
     "PASSWORD": "__BITCOIN_RPC_PASS__"
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:5000",
-    "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-mainnet"
+    "REST_API_URL": "http://127.0.0.1:4000"
   },
   "LIGHTNING": {
     "ENABLED": true,
@@ -42,7 +41,7 @@
   "DATABASE": {
     "ENABLED": true,
     "HOST": "127.0.0.1",
-    "SOCKET": "/var/run/mysql/mysql.sock",
+    "PORT": 3306,
     "USERNAME": "__MEMPOOL_MAINNET_LIGHTNING_USER__",
     "PASSWORD": "__MEMPOOL_MAINNET_LIGHTNING_PASS__",
     "DATABASE": "mempool_mainnet_lightning"

production/mempool-config.mainnet.json

@@ -31,13 +31,12 @@
     "PASSWORD": "__BITCOIN_RPC_PASS__"
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:5000",
     "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-mainnet"
   },
   "DATABASE": {
     "ENABLED": true,
     "HOST": "127.0.0.1",
-    "SOCKET": "/var/run/mysql/mysql.sock",
+    "PORT": 3306,
     "USERNAME": "__MEMPOOL_MAINNET_USER__",
     "PASSWORD": "__MEMPOOL_MAINNET_PASS__",
     "DATABASE": "mempool"

production/mempool-config.signet-lightning.json

@@ -15,8 +15,7 @@
     "PASSWORD": "__BITCOIN_RPC_PASS__"
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:5003",
-    "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-signet"
+    "REST_API_URL": "http://127.0.0.1:4003"
   },
   "LIGHTNING": {
     "ENABLED": true,
@@ -37,7 +36,7 @@
   "DATABASE": {
     "ENABLED": true,
     "HOST": "127.0.0.1",
-    "SOCKET": "/var/run/mysql/mysql.sock",
+    "PORT": 3306,
     "USERNAME": "__MEMPOOL_SIGNET_LIGHTNING_USER__",
     "PASSWORD": "__MEMPOOL_SIGNET_LIGHTNING_PASS__",
     "DATABASE": "mempool_signet_lightning"

production/mempool-config.signet.json

@@ -22,13 +22,12 @@
     "PASSWORD": "__BITCOIN_RPC_PASS__"
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:5003",
     "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-signet"
   },
   "DATABASE": {
     "ENABLED": true,
     "HOST": "127.0.0.1",
-    "SOCKET": "/var/run/mysql/mysql.sock",
+    "PORT": 3306,
     "USERNAME": "__MEMPOOL_SIGNET_USER__",
     "PASSWORD": "__MEMPOOL_SIGNET_PASS__",
     "DATABASE": "mempool_signet"

production/mempool-config.testnet-lightning.json

@@ -15,8 +15,7 @@
     "PASSWORD": "__BITCOIN_RPC_PASS__"
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:5002",
-    "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-testnet"
+    "REST_API_URL": "http://127.0.0.1:4002"
   },
   "LIGHTNING": {
     "ENABLED": true,
@@ -37,7 +36,7 @@
   "DATABASE": {
     "ENABLED": true,
     "HOST": "127.0.0.1",
-    "SOCKET": "/var/run/mysql/mysql.sock",
+    "PORT": 3306,
     "USERNAME": "__MEMPOOL_TESTNET_LIGHTNING_USER__",
     "PASSWORD": "__MEMPOOL_TESTNET_LIGHTNING_PASS__",
     "DATABASE": "mempool_testnet_lightning"

production/mempool-config.testnet.json

@@ -22,13 +22,12 @@
     "PASSWORD": "__BITCOIN_RPC_PASS__"
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:5002",
     "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-testnet"
   },
   "DATABASE": {
     "ENABLED": true,
     "HOST": "127.0.0.1",
-    "SOCKET": "/var/run/mysql/mysql.sock",
+    "PORT": 3306,
     "USERNAME": "__MEMPOOL_TESTNET_USER__",
     "PASSWORD": "__MEMPOOL_TESTNET_PASS__",
     "DATABASE": "mempool_testnet"

production/mempool.crontab

@@ -1,5 +1,5 @@
 # start on reboot
-@reboot sleep 90 ; $HOME/start
+@reboot sleep 10 ; $HOME/start
 
 # daily backup
 37 13 * * * sleep 30 ; /mempool/mempool.space/backup >/dev/null 2>&1 &

production/minfee.crontab

@@ -1 +1 @@
-@reboot /usr/local/bin/bitcoind >/dev/null 2>&1
+@reboot sleep 120 ; /usr/local/bin/bitcoind >/dev/null 2>&1