Merge pull request #1839 from mempool/wiz/fix-docker-backend-start-script

Add missing sed substitutions in docker/backend/start.sh

.github/ISSUE_TEMPLATE/config.yml

@@ -3,3 +3,6 @@ contact_links:
   - name: 🙋 Need help? Chat with us on Matrix
     url: https://matrix.to/#/#mempool.support:bitcoin.kyoto
     about: For support requests or general questions
+  - name: 🌐 Want to help with translations? Use Transifex
+    url: https://www.transifex.com/mempool/mempool
+    about: All translations work is done on Transifex

.github/pull_request_template.md

@@ -0,0 +1,6 @@
+<!--
+Please do not open pull requests for translations.
+
+All translations work is done on Transifex:
+https://www.transifex.com/mempool/mempool
+-->

.github/workflows/cypress.yml

@@ -1,10 +1,6 @@
 name: Cypress Tests
 
-on:
-  push:
-    branches:
-      - master
-  pull_request:
+on: [push, pull_request]
 jobs:
   cypress:
     runs-on: ${{ matrix.os }}
@@ -25,7 +21,7 @@ jobs:
           cache: 'npm'
           cache-dependency-path: frontend/package-lock.json
       - name: ${{ matrix.browser }} browser tests (Mempool)
-        uses: cypress-io/github-action@v2
+        uses: cypress-io/github-action@v4
         with:
           tag: ${{ github.event_name }}
           working-directory: frontend
@@ -36,9 +32,9 @@ jobs:
           record: true
           parallel: true
           spec: |
-            cypress/integration/mainnet/*.spec.ts
-            cypress/integration/signet/*.spec.ts
-            cypress/integration/testnet/*.spec.ts
+            cypress/e2e/mainnet/*.spec.ts
+            cypress/e2e/signet/*.spec.ts
+            cypress/e2e/testnet/*.spec.ts
           group: Tests on ${{ matrix.browser }} (Mempool)
           browser: ${{ matrix.browser }}
           ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'
@@ -49,7 +45,7 @@ jobs:
           CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
 
       - name: ${{ matrix.browser }} browser tests (Liquid)
-        uses: cypress-io/github-action@v2
+        uses: cypress-io/github-action@v4
         if: always()
         with:
           tag: ${{ github.event_name }}
@@ -61,8 +57,8 @@ jobs:
           record: true
           parallel: true
           spec: |
-            cypress/integration/liquid/liquid.spec.ts
-            cypress/integration/liquidtestnet/liquidtestnet.spec.ts
+            cypress/e2e/liquid/liquid.spec.ts
+            cypress/e2e/liquidtestnet/liquidtestnet.spec.ts
           group: Tests on ${{ matrix.browser }} (Liquid)
           browser: ${{ matrix.browser }}
           ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'
@@ -73,7 +69,7 @@ jobs:
           CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }}
 
       - name: ${{ matrix.browser }} browser tests (Bisq)
-        uses: cypress-io/github-action@v2
+        uses: cypress-io/github-action@v4
         if: always()
         with:
           tag: ${{ github.event_name }}
@@ -84,7 +80,7 @@ jobs:
           wait-on-timeout: 120
           record: true
           parallel: true
-          spec: cypress/integration/bisq/bisq.spec.ts
+          spec: cypress/e2e/bisq/bisq.spec.ts
           group: Tests on ${{ matrix.browser }} (Bisq)
           browser: ${{ matrix.browser }}
           ci-build-id: '${{ github.sha }}-${{ github.workflow }}-${{ github.event_name }}'

backend/mempool-config.sample.json

@@ -15,9 +15,10 @@
     "INDEXING_BLOCKS_AMOUNT": 11000,
     "PRICE_FEED_UPDATE_INTERVAL": 600,
     "USE_SECOND_NODE_FOR_MINFEE": false,
-    "EXTERNAL_ASSETS": [
-      "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json"
-    ],
+    "EXTERNAL_ASSETS": [],
+    "EXTERNAL_MAX_RETRY": 1,
+    "EXTERNAL_RETRY_INTERVAL": 0,
+    "USER_AGENT": "mempool",
     "STDOUT_LOG_MIN_PRIORITY": "debug"
   },
   "CORE_RPC": {
@@ -66,6 +67,7 @@
   },
   "SOCKS5PROXY": {
     "ENABLED": false,
+    "USE_ONION": true,
     "HOST": "127.0.0.1",
     "PORT": 9050,
     "USERNAME": "",
@@ -74,5 +76,13 @@
   "PRICE_DATA_SERVER": {
     "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices",
     "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices"
+  },
+  "EXTERNAL_DATA_SERVER": {
+    "MEMPOOL_API": "https://mempool.space/api/v1",
+    "MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1",
+    "LIQUID_API": "https://liquid.network/api/v1",
+    "LIQUID_ONION": "http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1",
+    "BISQ_URL": "https://bisq.markets/api",
+    "BISQ_ONION": "http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api"
   }
 }

backend/src/api/bisq/bisq.ts

@@ -1,10 +1,14 @@
 import config from '../../config';
 import * as fs from 'fs';
-import axios from 'axios';
+import axios, { AxiosResponse } from 'axios';
+import * as http from 'http';
+import * as https from 'https';
+import { SocksProxyAgent } from 'socks-proxy-agent';
 import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from './interfaces';
 import { Common } from '../common';
 import { BlockExtended } from '../../mempool.interfaces';
 import { StaticPool } from 'node-worker-threads-pool';
+import backendInfo from '../backend-info';
 import logger from '../../logger';
 
 class Bisq {
@@ -143,12 +147,59 @@ class Bisq {
       }, 2000);
     });
   }
+  private async updatePrice() {
+    type axiosOptions = {
+      headers: {
+        'User-Agent': string
+      };
+      timeout: number;
+      httpAgent?: http.Agent;
+      httpsAgent?: https.Agent;
+    }
+    const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
+    const BISQ_URL = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.EXTERNAL_DATA_SERVER.BISQ_ONION : config.EXTERNAL_DATA_SERVER.BISQ_URL;
+    const isHTTP = (new URL(BISQ_URL).protocol.split(':')[0] === 'http') ? true : false;
+    const axiosOptions: axiosOptions = {
+      headers: {
+        'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+      },
+      timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
+    };
+    let retry = 0;
 
-  private updatePrice() {
-    axios.get<BisqTrade[]>('https://bisq.markets/api/trades/?market=bsq_btc', { timeout: 10000 })
-      .then((response) => {
+    while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
+      try {
+        if (config.SOCKS5PROXY.ENABLED) {
+          const socksOptions: any = {
+            agentOptions: {
+              keepAlive: true,
+            },
+            hostname: config.SOCKS5PROXY.HOST,
+            port: config.SOCKS5PROXY.PORT
+          };
+
+          if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+            socksOptions.username = config.SOCKS5PROXY.USERNAME;
+            socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+          } else {
+            // Retry with different tor circuits https://stackoverflow.com/a/64960234
+            socksOptions.username = `circuit${retry}`;
+          }
+
+          // Handle proxy agent for onion addresses
+          if (isHTTP) {
+            axiosOptions.httpAgent = new SocksProxyAgent(socksOptions);
+          } else {
+            axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
+          }
+        }
+        
+        const data: AxiosResponse = await axios.get(`${BISQ_URL}/trades/?market=bsq_btc`, axiosOptions);
+        if (data.statusText === 'error' || !data.data) {
+          throw new Error(`Could not fetch data from Bisq market, Error: ${data.status}`);
+        }
         const prices: number[] = [];
-        response.data.forEach((trade) => {
+        data.data.forEach((trade) => {
           prices.push(parseFloat(trade.price) * 100000000);
         });
         prices.sort((a, b) => a - b);
@@ -156,9 +207,14 @@ class Bisq {
         if (this.priceUpdateCallbackFunction) {
           this.priceUpdateCallbackFunction(this.price);
         }
-    }).catch((err) => {
-      logger.err('Error updating Bisq market price: ' + err);
-    });
+        logger.debug('Successfully updated Bisq market price');
+        break;
+      } catch (e) {
+        logger.err('Error updating Bisq market price: '  + (e instanceof Error ? e.message : e));
+        await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
+        retry++;
+      }
+    }
   }
 
   private async loadBisqDumpFile(): Promise<void> {

backend/src/api/blocks.ts

@@ -18,6 +18,7 @@ import HashratesRepository from '../repositories/HashratesRepository';
 import indexer from '../indexer';
 import fiatConversion from './fiat-conversion';
 import RatesRepository from '../repositories/RatesRepository';
+import poolsParser from './pools-parser';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -139,7 +140,11 @@ class Blocks {
       if (blockExtended.extras?.coinbaseTx !== undefined) {
         pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
       } else {
-        pool = await poolsRepository.$getUnknownPool();
+        if (config.DATABASE.ENABLED === true) {
+          pool = await poolsRepository.$getUnknownPool();
+        } else {
+          pool = poolsParser.unknownPool;
+        }
       }
 
       if (!pool) { // We should never have this situation in practise
@@ -165,13 +170,22 @@ class Blocks {
    */
   private async $findBlockMiner(txMinerInfo: TransactionMinerInfo | undefined): Promise<PoolTag> {
     if (txMinerInfo === undefined || txMinerInfo.vout.length < 1) {
-      return await poolsRepository.$getUnknownPool();
+      if (config.DATABASE.ENABLED === true) {
+        return await poolsRepository.$getUnknownPool();
+      } else {
+        return poolsParser.unknownPool;
+      }
     }
 
     const asciiScriptSig = transactionUtils.hex2ascii(txMinerInfo.vin[0].scriptsig);
     const address = txMinerInfo.vout[0].scriptpubkey_address;
 
-    const pools: PoolTag[] = await poolsRepository.$getPools();
+    let pools: PoolTag[] = [];
+    if (config.DATABASE.ENABLED === true) {
+      pools = await poolsRepository.$getPools();
+    } else {
+      pools = poolsParser.miningPools;
+    }
     for (let i = 0; i < pools.length; ++i) {
       if (address !== undefined) {
         const addresses: string[] = JSON.parse(pools[i].addresses);
@@ -190,7 +204,11 @@ class Blocks {
       }
     }
 
-    return await poolsRepository.$getUnknownPool();
+    if (config.DATABASE.ENABLED === true) {
+      return await poolsRepository.$getUnknownPool();
+    } else {
+      return poolsParser.unknownPool;
+    }
   }
 
   /**

backend/src/api/disk-cache.ts

@@ -103,14 +103,14 @@ class DiskCache {
             }
           }
         } catch (e) {
-          logger.debug('Error parsing ' + fileName + '. Skipping.');
+          logger.info('Error parsing ' + fileName + '. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
         }
       }
 
       memPool.setMempool(data.mempool);
       blocks.setBlocks(data.blocks);
     } catch (e) {
-      logger.warn('Failed to parse mempoool and blocks cache. Skipping.');
+      logger.warn('Failed to parse mempoool and blocks cache. Skipping. Reason: ' + (e instanceof Error ? e.message : e));
     }
   }
 }

backend/src/api/fiat-conversion.ts

@@ -1,4 +1,6 @@
 import logger from '../logger';
+import * as http from 'http';
+import * as https from 'https';
 import axios, { AxiosResponse } from 'axios';
 import { IConversionRates } from '../mempool.interfaces';
 import config from '../config';
@@ -25,9 +27,10 @@ class FiatConversion {
   }
 
   public startService() {
+    const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL;
     logger.info('Starting currency rates service');
     if (config.SOCKS5PROXY.ENABLED) {
-      logger.info(`Currency rates service will be queried over the Tor network using ${config.PRICE_DATA_SERVER.TOR_URL}`);
+      logger.info(`Currency rates service will be queried over the Tor network using ${fiatConversionUrl}`);
     } else {
       logger.info(`Currency rates service will be queried over clearnet using ${config.PRICE_DATA_SERVER.CLEARNET_URL}`);
     }
@@ -40,49 +43,79 @@ class FiatConversion {
   }
 
   private async updateCurrency(): Promise<void> {
-    const headers = { 'User-Agent': `mempool/v${backendInfo.getBackendInfo().version}` };
-    let fiatConversionUrl: string;
-    let response: AxiosResponse;
+    type axiosOptions = {
+      headers: {
+        'User-Agent': string
+      };
+      timeout: number;
+      httpAgent?: http.Agent;
+      httpsAgent?: https.Agent;
+    }
+    const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
+    const fiatConversionUrl = (config.SOCKS5PROXY.ENABLED === true) && (config.SOCKS5PROXY.USE_ONION === true) ? config.PRICE_DATA_SERVER.TOR_URL : config.PRICE_DATA_SERVER.CLEARNET_URL;
+    const isHTTP = (new URL(fiatConversionUrl).protocol.split(':')[0] === 'http') ? true : false;
+    const axiosOptions: axiosOptions = {
+      headers: {
+        'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+      },
+      timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
+    };
 
-    try {
-      if (config.SOCKS5PROXY.ENABLED) {
-        let socksOptions: any = {
-          agentOptions: {
-            keepAlive: true,
-          },
-          hostname: config.SOCKS5PROXY.HOST,
-          port: config.SOCKS5PROXY.PORT
-        };
+    let retry = 0;
 
-        if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
-          socksOptions.username = config.SOCKS5PROXY.USERNAME;
-          socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+    while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
+      try {
+        if (config.SOCKS5PROXY.ENABLED) {
+          let socksOptions: any = {
+            agentOptions: {
+              keepAlive: true,
+            },
+            hostname: config.SOCKS5PROXY.HOST,
+            port: config.SOCKS5PROXY.PORT
+          };
+
+          if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+            socksOptions.username = config.SOCKS5PROXY.USERNAME;
+            socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+          } else {
+            // Retry with different tor circuits https://stackoverflow.com/a/64960234
+            socksOptions.username = `circuit${retry}`;
+          }
+
+          // Handle proxy agent for onion addresses
+          if (isHTTP) {
+            axiosOptions.httpAgent = new SocksProxyAgent(socksOptions);
+          } else {
+            axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
+          }
+        }
+        
+        logger.debug('Querying currency rates service...');
+
+        const response: AxiosResponse = await axios.get(`${fiatConversionUrl}`, axiosOptions);
+
+        if (response.statusText === 'error' || !response.data) {
+          throw new Error(`Could not fetch data from ${fiatConversionUrl}, Error: ${response.status}`);
         }
 
-        const agent = new SocksProxyAgent(socksOptions);
-        fiatConversionUrl = config.PRICE_DATA_SERVER.TOR_URL;
-        logger.debug('Querying currency rates service...');
-        response = await axios.get(fiatConversionUrl, { httpAgent: agent, headers: headers, timeout: 30000 });
-      } else {
-        fiatConversionUrl = config.PRICE_DATA_SERVER.CLEARNET_URL;
-        logger.debug('Querying currency rates service...');
-        response = await axios.get(fiatConversionUrl, { headers: headers, timeout: 10000 });
-      }
-
-      for (const rate of response.data.data) {
-        if (this.debasingFiatCurrencies.includes(rate.currencyCode) && rate.provider === 'Bisq-Aggregate') {
-          this.conversionRates[rate.currencyCode] = Math.round(100 * rate.price) / 100;
+        for (const rate of response.data.data) {
+          if (this.debasingFiatCurrencies.includes(rate.currencyCode) && rate.provider === 'Bisq-Aggregate') {
+            this.conversionRates[rate.currencyCode] = Math.round(100 * rate.price) / 100;
+          }
         }
-      }
 
-      this.ratesInitialized = true;
-      logger.debug(`USD Conversion Rate: ${this.conversionRates.USD}`);
+        this.ratesInitialized = true;
+        logger.debug(`USD Conversion Rate: ${this.conversionRates.USD}`);
 
-      if (this.ratesChangedCallback) {
-        this.ratesChangedCallback(this.conversionRates);
+        if (this.ratesChangedCallback) {
+          this.ratesChangedCallback(this.conversionRates);
+        }
+        break;
+      } catch (e) {
+        logger.err('Error updating fiat conversion rates: '  + (e instanceof Error ? e.message : e));
+        await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
+        retry++;
       }
-    } catch (e) {
-      logger.err('Error updating fiat conversion rates: ' + (e instanceof Error ? e.message : e));
     }
   }
 }

backend/src/api/pools-parser.ts

@@ -11,6 +11,14 @@ interface Pool {
 }
 
 class PoolsParser {
+  miningPools: any[] = [];
+  unknownPool: any = {
+    'name': "Unknown",
+    'link': "https://learnmeabitcoin.com/technical/coinbase-transaction",
+    'regexes': "[]",
+    'addresses': "[]",
+    'slug': 'unknown'
+  };
   slugWarnFlag = false;
 
   /**
@@ -60,12 +68,18 @@ class PoolsParser {
     // Get existing pools from the db
     let existingPools;
     try {
-      [existingPools] = await DB.query({ sql: 'SELECT * FROM pools;', timeout: 120000 });
+      if (config.DATABASE.ENABLED === true) {
+        [existingPools] = await DB.query({ sql: 'SELECT * FROM pools;', timeout: 120000 });
+      } else {
+        existingPools = [];
+      }
     } catch (e) {
       logger.err('Cannot get existing pools from the database, skipping pools.json import');
       return;
     }
 
+    this.miningPools = [];
+
     // Finally, we generate the final consolidated pools data
     const finalPoolDataAdd: Pool[] = [];
     const finalPoolDataUpdate: Pool[] = [];
@@ -97,24 +111,33 @@ class PoolsParser {
         logger.warn(`No slug found for '${poolNames[i]}', generating it => '${slug}'`);
       }
 
+      const poolObj = {
+        'name': finalPoolName,
+        'link': match[0].link,
+        'regexes': allRegexes,
+        'addresses': allAddresses,
+        'slug': slug
+      };
+
       if (existingPools.find((pool) => pool.name === poolNames[i]) !== undefined) {
-        finalPoolDataUpdate.push({
-          'name': finalPoolName,
-          'link': match[0].link,
-          'regexes': allRegexes,
-          'addresses': allAddresses,
-          'slug': slug
-        });
+        finalPoolDataUpdate.push(poolObj);
       } else {
         logger.debug(`Add '${finalPoolName}' mining pool`);
-        finalPoolDataAdd.push({
-          'name': finalPoolName,
-          'link': match[0].link,
-          'regexes': allRegexes,
-          'addresses': allAddresses,
-          'slug': slug
-        });
+        finalPoolDataAdd.push(poolObj);
       }
+
+      this.miningPools.push({
+        'name': finalPoolName,
+        'link': match[0].link,
+        'regexes': JSON.stringify(allRegexes),
+        'addresses': JSON.stringify(allAddresses),
+        'slug': slug
+      });
+    }
+
+    if (config.DATABASE.ENABLED === false) { // Don't run db operations
+      logger.info('Mining pools.json import completed (no database)');
+      return;
     }
 
     logger.debug(`Update pools table now`);
@@ -128,7 +151,7 @@ class PoolsParser {
     }
     queryAdd = queryAdd.slice(0, -1) + ';';
 
-    // Add new mining pools into the database
+    // Updated existing mining pools in the database
     const updateQueries: string[] = [];
     for (let i = 0; i < finalPoolDataUpdate.length; ++i) {
       updateQueries.push(`

backend/src/api/websocket-handler.ts

@@ -116,12 +116,10 @@ class WebsocketHandler {
               const index = parsedMessage['track-mempool-block'];
               client['track-mempool-block'] = index;
               const mBlocksWithTransactions = mempoolBlocks.getMempoolBlocksWithTransactions();
-              if (mBlocksWithTransactions[index]) {
-                response['projected-block-transactions'] = {
-                  index: index,
-                  blockTransactions: mBlocksWithTransactions[index].transactions
-                };
-              }
+              response['projected-block-transactions'] = {
+                index: index,
+                blockTransactions: mBlocksWithTransactions[index]?.transactions || [],
+              };
             } else {
               client['track-mempool-block'] = null;
             }

backend/src/config.ts

@@ -18,6 +18,9 @@ interface IConfig {
     PRICE_FEED_UPDATE_INTERVAL: number;
     USE_SECOND_NODE_FOR_MINFEE: boolean;
     EXTERNAL_ASSETS: string[];
+    EXTERNAL_MAX_RETRY: number;
+    EXTERNAL_RETRY_INTERVAL: number;
+    USER_AGENT: string;
     STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
   };
   ESPLORA: {
@@ -66,6 +69,7 @@ interface IConfig {
   };
   SOCKS5PROXY: {
     ENABLED: boolean;
+    USE_ONION: boolean;
     HOST: string;
     PORT: number;
     USERNAME: string;
@@ -75,6 +79,14 @@ interface IConfig {
     TOR_URL: string;
     CLEARNET_URL: string;
   };
+  EXTERNAL_DATA_SERVER: {
+    MEMPOOL_API: string;
+    MEMPOOL_ONION: string;
+    LIQUID_API: string;
+    LIQUID_ONION: string;
+    BISQ_URL: string;
+    BISQ_ONION: string;
+  };
 }
 
 const defaults: IConfig = {
@@ -94,9 +106,10 @@ const defaults: IConfig = {
     'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks
     'PRICE_FEED_UPDATE_INTERVAL': 600,
     'USE_SECOND_NODE_FOR_MINFEE': false,
-    'EXTERNAL_ASSETS': [
-      'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json'
-    ],
+    'EXTERNAL_ASSETS': [],
+    'EXTERNAL_MAX_RETRY': 1,
+    'EXTERNAL_RETRY_INTERVAL': 0,
+    'USER_AGENT': 'mempool',
     'STDOUT_LOG_MIN_PRIORITY': 'debug',
   },
   'ESPLORA': {
@@ -145,6 +158,7 @@ const defaults: IConfig = {
   },
   'SOCKS5PROXY': {
     'ENABLED': false,
+    'USE_ONION': true,
     'HOST': '127.0.0.1',
     'PORT': 9050,
     'USERNAME': '',
@@ -153,6 +167,14 @@ const defaults: IConfig = {
   "PRICE_DATA_SERVER": {
     'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
     'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices'
+  },
+  "EXTERNAL_DATA_SERVER": {
+    'MEMPOOL_API': 'https://mempool.space/api/v1',
+    'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
+    'LIQUID_API': 'https://liquid.network/api/v1',
+    'LIQUID_ONION': 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1',
+    'BISQ_URL': 'https://bisq.markets/api',
+    'BISQ_ONION': 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
   }
 };
 
@@ -168,6 +190,7 @@ class Config implements IConfig {
   BISQ: IConfig['BISQ'];
   SOCKS5PROXY: IConfig['SOCKS5PROXY'];
   PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
+  EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
 
   constructor() {
     const configs = this.merge(configFile, defaults);
@@ -182,6 +205,7 @@ class Config implements IConfig {
     this.BISQ = configs.BISQ;
     this.SOCKS5PROXY = configs.SOCKS5PROXY;
     this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
+    this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;
   }
 
   merge = (...objects: object[]): IConfig => {

backend/src/database.ts

@@ -22,12 +22,20 @@ import { PoolOptions } from 'mysql2/typings/mysql';
     timezone: '+00:00',
   };
 
+  private checkDBFlag() {
+    if (config.DATABASE.ENABLED === false) {
+      logger.err('Trying to use DB feature but config.DATABASE.ENABLED is set to false, please open an issue');
+    }
+  }
+
   public async query(query, params?) {
+    this.checkDBFlag();
     const pool = await this.getPool();
     return pool.query(query, params);
   }
 
   public async checkDbConnection() {
+    this.checkDBFlag();
     try {
       await this.query('SELECT ?', [1]);
       logger.info('Database connection established.');

backend/src/index.ts

@@ -205,7 +205,7 @@ class Server {
       .post(config.MEMPOOL.API_URL_PREFIX + 'tx/push', routes.$postTransactionForm)
       .get(config.MEMPOOL.API_URL_PREFIX + 'donations', async (req, res) => {
         try {
-          const response = await axios.get('https://mempool.space/api/v1/donations', { responseType: 'stream', timeout: 10000 });
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations`, { responseType: 'stream', timeout: 10000 });
           response.data.pipe(res);
         } catch (e) {
           res.status(500).end();
@@ -213,7 +213,7 @@ class Server {
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'donations/images/:id', async (req, res) => {
         try {
-          const response = await axios.get('https://mempool.space/api/v1/donations/images/' + req.params.id, {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/donations/images/${req.params.id}`, {
             responseType: 'stream', timeout: 10000
           });
           response.data.pipe(res);
@@ -223,7 +223,7 @@ class Server {
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'contributors', async (req, res) => {
         try {
-          const response = await axios.get('https://mempool.space/api/v1/contributors', { responseType: 'stream', timeout: 10000 });
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors`, { responseType: 'stream', timeout: 10000 });
           response.data.pipe(res);
         } catch (e) {
           res.status(500).end();
@@ -231,7 +231,7 @@ class Server {
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'contributors/images/:id', async (req, res) => {
         try {
-          const response = await axios.get('https://mempool.space/api/v1/contributors/images/' + req.params.id, {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/contributors/images/${req.params.id}`, {
             responseType: 'stream', timeout: 10000
           });
           response.data.pipe(res);
@@ -241,7 +241,7 @@ class Server {
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'translators', async (req, res) => {
         try {
-          const response = await axios.get('https://mempool.space/api/v1/translators', { responseType: 'stream', timeout: 10000 });
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators`, { responseType: 'stream', timeout: 10000 });
           response.data.pipe(res);
         } catch (e) {
           res.status(500).end();
@@ -249,7 +249,7 @@ class Server {
       })
       .get(config.MEMPOOL.API_URL_PREFIX + 'translators/images/:id', async (req, res) => {
         try {
-          const response = await axios.get('https://mempool.space/api/v1/translators/images/' + req.params.id, {
+          const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.MEMPOOL_API}/translators/images/${req.params.id}`, {
             responseType: 'stream', timeout: 10000
           });
           response.data.pipe(res);

backend/src/indexer.ts

@@ -13,7 +13,9 @@ class Indexer {
   }
 
   public reindex() {
-    this.runIndexer = true;
+    if (Common.indexingEnabled()) {
+      this.runIndexer = true;
+    }
   }
 
   public async $run() {

backend/src/routes.ts

@@ -990,7 +990,7 @@ class Routes {
 
   public async $getAllFeaturedLiquidAssets(req: Request, res: Response) {
     try {
-      const response = await axios.get('https://liquid.network/api/v1/assets/featured', { responseType: 'stream', timeout: 10000 });
+      const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/featured`, { responseType: 'stream', timeout: 10000 });
       response.data.pipe(res);
     } catch (e) {
       res.status(500).end();
@@ -999,7 +999,7 @@ class Routes {
 
   public async $getAssetGroup(req: Request, res: Response) {
     try {
-      const response = await axios.get('https://liquid.network/api/v1/assets/group/' + parseInt(req.params.id, 10),
+      const response = await axios.get(`${config.EXTERNAL_DATA_SERVER.LIQUID_API}/assets/group/${parseInt(req.params.id, 10)}`,
         { responseType: 'stream', timeout: 10000 });
       response.data.pipe(res);
     } catch (e) {

backend/src/sync-assets.ts

@@ -1,6 +1,7 @@
 import axios, { AxiosResponse } from 'axios';
 import * as fs from 'fs';
 import config from './config';
+import backendInfo from './api/backend-info';
 import logger from './logger';
 import { SocksProxyAgent } from 'socks-proxy-agent';
 
@@ -42,6 +43,9 @@ class SyncAssets {
 
           logger.info(`Downloading external asset ${fileName} over the Tor network...`);
           return axios.get(url, {
+            headers: {
+              'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+            },
             httpAgent: agent,
             httpsAgent: agent,
             responseType: 'stream',
@@ -57,6 +61,9 @@ class SyncAssets {
         } else {
           logger.info(`Downloading external asset ${fileName} over clearnet...`);
           return axios.get(url, {
+            headers: {
+              'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+            },
             responseType: 'stream',
             timeout: 30000
           }).then(function (response) {

backend/src/tasks/pools-updater.ts

@@ -1,7 +1,8 @@
-import axios from 'axios';
+import axios, { AxiosResponse } from 'axios';
 import poolsParser from '../api/pools-parser';
 import config from '../config';
 import DB from '../database';
+import backendInfo from '../api/backend-info';
 import logger from '../logger';
 import { SocksProxyAgent } from 'socks-proxy-agent';
 import * as https from 'https';
@@ -11,12 +12,13 @@ import * as https from 'https';
  */
 class PoolsUpdater {
   lastRun: number = 0;
+  currentSha: any = undefined;
 
   constructor() {
   }
 
   public async updatePoolsJson() {
-    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false || config.DATABASE.ENABLED === false) {
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
       return;
     }
 
@@ -38,14 +40,17 @@ class PoolsUpdater {
     }
 
     try {
-      const dbSha = await this.getShaFromDb();
       const githubSha = await this.fetchPoolsSha(); // Fetch pools.json sha from github
       if (githubSha === undefined) {
         return;
       }
 
-      logger.debug(`Pools.json sha | Current: ${dbSha} | Github: ${githubSha}`);
-      if (dbSha !== undefined && dbSha === githubSha) {
+      if (config.DATABASE.ENABLED === true) {
+        this.currentSha = await this.getShaFromDb();      
+      }
+
+      logger.debug(`Pools.json sha | Current: ${this.currentSha} | Github: ${githubSha}`);
+      if (this.currentSha !== undefined && this.currentSha === githubSha) {
         return;
       }
 
@@ -68,12 +73,14 @@ class PoolsUpdater {
    * Fetch our latest pools.json sha from the db
    */
   private async updateDBSha(githubSha: string) {
-    try {
-      await DB.query('DELETE FROM state where name="pools_json_sha"');
-      await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`);
-    } catch (e) {
-      logger.err('Cannot save github pools.json sha into the db. Reason: '  + (e instanceof Error ? e.message : e));
-      return undefined;
+    this.currentSha = githubSha;
+    if (config.DATABASE.ENABLED === true) {
+      try {
+        await DB.query('DELETE FROM state where name="pools_json_sha"');
+        await DB.query(`INSERT INTO state VALUES('pools_json_sha', NULL, '${githubSha}')`);
+      } catch (e) {
+        logger.err('Cannot save github pools.json sha into the db. Reason: '  + (e instanceof Error ? e.message : e));
+      }
     }
   }
 
@@ -113,33 +120,45 @@ class PoolsUpdater {
    */
   private async query(path): Promise<object | undefined> {
     type axiosOptions = {
+      headers: {
+        'User-Agent': string
+      };
+      timeout: number;
       httpsAgent?: https.Agent;
     }
     const setDelay = (secs: number = 1): Promise<void> => new Promise(resolve => setTimeout(() => resolve(), secs * 1000));
-    const axiosOptions: axiosOptions = {};
+    const axiosOptions: axiosOptions = {
+      headers: {
+        'User-Agent': (config.MEMPOOL.USER_AGENT === 'mempool') ? `mempool/v${backendInfo.getBackendInfo().version}` : `${config.MEMPOOL.USER_AGENT}`
+      },
+      timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
+    };
     let retry = 0;
 
-    if (config.SOCKS5PROXY.ENABLED) {
-      const socksOptions: any = {
-        agentOptions: {
-          keepAlive: true,
-        },
-        hostname: config.SOCKS5PROXY.HOST,
-        port: config.SOCKS5PROXY.PORT
-      };
-
-      if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
-        socksOptions.username = config.SOCKS5PROXY.USERNAME;
-        socksOptions.password = config.SOCKS5PROXY.PASSWORD;
-      }
-
-      axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
-    }
-
-    while(retry < 5) {
+    while(retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
       try {
-        const data = await axios.get(path, axiosOptions);
-        if (data.statusText !== 'OK' || !data.data) {
+        if (config.SOCKS5PROXY.ENABLED) {
+          const socksOptions: any = {
+            agentOptions: {
+              keepAlive: true,
+            },
+            hostname: config.SOCKS5PROXY.HOST,
+            port: config.SOCKS5PROXY.PORT
+          };
+
+          if (config.SOCKS5PROXY.USERNAME && config.SOCKS5PROXY.PASSWORD) {
+            socksOptions.username = config.SOCKS5PROXY.USERNAME;
+            socksOptions.password = config.SOCKS5PROXY.PASSWORD;
+          } else {
+            // Retry with different tor circuits https://stackoverflow.com/a/64960234
+            socksOptions.username = `circuit${retry}`;
+          }
+
+          axiosOptions.httpsAgent = new SocksProxyAgent(socksOptions);
+        }
+        
+        const data: AxiosResponse = await axios.get(path, axiosOptions);
+        if (data.statusText === 'error' || !data.data) {
           throw new Error(`Could not fetch data from Github, Error: ${data.status}`);
         }
         return data.data;
@@ -147,7 +166,7 @@ class PoolsUpdater {
         logger.err('Could not connect to Github. Reason: '  + (e instanceof Error ? e.message : e));
         retry++;
       }
-      await setDelay();
+      await setDelay(config.MEMPOOL.EXTERNAL_RETRY_INTERVAL);
     }
     return undefined;
   }

docker/backend/mempool-config.json

@@ -15,6 +15,9 @@
     "PRICE_FEED_UPDATE_INTERVAL": __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__,
     "USE_SECOND_NODE_FOR_MINFEE": __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__,
     "EXTERNAL_ASSETS": __MEMPOOL_EXTERNAL_ASSETS__,
+    "EXTERNAL_MAX_RETRY": __MEMPOOL_EXTERNAL_MAX_RETRY__,
+    "EXTERNAL_RETRY_INTERVAL": __MEMPOOL_EXTERNAL_RETRY_INTERVAL__,
+    "USER_AGENT": "__MEMPOOL_USER_AGENT__",
     "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
     "INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__
   },
@@ -64,6 +67,7 @@
   },
   "SOCKS5PROXY": {
     "ENABLED": __SOCKS5PROXY_ENABLED__,
+    "USE_ONION": __SOCKS5PROXY_USE_ONION__,
     "HOST": "__SOCKS5PROXY_HOST__",
     "PORT": "__SOCKS5PROXY_PORT__",
     "USERNAME": "__SOCKS5PROXY_USERNAME__",
@@ -72,5 +76,13 @@
   "PRICE_DATA_SERVER": {
     "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__",
     "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__"
+  },
+  "EXTERNAL_DATA_SERVER": {
+    "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__",
+    "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__",
+    "LIQUID_API": "__EXTERNAL_DATA_SERVER_LIQUID_API__",
+    "LIQUID_ONION": "__EXTERNAL_DATA_SERVER_LIQUID_ONION__",
+    "BISQ_URL": "__EXTERNAL_DATA_SERVER_BISQ_URL__",
+    "BISQ_ONION": "__EXTERNAL_DATA_SERVER_BISQ_ONION__"
   }
 }

docker/backend/start.sh

@@ -16,7 +16,10 @@ __MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_MEMPOOL_BLOCKS_AMOUNT:=8}
 __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=11000}
 __MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__=${MEMPOOL_PRICE_FEED_UPDATE_INTERVAL:=600}
 __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__=${MEMPOOL_USE_SECOND_NODE_FOR_MINFEE:=false}
-__MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[\"https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json\"]}
+__MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[]}
+__MEMPOOL_EXTERNAL_MAX_RETRY__=${MEMPOOL_EXTERNAL_MAX_RETRY:=1}
+__MEMPOOL_EXTERNAL_RETRY_INTERVAL__=${MEMPOOL_EXTERNAL_RETRY_INTERVAL:=0}
+__MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool}
 __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info}
 
 # CORE_RPC
@@ -65,6 +68,7 @@ __BISQ_DATA_PATH__=${BISQ_DATA_PATH:=/bisq/statsnode-data/btc_mainnet/db}
 
 # SOCKS5PROXY
 __SOCKS5PROXY_ENABLED__=${SOCKS5PROXY_ENABLED:=false}
+__SOCKS5PROXY_USE_ONION__=${SOCKS5PROXY_USE_ONION:=true}
 __SOCKS5PROXY_HOST__=${SOCKS5PROXY_HOST:=localhost}
 __SOCKS5PROXY_PORT__=${SOCKS5PROXY_PORT:=9050}
 __SOCKS5PROXY_USERNAME__=${SOCKS5PROXY_USERNAME:=""}
@@ -74,6 +78,14 @@ __SOCKS5PROXY_PASSWORD__=${SOCKS5PROXY_PASSWORD:=""}
 __PRICE_DATA_SERVER_TOR_URL__=${PRICE_DATA_SERVER_TOR_URL:=http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices}
 __PRICE_DATA_SERVER_CLEARNET_URL__=${PRICE_DATA_SERVER_CLEARNET_URL:=https://price.bisq.wiz.biz/getAllMarketPrices}
 
+# EXTERNAL_DATA_SERVER
+__EXTERNAL_DATA_SERVER_MEMPOOL_API__=${EXTERNAL_DATA_SERVER_MEMPOOL_API:=https://mempool.space/api/v1}
+__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__=${EXTERNAL_DATA_SERVER_MEMPOOL_ONION:=http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1}
+__EXTERNAL_DATA_SERVER_LIQUID_API__=${EXTERNAL_DATA_SERVER_LIQUID_API:=https://liquid.network/api/v1}
+__EXTERNAL_DATA_SERVER_LIQUID_ONION__=${EXTERNAL_DATA_SERVER_LIQUID_ONION:=http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1}
+__EXTERNAL_DATA_SERVER_BISQ_URL__=${EXTERNAL_DATA_SERVER_BISQ_URL:=https://bisq.markets/api}
+__EXTERNAL_DATA_SERVER_BISQ_ONION__=${EXTERNAL_DATA_SERVER_BISQ_ONION:=http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api}
+
 mkdir -p "${__MEMPOOL_CACHE_DIR__}"
 
 sed -i "s/__MEMPOOL_NETWORK__/${__MEMPOOL_NETWORK__}/g" mempool-config.json
@@ -92,6 +104,9 @@ sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT_
 sed -i "s/__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__/${__MEMPOOL_PRICE_FEED_UPDATE_INTERVAL__}/g" mempool-config.json
 sed -i "s/__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__/${__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__}/g" mempool-config.json
 sed -i "s!__MEMPOOL_EXTERNAL_ASSETS__!${__MEMPOOL_EXTERNAL_ASSETS__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_EXTERNAL_MAX_RETRY__!${__MEMPOOL_EXTERNAL_MAX_RETRY__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_EXTERNAL_RETRY_INTERVAL__!${__MEMPOOL_EXTERNAL_RETRY_INTERVAL__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_USER_AGENT__!${__MEMPOOL_USER_AGENT__}!g" mempool-config.json
 sed -i "s/__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__/${__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__}/g" mempool-config.json
 
 sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
@@ -132,6 +147,7 @@ sed -i "s/__BISQ_ENABLED__/${__BISQ_ENABLED__}/g" mempool-config.json
 sed -i "s!__BISQ_DATA_PATH__!${__BISQ_DATA_PATH__}!g" mempool-config.json
 
 sed -i "s/__SOCKS5PROXY_ENABLED__/${__SOCKS5PROXY_ENABLED__}/g" mempool-config.json
+sed -i "s/__SOCKS5PROXY_USE_ONION__/${__SOCKS5PROXY_USE_ONION__}/g" mempool-config.json
 sed -i "s/__SOCKS5PROXY_HOST__/${__SOCKS5PROXY_HOST__}/g" mempool-config.json
 sed -i "s/__SOCKS5PROXY_PORT__/${__SOCKS5PROXY_PORT__}/g" mempool-config.json
 sed -i "s/__SOCKS5PROXY_USERNAME__/${__SOCKS5PROXY_USERNAME__}/g" mempool-config.json
@@ -140,4 +156,11 @@ sed -i "s/__SOCKS5PROXY_PASSWORD__/${__SOCKS5PROXY_PASSWORD__}/g" mempool-config
 sed -i "s!__PRICE_DATA_SERVER_TOR_URL__!${__PRICE_DATA_SERVER_TOR_URL__}!g" mempool-config.json
 sed -i "s!__PRICE_DATA_SERVER_CLEARNET_URL__!${__PRICE_DATA_SERVER_CLEARNET_URL__}!g" mempool-config.json
 
+sed -i "s!__EXTERNAL_DATA_SERVER_MEMPOOL_API__!${__EXTERNAL_DATA_SERVER_MEMPOOL_API__}!g" mempool-config.json
+sed -i "s!__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__!${__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__}!g" mempool-config.json
+sed -i "s!__EXTERNAL_DATA_SERVER_LIQUID_API__!${__EXTERNAL_DATA_SERVER_LIQUID_API__}!g" mempool-config.json
+sed -i "s!__EXTERNAL_DATA_SERVER_LIQUID_ONION__!${__EXTERNAL_DATA_SERVER_LIQUID_ONION__}!g" mempool-config.json
+sed -i "s!__EXTERNAL_DATA_SERVER_BISQ_API__!${__EXTERNAL_DATA_SERVER_BISQ_API__}!g" mempool-config.json
+sed -i "s!__EXTERNAL_DATA_SERVER_BISQ_ONION__!${__EXTERNAL_DATA_SERVER_BISQ_ONION__}!g" mempool-config.json
+
 node /backend/dist/index.js

frontend/cypress.config.ts

@@ -0,0 +1,23 @@
+import { defineConfig } from 'cypress'
+
+export default defineConfig({
+  projectId: 'ry4br7',
+  videosFolder: 'cypress/videos',
+  screenshotsFolder: 'cypress/screenshots',
+  fixturesFolder: 'cypress/fixtures',
+  video: false,
+  retries: {
+    runMode: 3,
+    openMode: 0,
+  },
+  chromeWebSecurity: false,
+  e2e: {
+    // We've imported your old cypress plugins here.
+    // You may want to clean this up later by importing these.
+    setupNodeEvents(on, config) {
+      return require('./cypress/plugins/index.js')(on, config)
+    },
+    baseUrl: 'http://localhost:4200',
+    specPattern: 'cypress/e2e/**/*.{js,jsx,ts,tsx}',
+  },
+})

frontend/cypress.json

@@ -1,16 +0,0 @@
-{
-  "projectId": "ry4br7",
-  "integrationFolder": "cypress/integration",
-  "supportFile": "cypress/support/index.ts",
-  "videosFolder": "cypress/videos",
-  "screenshotsFolder": "cypress/screenshots",
-  "pluginsFile": "cypress/plugins/index.js",
-  "fixturesFolder": "cypress/fixtures",
-  "baseUrl": "http://localhost:4200",
-  "video": false,
-  "retries": {
-    "runMode": 3,
-    "openMode": 0
-  },
-  "chromeWebSecurity": false
-}

frontend/cypress/e2e/bisq/bisq.spec.ts

@@ -35,13 +35,14 @@ describe('Bisq', () => {
         "Proposal", "Reimbursement request", "Transfer BSQ", "Unlock", "Vote reveal"
       ];
       filters.forEach((filter) => {
-        it(`filters the transaction screen by ${filter}`, () => {
+        it.only(`filters the transaction screen by ${filter}`, () => {
           cy.visit(`${basePath}/transactions`);
+          cy.wait('@txs');
           cy.waitForSkeletonGone();
           cy.get('#filter').click();
           cy.contains(filter).find('input').click();
-          //TODO: change this waiter
-          cy.wait(1000);
+          cy.wait('@txs');
+          cy.wait(500);
           cy.get('td:nth-of-type(2)').each(($td) => {
             expect($td.text().trim()).to.eq(filter);
           });
@@ -56,7 +57,7 @@ describe('Bisq', () => {
         filters.forEach((filter) => {
           cy.contains(filter).find('input').click();
           //TODO: change this waiter
-          cy.wait(1000);
+          cy.wait(1500);
         });
         cy.get('td:nth-of-type(2)').each(($td) => {
           const regex = new RegExp(`${filters.join('|')}`, 'g');

frontend/cypress/e2e/liquid/liquid.spec.ts

@@ -124,7 +124,7 @@ describe('Liquid', () => {
         cy.visit(`${basePath}/assets`);
         cy.waitForSkeletonGone();
         cy.get('.container-xl input').click().type('Liquid Bitcoin').then(() => {
-          cy.get('ngb-typeahead-window').should('have.length', 1);
+          cy.get('ngb-typeahead-window', { timeout: 30000 }).should('have.length', 1);
         });
       });
 
@@ -132,7 +132,7 @@ describe('Liquid', () => {
         cy.visit(`${basePath}/assets`);
         cy.waitForSkeletonGone();
         cy.get('.container-xl input').click().type('Liquid AUD').then(() => {
-          cy.get('ngb-typeahead-window:nth-of-type(1) button').click();
+          cy.get('ngb-typeahead-window:nth-of-type(1) button', { timeout: 30000 }).click();
         });
       });
     });

frontend/cypress/e2e/mainnet/mainnet.spec.ts

@@ -189,7 +189,7 @@ describe('Mainnet', () => {
         cy.get('[data-cy="tx-2"] .table-tx-vin .highlight').invoke('text').should('contain', `${address}`);
       });
 
-      it.only('highlights both input and output addresses in the same transaction', () => {
+      it('highlights both input and output addresses in the same transaction', () => {
         const address = 'bc1q03u63r6hm7a3v6em58zdqtp446w2pw30nm63mv';
         cy.visit(`/address/${address}`);
         cy.waitForSkeletonGone();
@@ -241,7 +241,7 @@ describe('Mainnet', () => {
             cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('not.exist');
             cy.get('[ngbtooltip="Previous Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
             cy.document().left();
-            cy.get('.title-block h1').invoke('text').should('equal', 'Next block');
+            cy.get('.title-block h1').invoke('text').should('equal', 'Next Block');
           });
         });
 

frontend/cypress/support/commands.ts

@@ -50,98 +50,98 @@ import { mockWebSocket } from './websocket';
 
 /* global Cypress */
 const codes = {
-    ArrowLeft: 37,
-    ArrowUp: 38,
-    ArrowRight: 39,
-    ArrowDown: 40
+  ArrowLeft: 37,
+  ArrowUp: 38,
+  ArrowRight: 39,
+  ArrowDown: 40
 }
 
 Cypress.Commands.add('waitForSkeletonGone', () => {
-    cy.waitUntil(() => {
-        return Cypress.$('.skeleton-loader').length === 0;
-    }, { verbose: true, description: "waitForSkeletonGone", errorMsg: "skeleton loaders never went away", timeout: 15000, interval: 50});
+  cy.waitUntil(() => {
+    return Cypress.$('.skeleton-loader').length === 0;
+  }, { verbose: true, description: "waitForSkeletonGone", errorMsg: "skeleton loaders never went away", timeout: 15000, interval: 50 });
 });
 
 Cypress.Commands.add(
-    "waitForPageIdle",
-    () => {
-        console.warn("Waiting for page idle state");
-        const pageIdleDetector = new PageIdleDetector();
-        pageIdleDetector.WaitForPageToBeIdle();
-    }
+  "waitForPageIdle",
+  () => {
+    console.warn("Waiting for page idle state");
+    const pageIdleDetector = new PageIdleDetector();
+    pageIdleDetector.WaitForPageToBeIdle();
+  }
 );
 
 Cypress.Commands.add('mockMempoolSocket', () => {
   mockWebSocket();
 });
 
-Cypress.Commands.add('changeNetwork', (network: "testnet"|"signet"|"liquid"|"bisq"|"mainnet" ) => {
-    cy.get('.dropdown-toggle').click().then(() => {
-        cy.get(`.${network}`).click().then(() => {
-            cy.waitForPageIdle();
-            if(network !== 'bisq'){
-                cy.waitForSkeletonGone();
-            }
-        });
+Cypress.Commands.add('changeNetwork', (network: "testnet" | "signet" | "liquid" | "bisq" | "mainnet") => {
+  cy.get('.dropdown-toggle').click().then(() => {
+    cy.get(`a.${network}`).click().then(() => {
+      cy.waitForPageIdle();
+      if (network !== 'bisq') {
+        cy.waitForSkeletonGone();
+      }
     });
+  });
 });
 
 // https://github.com/bahmutov/cypress-arrows/blob/8f0303842a343550fbeaf01528d01d1ff213b70c/src/index.js
-function keydownCommand ($el, key) {
-    const message = `sending the "${key}" keydown event`
-    const log = Cypress.log({
-      name: `keydown: ${key}`,
-      message: message,
-      consoleProps: function () {
-        return {
-          Subject: $el
-        }
+function keydownCommand($el, key) {
+  const message = `sending the "${key}" keydown event`
+  const log = Cypress.log({
+    name: `keydown: ${key}`,
+    message: message,
+    consoleProps: function () {
+      return {
+        Subject: $el
       }
-    })
-  
-    const e = $el.createEvent('KeyboardEvent')
-  
-    Object.defineProperty(e, 'key', {
-      get: function () {
-        return key
-      }
-    })
-  
-    Object.defineProperty(e, 'keyCode', {
-      get: function () {
-        return this.keyCodeVal
-      }
-    })
-    Object.defineProperty(e, 'which', {
-      get: function () {
-        return this.keyCodeVal
-      }
-    })
-    var metaKey = false
-  
-    Object.defineProperty(e, 'metaKey', {
-      get: function () {
-        return metaKey
-      }
-    })
-  
-    Object.defineProperty(e, 'shiftKey', {
-      get: function () {
-        return false
-      }
-    })
-    e.keyCodeVal = codes[key]
-  
-    e.initKeyboardEvent('keydown', true, true,
-      $el.defaultView, false, false, false, false, e.keyCodeVal, e.keyCodeVal)
-  
-    $el.dispatchEvent(e)
-    log.snapshot().end()
-    return $el
-  }
-  
-  Cypress.Commands.add('keydown', { prevSubject: "dom" }, keydownCommand)
-  Cypress.Commands.add('left', { prevSubject: "dom" }, $el => keydownCommand($el, 'ArrowLeft'))
-  Cypress.Commands.add('right', { prevSubject: "dom" }, $el => keydownCommand($el, 'ArrowRight'))
-  Cypress.Commands.add('up', { prevSubject: "dom" }, $el => keydownCommand($el, 'ArrowUp'))
-  Cypress.Commands.add('down', { prevSubject: "dom" }, $el => keydownCommand($el, 'ArrowDown'))
+    }
+  })
+
+  const e = $el.createEvent('KeyboardEvent')
+
+  Object.defineProperty(e, 'key', {
+    get: function () {
+      return key
+    }
+  })
+
+  Object.defineProperty(e, 'keyCode', {
+    get: function () {
+      return this.keyCodeVal
+    }
+  })
+  Object.defineProperty(e, 'which', {
+    get: function () {
+      return this.keyCodeVal
+    }
+  })
+  var metaKey = false
+
+  Object.defineProperty(e, 'metaKey', {
+    get: function () {
+      return metaKey
+    }
+  })
+
+  Object.defineProperty(e, 'shiftKey', {
+    get: function () {
+      return false
+    }
+  })
+  e.keyCodeVal = codes[key]
+
+  e.initKeyboardEvent('keydown', true, true,
+    $el.defaultView, false, false, false, false, e.keyCodeVal, e.keyCodeVal)
+
+  $el.dispatchEvent(e)
+  log.snapshot().end()
+  return $el
+}
+
+Cypress.Commands.add('keydown', { prevSubject: "dom" }, keydownCommand)
+Cypress.Commands.add('left', { prevSubject: "dom" }, $el => keydownCommand($el, 'ArrowLeft'))
+Cypress.Commands.add('right', { prevSubject: "dom" }, $el => keydownCommand($el, 'ArrowRight'))
+Cypress.Commands.add('up', { prevSubject: "dom" }, $el => keydownCommand($el, 'ArrowUp'))
+Cypress.Commands.add('down', { prevSubject: "dom" }, $el => keydownCommand($el, 'ArrowDown'))

frontend/package-lock.json

@@ -70,7 +70,7 @@
       },
       "optionalDependencies": {
         "@cypress/schematic": "^1.3.0",
-        "cypress": "^9.6.1",
+        "cypress": "^10.0.2",
         "cypress-fail-on-console-error": "^2.1.3",
         "cypress-wait-until": "^1.7.1",
         "mock-socket": "^9.0.3",
@@ -6901,9 +6901,9 @@
       "devOptional": true
     },
     "node_modules/cypress": {
-      "version": "9.6.1",
-      "resolved": "https://registry.npmjs.org/cypress/-/cypress-9.6.1.tgz",
-      "integrity": "sha512-ECzmV7pJSkk+NuAhEw6C3D+RIRATkSb2VAHXDY6qGZbca/F9mv5pPsj2LO6Ty6oIFVBTrwCyL9agl28MtJMe2g==",
+      "version": "10.0.2",
+      "resolved": "https://registry.npmjs.org/cypress/-/cypress-10.0.2.tgz",
+      "integrity": "sha512-7+C4KHYBcfZrawss+Gt5PlS35rfc6ySc59JcHDVsIMm1E/J35dqE41UEXpdtwIq3549umCerNWnFADzqib4kcA==",
       "hasInstallScript": true,
       "optional": true,
       "dependencies": {
@@ -22471,9 +22471,9 @@
       "devOptional": true
     },
     "cypress": {
-      "version": "9.6.1",
-      "resolved": "https://registry.npmjs.org/cypress/-/cypress-9.6.1.tgz",
-      "integrity": "sha512-ECzmV7pJSkk+NuAhEw6C3D+RIRATkSb2VAHXDY6qGZbca/F9mv5pPsj2LO6Ty6oIFVBTrwCyL9agl28MtJMe2g==",
+      "version": "10.0.2",
+      "resolved": "https://registry.npmjs.org/cypress/-/cypress-10.0.2.tgz",
+      "integrity": "sha512-7+C4KHYBcfZrawss+Gt5PlS35rfc6ySc59JcHDVsIMm1E/J35dqE41UEXpdtwIq3549umCerNWnFADzqib4kcA==",
       "optional": true,
       "requires": {
         "@cypress/request": "^2.88.10",

frontend/package.json

@@ -122,7 +122,7 @@
   },
   "optionalDependencies": {
     "@cypress/schematic": "^1.3.0",
-    "cypress": "^9.6.1",
+    "cypress": "^10.0.2",
     "cypress-fail-on-console-error": "^2.1.3",
     "cypress-wait-until": "^1.7.1",
     "mock-socket": "^9.0.3",

frontend/src/app/components/about/about.component.html

@@ -48,11 +48,13 @@
         <span>Spiral</span>
       </a>
       <a href="https://foundrydigital.com/" target="_blank" title="Foundry">
-        <svg xmlns="http://www.w3.org/2000/svg" version="1.1" width="400px" height="400px" viewBox="0 0 400 400" class="image">
+        <svg xmlns="http://www.w3.org/2000/svg" version="1.1" viewBox="-10 -10 100 100" class="image">
           <g stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
-            <g>
-              <rect fill="#87E1A1" fill-rule="nonzero" x="0" y="0" width="400" height="400"></rect>
-              <path d="M124,149.256434 L169.106586,149.256434 L169.106586,128.378728 C169.106586,102.958946 183.316852,90 207.489341,90 L276.773787,90 L276.773787,119.404671 L222.192348,119.404671 C216.458028,119.404671 213.968815,122.397366 213.968815,127.633575 L213.968815,149.256434 L276.023264,149.256434 L276.023264,181.902184 L213.968815,181.902184 L213.968815,310 L169.106586,310 L169.106586,181.902184 L124,181.902184 L124,149.256434" fill="#000000"></path>
+            <g transform="translate(-186.000000, -2316.000000)">
+              <g transform="translate(186.000000, 2316.000000)">
+                <rect id="" fill="#023D32" x="-10" y="-10" width="100" height="100" rx="8"></rect>
+                <path d="M61.6666667,9.16666667 L61.6666667,17.0041667 L46.2625,17.0041667 C46.2625,17.0041667 44.1666667,16.6666667 44.1666667,18.3333333 L44.1666667,25.8025 L61.6666667,25.8025 L61.6666667,34.7391667 L44.1666667,34.7391667 L44.1666667,70.5575 L31.7825,70.5575 L31.7825,35 L19.1666667,35 L19.1666667,25.595 L31.6666667,25.595 L31.6666667,17.5 C31.6666667,17.5 32.5,9.16666667 40.4166667,9.16666667 L61.6666667,9.16666667 Z" id="Fill-1" fill="#86E2A0"></path>
+              </g>
             </g>
           </g>
         </svg>

frontend/src/app/components/address-labels/address-labels.component.ts

@@ -85,6 +85,8 @@ export class AddressLabelsComponent implements OnInit {
     }
 
     this.detectMultisig(this.vin.inner_redeemscript_asm);
+
+    this.detectMultisig(this.vin.prevout.scriptpubkey_asm);
   }
 
   detectMultisig(script: string) {
@@ -118,7 +120,11 @@ export class AddressLabelsComponent implements OnInit {
     }
     const m = parseInt(opM.match(/[0-9]+/)[0], 10);
 
-    this.label = $localize`:@@address-label.multisig:Multisig ${m}:multisigM: of ${n}:multisigN:`
+    if (ops.length) {
+      return;
+    }
+
+    this.label = $localize`:@@address-label.multisig:Multisig ${m}:multisigM: of ${n}:multisigN:`;
   }
 
   handleVout() {

frontend/src/app/components/blocks-list/blocks-list.component.html

@@ -22,7 +22,7 @@
       <tbody *ngIf="blocks$ | async as blocks; else skeleton" [style]="isLoading ? 'opacity: 0.75' : ''">
         <tr *ngFor="let block of blocks; let i= index; trackBy: trackByBlock">
           <td class="text-left" [class]="widget ? 'widget' : ''">
-            <a [routerLink]="['/block' | relativeUrl, block.id]" [state]="{ data: { block: block } }">{{ block.height }}</a>
+            <a [routerLink]="['/block' | relativeUrl, block.id]">{{ block.height }}</a>
           </td>
           <td  *ngIf="indexingAvailable" class="pool text-left" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
             <div class="tooltip-custom">

frontend/src/app/components/blocks-list/blocks-list.component.ts

@@ -80,7 +80,7 @@ export class BlocksList implements OnInit {
       this.stateService.blocks$
         .pipe(
           switchMap((block) => {
-            if (block[0].height <= this.lastBlockHeight) {
+            if (block[0].height < this.lastBlockHeight) {
               return []; // Return an empty stream so the last pipe is not executed
             }
             this.lastBlockHeight = block[0].height;

frontend/src/app/components/fees-box/fees-box.component.html

@@ -1,35 +1,35 @@
 <div class="fee-estimation-wrapper" *ngIf="(isLoadingWebSocket$ | async) === false && (recommendedFees$ | async) as recommendedFees; else loadingFees">
   <div class="d-flex">
     <div class="fee-progress-bar" [style.background]="noPriority">
-      <span class="fee-label" i18n="fees-box.no-priority">No Priority</span>
+      <span class="fee-label" i18n="fees-box.no-priority" i18n-ngbTooltip="Transaction feerate tooltip (economy)" ngbTooltip="Either 2x the minimum, or the Low Priority rate (whichever is lower)" placement="top">No Priority</span>
     </div>
     <div class="band-separator fill"></div>
     <div class="fee-progress-bar priority" [style.background]="gradient">
-      <span class="fee-label prority" i18n="fees-box.low-priority">Low Priority</span>
-      <span class="fee-label prority" i18n="fees-box.medium-priority">Medium Priority</span>
-      <span class="fee-label prority" i18n="fees-box.high-priority">High Priority</span>
+      <span class="fee-label prority" i18n="fees-box.low-priority" i18n-ngbTooltip="Transaction feerate tooltip (low priority)" ngbTooltip="Places your transaction in between the second and third mempool blocks" placement="top">Low Priority</span>
+      <span class="fee-label prority" i18n="fees-box.medium-priority" i18n-ngbTooltip="Transaction feerate tooltip (medium priority)" ngbTooltip="Places your transaction in between the first and second mempool blocks" placement="top">Medium Priority</span>
+      <span class="fee-label prority" i18n="fees-box.high-priority" i18n-ngbTooltip="Transaction feerate tooltip (high priority)" ngbTooltip="Places your transaction in the first mempool block" placement="top">High Priority</span>
     </div>
   </div>
   <div class="fee-estimation-container">
     <div class="item">
-      <div class="card-text" i18n-ngbTooltip="Transaction fee tooltip" ngbTooltip="Based on average native segwit transaction of 140 vBytes" placement="bottom">
-        <div class="fee-text">{{ recommendedFees.economyFee }} <span i18n="shared.sat-vbyte|sat/vB">sat/vB</span></div> <span class="fiat"><app-fiat [value]="recommendedFees.economyFee * 140" ></app-fiat></span>
+      <div class="card-text">
+        <div class="fee-text">{{ recommendedFees.economyFee }} <span i18n="shared.sat-vbyte|sat/vB">sat/vB</span></div> <span class="fiat"><app-fiat i18n-ngbTooltip="Transaction fee tooltip" ngbTooltip="Based on average native segwit transaction of 140 vBytes" placement="bottom" [value]="recommendedFees.economyFee * 140" ></app-fiat></span>
       </div>
     </div>
     <div class="band-separator"></div>
     <div class="item">
-      <div class="card-text" i18n-ngbTooltip="Transaction fee tooltip" ngbTooltip="Based on average native segwit transaction of 140 vBytes" placement="bottom">
-        <div class="fee-text">{{ recommendedFees.hourFee }} <span i18n="shared.sat-vbyte|sat/vB">sat/vB</span></div> <span class="fiat"><app-fiat [value]="recommendedFees.hourFee * 140" ></app-fiat></span>
+      <div class="card-text">
+        <div class="fee-text">{{ recommendedFees.hourFee }} <span i18n="shared.sat-vbyte|sat/vB">sat/vB</span></div> <span class="fiat"><app-fiat i18n-ngbTooltip="Transaction fee tooltip" ngbTooltip="Based on average native segwit transaction of 140 vBytes" placement="bottom" [value]="recommendedFees.hourFee * 140" ></app-fiat></span>
       </div>
     </div>
     <div class="item">
-      <div class="card-text" i18n-ngbTooltip="Transaction fee tooltip" ngbTooltip="Based on average native segwit transaction of 140 vBytes" placement="bottom">
-        <div class="fee-text">{{ recommendedFees.halfHourFee }} <span i18n="shared.sat-vbyte|sat/vB">sat/vB</span></div> <span class="fiat"><app-fiat [value]="recommendedFees.halfHourFee * 140" ></app-fiat></span>
+      <div class="card-text">
+        <div class="fee-text">{{ recommendedFees.halfHourFee }} <span i18n="shared.sat-vbyte|sat/vB">sat/vB</span></div> <span class="fiat"><app-fiat i18n-ngbTooltip="Transaction fee tooltip" ngbTooltip="Based on average native segwit transaction of 140 vBytes" placement="bottom" [value]="recommendedFees.halfHourFee * 140" ></app-fiat></span>
       </div>
     </div>
     <div class="item">
-      <div class="card-text" i18n-ngbTooltip="Transaction fee tooltip" ngbTooltip="Based on average native segwit transaction of 140 vBytes" placement="bottom">
-        <div class="fee-text">{{ recommendedFees.fastestFee }} <span i18n="shared.sat-vbyte|sat/vB">sat/vB</span></div> <span class="fiat"><app-fiat [value]="recommendedFees.fastestFee * 140" ></app-fiat></span>
+      <div class="card-text">
+        <div class="fee-text">{{ recommendedFees.fastestFee }} <span i18n="shared.sat-vbyte|sat/vB">sat/vB</span></div> <span class="fiat"><app-fiat i18n-ngbTooltip="Transaction fee tooltip" ngbTooltip="Based on average native segwit transaction of 140 vBytes" placement="bottom" [value]="recommendedFees.fastestFee * 140" ></app-fiat></span>
       </div>
     </div>
   </div>

frontend/src/app/components/mempool-block-overview/mempool-block-overview.component.ts

@@ -1,8 +1,9 @@
 import { Component, ElementRef, ViewChild, HostListener, Input, Output, EventEmitter, OnInit,
   OnDestroy, OnChanges, ChangeDetectionStrategy, NgZone, AfterViewInit } from '@angular/core';
 import { StateService } from 'src/app/services/state.service';
-import { MempoolBlockWithTransactions, MempoolBlockDelta, TransactionStripped } from 'src/app/interfaces/websocket.interface';
-import { Subscription, BehaviorSubject } from 'rxjs';
+import { MempoolBlockDelta, TransactionStripped } from 'src/app/interfaces/websocket.interface';
+import { Subscription, BehaviorSubject, merge, of } from 'rxjs';
+import { switchMap, filter } from 'rxjs/operators';
 import { WebsocketService } from 'src/app/services/websocket.service';
 import { FastVertexArray } from './fast-vertex-array';
 import BlockScene from './block-scene';
@@ -48,9 +49,14 @@ export class MempoolBlockOverviewComponent implements OnInit, OnDestroy, OnChang
   }
 
   ngOnInit(): void {
-    this.blockSub = this.stateService.mempoolBlockTransactions$.subscribe((transactionsStripped) => {
-      this.replaceBlock(transactionsStripped);
-    });
+    this.blockSub = merge(
+        of(true),
+        this.stateService.connectionState$.pipe(filter((state) => state === 2))
+      )
+      .pipe(switchMap(() => this.stateService.mempoolBlockTransactions$))
+      .subscribe((transactionsStripped) => {
+        this.replaceBlock(transactionsStripped);
+      });
     this.deltaSub = this.stateService.mempoolBlockDelta$.subscribe((delta) => {
       this.updateBlock(delta);
     });

frontend/src/app/components/svg-images/svg-images.component.html

@@ -15,10 +15,10 @@
       <path d="M464.598 56.5679C452.881 56.5679 444.371 48.0576 444.371 36.32C444.371 24.644 452.881 16.1748 464.598 16.1748C476.254 16.1748 484.723 24.644 484.723 36.32C484.723 48.0576 476.254 56.5679 464.598 56.5679ZM464.598 23.1023C457.198 23.1023 452.018 28.5291 452.018 36.32C452.018 44.1108 457.198 49.5377 464.598 49.5377C471.937 49.5377 477.076 44.1108 477.076 36.32C477.076 28.5291 471.958 23.1023 464.598 23.1023Z" fill="white"/>
       <path d="M499.996 1.14844H492.391V56.1982H499.996V1.14844Z" fill="white"/>
       <path d="M124.706 110.25C124.706 118.849 117.772 125.791 109.183 125.791H15.5236C6.93387 125.791 0 118.849 0 110.25V16.4837C0 7.88416 6.98561 0.942383 15.5236 0.942383H109.183C117.772 0.942383 124.706 7.88416 124.706 16.4837V110.25Z" fill="#2E3349"/>
-      <path d="M0 63.5225V110.25C0 118.849 6.98561 125.791 15.5753 125.791H109.183C117.772 125.791 124.758 118.849 124.758 110.25V63.5225H0Z" fill="url(#paint0_linear)"/>
+      <path d="M0 63.5225V110.25C0 118.849 6.98561 125.791 15.5753 125.791H109.183C117.772 125.791 124.758 118.849 124.758 110.25V63.5225H0Z" [attr.fill]="'url(#paint0_linear' + randomId + ')'"/>
       <path opacity="0.3" d="M109.909 109.11C109.909 111.026 108.615 112.581 107.011 112.581H90.8665C89.2624 112.581 87.9688 111.026 87.9688 109.11V17.6232C87.9688 15.7065 89.2624 14.1523 90.8665 14.1523H107.011C108.615 14.1523 109.909 15.7065 109.909 17.6232V109.11Z" fill="white"/>
       <defs>
-        <linearGradient id="paint0_linear" x1="62.3768" y1="36.3949" x2="62.3768" y2="156.837" gradientUnits="userSpaceOnUse">
+        <linearGradient [id]="'paint0_linear' + randomId" x1="62.3768" y1="36.3949" x2="62.3768" y2="156.837" gradientUnits="userSpaceOnUse">
           <stop stop-color="#AE61FF"/>
           <stop offset="1" stop-color="#13EFD8"/>
         </linearGradient>

frontend/src/app/components/svg-images/svg-images.component.ts

@@ -6,6 +6,7 @@ import { ChangeDetectionStrategy, Component, Input } from '@angular/core';
   changeDetection: ChangeDetectionStrategy.OnPush,
 })
 export class SvgImagesComponent {
+  randomId = Math.floor(Math.random() * 10000);
   @Input() name: string;
   @Input() class: string;
   @Input() style: string;

frontend/src/app/docs/api-docs/api-docs-data.ts

@@ -6107,7 +6107,7 @@ export const faqData = [
     showConditions: bitcoinNetworks,
     fragment: "looking-up-fee-estimates",
     title: "How can I look up fee estimates?",
-    answer: "<p>See real-time fee estimates on <a href='/'>the main dashboard</a>.</p><p>Low priority is suggested for confirmation within 6 blocks (~1 hour), Medium priority is suggested for confirmation within 3 blocks (~30 minutes), and High priority is suggested for confirmation in the next block (~10 minutes).</p>"
+    answer: "<p>See real-time fee estimates on <a href='/'>the main dashboard</a>.</p><p>Here is an overview of Mempool's feerate suggestions:</p><ul> <li><b>High Priority.</b> This figure is the median feerate of transactions in the <a href='/mempool-block/0'>first mempool block</a>. Consider using this feerate if you want confirmation as soon as possible.</li><li><b>Medium Priority.</b> This figure is the average of the median feerate of the <a href='/mempool-block/0'>first mempool block</a> and the median feerate of the <a href='/mempool-block/1'>second mempool block</a>.</li><li><b>Low Priority.</b> This figure is the average of the Medium Priority feerate and the median feerate of the <a href='/mempool-block/2'>third mempool block</a>. Consider using this feerate if you want confirmation soon but don't need it particularly quickly.</li><li><b>No Priority.</b> This figure is either 2x the minimum feerate, or the Low Priority feerate (whichever is lower). Consider using this feerate if you are in no rush and don't mind if confirmation takes a while.</li></ul><p>In all cases, the suggested feerate is adjusted lower if any of the mempool blocks involved in the calculation are not full (example: if there is only 1 mempool block that's less than half-full, Mempool will suggest a feerate of 1 sat/vB—not the median feerate of transactions in the block).</p><p>Mempool blocks use feerates, transaction sizes, and other metrics to <b>forecast</b> which transactions will be in future blocks. Actual blocks will turn out to be different: miners have their own views of the mempool, their own algorithms for determining which transactions to include in a block, etc.</p><p>Ultimately, the Bitcoin network is not perfectly predictable, so fee estimation cannot be perfectly precise.</p><p><b>Use Mempool's feerate suggestions as a guide, and understand that they do not guarantee transaction confirmation in any period of time.</b></p>"
   },
   {
     type: "endpoint",

frontend/src/locale/messages.xlf

@@ -699,7 +699,7 @@
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/about/about.component.html</context>
-          <context context-type="linenumber">383,387</context>
+          <context context-type="linenumber">385,389</context>
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/dashboard/dashboard.component.html</context>
@@ -1332,7 +1332,7 @@
         <source>Community Sponsors ❤️</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/about/about.component.html</context>
-          <context context-type="linenumber">181,184</context>
+          <context context-type="linenumber">183,186</context>
         </context-group>
         <note priority="1" from="description">about.sponsors.withHeart</note>
       </trans-unit>
@@ -1340,7 +1340,7 @@
         <source>Self-Hosted Integrations</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/about/about.component.html</context>
-          <context context-type="linenumber">195,197</context>
+          <context context-type="linenumber">197,199</context>
         </context-group>
         <note priority="1" from="description">about.self-hosted-integrations</note>
       </trans-unit>
@@ -1348,7 +1348,7 @@
         <source>Wallet Integrations</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/about/about.component.html</context>
-          <context context-type="linenumber">229,231</context>
+          <context context-type="linenumber">231,233</context>
         </context-group>
         <note priority="1" from="description">about.wallet-integrations</note>
       </trans-unit>
@@ -1356,7 +1356,7 @@
         <source>Community Alliances</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/about/about.component.html</context>
-          <context context-type="linenumber">283,285</context>
+          <context context-type="linenumber">285,287</context>
         </context-group>
         <note priority="1" from="description">about.alliances</note>
       </trans-unit>
@@ -1364,7 +1364,7 @@
         <source>Project Translators</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/about/about.component.html</context>
-          <context context-type="linenumber">299,301</context>
+          <context context-type="linenumber">301,303</context>
         </context-group>
         <note priority="1" from="description">about.translators</note>
       </trans-unit>
@@ -1372,7 +1372,7 @@
         <source>Project Contributors</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/about/about.component.html</context>
-          <context context-type="linenumber">313,315</context>
+          <context context-type="linenumber">315,317</context>
         </context-group>
         <note priority="1" from="description">about.contributors</note>
       </trans-unit>
@@ -1380,7 +1380,7 @@
         <source>Project Members</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/about/about.component.html</context>
-          <context context-type="linenumber">325,327</context>
+          <context context-type="linenumber">327,329</context>
         </context-group>
         <note priority="1" from="description">about.project_members</note>
       </trans-unit>
@@ -1388,7 +1388,7 @@
         <source>Project Maintainers</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/about/about.component.html</context>
-          <context context-type="linenumber">338,340</context>
+          <context context-type="linenumber">340,342</context>
         </context-group>
         <note priority="1" from="description">about.maintainers</note>
       </trans-unit>
@@ -1415,7 +1415,7 @@
         <source>Multisig <x id="multisigM" equiv-text="m"/> of <x id="multisigN" equiv-text="n"/></source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/address-labels/address-labels.component.ts</context>
-          <context context-type="linenumber">121</context>
+          <context context-type="linenumber">127</context>
         </context-group>
       </trans-unit>
       <trans-unit id="04ffd930e7a2dc086c952a3a51b42c836bf21cc1" datatype="html">
@@ -2122,19 +2122,19 @@
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/fees-box/fees-box.component.html</context>
-          <context context-type="linenumber">15,16</context>
+          <context context-type="linenumber">16,18</context>
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/fees-box/fees-box.component.html</context>
-          <context context-type="linenumber">21,22</context>
+          <context context-type="linenumber">22,24</context>
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/fees-box/fees-box.component.html</context>
-          <context context-type="linenumber">26,27</context>
+          <context context-type="linenumber">27,28</context>
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/fees-box/fees-box.component.html</context>
-          <context context-type="linenumber">31,32</context>
+          <context context-type="linenumber">32,34</context>
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/mempool-block/mempool-block.component.html</context>
@@ -2457,6 +2457,14 @@
         </context-group>
         <note priority="1" from="description">difficulty-box.next-halving</note>
       </trans-unit>
+      <trans-unit id="6ff9e8b67bc2cda7569dc0996d4c2fd858c5d4e6" datatype="html">
+        <source>Either 2x the minimum, or the Low Priority rate (whichever is lower)</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/fees-box/fees-box.component.html</context>
+          <context context-type="linenumber">4,7</context>
+        </context-group>
+        <note priority="1" from="description">Transaction feerate tooltip (economy)</note>
+      </trans-unit>
       <trans-unit id="eef30290726d3d569232f4c136082bb9daaf490b" datatype="html">
         <source>No Priority</source>
         <context-group purpose="location">
@@ -2469,6 +2477,14 @@
         </context-group>
         <note priority="1" from="description">fees-box.no-priority</note>
       </trans-unit>
+      <trans-unit id="3be54a643a1ac01e9f57f6ffe75717215c59e90d" datatype="html">
+        <source>Places your transaction in between the second and third mempool blocks</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/fees-box/fees-box.component.html</context>
+          <context context-type="linenumber">8,9</context>
+        </context-group>
+        <note priority="1" from="description">Transaction feerate tooltip (low priority)</note>
+      </trans-unit>
       <trans-unit id="29949587189ee02db19274db4ac656913cb243c3" datatype="html">
         <source>Low Priority</source>
         <context-group purpose="location">
@@ -2481,11 +2497,19 @@
         </context-group>
         <note priority="1" from="description">fees-box.low-priority</note>
       </trans-unit>
+      <trans-unit id="e095ca196e0b6331585f35ae7373dab9bb772399" datatype="html">
+        <source>Places your transaction in between the first and second mempool blocks</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/fees-box/fees-box.component.html</context>
+          <context context-type="linenumber">9,10</context>
+        </context-group>
+        <note priority="1" from="description">Transaction feerate tooltip (medium priority)</note>
+      </trans-unit>
       <trans-unit id="ee847b69ef2dc81bb3e9b8cd30f02f8d63adbe07" datatype="html">
         <source>Medium Priority</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/fees-box/fees-box.component.html</context>
-          <context context-type="linenumber">9,11</context>
+          <context context-type="linenumber">9,10</context>
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/fees-box/fees-box.component.html</context>
@@ -2493,6 +2517,14 @@
         </context-group>
         <note priority="1" from="description">fees-box.medium-priority</note>
       </trans-unit>
+      <trans-unit id="a5a4d2c76b74faddf1aab8dc6e092cddee5a6142" datatype="html">
+        <source>Places your transaction in the first mempool block</source>
+        <context-group purpose="location">
+          <context context-type="sourcefile">src/app/components/fees-box/fees-box.component.html</context>
+          <context context-type="linenumber">10,14</context>
+        </context-group>
+        <note priority="1" from="description">Transaction feerate tooltip (high priority)</note>
+      </trans-unit>
       <trans-unit id="d1d0bb0a34b216be66137562a0b18eaaca546113" datatype="html">
         <source>High Priority</source>
         <context-group purpose="location">
@@ -2932,11 +2964,11 @@
         <source><x id="PH" equiv-text="i"/> blocks</source>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/pool-ranking/pool-ranking.component.ts</context>
-          <context context-type="linenumber">160,158</context>
+          <context context-type="linenumber">153,151</context>
         </context-group>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/pool-ranking/pool-ranking.component.ts</context>
-          <context context-type="linenumber">163,162</context>
+          <context context-type="linenumber">156,155</context>
         </context-group>
       </trans-unit>
       <trans-unit id="cafc87479686947e2590b9f588a88040aeaf660b" datatype="html">

production/mempool-build-all

@@ -16,7 +16,8 @@ if [ -f "${LOCKFILE}" ];then
     exit 1
 fi
 
-trap 'rm -f "${LOCKFILE}"; exit $?' INT TERM EXIT
+trap "rv=\$?; rm -rf "${LOCKFILE}"; exit \$rv" INT TERM EXIT
+
 touch "${LOCKFILE}"
 
 echo "Upgrading mempool to ${REF}" | wall