Merge branch 'master' into update_gha

.github/dependabot.yml

@@ -1,20 +1,28 @@
 version: 2
 updates:
-- package-ecosystem: npm
+  - package-ecosystem: npm
-  directory: "/backend"
+    directory: "/backend"
-  schedule:
+    schedule:
-    interval: daily
+      interval: daily
-  open-pull-requests-limit: 10
+    open-pull-requests-limit: 10
-- package-ecosystem: npm
+    ignore:
-  directory: "/frontend"
+      - update-types: ["version-update:semver-major"]
-  schedule:
+  - package-ecosystem: npm
-    interval: daily
+    directory: "/frontend"
-  open-pull-requests-limit: 10
+    schedule:
-- package-ecosystem: docker
+      interval: daily
-  directory: "/docker/backend"
+    open-pull-requests-limit: 10
-  schedule:
+    ignore:
-    interval: weekly
+      - update-types: ["version-update:semver-major"]
-- package-ecosystem: "github-actions"
+  - package-ecosystem: docker
-  directory: "/"
+    directory: "/docker/backend"
-  schedule:
+    schedule:
-      interval: "weekly"
+      interval: daily
+    ignore:
+      - update-types: ["version-update:semver-major"]
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: daily
+    ignore:
+      - update-types: ["version-update:semver-major"]

backend/mempool-config.sample.json

@@ -85,7 +85,8 @@
     "STATS_REFRESH_INTERVAL": 600,
     "GRAPH_REFRESH_INTERVAL": 600,
     "LOGGER_UPDATE_INTERVAL": 30,
-    "FORENSICS_INTERVAL": 43200
+    "FORENSICS_INTERVAL": 43200,
+    "FORENSICS_RATE_LIMIT": 20
   },
   "LND": {
     "TLS_CERT_PATH": "tls.cert",

backend/src/__fixtures__/mempool-config.template.json

@@ -101,7 +101,8 @@
     "STATS_REFRESH_INTERVAL": 600,
     "GRAPH_REFRESH_INTERVAL": 600,
     "LOGGER_UPDATE_INTERVAL": 30,
-    "FORENSICS_INTERVAL": 43200
+    "FORENSICS_INTERVAL": 43200,
+    "FORENSICS_RATE_LIMIT": "__FORENSICS_RATE_LIMIT__"
   },
   "LND": {
     "TLS_CERT_PATH": "",

backend/src/api/explorer/nodes.api.ts

@@ -538,6 +538,10 @@ class NodesApi {
 
       const IPSIds = ISPId.split(',');
       const [rows]: any = await DB.query(query, [IPSIds, IPSIds]);
+      if (!rows || rows.length === 0) {
+        return [];
+      }
+
       const nodes = {};
 
       const intISPIds: number[] = [];

backend/src/api/tx-selection-worker.ts

@@ -106,44 +106,44 @@ function makeBlockTemplates({ mempool, blockLimit, weightLimit, condenseRest }:
     if (nextTx && !nextTx?.used) {
       // Check if the package fits into this block
       if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
-        blockWeight += nextTx.ancestorWeight;
         const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values());
         const descendants: AuditTransaction[] = [];
         // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count)
         const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx];
         const effectiveFeeRate = nextTx.ancestorFee / (nextTx.ancestorWeight / 4);
-
+        const used: AuditTransaction[] = [];
         while (sortedTxSet.length) {
           const ancestor = sortedTxSet.pop();
           const mempoolTx = mempool[ancestor.txid];
-          if (ancestor && !ancestor?.used) {
+          ancestor.used = true;
-            ancestor.used = true;
+          ancestor.usedBy = nextTx.txid;
-            // update original copy of this tx with effective fee rate & relatives data
+          // update original copy of this tx with effective fee rate & relatives data
-            mempoolTx.effectiveFeePerVsize = effectiveFeeRate;
+          mempoolTx.effectiveFeePerVsize = effectiveFeeRate;
-            mempoolTx.ancestors = sortedTxSet.map((a) => {
+          mempoolTx.ancestors = sortedTxSet.map((a) => {
-              return {
+            return {
-                txid: a.txid,
+              txid: a.txid,
-                fee: a.fee,
+              fee: a.fee,
-                weight: a.weight,
+              weight: a.weight,
-              };
+            };
-            }).reverse();
+          }).reverse();
-            mempoolTx.descendants = descendants.map((a) => {
+          mempoolTx.descendants = descendants.map((a) => {
-              return {
+            return {
-                txid: a.txid,
+              txid: a.txid,
-                fee: a.fee,
+              fee: a.fee,
-                weight: a.weight,
+              weight: a.weight,
-              };
+            };
-            });
+          });
-            descendants.push(ancestor);
+          descendants.push(ancestor);
-            mempoolTx.cpfpChecked = true;
+          mempoolTx.cpfpChecked = true;
-            transactions.push(ancestor);
+          transactions.push(ancestor);
-            blockSize += ancestor.size;
+          blockSize += ancestor.size;
-          }
+          blockWeight += ancestor.weight;
+          used.push(ancestor);
         }
 
         // remove these as valid package ancestors for any descendants remaining in the mempool
-        if (sortedTxSet.length) {
+        if (used.length) {
-          sortedTxSet.forEach(tx => {
+          used.forEach(tx => {
             updateDescendants(tx, auditPool, modified);
           });
         }

backend/src/api/websocket-handler.ts

@@ -439,7 +439,7 @@ class WebsocketHandler {
         };
       }) : [];
 
-      BlocksSummariesRepository.$saveSummary({
+      BlocksSummariesRepository.$saveTemplate({
         height: block.height,
         template: {
           id: block.id,
@@ -468,7 +468,7 @@ class WebsocketHandler {
     }
 
     if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) {
-      await mempoolBlocks.makeBlockTemplates(_memPool, 2);
+      await mempoolBlocks.makeBlockTemplates(_memPool, 8, null, true);
     } else {
       mempoolBlocks.updateMempoolBlocks(_memPool);
     }

backend/src/config.ts

@@ -44,6 +44,7 @@ interface IConfig {
     GRAPH_REFRESH_INTERVAL: number;
     LOGGER_UPDATE_INTERVAL: number;
     FORENSICS_INTERVAL: number;
+    FORENSICS_RATE_LIMIT: number;
   };
   LND: {
     TLS_CERT_PATH: string;
@@ -205,6 +206,7 @@ const defaults: IConfig = {
     'GRAPH_REFRESH_INTERVAL': 600,
     'LOGGER_UPDATE_INTERVAL': 30,
     'FORENSICS_INTERVAL': 43200,
+    'FORENSICS_RATE_LIMIT': 20,
   },
   'LND': {
     'TLS_CERT_PATH': '',

backend/src/repositories/BlocksSummariesRepository.ts

@@ -17,19 +17,16 @@ class BlocksSummariesRepository {
     return undefined;
   }
 
-  public async $saveSummary(params: { height: number, mined?: BlockSummary, template?: BlockSummary}) {
+  public async $saveSummary(params: { height: number, mined?: BlockSummary}) {
-    const blockId = params.mined?.id ?? params.template?.id;
+    const blockId = params.mined?.id;
     try {
-      const [dbSummary]: any[] = await DB.query(`SELECT * FROM blocks_summaries WHERE id = "${blockId}"`);
+      const transactions = JSON.stringify(params.mined?.transactions || []);
-      if (dbSummary.length === 0) { // First insertion
+      await DB.query(`
-        await DB.query(`INSERT INTO blocks_summaries VALUE (?, ?, ?, ?)`, [
+        INSERT INTO blocks_summaries (height, id, transactions, template)
-          params.height, blockId, JSON.stringify(params.mined?.transactions ?? []), JSON.stringify(params.template?.transactions ?? [])
+        VALUE (?, ?, ?, ?)
-        ]);
+        ON DUPLICATE KEY UPDATE
-      } else if (params.mined !== undefined) { // Update mined block summary
+          transactions = ?
-        await DB.query(`UPDATE blocks_summaries SET transactions = ? WHERE id = "${params.mined.id}"`, [JSON.stringify(params.mined.transactions)]);
+      `, [params.height, blockId, transactions, '[]', transactions]);
-      } else if (params.template !== undefined) { // Update template block summary
-        await DB.query(`UPDATE blocks_summaries SET template = ? WHERE id = "${params.template.id}"`, [JSON.stringify(params.template?.transactions)]);
-      }
     } catch (e: any) {
       if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
         logger.debug(`Cannot save block summary for ${blockId} because it has already been indexed, ignoring`);
@@ -40,6 +37,26 @@ class BlocksSummariesRepository {
     }
   }
 
+  public async $saveTemplate(params: { height: number, template: BlockSummary}) {
+    const blockId = params.template?.id;
+    try {
+      const transactions = JSON.stringify(params.template?.transactions || []);
+      await DB.query(`
+        INSERT INTO blocks_summaries (height, id, transactions, template)
+        VALUE (?, ?, ?, ?)
+        ON DUPLICATE KEY UPDATE
+          template = ?
+      `, [params.height, blockId, '[]', transactions, transactions]);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`Cannot save block template for ${blockId} because it has already been indexed, ignoring`);
+      } else {
+        logger.debug(`Cannot save block template for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`);
+        throw e;
+      }
+    }
+  }
+
   public async $getIndexedSummariesId(): Promise<string[]> {
     try {
       const [rows]: any[] = await DB.query(`SELECT id from blocks_summaries`);

backend/src/tasks/lightning/forensics.service.ts

@@ -7,7 +7,6 @@ import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
 import { Common } from '../../api/common';
 import { ILightningApi } from '../../api/lightning/lightning-api.interface';
 
-const throttleDelay = 20; //ms
 const tempCacheSize = 10000;
 
 class ForensicsService {
@@ -91,7 +90,7 @@ class ForensicsService {
           let outspends: IEsploraApi.Outspend[] | undefined;
           try {
             outspends = await bitcoinApi.$getOutspends(channel.closing_transaction_id);
-            await Common.sleep$(throttleDelay);
+            await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT);
           } catch (e) {
             logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
             continue;
@@ -340,7 +339,7 @@ class ForensicsService {
       let outspends: IEsploraApi.Outspend[] | undefined;
       try {
         outspends = await bitcoinApi.$getOutspends(input.txid);
-        await Common.sleep$(throttleDelay);
+        await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT);
       } catch (e) {
         logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + input.txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
       }
@@ -429,7 +428,7 @@ class ForensicsService {
         if (temp) {
           this.tempCached.push(txid);
         }
-        await Common.sleep$(throttleDelay);
+        await Common.sleep$(config.LIGHTNING.FORENSICS_RATE_LIMIT);
       } catch (e) {
         logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
         return null;

frontend/proxy.conf.staging.js

@@ -5,7 +5,7 @@ let PROXY_CONFIG = require('./proxy.conf');
 PROXY_CONFIG.forEach(entry => {
   entry.target = entry.target.replace("mempool.space", "mempool-staging.tk7.mempool.space");
   entry.target = entry.target.replace("liquid.network", "liquid-staging.tk7.mempool.space");
-  entry.target = entry.target.replace("bisq.markets", "bisq-staging.tk7.mempool.space");
+  entry.target = entry.target.replace("bisq.markets", "bisq-staging.fra.mempool.space");
 });
 
 module.exports = PROXY_CONFIG;