Merge branch 'simon/cpfp-frontend' into simon/bisq-dashboard

* simon/cpfp-frontend: (46 commits)
  Bugfix: Don't extend already extended transactions to not override the firstSeen property. fixes #390
  Shuffle mempool transactions before saving disk cache. (#398)
  Adding missing return after expressjs response.
  CPFP support (#395)
  Round sat/vB in fee rating tooltip. fixes #364
  Add the GNU AGPLv3 logo to About page
  Update package.json license tags
  Add recommended fee percentile config (#394)
  Fix typo in README (#392)
  Fix icon for Specter Wallet on About page
  Add link to Specter Wallet on our About page
  Add link to WARden Portfolio app as Community Integration on About page
  Delete MIT+CC license from Terms of Service, add AGPLv3 to About page
  Change mempool project license to GNU Affero General Public License v3
  Lower volume for sound effects (#385)
  Improve grammar, layout, and formatting of Terms of Service page
  Display all Project Contributors on About page using GitHub API (#382)
  Modify nginx.conf to cache HTML for 10m and static resources for 1h
  Proxy /api/v1/contributors from mempool.space, also fix HTTP headers
  Add link to Bisq's GitHub repo on About page
  ...

backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts

@@ -3,7 +3,6 @@ import { IEsploraApi } from './esplora-api.interface';
 export interface AbstractBitcoinApi {
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]>;
   $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean): Promise<IEsploraApi.Transaction>;
-  $getRawTransactionBitcoind(txId: string, skipConversion?: boolean, addPrevout?: boolean): Promise<IEsploraApi.Transaction>;
   $getBlockHeightTip(): Promise<number>;
   $getTxIdsForBlock(hash: string): Promise<string[]>;
   $getBlockHash(height: number): Promise<string>;

backend/src/api/bitcoin/bitcoin-api.ts

@@ -22,16 +22,6 @@ class BitcoinApi implements AbstractBitcoinApi {
     });
   }
 
-  $getRawTransactionBitcoind(txId: string, skipConversion = false, addPrevout = false): Promise<IEsploraApi.Transaction> {
-    return this.bitcoindClient.getRawTransaction(txId, true)
-      .then((transaction: IBitcoinApi.Transaction) => {
-        if (skipConversion) {
-          return transaction;
-        }
-        return this.$convertTransaction(transaction, addPrevout);
-      });
-  }
-
   $getRawTransaction(txId: string, skipConversion = false, addPrevout = false): Promise<IEsploraApi.Transaction> {
     // If the transaction is in the mempool we already converted and fetched the fee. Only prevouts are missing
     const txInMempool = mempool.getMempool()[txId];
@@ -47,6 +37,9 @@ class BitcoinApi implements AbstractBitcoinApi {
     return this.bitcoindClient.getRawTransaction(txId, true)
       .then((transaction: IBitcoinApi.Transaction) => {
         if (skipConversion) {
+          transaction.vout.forEach((vout) => {
+            vout.value = vout.value * 100000000;
+          });
           return transaction;
         }
         return this.$convertTransaction(transaction, addPrevout);

backend/src/api/bitcoin/esplora-api.ts

@@ -48,11 +48,6 @@ class ElectrsApi implements AbstractBitcoinApi {
     throw new Error('Method getAddressTransactions not implemented.');
   }
 
-  $getRawTransactionBitcoind(txId: string): Promise<IEsploraApi.Transaction> {
-    return axios.get<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig)
-      .then((response) => response.data);
-  }
-
   $getAddressPrefix(prefix: string): string[] {
     throw new Error('Method not implemented.');
   }

backend/src/api/blocks.ts

@@ -84,14 +84,21 @@ class Blocks {
         }
       }
 
+      transactions.forEach((tx) => {
+        if (!tx.cpfpChecked) {
+          Common.setRelativesAndGetCpfpInfo(tx, mempool);
+        }
+      });
+
       logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${txIds.length - transactionsFound} not found.`);
 
       const blockExtended: BlockExtended = Object.assign({}, block);
       blockExtended.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
       blockExtended.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
-      transactions.sort((a, b) => b.feePerVsize - a.feePerVsize);
-      blockExtended.medianFee = transactions.length > 1 ? Common.median(transactions.map((tx) => tx.feePerVsize)) : 0;
-      blockExtended.feeRange = transactions.length > 1 ? Common.getFeesInRange(transactions.slice(0, transactions.length - 1), 8) : [0, 0];
+      transactions.shift();
+      transactions.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize);
+      blockExtended.medianFee = transactions.length > 1 ? Common.median(transactions.map((tx) => tx.effectiveFeePerVsize)) : 0;
+      blockExtended.feeRange = transactions.length > 1 ? Common.getFeesInRange(transactions, 8) : [0, 0];
 
       if (block.height % 2016 === 0) {
         this.lastDifficultyAdjustmentTime = block.timestamp;

backend/src/api/common.ts

@@ -1,4 +1,4 @@
-import { TransactionExtended, TransactionStripped } from '../mempool.interfaces';
+import { CpfpInfo, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
 
 export class Common {
   static median(numbers: number[]) {
@@ -12,17 +12,24 @@ export class Common {
     return medianNr;
   }
 
+  static percentile(numbers: number[], percentile: number) {
+    if (percentile === 50) return this.median(numbers);
+    const index = Math.ceil(numbers.length * (100 - percentile) * 1e-2);
+    if (index < 0 || index > numbers.length - 1) return 0;
+    return numbers[index];
+  }
+
   static getFeesInRange(transactions: TransactionExtended[], rangeLength: number) {
-    const arr = [transactions[transactions.length - 1].feePerVsize];
+    const arr = [transactions[transactions.length - 1].effectiveFeePerVsize];
     const chunk = 1 / (rangeLength - 1);
     let itemsToAdd = rangeLength - 2;
 
     while (itemsToAdd > 0) {
-      arr.push(transactions[Math.floor(transactions.length * chunk * itemsToAdd)].feePerVsize);
+      arr.push(transactions[Math.floor(transactions.length * chunk * itemsToAdd)].effectiveFeePerVsize);
       itemsToAdd--;
     }
 
-    arr.push(transactions[0].feePerVsize);
+    arr.push(transactions[0].effectiveFeePerVsize);
     return arr;
   }
 
@@ -64,4 +71,69 @@ export class Common {
        }, ms);
     });
   }
+  
+  static shuffleArray(array: any[]) {
+    for (let i = array.length - 1; i > 0; i--) {
+        const j = Math.floor(Math.random() * (i + 1));
+        [array[i], array[j]] = [array[j], array[i]];
+    }
+  }
+
+  static setRelativesAndGetCpfpInfo(tx: TransactionExtended, memPool: { [txid: string]: TransactionExtended }): CpfpInfo {
+    const parents = this.findAllParents(tx, memPool);
+
+    let totalWeight = tx.weight + parents.reduce((prev, val) => prev + val.weight, 0);
+    let totalFees = tx.fee + parents.reduce((prev, val) => prev + val.fee, 0);
+
+    tx.ancestors = parents
+      .map((t) => {
+        return {
+          txid: t.txid,
+          weight: t.weight,
+          fee: t.fee,
+        };
+      });
+
+    // Add high (high fee) decendant weight and fees
+    if (tx.bestDescendant) {
+      totalWeight += tx.bestDescendant.weight;
+      totalFees += tx.bestDescendant.fee;
+    }
+
+    tx.effectiveFeePerVsize = totalFees / (totalWeight / 4);
+    tx.cpfpChecked = true;
+
+    return {
+      ancestors: tx.ancestors,
+      bestDescendant: tx.bestDescendant || null,
+    };
+  }
+
+
+  private static findAllParents(tx: TransactionExtended, memPool: { [txid: string]: TransactionExtended }): TransactionExtended[] {
+    let parents: TransactionExtended[] = [];
+    tx.vin.forEach((parent) => {
+      const parentTx = memPool[parent.txid];
+      if (parentTx) {
+        if (tx.bestDescendant && tx.bestDescendant.fee / (tx.bestDescendant.weight / 4) > parentTx.feePerVsize) {
+          if (parentTx.bestDescendant && parentTx.bestDescendant.fee < tx.fee + tx.bestDescendant.fee) {
+            parentTx.bestDescendant = {
+              weight: tx.weight + tx.bestDescendant.weight,
+              fee: tx.fee + tx.bestDescendant.fee,
+              txid: tx.txid,
+            };
+          }
+        } else if (tx.feePerVsize > parentTx.feePerVsize) {
+          parentTx.bestDescendant = {
+            weight: tx.weight,
+            fee: tx.fee,
+            txid: tx.txid
+          };
+        }
+        parents.push(parentTx);
+        parents = parents.concat(this.findAllParents(parentTx, memPool));
+      }
+    });
+    return parents;
+  }
 }

backend/src/api/disk-cache.ts

@@ -6,6 +6,7 @@ import blocks from './blocks';
 import logger from '../logger';
 import config from '../config';
 import { TransactionExtended } from '../mempool.interfaces';
+import { Common } from './common';
 
 class DiskCache {
   private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
@@ -27,6 +28,8 @@ class DiskCache {
         mempoolArray.push(mempool[tx]);
       }
 
+      Common.shuffleArray(mempoolArray);
+
       const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
 
       await fsPromises.writeFile(DiskCache.FILE_NAME, JSON.stringify({

backend/src/api/mempool-blocks.ts

@@ -1,5 +1,7 @@
+import logger from '../logger';
 import { MempoolBlock, TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
 import { Common } from './common';
+import config from '../config';
 
 class MempoolBlocks {
   private static DEFAULT_PROJECTED_BLOCKS_AMOUNT = 8;
@@ -32,9 +34,40 @@ class MempoolBlocks {
         memPoolArray.push(latestMempool[i]);
       }
     }
+    const start = new Date().getTime();
+
+    // Clear bestDescendants & ancestors
+    memPoolArray.forEach((tx) => {
+      tx.bestDescendant = null;
+      tx.ancestors = [];
+      tx.cpfpChecked = false;
+      if (!tx.effectiveFeePerVsize) {
+        tx.effectiveFeePerVsize = tx.feePerVsize;
+      }
+    });
+
+    // First sort
     memPoolArray.sort((a, b) => b.feePerVsize - a.feePerVsize);
-    const transactionsSorted = memPoolArray.filter((tx) => tx.feePerVsize);
-    this.mempoolBlocks = this.calculateMempoolBlocks(transactionsSorted);
+
+    // Loop through and traverse all ancestors and sum up all the sizes + fees
+    // Pass down size + fee to all unconfirmed children
+    let sizes = 0;
+    memPoolArray.forEach((tx, i) => {
+      sizes += tx.weight
+      if (sizes > 4000000 * 8) {
+        return;
+      }
+      Common.setRelativesAndGetCpfpInfo(tx, memPool);
+    });
+
+    // Final sort, by effective fee
+    memPoolArray.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize);
+
+    const end = new Date().getTime();
+    const time = end - start;
+    logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds');
+
+    this.mempoolBlocks = this.calculateMempoolBlocks(memPoolArray);
   }
 
   private calculateMempoolBlocks(transactionsSorted: TransactionExtended[]): MempoolBlockWithTransactions[] {
@@ -76,7 +109,7 @@ class MempoolBlocks {
       blockVSize: blockVSize,
       nTx: transactions.length,
       totalFees: transactions.reduce((acc, cur) => acc + cur.fee, 0),
-      medianFee: Common.median(transactions.map((tx) => tx.feePerVsize)),
+      medianFee: Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
       feeRange: Common.getFeesInRange(transactions, rangeLength),
       transactionIds: transactions.map((tx) => tx.txid),
     };

backend/src/api/statistics.ts

@@ -68,13 +68,13 @@ class Statistics {
       }
     }
     // Remove 0 and undefined
-    memPoolArray = memPoolArray.filter((tx) => tx.feePerVsize);
+    memPoolArray = memPoolArray.filter((tx) => tx.effectiveFeePerVsize);
 
     if (!memPoolArray.length) {
       return;
     }
 
-    memPoolArray.sort((a, b) => a.feePerVsize - b.feePerVsize);
+    memPoolArray.sort((a, b) => a.effectiveFeePerVsize - b.effectiveFeePerVsize);
     const totalWeight = memPoolArray.map((tx) => tx.vsize).reduce((acc, curr) => acc + curr) * 4;
     const totalFee = memPoolArray.map((tx) => tx.fee).reduce((acc, curr) => acc + curr);
 
@@ -85,7 +85,7 @@ class Statistics {
 
     memPoolArray.forEach((transaction) => {
       for (let i = 0; i < logFees.length; i++) {
-        if ((logFees[i] === 2000 && transaction.feePerVsize >= 2000) || transaction.feePerVsize <= logFees[i]) {
+        if ((logFees[i] === 2000 && transaction.effectiveFeePerVsize >= 2000) || transaction.effectiveFeePerVsize <= logFees[i]) {
           if (weightVsizeFees[logFees[i]]) {
             weightVsizeFees[logFees[i]] += transaction.vsize;
           } else {

backend/src/api/transaction-utils.ts

@@ -26,9 +26,16 @@ class TransactionUtils {
   }
 
   private extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended {
+    // @ts-ignore
+    if (transaction.vsize) {
+      // @ts-ignore
+      return transaction;
+    }
+    const feePerVbytes = Math.max(1, (transaction.fee || 0) / (transaction.weight / 4));
     const transactionExtended: TransactionExtended = Object.assign({
       vsize: Math.round(transaction.weight / 4),
-      feePerVsize: Math.max(1, (transaction.fee || 0) / (transaction.weight / 4)),
+      feePerVsize: feePerVbytes,
+      effectiveFeePerVsize: feePerVbytes,
     }, transaction);
     if (!transaction.status.confirmed) {
       transactionExtended.firstSeen = Math.round((new Date().getTime() / 1000));