Merge branch 'master' into simon/mempool-break-limit

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -399,9 +399,13 @@ class BitcoinRoutes {
 
   private async getBlockAuditSummary(req: Request, res: Response) {
     try {
-      const transactions = await blocks.$getBlockAuditSummary(req.params.hash);
+      const auditSummary = await blocks.$getBlockAuditSummary(req.params.hash);
+      if (auditSummary) {
         res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24 * 30).toUTCString());
-      res.json(transactions);
+        res.json(auditSummary);
+      } else {
+        return res.status(404).send(`audit not available`);
+      }
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
     }

backend/src/api/blocks.ts

@@ -158,6 +158,13 @@ class Blocks {
     };
   }
 
+  public summarizeBlockTransactions(hash: string, transactions: TransactionExtended[]): BlockSummary {
+    return {
+      id: hash,
+      transactions: Common.stripTransactions(transactions),
+    };
+  }
+
   private convertLiquidFees(block: IBitcoinApi.VerboseBlock): IBitcoinApi.VerboseBlock {
     block.tx.forEach(tx => {
       tx.fee = Object.values(tx.fee || {}).reduce((total, output) => total + output, 0);
@@ -646,7 +653,7 @@ class Blocks {
       }
       const cpfpSummary: CpfpSummary = Common.calculateCpfp(block.height, transactions);
       const blockExtended: BlockExtended = await this.$getBlockExtended(block, cpfpSummary.transactions);
-      const blockSummary: BlockSummary = this.summarizeBlock(verboseBlock);
+      const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, cpfpSummary.transactions);
       this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`);
 
       // start async callbacks
@@ -668,12 +675,13 @@ class Blocks {
             for (let i = 10; i >= 0; --i) {
               const newBlock = await this.$indexBlock(lastBlock.height - i);
               this.updateTimerProgress(timer, `reindexed block`);
-              await this.$getStrippedBlockTransactions(newBlock.id, true, true);
+              let cpfpSummary;
-              this.updateTimerProgress(timer, `reindexed block summary`);
               if (config.MEMPOOL.CPFP_INDEXING) {
-                await this.$indexCPFP(newBlock.id, lastBlock.height - i);
+                cpfpSummary = await this.$indexCPFP(newBlock.id, lastBlock.height - i);
                 this.updateTimerProgress(timer, `reindexed block cpfp`);
               }
+              await this.$getStrippedBlockTransactions(newBlock.id, true, true, cpfpSummary, newBlock.height);
+              this.updateTimerProgress(timer, `reindexed block summary`);
             }
             await mining.$indexDifficultyAdjustments();
             await DifficultyAdjustmentsRepository.$deleteLastAdjustment();
@@ -704,7 +712,7 @@ class Blocks {
 
           // Save blocks summary for visualization if it's enabled
           if (Common.blocksSummariesIndexingEnabled() === true) {
-            await this.$getStrippedBlockTransactions(blockExtended.id, true);
+            await this.$getStrippedBlockTransactions(blockExtended.id, true, false, cpfpSummary, blockExtended.height);
             this.updateTimerProgress(timer, `saved block summary for ${this.currentBlockHeight}`);
           }
           if (config.MEMPOOL.CPFP_INDEXING) {
@@ -730,6 +738,11 @@ class Blocks {
         this.currentDifficulty = block.difficulty;
       }
 
+      // wait for pending async callbacks to finish
+      this.updateTimerProgress(timer, `waiting for async callbacks to complete for ${this.currentBlockHeight}`);
+      await Promise.all(callbackPromises);
+      this.updateTimerProgress(timer, `async callbacks completed for ${this.currentBlockHeight}`);
+
       this.blocks.push(blockExtended);
       if (this.blocks.length > config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4) {
         this.blocks = this.blocks.slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT * 4);
@@ -746,11 +759,6 @@ class Blocks {
         diskCache.$saveCacheToDisk();
       }
 
-      // wait for pending async callbacks to finish
-      this.updateTimerProgress(timer, `waiting for async callbacks to complete for ${this.currentBlockHeight}`);
-      await Promise.all(callbackPromises);
-      this.updateTimerProgress(timer, `async callbacks completed for ${this.currentBlockHeight}`);
-
       handledBlocks++;
     }
 
@@ -827,7 +835,7 @@ class Blocks {
   }
 
   public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false,
-    skipDBLookup = false): Promise<TransactionStripped[]>
+    skipDBLookup = false, cpfpSummary?: CpfpSummary, blockHeight?: number): Promise<TransactionStripped[]>
   {
     if (skipMemoryCache === false) {
       // Check the memory cache
@@ -845,13 +853,35 @@ class Blocks {
       }
     }
 
+    let height = blockHeight;
+    let summary: BlockSummary;
+    if (cpfpSummary) {
+      summary = {
+        id: hash,
+        transactions: cpfpSummary.transactions.map(tx => {
+          return {
+            txid: tx.txid,
+            fee: tx.fee,
+            vsize: tx.vsize,
+            value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0)),
+            rate: tx.effectiveFeePerVsize
+          };
+        }),
+      };
+    } else {
       // Call Core RPC
       const block = await bitcoinClient.getBlock(hash, 2);
-    const summary = this.summarizeBlock(block);
+      summary = this.summarizeBlock(block);
+      height = block.height;
+    }
+    if (height == null) {
+      const block = await bitcoinApi.$getBlock(hash);
+      height = block.height;
+    }
 
     // Index the response if needed
     if (Common.blocksSummariesIndexingEnabled() === true) {
-      await BlocksSummariesRepository.$saveTransactions(block.height, block.hash, summary.transactions);
+      await BlocksSummariesRepository.$saveTransactions(height, hash, summary.transactions);
     }
 
     return summary.transactions;
@@ -1007,19 +1037,11 @@ class Blocks {
   }
 
   public async $getBlockAuditSummary(hash: string): Promise<any> {
-    let summary;
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
-      summary = await BlocksAuditsRepository.$getBlockAudit(hash);
+      return BlocksAuditsRepository.$getBlockAudit(hash);
+    } else {
+      return null;
     }
-
-    // fallback to non-audited transaction summary
-    if (!summary?.transactions?.length) {
-      const strippedTransactions = await this.$getStrippedBlockTransactions(hash);
-      summary = {
-        transactions: strippedTransactions
-      };
-    }
-    return summary;
   }
 
   public getLastDifficultyAdjustmentTime(): number {
@@ -1050,10 +1072,14 @@ class Blocks {
   }
 
   public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise<void> {
+    try {
       const result = await cpfpRepository.$batchSaveClusters(cpfpSummary.clusters);
       if (!result) {
         await cpfpRepository.$insertProgressMarker(height);
       }
+    } catch (e) {
+      // not a fatal error, we'll try again next time the indexer runs
+    }
   }
 }
 

backend/src/api/common.ts

@@ -113,6 +113,10 @@ export class Common {
     };
   }
 
+  static stripTransactions(txs: TransactionExtended[]): TransactionStripped[] {
+    return txs.map(this.stripTransaction);
+  }
+
   static sleep$(ms: number): Promise<void> {
     return new Promise((resolve) => {
        setTimeout(() => {

backend/src/api/mempool-blocks.ts

@@ -143,7 +143,7 @@ class MempoolBlocks {
           const stackWeight = transactionsSorted.slice(index).reduce((total, tx) => total + (tx.weight || 0), 0);
           if (stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS) {
             onlineStats = true;
-            feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5);
+            feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]);
             feeStatsCalculator.processNext(tx);
           }
         }
@@ -334,7 +334,7 @@ class MempoolBlocks {
     if (hasBlockStack) {
       stackWeight = blocks[blocks.length - 1].reduce((total, tx) => total + (mempool[tx]?.weight || 0), 0);
       hasBlockStack = stackWeight > config.MEMPOOL.BLOCK_WEIGHT_UNITS;
-      feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5);
+      feeStatsCalculator = new OnlineFeeStatsCalculator(stackWeight, 0.5, [10, 20, 30, 40, 50, 60, 70, 80, 90]);
     }
 
     const readyBlocks: { transactionIds, transactions, totalSize, totalWeight, totalFees, feeStats }[] = [];

backend/src/api/statistics/statistics-api.ts

@@ -211,7 +211,7 @@ class StatisticsApi {
       CAST(avg(vsize_1800) as DOUBLE) as vsize_1800,
       CAST(avg(vsize_2000) as DOUBLE) as vsize_2000 \
       FROM statistics \
-      WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() \
+      ${interval === 'all' ? '' : `WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`} \
       GROUP BY UNIX_TIMESTAMP(added) DIV ${div} \
       ORDER BY statistics.added DESC;`;
   }
@@ -259,7 +259,7 @@ class StatisticsApi {
       vsize_1800,
       vsize_2000 \
       FROM statistics \
-      WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW() \
+      ${interval === 'all' ? '' : `WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`} \
       GROUP BY UNIX_TIMESTAMP(added) DIV ${div} \
       ORDER BY statistics.added DESC;`;
   }
@@ -386,6 +386,17 @@ class StatisticsApi {
     }
   }
 
+  public async $listAll(): Promise<OptimizedStatistic[]> {
+    try {
+      const query = this.getQueryForDays(43200, 'all'); // 12h interval
+      const [rows] = await DB.query({ sql: query, timeout: this.queryTimeout });
+      return this.mapStatisticToOptimizedStatistic(rows as Statistic[]);
+    } catch (e) {
+      logger.err('$listAll() error' + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
   private mapStatisticToOptimizedStatistic(statistic: Statistic[]): OptimizedStatistic[] {
     return statistic.map((s) => {
       return {

backend/src/api/statistics/statistics.routes.ts

@@ -15,10 +15,11 @@ class StatisticsRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2y', this.$getStatisticsByTime.bind(this, '2y'))
       .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3y', this.$getStatisticsByTime.bind(this, '3y'))
       .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/4y', this.$getStatisticsByTime.bind(this, '4y'))
+      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/all', this.$getStatisticsByTime.bind(this, 'all'))
     ;
   }
 
-  private async $getStatisticsByTime(time: '2h' | '24h' | '1w' | '1m' | '3m' | '6m' | '1y' | '2y' | '3y' | '4y', req: Request, res: Response) {
+  private async $getStatisticsByTime(time: '2h' | '24h' | '1w' | '1m' | '3m' | '6m' | '1y' | '2y' | '3y' | '4y' | 'all', req: Request, res: Response) {
     res.header('Pragma', 'public');
     res.header('Cache-control', 'public');
     res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
@@ -26,10 +27,6 @@ class StatisticsRoutes {
     try {
       let result;
       switch (time as string) {
-        case '2h':
-          result = await statisticsApi.$list2H();
-          res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
-          break;
         case '24h':
           result = await statisticsApi.$list24H();
           res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
@@ -58,8 +55,13 @@ class StatisticsRoutes {
         case '4y':
           result = await statisticsApi.$list4Y();
           break;
+        case 'all':
+          result = await statisticsApi.$listAll();
+          break;
         default:
           result = await statisticsApi.$list2H();
+          res.setHeader('Expires', new Date(Date.now() + 1000 * 30).toUTCString());
+          break;
       }
       res.json(result);
     } catch (e) {

backend/src/api/websocket-handler.ts

@@ -22,6 +22,14 @@ import { deepClone } from '../utils/clone';
 import priceUpdater from '../tasks/price-updater';
 import { ApiPrice } from '../repositories/PricesRepository';
 
+// valid 'want' subscriptions
+const wantable = [
+  'blocks',
+  'mempool-blocks',
+  'live-2h-chart',
+  'stats',
+];
+
 class WebsocketHandler {
   private wss: WebSocket.Server | undefined;
   private extraInitProperties = {};
@@ -30,7 +38,7 @@ class WebsocketHandler {
   private numConnected = 0;
   private numDisconnected = 0;
 
-  private initData: { [key: string]: string } = {};
+  private socketData: { [key: string]: string } = {};
   private serializedInitData: string = '{}';
 
   constructor() { }
@@ -39,28 +47,28 @@ class WebsocketHandler {
     this.wss = wss;
   }
 
-  setExtraInitProperties(property: string, value: any) {
+  setExtraInitData(property: string, value: any) {
     this.extraInitProperties[property] = value;
-    this.setInitDataFields(this.extraInitProperties);
+    this.updateSocketDataFields(this.extraInitProperties);
   }
 
-  private setInitDataFields(data: { [property: string]: any }): void {
+  private updateSocketDataFields(data: { [property: string]: any }): void {
     for (const property of Object.keys(data)) {
       if (data[property] != null) {
-        this.initData[property] = JSON.stringify(data[property]);
+        this.socketData[property] = JSON.stringify(data[property]);
       } else {
-        delete this.initData[property];
+        delete this.socketData[property];
       }
     }
     this.serializedInitData = '{'
-      + Object.keys(this.initData).map(key => `"${key}": ${this.initData[key]}`).join(', ')
+    + Object.keys(this.socketData).map(key => `"${key}": ${this.socketData[key]}`).join(', ')
     + '}';
   }
 
-  private updateInitData(): void {
+  private updateSocketData(): void {
     const _blocks = blocks.getBlocks().slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT);
     const da = difficultyAdjustment.getDifficultyAdjustment();
-    this.setInitDataFields({
+    this.updateSocketDataFields({
       'mempoolInfo': memPool.getMempoolInfo(),
       'vBytesPerSecond': memPool.getVBytesPerSecond(),
       'blocks': _blocks,
@@ -94,11 +102,33 @@ class WebsocketHandler {
           const parsedMessage: WebsocketResponse = JSON.parse(message);
           const response = {};
 
-          if (parsedMessage.action === 'want') {
+          const wantNow = {};
-            client['want-blocks'] = parsedMessage.data.indexOf('blocks') > -1;
+          if (parsedMessage && parsedMessage.action === 'want' && Array.isArray(parsedMessage.data)) {
-            client['want-mempool-blocks'] = parsedMessage.data.indexOf('mempool-blocks') > -1;
+            for (const sub of wantable) {
-            client['want-live-2h-chart'] = parsedMessage.data.indexOf('live-2h-chart') > -1;
+              const key = `want-${sub}`;
-            client['want-stats'] = parsedMessage.data.indexOf('stats') > -1;
+              const wants = parsedMessage.data.includes(sub);
+              if (wants && client['wants'] && !client[key]) {
+                wantNow[key] = true;
+              }
+              client[key] = wants;
+            }
+            client['wants'] = true;
+          }
+
+          // send initial data when a client first starts a subscription
+          if (wantNow['want-blocks'] || (parsedMessage && parsedMessage['refresh-blocks'])) {
+            response['blocks'] = this.socketData['blocks'];
+          }
+
+          if (wantNow['want-mempool-blocks']) {
+            response['mempool-blocks'] = this.socketData['mempool-blocks'];
+          }
+
+          if (wantNow['want-stats']) {
+            response['mempoolInfo'] = this.socketData['mempoolInfo'];
+            response['vBytesPerSecond'] = this.socketData['vBytesPerSecond'];
+            response['fees'] = this.socketData['fees'];
+            response['da'] = this.socketData['da'];
           }
 
           if (parsedMessage && parsedMessage['track-tx']) {
@@ -109,21 +139,21 @@ class WebsocketHandler {
               if (parsedMessage['watch-mempool']) {
                 const rbfCacheTxid = rbfCache.getReplacedBy(trackTxid);
                 if (rbfCacheTxid) {
-                  response['txReplaced'] = {
+                  response['txReplaced'] = JSON.stringify({
                     txid: rbfCacheTxid,
-                  };
+                  });
                   client['track-tx'] = null;
                 } else {
                   // It might have appeared before we had the time to start watching for it
                   const tx = memPool.getMempool()[trackTxid];
                   if (tx) {
                     if (config.MEMPOOL.BACKEND === 'esplora') {
-                      response['tx'] = tx;
+                      response['tx'] = JSON.stringify(tx);
                     } else {
                       // tx.prevout is missing from transactions when in bitcoind mode
                       try {
                         const fullTx = await transactionUtils.$getMempoolTransactionExtended(tx.txid, true);
-                        response['tx'] = fullTx;
+                        response['tx'] = JSON.stringify(fullTx);
                       } catch (e) {
                         logger.debug('Error finding transaction: ' + (e instanceof Error ? e.message : e));
                       }
@@ -131,7 +161,7 @@ class WebsocketHandler {
                   } else {
                     try {
                       const fullTx = await transactionUtils.$getMempoolTransactionExtended(client['track-tx'], true);
-                      response['tx'] = fullTx;
+                      response['tx'] = JSON.stringify(fullTx);
                     } catch (e) {
                       logger.debug('Error finding transaction. ' + (e instanceof Error ? e.message : e));
                       client['track-mempool-tx'] = parsedMessage['track-tx'];
@@ -141,10 +171,10 @@ class WebsocketHandler {
               }
               const tx = memPool.getMempool()[trackTxid];
               if (tx && tx.position) {
-                response['txPosition'] = {
+                response['txPosition'] = JSON.stringify({
                   txid: trackTxid,
                   position: tx.position,
-                };
+                });
               }
             } else {
               client['track-tx'] = null;
@@ -177,10 +207,10 @@ class WebsocketHandler {
               const index = parsedMessage['track-mempool-block'];
               client['track-mempool-block'] = index;
               const mBlocksWithTransactions = mempoolBlocks.getMempoolBlocksWithTransactions();
-              response['projected-block-transactions'] = {
+              response['projected-block-transactions'] = JSON.stringify({
                 index: index,
                 blockTransactions: mBlocksWithTransactions[index]?.transactions || [],
-              };
+              });
             } else {
               client['track-mempool-block'] = null;
             }
@@ -189,23 +219,24 @@ class WebsocketHandler {
           if (parsedMessage && parsedMessage['track-rbf'] !== undefined) {
             if (['all', 'fullRbf'].includes(parsedMessage['track-rbf'])) {
               client['track-rbf'] = parsedMessage['track-rbf'];
+              response['rbfLatest'] = JSON.stringify(rbfCache.getRbfTrees(parsedMessage['track-rbf'] === 'fullRbf'));
             } else {
               client['track-rbf'] = false;
             }
           }
 
           if (parsedMessage.action === 'init') {
-            if (!this.initData['blocks']?.length || !this.initData['da']) {
+            if (!this.socketData['blocks']?.length || !this.socketData['da']) {
-              this.updateInitData();
+              this.updateSocketData();
             }
-            if (!this.initData['blocks']?.length) {
+            if (!this.socketData['blocks']?.length) {
               return;
             }
             client.send(this.serializedInitData);
           }
 
           if (parsedMessage.action === 'ping') {
-            response['pong'] = true;
+            response['pong'] = JSON.stringify(true);
           }
 
           if (parsedMessage['track-donation'] && parsedMessage['track-donation'].length === 22) {
@@ -221,7 +252,8 @@ class WebsocketHandler {
           }
 
           if (Object.keys(response).length) {
-            client.send(JSON.stringify(response));
+            const serializedResponse = this.serializeResponse(response);
+            client.send(serializedResponse);
           }
         } catch (e) {
           logger.debug('Error parsing websocket message: ' + (e instanceof Error ? e.message : e));
@@ -250,7 +282,7 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
-    this.setInitDataFields({ 'loadingIndicators': indicators });
+    this.updateSocketDataFields({ 'loadingIndicators': indicators });
 
     const response = JSON.stringify({ loadingIndicators: indicators });
     this.wss.clients.forEach((client) => {
@@ -266,7 +298,7 @@ class WebsocketHandler {
       throw new Error('WebSocket.Server is not set');
     }
 
-    this.setInitDataFields({ 'conversions': conversionRates });
+    this.updateSocketDataFields({ 'conversions': conversionRates });
 
     const response = JSON.stringify({ conversions: conversionRates });
     this.wss.clients.forEach((client) => {
@@ -336,11 +368,21 @@ class WebsocketHandler {
     memPool.addToSpendMap(newTransactions);
     const recommendedFees = feeApi.getRecommendedFee();
 
+    const latestTransactions = newTransactions.slice(0, 6).map((tx) => Common.stripTransaction(tx));
+
     // update init data
-    this.updateInitData();
+    this.updateSocketDataFields({
+      'mempoolInfo': mempoolInfo,
+      'vBytesPerSecond': vBytesPerSecond,
+      'mempool-blocks': mBlocks,
+      'transactions': latestTransactions,
+      'loadingIndicators': loadingIndicators.getLoadingIndicators(),
+      'da': da?.previousTime ? da : undefined,
+      'fees': recommendedFees,
+    });
 
     // cache serialized objects to avoid stringify-ing the same thing for every client
-    const responseCache = { ...this.initData };
+    const responseCache = { ...this.socketData };
     function getCachedResponse(key: string,  data): string {
       if (!responseCache[key]) {
         responseCache[key] = JSON.stringify(data);
@@ -371,8 +413,6 @@ class WebsocketHandler {
       }
     }
 
-    const latestTransactions = newTransactions.slice(0, 6).map((tx) => Common.stripTransaction(tx));
-
     this.wss.clients.forEach(async (client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
@@ -490,7 +530,7 @@ class WebsocketHandler {
         if (rbfReplacedBy) {
           response['rbfTransaction'] = JSON.stringify({
             txid: rbfReplacedBy,
-          })
+          });
         }
 
         const rbfChange = rbfChanges.map[client['track-tx']];
@@ -524,9 +564,7 @@ class WebsocketHandler {
       }
 
       if (Object.keys(response).length) {
-        const serializedResponse = '{'
+        const serializedResponse = this.serializeResponse(response);
-          + Object.keys(response).map(key => `"${key}": ${response[key]}`).join(', ')
-          + '}';
         client.send(serializedResponse);
       }
     });
@@ -562,14 +600,7 @@ class WebsocketHandler {
         const { censored, added, fresh, sigop, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
         const matchRate = Math.round(score * 100 * 100) / 100;
 
-        const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
+        const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : [];
-          return {
-            txid: tx.txid,
-            vsize: tx.vsize,
-            fee: tx.fee ? Math.round(tx.fee) : 0,
-            value: tx.value,
-          };
-        }) : [];
 
         let totalFees = 0;
         let totalWeight = 0;
@@ -633,11 +664,19 @@ class WebsocketHandler {
 
     const da = difficultyAdjustment.getDifficultyAdjustment();
     const fees = feeApi.getRecommendedFee();
+    const mempoolInfo = memPool.getMempoolInfo();
 
     // update init data
-    this.updateInitData();
+    this.updateSocketDataFields({
+      'mempoolInfo': mempoolInfo,
+      'blocks': [...blocks.getBlocks(), block].slice(-config.MEMPOOL.INITIAL_BLOCKS_AMOUNT),
+      'mempool-blocks': mBlocks,
+      'loadingIndicators': loadingIndicators.getLoadingIndicators(),
+      'da': da?.previousTime ? da : undefined,
+      'fees': fees,
+    });
 
-    const responseCache = { ...this.initData };
+    const responseCache = { ...this.socketData };
     function getCachedResponse(key, data): string {
       if (!responseCache[key]) {
         responseCache[key] = JSON.stringify(data);
@@ -645,23 +684,27 @@ class WebsocketHandler {
       return responseCache[key];
     }
 
-    const mempoolInfo = memPool.getMempoolInfo();
-
     this.wss.clients.forEach((client) => {
       if (client.readyState !== WebSocket.OPEN) {
         return;
       }
 
-      if (!client['want-blocks']) {
+      const response = {};
-        return;
+
+      if (client['want-blocks']) {
+        response['block'] = getCachedResponse('block', block);
       }
 
-      const response = {};
+      if (client['want-stats']) {
-      response['block'] = getCachedResponse('block', block);
         response['mempoolInfo'] = getCachedResponse('mempoolInfo', mempoolInfo);
-      response['da'] = getCachedResponse('da', da?.previousTime ? da : undefined);
+        response['vBytesPerSecond'] = getCachedResponse('vBytesPerSecond', memPool.getVBytesPerSecond());
         response['fees'] = getCachedResponse('fees', fees);
 
+        if (da?.previousTime) {
+          response['da'] = getCachedResponse('da', da);
+        }
+      }
+
       if (mBlocks && client['want-mempool-blocks']) {
         response['mempool-blocks'] = getCachedResponse('mempool-blocks', mBlocks);
       }
@@ -755,11 +798,19 @@ class WebsocketHandler {
         }
       }
 
-      const serializedResponse = '{'
+      if (Object.keys(response).length) {
+        const serializedResponse = this.serializeResponse(response);
+        client.send(serializedResponse);
+      }
+    });
+  }
+
+  // takes a dictionary of JSON serialized values
+  // and zips it together into a valid JSON object
+  private serializeResponse(response): string {
+    return '{'
         + Object.keys(response).map(key => `"${key}": ${response[key]}`).join(', ')
         + '}';
-      client.send(serializedResponse);
-    });
   }
 
   private printLogs(): void {

backend/src/database.ts

@@ -30,7 +30,7 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr
   }
 
   public async query<T extends RowDataPacket[][] | RowDataPacket[] | OkPacket |
-    OkPacket[] | ResultSetHeader>(query, params?): Promise<[T, FieldPacket[]]>
+    OkPacket[] | ResultSetHeader>(query, params?, connection?: PoolConnection): Promise<[T, FieldPacket[]]>
   {
     this.checkDBFlag();
     let hardTimeout;
@@ -45,7 +45,9 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr
           reject(new Error(`DB query failed to return, reject or time out within ${hardTimeout / 1000}s - ${query?.sql?.slice(0, 160) || (typeof(query) === 'string' || query instanceof String ? query?.slice(0, 160) : 'unknown query')}`));
         }, hardTimeout);
 
-        this.getPool().then(pool => {
+        // Use a specific connection if provided, otherwise delegate to the pool
+        const connectionPromise = connection ? Promise.resolve(connection) : this.getPool();
+        connectionPromise.then((pool: PoolConnection | Pool) => {
           return pool.query(query, params) as Promise<[T, FieldPacket[]]>;
         }).then(result => {
           resolve(result);
@@ -61,6 +63,33 @@ import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } fr
     }
   }
 
+  public async $atomicQuery<T extends RowDataPacket[][] | RowDataPacket[] | OkPacket |
+    OkPacket[] | ResultSetHeader>(queries: { query, params }[]): Promise<[T, FieldPacket[]][]>
+  {
+    const pool = await this.getPool();
+    const connection = await pool.getConnection();
+    try {
+      await connection.beginTransaction();
+
+      const results: [T, FieldPacket[]][]  = [];
+      for (const query of queries) {
+        const result = await this.query(query.query, query.params, connection) as [T, FieldPacket[]];
+        results.push(result);
+      }
+
+      await connection.commit();
+
+      return results;
+    } catch (e) {
+      logger.err('Could not complete db transaction, rolling back: ' + (e instanceof Error ? e.message : e));
+      connection.rollback();
+      connection.release();
+      throw e;
+    } finally {
+      connection.release();
+    }
+  }
+
   public async checkDbConnection() {
     this.checkDBFlag();
     try {

backend/src/index.ts

@@ -150,7 +150,7 @@ class Server {
 
     if (config.BISQ.ENABLED) {
       bisq.startBisqService();
-      bisq.setPriceCallbackFunction((price) => websocketHandler.setExtraInitProperties('bsq-price', price));
+      bisq.setPriceCallbackFunction((price) => websocketHandler.setExtraInitData('bsq-price', price));
       blocks.setNewBlockCallback(bisq.handleNewBitcoinBlock.bind(bisq));
       bisqMarkets.startBisqService();
     }

backend/src/repositories/BlocksAuditsRepository.ts

@@ -64,7 +64,6 @@ class BlocksAuditRepositories {
       const [rows]: any[] = await DB.query(
         `SELECT blocks.height, blocks.hash as id, UNIX_TIMESTAMP(blocks.blockTimestamp) as timestamp, blocks.size,
         blocks.weight, blocks.tx_count,
-        transactions,
         template,
         missing_txs as missingTxs,
         added_txs as addedTxs,
@@ -76,7 +75,6 @@ class BlocksAuditRepositories {
         FROM blocks_audits
         JOIN blocks ON blocks.hash = blocks_audits.hash
         JOIN blocks_templates ON blocks_templates.id = blocks_audits.hash
-        JOIN blocks_summaries ON blocks_summaries.id = blocks_audits.hash
         WHERE blocks_audits.hash = "${hash}"
       `);
       
@@ -85,13 +83,10 @@ class BlocksAuditRepositories {
         rows[0].addedTxs = JSON.parse(rows[0].addedTxs);
         rows[0].freshTxs = JSON.parse(rows[0].freshTxs);
         rows[0].sigopTxs = JSON.parse(rows[0].sigopTxs);
-        rows[0].transactions = JSON.parse(rows[0].transactions);
         rows[0].template = JSON.parse(rows[0].template);
 
-        if (rows[0].transactions.length) {
         return rows[0];
       }
-      }
       return null;
     } catch (e: any) {
       logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e));

backend/src/repositories/CpfpRepository.ts

@@ -5,52 +5,10 @@ import { Ancestor, CpfpCluster } from '../mempool.interfaces';
 import transactionRepository from '../repositories/TransactionRepository';
 
 class CpfpRepository {
-  public async $saveCluster(clusterRoot: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number): Promise<boolean> {
-    if (!txs[0]) {
-      return false;
-    }
-    // skip clusters of transactions with the same fees
-    const roundedEffectiveFee = Math.round(effectiveFeePerVsize * 100) / 100;
-    const equalFee = txs.length > 1 && txs.reduce((acc, tx) => {
-      return (acc && Math.round(((tx.fee || 0) / (tx.weight / 4)) * 100) / 100 === roundedEffectiveFee);
-    }, true);
-    if (equalFee) {
-      return false;
-    }
-
-    try {
-      const packedTxs = Buffer.from(this.pack(txs));
-      await DB.query(
-        `
-          INSERT INTO compact_cpfp_clusters(root, height, txs, fee_rate)
-          VALUE (UNHEX(?), ?, ?, ?)
-          ON DUPLICATE KEY UPDATE
-            height = ?,
-            txs = ?,
-            fee_rate = ?
-        `,
-        [clusterRoot, height, packedTxs, effectiveFeePerVsize, height, packedTxs, effectiveFeePerVsize]
-      );
-      const maxChunk = 10;
-      let chunkIndex = 0;
-      while (chunkIndex < txs.length) {
-        const chunk = txs.slice(chunkIndex, chunkIndex + maxChunk).map(tx => {
-          return { txid: tx.txid, cluster: clusterRoot };
-        });
-        await transactionRepository.$batchSetCluster(chunk);
-        chunkIndex += maxChunk;
-      }
-      return true;
-    } catch (e: any) {
-      logger.err(`Cannot save cpfp cluster into db. Reason: ` + (e instanceof Error ? e.message : e));
-      throw e;
-    }
-  }
-
   public async $batchSaveClusters(clusters: { root: string, height: number, txs: Ancestor[], effectiveFeePerVsize: number }[]): Promise<boolean> {
     try {
-      const clusterValues: any[] = [];
+      const clusterValues: [string, number, Buffer, number][] = [];
-      const txs: any[] = [];
+      const txs: { txid: string, cluster: string }[] = [];
 
       for (const cluster of clusters) {
         if (cluster.txs?.length) {
@@ -76,6 +34,8 @@ class CpfpRepository {
         return false;
       }
 
+      const queries: { query, params }[] = [];
+
       const maxChunk = 100;
       let chunkIndex = 0;
       // insert clusters in batches of up to 100 rows
@@ -89,10 +49,10 @@ class CpfpRepository {
           return (' (UNHEX(?), ?, ?, ?)');
         }) + ';';
         const values = chunk.flat();
-        await DB.query(
+        queries.push({
           query,
-          values
+          params: values,
-        );
+        });
         chunkIndex += maxChunk;
       }
 
@@ -100,10 +60,12 @@ class CpfpRepository {
       // insert transactions in batches of up to 100 rows
       while (chunkIndex < txs.length) {
         const chunk = txs.slice(chunkIndex, chunkIndex + maxChunk);
-        await transactionRepository.$batchSetCluster(chunk);
+        queries.push(transactionRepository.buildBatchSetQuery(chunk));
         chunkIndex += maxChunk;
       }
 
+      await DB.$atomicQuery(queries);
+
       return true;
     } catch (e: any) {
       logger.err(`Cannot save cpfp clusters into db. Reason: ` + (e instanceof Error ? e.message : e));

backend/src/repositories/TransactionRepository.ts

@@ -25,8 +25,7 @@ class TransactionRepository {
     }
   }
 
-  public async $batchSetCluster(txs): Promise<void> {
+  public buildBatchSetQuery(txs: { txid: string, cluster: string }[]): { query, params } {
-    try {
     let query = `
           INSERT IGNORE INTO compact_transactions
           (
@@ -39,9 +38,18 @@ class TransactionRepository {
       return (' (UNHEX(?), UNHEX(?))');
     }) + ';';
     const values = txs.map(tx => [tx.txid, tx.cluster]).flat();
-      await DB.query(
+    return {
       query,
-        values
+      params: values,
+    };
+  }
+
+  public async $batchSetCluster(txs): Promise<void> {
+    try {
+      const query = this.buildBatchSetQuery(txs);
+      await DB.query(
+        query.query,
+        query.params,
       );
     } catch (e: any) {
       logger.err(`Cannot save cpfp transactions into db. Reason: ` + (e instanceof Error ? e.message : e));

frontend/src/app/components/about/about.component.html

@@ -220,7 +220,7 @@
         <img class="image" src="/resources/profile/mynodebtc.png" />
         <span>myNode</span>
       </a>
-      <a href="https://github.com/RoninDojo/RoninDojo" target="_blank" title="RoninDojo">
+      <a href="https://code.samourai.io/ronindojo/RoninDojo" target="_blank" title="RoninDojo">
         <img class="image" src="/resources/profile/ronindojo.png" />
         <span>RoninDojo</span>
       </a>

frontend/src/app/components/block/block.component.html

@@ -63,7 +63,7 @@
                     *ngIf="blockAudit?.matchRate != null; else nullHealth"
                   >{{ blockAudit?.matchRate }}%</span>
                   <ng-template #nullHealth>
-                    <ng-container *ngIf="!isLoadingAudit; else loadingHealth">
+                    <ng-container *ngIf="!isLoadingOverview; else loadingHealth">
                       <span class="health-badge badge badge-secondary" i18n="unknown">Unknown</span>
                     </ng-container>
                   </ng-template>

frontend/src/app/components/block/block.component.ts

@@ -2,9 +2,9 @@ import { Component, OnInit, OnDestroy, ViewChildren, QueryList } from '@angular/
 import { Location } from '@angular/common';
 import { ActivatedRoute, ParamMap, Router } from '@angular/router';
 import { ElectrsApiService } from '../../services/electrs-api.service';
-import { switchMap, tap, throttleTime, catchError, map, shareReplay, startWith, pairwise, filter } from 'rxjs/operators';
+import { switchMap, tap, throttleTime, catchError, map, shareReplay, startWith } from 'rxjs/operators';
 import { Transaction, Vout } from '../../interfaces/electrs.interface';
-import { Observable, of, Subscription, asyncScheduler, EMPTY, combineLatest } from 'rxjs';
+import { Observable, of, Subscription, asyncScheduler, EMPTY, combineLatest, forkJoin } from 'rxjs';
 import { StateService } from '../../services/state.service';
 import { SeoService } from '../../services/seo.service';
 import { WebsocketService } from '../../services/websocket.service';
@@ -44,7 +44,6 @@ export class BlockComponent implements OnInit, OnDestroy {
   strippedTransactions: TransactionStripped[];
   overviewTransitionDirection: string;
   isLoadingOverview = true;
-  isLoadingAudit = true;
   error: any;
   blockSubsidy: number;
   fees: number;
@@ -281,57 +280,35 @@ export class BlockComponent implements OnInit, OnDestroy {
       this.isLoadingOverview = false;
     });
 
-    if (!this.auditSupported) {
     this.overviewSubscription = block$.pipe(
-        startWith(null),
+      switchMap((block) => {
-        pairwise(),
+        return forkJoin([
-        switchMap(([prevBlock, block]) => this.apiService.getStrippedBlockTransactions$(block.id)
+          this.apiService.getStrippedBlockTransactions$(block.id)
             .pipe(
               catchError((err) => {
                 this.overviewError = err;
-              return of([]);
+                return of(null);
-            }),
+              })
-            switchMap((transactions) => {
+            ),
-              if (prevBlock) {
+          !this.isAuditAvailableFromBlockHeight(block.height) ? of(null) : this.apiService.getBlockAudit$(block.id)
-                return of({ transactions, direction: (prevBlock.height < block.height) ? 'right' : 'left' });
+            .pipe(
-              } else {
+              catchError((err) => {
-                return of({ transactions, direction: 'down' });
+                this.overviewError = err;
-              }
+                return of(null);
               })
             )
-        ),
+        ]);
+      })
     )
-      .subscribe(({transactions, direction}: {transactions: TransactionStripped[], direction: string}) => {
+    .subscribe(([transactions, blockAudit]) => {      
+      if (transactions) {
         this.strippedTransactions = transactions;
-        this.isLoadingOverview = false;
+      } else {
-        this.setupBlockGraphs();
+        this.strippedTransactions = [];
-      },
-      (error) => {
-        this.error = error;
-        this.isLoadingOverview = false;
-      });
       }
 
-    if (this.auditSupported) {
-      this.auditSubscription = block$.pipe(
-        startWith(null),
-        pairwise(),
-        switchMap(([prevBlock, block]) => {
-          this.isLoadingAudit = true;
       this.blockAudit = null;
-          return this.apiService.getBlockAudit$(block.id)
+      if (transactions && blockAudit) {
-            .pipe(
-              catchError((err) => {
-                this.overviewError = err;
-                this.isLoadingAudit = false;
-                return of([]);
-              })
-            );
-        }
-        ),
-        filter((response) => response != null),
-        map((response) => {
-          const blockAudit = response.body;
         const inTemplate = {};
         const inBlock = {};
         const isAdded = {};
@@ -347,7 +324,7 @@ export class BlockComponent implements OnInit, OnDestroy {
           for (const tx of blockAudit.template) {
             inTemplate[tx.txid] = true;
           }
-            for (const tx of blockAudit.transactions) {
+          for (const tx of transactions) {
             inBlock[tx.txid] = true;
           }
           for (const txid of blockAudit.addedTxs) {
@@ -375,7 +352,7 @@ export class BlockComponent implements OnInit, OnDestroy {
               this.numMissing++;
             }
           }
-            for (const [index, tx] of blockAudit.transactions.entries()) {
+          for (const [index, tx] of transactions.entries()) {
             tx.context = 'actual';
             if (index === 0) {
               tx.status = null;
@@ -389,36 +366,26 @@ export class BlockComponent implements OnInit, OnDestroy {
               this.numUnexpected++;
             }
           }
-            for (const tx of blockAudit.transactions) {
+          for (const tx of transactions) {
             inBlock[tx.txid] = true;
           }
 
           blockAudit.feeDelta = blockAudit.expectedFees > 0 ? (blockAudit.expectedFees - this.block.extras.totalFees) / blockAudit.expectedFees : 0;
           blockAudit.weightDelta = blockAudit.expectedWeight > 0 ? (blockAudit.expectedWeight - this.block.weight) / blockAudit.expectedWeight : 0;
           blockAudit.txDelta = blockAudit.template.length > 0 ? (blockAudit.template.length - this.block.tx_count) / blockAudit.template.length : 0;
-
+          this.blockAudit = blockAudit;
           this.setAuditAvailable(true);
         } else {
           this.setAuditAvailable(false);
         }
-          return blockAudit;
+      } else {
-        }),
-        catchError((err) => {
-          console.log(err);
-          this.error = err;
-          this.isLoadingOverview = false;
-          this.isLoadingAudit = false;
         this.setAuditAvailable(false);
-          return of(null);
-        }),
-      ).subscribe((blockAudit) => {
-        this.blockAudit = blockAudit;
-        this.setupBlockGraphs();
-        this.isLoadingOverview = false;
-        this.isLoadingAudit = false;
-      });
       }
 
+      this.isLoadingOverview = false;
+      this.setupBlockGraphs();
+    });
+
     this.networkChangedSubscription = this.stateService.networkChanged$
       .subscribe((network) => this.network = network);
 
@@ -652,25 +619,32 @@ export class BlockComponent implements OnInit, OnDestroy {
   }
 
   updateAuditAvailableFromBlockHeight(blockHeight: number): void {
-    if (!this.auditSupported) {
+    if (!this.isAuditAvailableFromBlockHeight(blockHeight)) {
       this.setAuditAvailable(false);
     }
+  }
+  
+  isAuditAvailableFromBlockHeight(blockHeight: number): boolean {
+    if (!this.auditSupported) {
+      return false;
+    }
     switch (this.stateService.network) {
       case 'testnet':
         if (blockHeight < this.stateService.env.TESTNET_BLOCK_AUDIT_START_HEIGHT) {
-          this.setAuditAvailable(false);
+          return false;
         }
         break;
       case 'signet':
         if (blockHeight < this.stateService.env.SIGNET_BLOCK_AUDIT_START_HEIGHT) {
-          this.setAuditAvailable(false);
+          return false;
         }
         break;
       default:
         if (blockHeight < this.stateService.env.MAINNET_BLOCK_AUDIT_START_HEIGHT) {
-          this.setAuditAvailable(false);
+          return false;
         }
     }
+    return true;
   }
 
   getMinBlockFee(block: BlockExtended): number {

frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts

@@ -1,5 +1,9 @@
 import { OnChanges } from '@angular/core';
 import { Component, Input, OnInit, ChangeDetectionStrategy } from '@angular/core';
+import { TransactionStripped } from '../../interfaces/websocket.interface';
+import { StateService } from '../../services/state.service';
+import { VbytesPipe } from '../../shared/pipes/bytes-pipe/vbytes.pipe';
+import { selectPowerOfTen } from '../../bitcoin.utils';
 
 @Component({
   selector: 'app-fee-distribution-graph',
@@ -7,47 +11,121 @@ import { Component, Input, OnInit, ChangeDetectionStrategy } from '@angular/core
   changeDetection: ChangeDetectionStrategy.OnPush,
 })
 export class FeeDistributionGraphComponent implements OnInit, OnChanges {
-  @Input() data: any;
+  @Input() feeRange: number[];
+  @Input() vsize: number;
+  @Input() transactions: TransactionStripped[];
   @Input() height: number | string = 210;
   @Input() top: number | string = 20;
   @Input() right: number | string = 22;
   @Input() left: number | string = 30;
+  @Input() numSamples: number = 200;
+  @Input() numLabels: number = 10;
+
+  simple: boolean = false;
+  data: number[][];
+  labelInterval: number = 50;
 
   mempoolVsizeFeesOptions: any;
   mempoolVsizeFeesInitOptions = {
     renderer: 'svg'
   };
 
-  constructor() { }
+  constructor(
+    private stateService: StateService,
+    private vbytesPipe: VbytesPipe,
+  ) { }
 
-  ngOnInit() {
+  ngOnInit(): void {
     this.mountChart();
   }
 
-  ngOnChanges() {
+  ngOnChanges(): void {
+    this.simple = !!this.feeRange?.length;
+    this.prepareChart();
     this.mountChart();
   }
 
-  mountChart() {
+  prepareChart(): void {
+    if (this.simple) {
+      this.data = this.feeRange.map((rate, index) => [index * 10, rate]);
+      this.labelInterval = 1;
+      return;
+    }
+    this.data = [];
+    if (!this.transactions?.length) {
+      return;
+    }
+    const samples = [];
+    const txs = this.transactions.map(tx => { return { vsize: tx.vsize, rate: tx.rate || (tx.fee / tx.vsize) }; }).sort((a, b) => { return b.rate - a.rate; });
+    const maxBlockVSize = this.stateService.env.BLOCK_WEIGHT_UNITS / 4;
+    const sampleInterval = maxBlockVSize / this.numSamples;
+    let cumVSize = 0;
+    let sampleIndex = 0;
+    let nextSample = 0;
+    let txIndex = 0;
+    this.labelInterval = this.numSamples / this.numLabels;
+    while (nextSample <= maxBlockVSize) {
+      if (txIndex >= txs.length) {
+        samples.push([(1 - (sampleIndex / this.numSamples)) * 100, 0]);
+        nextSample += sampleInterval;
+        sampleIndex++;
+        continue;
+      }
+
+      while (txs[txIndex] && nextSample < cumVSize + txs[txIndex].vsize) {
+        samples.push([(1 - (sampleIndex / this.numSamples)) * 100, txs[txIndex].rate]);
+        nextSample += sampleInterval;
+        sampleIndex++;
+      }
+      cumVSize += txs[txIndex].vsize;
+      txIndex++;
+    }
+    this.data = samples.reverse();
+  }
+
+  mountChart(): void {
     this.mempoolVsizeFeesOptions = {
       grid: {
         height: '210',
         right: '20',
         top: '22',
-        left: '30',
+        left: '40',
       },
       xAxis: {
         type: 'category',
         boundaryGap: false,
+        name: '% Weight',
+        nameLocation: 'middle',
+        nameGap: 0,
+        nameTextStyle: {
+          verticalAlign: 'top',
+          padding: [30, 0, 0, 0],
+        },
+        axisLabel: {
+          interval: (index: number): boolean => { return index && (index % this.labelInterval === 0); },
+          formatter: (value: number): string => { return Number(value).toFixed(0); },
+        },
+        axisTick: {
+          interval: (index:number): boolean => { return (index % this.labelInterval === 0); },
+        },
       },
       yAxis: {
         type: 'value',
+        // name: 'Effective Fee Rate s/vb',
+        // nameLocation: 'middle',
         splitLine: {
           lineStyle: {
             type: 'dotted',
             color: '#ffffff66',
             opacity: 0.25,
           }
+        },
+        axisLabel: {
+          formatter: (value: number): string => {
+            const selectedPowerOfTen = selectPowerOfTen(value);
+            const newVal = Math.round(value / selectedPowerOfTen.divider);
+            return `${newVal}${selectedPowerOfTen.unit}`;
+          },
         }
       },
       series: [{
@@ -58,14 +136,18 @@ export class FeeDistributionGraphComponent implements OnInit, OnChanges {
           position: 'top',
           color: '#ffffff',
           textShadowBlur: 0,
-          formatter: (label: any) => {
+          formatter: (label: { data: number[] }): string => {
-            return Math.floor(label.data);
+            const value = label.data[1];
+            const selectedPowerOfTen = selectPowerOfTen(value);
+            const newVal = Math.round(value / selectedPowerOfTen.divider);
+            return `${newVal}${selectedPowerOfTen.unit}`;
           },
         },
+        showAllSymbol: false,
         smooth: true,
         lineStyle: {
           color: '#D81B60',
-          width: 4,
+          width: 1,
         },
         itemStyle: {
           color: '#b71c1c',

frontend/src/app/components/mempool-block/mempool-block.component.html

@@ -39,11 +39,11 @@
             </tr>
           </tbody>
         </table>
-        <app-fee-distribution-graph *ngIf="webGlEnabled" [data]="mempoolBlock.feeRange" ></app-fee-distribution-graph>
+        <app-fee-distribution-graph *ngIf="webGlEnabled" [transactions]="mempoolBlockTransactions$ | async" [feeRange]="mempoolBlock.isStack ? mempoolBlock.feeRange : []" [vsize]="mempoolBlock.blockVSize" ></app-fee-distribution-graph>
       </div>
       <div class="col-md chart-container">
         <app-mempool-block-overview *ngIf="webGlEnabled" [index]="mempoolBlockIndex" (txPreviewEvent)="setTxPreview($event)"></app-mempool-block-overview>
-        <app-fee-distribution-graph *ngIf="!webGlEnabled" [data]="mempoolBlock.feeRange" ></app-fee-distribution-graph>
+        <app-fee-distribution-graph *ngIf="!webGlEnabled" [transactions]="mempoolBlockTransactions$ | async" [feeRange]="mempoolBlock.isStack ? mempoolBlock.feeRange : []" [vsize]="mempoolBlock.blockVSize" ></app-fee-distribution-graph>
       </div>
     </div>
   </div>

frontend/src/app/components/mempool-block/mempool-block.component.ts

@@ -17,6 +17,7 @@ export class MempoolBlockComponent implements OnInit, OnDestroy {
   network$: Observable<string>;
   mempoolBlockIndex: number;
   mempoolBlock$: Observable<MempoolBlock>;
+  mempoolBlockTransactions$: Observable<TransactionStripped[]>;
   ordinal$: BehaviorSubject<string> = new BehaviorSubject('');
   previewTx: TransactionStripped | void;
   webGlEnabled: boolean;
@@ -53,6 +54,7 @@ export class MempoolBlockComponent implements OnInit, OnDestroy {
                 const ordinal = this.getOrdinal(mempoolBlocks[this.mempoolBlockIndex]);
                 this.ordinal$.next(ordinal);
                 this.seoService.setTitle(ordinal);
+                mempoolBlocks[this.mempoolBlockIndex].isStack = mempoolBlocks[this.mempoolBlockIndex].blockVSize > this.stateService.blockVSize;
                 return mempoolBlocks[this.mempoolBlockIndex];
               })
             );
@@ -62,6 +64,8 @@ export class MempoolBlockComponent implements OnInit, OnDestroy {
         })
       );
 
+    this.mempoolBlockTransactions$ = this.stateService.liveMempoolBlockTransactions$.pipe(map(txMap => Object.values(txMap)));
+
     this.network$ = this.stateService.networkChanged$;
   }
 

frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts

@@ -143,6 +143,8 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy {
           this.mempoolBlocksFull = JSON.parse(stringifiedBlocks);
           this.mempoolBlocks = this.reduceMempoolBlocksToFitScreen(JSON.parse(stringifiedBlocks));
 
+          this.now = Date.now();
+
           this.updateMempoolBlockStyles();
           this.calculateTransactionPosition();
           return this.mempoolBlocks;
@@ -152,7 +154,8 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy {
     this.difficultyAdjustments$ = this.stateService.difficultyAdjustment$
       .pipe(
         map((da) => {
-          this.now = new Date().getTime();
+          this.now = Date.now();
+          this.cd.markForCheck();
           return da;
         })
       );

frontend/src/app/components/statistics/statistics.component.html

@@ -20,7 +20,8 @@
                 <fa-icon [icon]="['fas', 'tv']" [fixedWidth]="true" i18n-title="master-page.tvview" title="TV view"></fa-icon>
               </a>
             </div>
-            <div class="btn-group btn-group-toggle" name="radioBasic">
+            <div class="btn-toggle-rows" name="radioBasic">
+              <div class="btn-group btn-group-toggle">
                 <label class="btn btn-primary btn-sm" [class.active]="radioGroupForm.get('dateSpan').value === '2h'">
                   <input type="radio" [value]="'2h'" [routerLink]="['/graphs' | relativeUrl]" fragment="2h" formControlName="dateSpan"> 2H
                   (LIVE)
@@ -38,6 +39,8 @@
                 <label class="btn btn-primary btn-sm" [class.active]="radioGroupForm.get('dateSpan').value === '3m'">
                   <input type="radio" [value]="'3m'" [routerLink]="['/graphs' | relativeUrl]" fragment="3m" formControlName="dateSpan"> 3M
                 </label>
+              </div>
+              <div class="btn-group btn-group-toggle">
                 <label class="btn btn-primary btn-sm" [class.active]="radioGroupForm.get('dateSpan').value === '6m'">
                   <input type="radio" [value]="'6m'" [routerLink]="['/graphs' | relativeUrl]" fragment="6m" formControlName="dateSpan"> 6M
                 </label>
@@ -53,6 +56,10 @@
                 <label class="btn btn-primary btn-sm" [class.active]="radioGroupForm.get('dateSpan').value === '4y'">
                   <input type="radio" [value]="'4y'" [routerLink]="['/graphs' | relativeUrl]" fragment="4y" formControlName="dateSpan"> 4Y
                 </label>
+                <label class="btn btn-primary btn-sm" [class.active]="radioGroupForm.get('dateSpan').value === 'all'">
+                  <input type="radio" [value]="'all'" [routerLink]="['/graphs' | relativeUrl]" fragment="all" formControlName="dateSpan"><span i18n="all">All</span>
+                </label>
+              </div>
             </div>
             <div class="small-buttons">
               <div ngbDropdown #myDrop="ngbDropdown">

frontend/src/app/components/statistics/statistics.component.scss

@@ -53,17 +53,17 @@
   }
 }
 .formRadioGroup.mining {
-  @media (min-width: 991px) {
+  @media (min-width: 1035px) {
     position: relative;
     top: -100px;
   }
-  @media (min-width: 830px) and (max-width: 991px) {
+  @media (min-width: 830px) and (max-width: 1035px) {
     position: relative;
     top: 0px;
   }
 }
 .formRadioGroup.no-menu {
-  @media (min-width: 991px) {
+  @media (min-width: 1035px) {
     position: relative;
     top: -33px;
   }
@@ -183,3 +183,43 @@
     }
   }
 }
+
+.btn-toggle-rows {
+  display: flex;
+  flex-direction: row;
+  align-items: stretch;
+  justify-content: stretch;
+
+  .btn-group {
+    flex-grow: 1;
+    flex-shrink: 1;
+  }
+
+  @media (min-width: 500px) {
+    .btn-group:first-child > .btn:last-child {
+      border-top-right-radius: 0;
+      border-bottom-right-radius: 0;
+    }
+    .btn-group:last-child > .btn:first-child {
+      border-top-left-radius: 0;
+      border-bottom-left-radius: 0;
+    }
+  }
+
+  @media (max-width: 499px) {
+    flex-direction: column;
+
+    .btn-group:first-child > .btn:first-child {
+      border-bottom-left-radius: 0;
+    }
+    .btn-group:first-child > .btn:last-child {
+      border-bottom-right-radius: 0;
+    }
+    .btn-group:last-child > .btn:first-child {
+      border-top-left-radius: 0;
+    }
+    .btn-group:last-child > .btn:last-child {
+      border-top-right-radius: 0;
+    }
+  }
+}

frontend/src/app/components/statistics/statistics.component.ts

@@ -72,8 +72,10 @@ export class StatisticsComponent implements OnInit {
     this.route
       .fragment
       .subscribe((fragment) => {
-        if (['2h', '24h', '1w', '1m', '3m', '6m', '1y', '2y', '3y', '4y'].indexOf(fragment) > -1) {
+        if (['2h', '24h', '1w', '1m', '3m', '6m', '1y', '2y', '3y', '4y', 'all'].indexOf(fragment) > -1) {
           this.radioGroupForm.controls.dateSpan.setValue(fragment, { emitEvent: false });
+        } else {
+          this.radioGroupForm.controls.dateSpan.setValue('2h', { emitEvent: false });
         }
       });
 
@@ -114,7 +116,12 @@ export class StatisticsComponent implements OnInit {
         if (this.radioGroupForm.controls.dateSpan.value === '3y') {
           return this.apiService.list3YStatistics$();
         }
+        if (this.radioGroupForm.controls.dateSpan.value === '4y') {
           return this.apiService.list4YStatistics$();
+        }
+        if (this.radioGroupForm.controls.dateSpan.value === 'all') {
+          return this.apiService.listAllTimeStatistics$();
+        }
       })
     )
     .subscribe((mempoolStats: any) => {

frontend/src/app/components/transaction/transaction.component.html

@@ -105,7 +105,7 @@
                           <app-time kind="until" [time]="(60 * 1000 * this.mempoolPosition.block) + now" [fastRender]="false" [fixedRender]="true"></app-time>
                         </ng-template>
                         <ng-template #timeEstimateDefault>
-                          <app-time kind="until" *ngIf="(timeAvg$ | async) as timeAvg;" [time]="(timeAvg * this.mempoolPosition.block) + now + timeAvg" [fastRender]="false" [fixedRender]="true"></app-time>
+                          <app-time kind="until" *ngIf="(da$ | async) as da;" [time]="da.timeAvg * (this.mempoolPosition.block + 1) + now + da.timeOffset" [fastRender]="false" [fixedRender]="true"></app-time>
                         </ng-template>
                       </ng-template>
                     </ng-template>

frontend/src/app/components/transaction/transaction.component.ts

@@ -19,7 +19,7 @@ import { WebsocketService } from '../../services/websocket.service';
 import { AudioService } from '../../services/audio.service';
 import { ApiService } from '../../services/api.service';
 import { SeoService } from '../../services/seo.service';
-import { BlockExtended, CpfpInfo, RbfTree, MempoolPosition } from '../../interfaces/node-api.interface';
+import { BlockExtended, CpfpInfo, RbfTree, MempoolPosition, DifficultyAdjustment } from '../../interfaces/node-api.interface';
 import { LiquidUnblinding } from './liquid-ublinding';
 import { RelativeUrlPipe } from '../../shared/pipes/relative-url/relative-url.pipe';
 import { Price, PriceService } from '../../services/price.service';
@@ -65,7 +65,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
   fetchCachedTx$ = new Subject<string>();
   isCached: boolean = false;
   now = Date.now();
-  timeAvg$: Observable<number>;
+  da$: Observable<DifficultyAdjustment>;
   liquidUnblinding = new LiquidUnblinding();
   inputIndex: number;
   outputIndex: number;
@@ -117,10 +117,10 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
       this.setFlowEnabled();
     });
 
-    this.timeAvg$ = timer(0, 1000)
+    this.da$ = this.stateService.difficultyAdjustment$.pipe(
-      .pipe(
+      tap(() => {
-        switchMap(() => this.stateService.difficultyAdjustment$),
+        this.now = Date.now();
-        map((da) => da.timeAvg)
+      })
     );
 
     this.urlFragmentSubscription = this.route.fragment.subscribe((fragment) => {
@@ -236,6 +236,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
     });
 
     this.mempoolPositionSubscription = this.stateService.mempoolTxPosition$.subscribe(txPosition => {
+      this.now = Date.now();
       if (txPosition && txPosition.txid === this.txId && txPosition.position) {
         this.mempoolPosition = txPosition.position;
         if (this.tx && !this.tx.status.confirmed) {
@@ -434,12 +435,12 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
     });
 
     this.mempoolBlocksSubscription = this.stateService.mempoolBlocks$.subscribe((mempoolBlocks) => {
+      this.now = Date.now();
+
       if (!this.tx || this.mempoolPosition) {
         return;
       }
 
-      this.now = Date.now();
-
       const txFeePerVSize =
         this.tx.effectiveFeePerVsize || this.tx.fee / (this.tx.weight / 4);
 

frontend/src/app/interfaces/node-api.interface.ts

@@ -153,6 +153,8 @@ export interface BlockExtended extends Block {
 export interface BlockAudit extends BlockExtended {
   missingTxs: string[],
   addedTxs: string[],
+  freshTxs: string[],
+  sigopTxs: string[],
   matchRate: number,
   expectedFees: number,
   expectedWeight: number,
@@ -169,6 +171,7 @@ export interface TransactionStripped {
   vsize: number;
   value: number;
   status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected';
+  context?: 'projected' | 'actual';
 }
 
 interface RbfTransaction extends TransactionStripped {

frontend/src/app/interfaces/websocket.interface.ts

@@ -31,6 +31,7 @@ export interface WebsocketResponse {
   'track-rbf'?: string;
   'watch-mempool'?: boolean;
   'track-bisq-market'?: string;
+  'refresh-blocks'?: boolean;
 }
 
 export interface ReplacedTransaction extends Transaction {

frontend/src/app/services/api.service.ts

@@ -1,7 +1,7 @@
 import { Injectable } from '@angular/core';
 import { HttpClient, HttpParams, HttpResponse } from '@angular/common/http';
 import { CpfpInfo, OptimizedMempoolStats, AddressInformation, LiquidPegs, ITranslators,
-  PoolStat, BlockExtended, TransactionStripped, RewardStats, AuditScore, BlockSizesAndWeights, RbfTree } from '../interfaces/node-api.interface';
+  PoolStat, BlockExtended, TransactionStripped, RewardStats, AuditScore, BlockSizesAndWeights, RbfTree, BlockAudit } from '../interfaces/node-api.interface';
 import { Observable } from 'rxjs';
 import { StateService } from './state.service';
 import { WebsocketResponse } from '../interfaces/websocket.interface';
@@ -72,6 +72,10 @@ export class ApiService {
     return this.httpClient.get<OptimizedMempoolStats[]>(this.apiBaseUrl + this.apiBasePath + '/api/v1/statistics/4y');
   }
 
+  listAllTimeStatistics$(): Observable<OptimizedMempoolStats[]> {
+    return this.httpClient.get<OptimizedMempoolStats[]>(this.apiBaseUrl + this.apiBasePath + '/api/v1/statistics/all');
+  }
+
   getTransactionTimes$(txIds: string[]): Observable<number[]> {
     let params = new HttpParams();
     txIds.forEach((txId: string) => {
@@ -245,9 +249,9 @@ export class ApiService {
     );
   }
 
-  getBlockAudit$(hash: string) : Observable<any> {
+  getBlockAudit$(hash: string) : Observable<BlockAudit> {
-    return this.httpClient.get<any>(
+    return this.httpClient.get<BlockAudit>(
-      this.apiBaseUrl + this.apiBasePath + `/api/v1/block/${hash}/audit-summary`, { observe: 'response' }
+      this.apiBaseUrl + this.apiBasePath + `/api/v1/block/${hash}/audit-summary`
     );
   }
 

frontend/src/app/services/state.service.ts

@@ -1,11 +1,11 @@
 import { Inject, Injectable, PLATFORM_ID, LOCALE_ID } from '@angular/core';
-import { ReplaySubject, BehaviorSubject, Subject, fromEvent, Observable } from 'rxjs';
+import { ReplaySubject, BehaviorSubject, Subject, fromEvent, Observable, merge } from 'rxjs';
 import { Transaction } from '../interfaces/electrs.interface';
 import { IBackendInfo, MempoolBlock, MempoolBlockWithTransactions, MempoolBlockDelta, MempoolInfo, Recommendedfees, ReplacedTransaction, TransactionStripped } from '../interfaces/websocket.interface';
 import { BlockExtended, DifficultyAdjustment, MempoolPosition, OptimizedMempoolStats, RbfTree } from '../interfaces/node-api.interface';
 import { Router, NavigationStart } from '@angular/router';
 import { isPlatformBrowser } from '@angular/common';
-import { map, shareReplay } from 'rxjs/operators';
+import { map, scan, shareReplay, tap } from 'rxjs/operators';
 import { StorageService } from './storage.service';
 
 interface MarkBlockState {
@@ -100,6 +100,7 @@ export class StateService {
   mempoolBlocks$ = new ReplaySubject<MempoolBlock[]>(1);
   mempoolBlockTransactions$ = new Subject<TransactionStripped[]>();
   mempoolBlockDelta$ = new Subject<MempoolBlockDelta>();
+  liveMempoolBlockTransactions$: Observable<{ [txid: string]: TransactionStripped}>;
   txReplaced$ = new Subject<ReplacedTransaction>();
   txRbfInfo$ = new Subject<RbfTree>();
   rbfLatest$ = new Subject<RbfTree[]>();
@@ -166,6 +167,30 @@ export class StateService {
 
     this.blocks$ = new ReplaySubject<[BlockExtended, string]>(this.env.KEEP_BLOCKS_AMOUNT);
 
+    this.liveMempoolBlockTransactions$ = merge(
+      this.mempoolBlockTransactions$.pipe(map(transactions => { return { transactions }; })),
+      this.mempoolBlockDelta$.pipe(map(delta => { return { delta }; })),
+    ).pipe(scan((transactions: { [txid: string]: TransactionStripped }, change: any): { [txid: string]: TransactionStripped } => {
+      if (change.transactions) {
+        const txMap = {}
+        change.transactions.forEach(tx => {
+          txMap[tx.txid] = tx;
+        })
+        return txMap;
+      } else {
+        change.delta.changed.forEach(tx => {
+          transactions[tx.txid].rate = tx.rate;
+        })
+        change.delta.removed.forEach(txid => {
+          delete transactions[txid];
+        });
+        change.delta.added.forEach(tx => {
+          transactions[tx.txid] = tx;
+        });
+        return transactions;
+      }
+    }, {}));
+
     if (this.env.BASE_MODULE === 'bisq') {
       this.network = this.env.BASE_MODULE;
       this.networkChanged$.next(this.env.BASE_MODULE);

frontend/src/app/services/websocket.service.ts

@@ -235,6 +235,8 @@ export class WebsocketService {
   }
 
   handleResponse(response: WebsocketResponse) {
+    let reinitBlocks = false;
+
     if (response.blocks && response.blocks.length) {
       const blocks = response.blocks;
       let maxHeight = 0;
@@ -256,9 +258,11 @@ export class WebsocketService {
     }
 
     if (response.block) {
-      if (response.block.height > this.stateService.latestBlockHeight) {
+      if (response.block.height === this.stateService.latestBlockHeight + 1) {
         this.stateService.updateChainTip(response.block.height);
         this.stateService.blocks$.next([response.block, response.txConfirmed || '']);
+      } else if (response.block.height > this.stateService.latestBlockHeight + 1) {
+        reinitBlocks = true;
       }
 
       if (response.txConfirmed) {
@@ -369,5 +373,9 @@ export class WebsocketService {
     if (response['git-commit']) {
       this.stateService.backendInfo$.next(response['git-commit']);
     }
+
+    if (reinitBlocks) {
+      this.websocketSubject.next({'refresh-blocks': true});
+    }
   }
 }

frontend/src/styles.scss

@@ -500,7 +500,7 @@ html:lang(ru) .card-title {
 }
 
 .fee-distribution-chart {
-  height: 250px;
+  height: 265px;
 }
 
 .fees-wrapper-tooltip-chart {

production/nginx-cache-warmer

@@ -21,6 +21,7 @@ do for url in / \
 	'/api/v1/statistics/2y' \
 	'/api/v1/statistics/3y' \
 	'/api/v1/statistics/4y' \
+	'/api/v1/statistics/all' \
 	'/api/v1/mining/pools/24h' \
 	'/api/v1/mining/pools/3d' \
 	'/api/v1/mining/pools/1w' \