Merge branch 'master' into nymkappa/health-check

Add missing fake_scripthash to the data filter

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -6,7 +6,7 @@ import websocketHandler from '../websocket-handler';
 import mempool from '../mempool';
 import feeApi from '../fee-api';
 import mempoolBlocks from '../mempool-blocks';
-import bitcoinApi from './bitcoin-api-factory';
+import bitcoinApi, { bitcoinCoreApi } from './bitcoin-api-factory';
 import { Common } from '../common';
 import backendInfo from '../backend-info';
 import transactionUtils from '../transaction-utils';
@@ -21,6 +21,7 @@ import transactionRepository from '../../repositories/TransactionRepository';
 import rbfCache from '../rbf-cache';
 import { calculateMempoolTxCpfp } from '../cpfp';
 import { handleError } from '../../utils/api';
+import BlocksRepository from '../../repositories/BlocksRepository';
 
 class BitcoinRoutes {
   public initRoutes(app: Application) {
@@ -80,8 +81,87 @@ class BitcoinRoutes {
           .get(config.MEMPOOL.API_URL_PREFIX + 'address-prefix/:prefix', this.getAddressPrefix)
           ;
       }
+
+      app.get('/api/internal/health', this.generateHealthReport);
   }
 
+  private async generateHealthReport(req: Request, res: Response): Promise<void> {
+    let response = {
+      core: {
+        height: -1
+      },
+      mempool: {
+        height: -1,
+        indexing: {
+          enabled: Common.indexingEnabled(),
+          blocks: {
+            count: -1,
+            progress: -1,
+            withCpfp: {
+              count: -1,
+              progress: -1,
+            },
+            withCoinStats: {
+              count: -1,
+              progress: -1,
+            }
+          },
+        }
+      },
+    };
+    
+    try {
+      // Bitcoin Core
+      let bitcoinCoreIndexes: number | string;
+      try {
+        bitcoinCoreIndexes = await bitcoinClient.getIndexInfo();
+        for (const indexName in bitcoinCoreIndexes as any) {
+          response.core[indexName.replace(/ /g,'_')] = bitcoinCoreIndexes[indexName];
+        }
+      } catch (e: any) {
+        response.core['error'] = e.message;
+      }
+      try {
+        response.core.height = await bitcoinCoreApi.$getBlockHeightTip();
+      } catch (e: any) {
+        response.core['error'] = e.message;
+      }
+
+      // Mempool
+      response.mempool.height = blocks.getCurrentBlockHeight();
+      if (Common.indexingEnabled()) {
+        const indexingBlockAmount = (config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === -1 ? response.core.height : config.MEMPOOL.INDEXING_BLOCKS_AMOUNT);
+        const computeProgress = (count: number): number => Math.min(1.0, Math.round(count / indexingBlockAmount * 100) / 100);
+        response.mempool.indexing.blocks.count = await BlocksRepository.$getIndexedBlockCount();
+        response.mempool.indexing.blocks.progress = computeProgress(response.mempool.indexing.blocks.count);
+        response.mempool.indexing.blocks.withCpfp.count = await BlocksRepository.$getIndexedCpfpBlockCount();
+        response.mempool.indexing.blocks.withCpfp.progress = computeProgress(response.mempool.indexing.blocks.withCpfp.count);
+        response.mempool.indexing.blocks.withCoinStats.count = await BlocksRepository.$getIndexedCoinStatsBlockCount();
+        response.mempool.indexing.blocks.withCoinStats.progress = computeProgress(response.mempool.indexing.blocks.withCoinStats.count);
+      }
+
+      // Esplora
+      if (config.MEMPOOL.BACKEND === 'esplora') {
+        try {
+          response['esplora'] = {
+            height: await bitcoinApi.$getBlockHeightTip()
+          };
+        } catch (e: any) {
+          response['esplora'] = {
+            height: -1,
+            error: e.message
+          };
+        }
+      }
+
+      res.json(response);
+
+    } catch (e: any) {
+      logger.err(`Unable to generate health report. Exception: ${JSON.stringify(e)}`);
+      logger.err(e.stack);
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
 
   private getInitData(req: Request, res: Response) {
     try {

backend/src/api/blocks.ts

@@ -412,8 +412,16 @@ class Blocks {
     }
 
     try {
+      const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+      const currentBlockHeight = blockchainInfo.blocks;
+      let indexingBlockAmount = Math.min(config.MEMPOOL.INDEXING_BLOCKS_AMOUNT, currentBlockHeight);
+      if (indexingBlockAmount <= -1) {
+        indexingBlockAmount = currentBlockHeight + 1;
+      }
+      const lastBlockToIndex = Math.max(0, currentBlockHeight - indexingBlockAmount + 1);
+
       // Get all indexed block hash
-      const indexedBlocks = await blocksRepository.$getIndexedBlocks();
+      const indexedBlocks = (await blocksRepository.$getIndexedBlocks()).filter(block => block.height >= lastBlockToIndex);
       const indexedBlockSummariesHashesArray = await BlocksSummariesRepository.$getIndexedSummariesId();
 
       const indexedBlockSummariesHashes = {}; // Use a map for faster seek during the indexing loop

backend/src/api/mempool.ts

@@ -10,6 +10,7 @@ import bitcoinClient from './bitcoin/bitcoin-client';
 import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
 import rbfCache from './rbf-cache';
 import { Acceleration } from './services/acceleration';
+import accelerationApi from './services/acceleration';
 import redisCache from './redis-cache';
 import blocks from './blocks';
 
@@ -207,7 +208,7 @@ class Mempool {
     return txTimes;
   }
 
-  public async $updateMempool(transactions: string[], accelerations: Acceleration[] | null, minFeeMempool: string[], minFeeTip: number, pollRate: number): Promise<void> {
+  public async $updateMempool(transactions: string[], accelerations: Record<string, Acceleration> | null, minFeeMempool: string[], minFeeTip: number, pollRate: number): Promise<void> {
     logger.debug(`Updating mempool...`);
 
     // warn if this run stalls the main loop for more than 2 minutes
@@ -354,7 +355,7 @@ class Mempool {
     const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx));
     this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6);
 
-    const accelerationDelta = accelerations != null ? await this.$updateAccelerations(accelerations) : [];
+    const accelerationDelta = accelerations != null ? await this.updateAccelerations(accelerations) : [];
     if (accelerationDelta.length) {
       hasChange = true;
     }
@@ -399,58 +400,11 @@ class Mempool {
     return this.accelerations;
   }
 
-  public $updateAccelerations(newAccelerations: Acceleration[]): string[] {
+  public updateAccelerations(newAccelerationMap: Record<string, Acceleration>): string[] {
     try {
-      const changed: string[] = [];
-
-      const newAccelerationMap: { [txid: string]: Acceleration } = {};
-      for (const acceleration of newAccelerations) {
-        // skip transactions we don't know about
-        if (!this.mempoolCache[acceleration.txid]) {
-          continue;
-        }
-        newAccelerationMap[acceleration.txid] = acceleration;
-        if (this.accelerations[acceleration.txid] == null) {
-          // new acceleration
-          changed.push(acceleration.txid);
-        } else {
-          if (this.accelerations[acceleration.txid].feeDelta !== acceleration.feeDelta) {
-            // feeDelta changed
-            changed.push(acceleration.txid);
-          } else if (this.accelerations[acceleration.txid].pools?.length) {
-            let poolsChanged = false;
-            const pools = new Set();
-            this.accelerations[acceleration.txid].pools.forEach(pool => {
-              pools.add(pool);
-            });
-            acceleration.pools.forEach(pool => {
-              if (!pools.has(pool)) {
-                poolsChanged = true;
-              } else {
-                pools.delete(pool);
-              }
-            });
-            if (pools.size > 0) {
-              poolsChanged = true;
-            }
-            if (poolsChanged) {
-              // pools changed
-              changed.push(acceleration.txid);
-            }
-          }
-        }
-      }
-
-      for (const oldTxid of Object.keys(this.accelerations)) {
-        if (!newAccelerationMap[oldTxid]) {
-          // removed
-          changed.push(oldTxid);
-        }
-      }
-
+      const accelerationDelta = accelerationApi.getAccelerationDelta(this.accelerations, newAccelerationMap);
       this.accelerations = newAccelerationMap;
-
-      return changed;
+      return accelerationDelta;
     } catch (e: any) {
       logger.debug(`Failed to update accelerations: ` + (e instanceof Error ? e.message : e));
       return [];

backend/src/api/mining/mining-routes.ts

@@ -459,7 +459,7 @@ class MiningRoutes {
         handleError(req, res, 400, 'Acceleration data is not available.');
         return;
       }
-      res.status(200).send(accelerationApi.accelerations || []);
+      res.status(200).send(Object.values(accelerationApi.getAccelerations() || {}));
     } catch (e) {
       handleError(req, res, 500, e instanceof Error ? e.message : e);
     }

backend/src/api/mining/mining.ts

@@ -136,9 +136,13 @@ class Mining {
     poolsStatistics['blockCount'] = blockCount;
 
     const totalBlock24h: number = await BlocksRepository.$blockCount(null, '24h');
+    const totalBlock3d: number = await BlocksRepository.$blockCount(null, '3d');
+    const totalBlock1w: number = await BlocksRepository.$blockCount(null, '1w');
 
     try {
       poolsStatistics['lastEstimatedHashrate'] = await bitcoinClient.getNetworkHashPs(totalBlock24h);
+      poolsStatistics['lastEstimatedHashrate3d'] = await bitcoinClient.getNetworkHashPs(totalBlock3d);
+      poolsStatistics['lastEstimatedHashrate1w'] = await bitcoinClient.getNetworkHashPs(totalBlock1w);
     } catch (e) {
       poolsStatistics['lastEstimatedHashrate'] = 0;
       logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate', logger.tags.mining);

backend/src/api/services/acceleration.ts

@@ -1,7 +1,10 @@
+import { WebSocket } from 'ws';
 import config from '../../config';
 import logger from '../../logger';
 import { BlockExtended } from '../../mempool.interfaces';
 import axios from 'axios';
+import mempool from '../mempool';
+import websocketHandler from '../websocket-handler';
 
 type MyAccelerationStatus = 'requested' | 'accelerating' | 'done';
 
@@ -37,14 +40,20 @@ export interface AccelerationHistory {
 };
 
 class AccelerationApi {
+  private ws: WebSocket | null = null;
+  private useWebsocket: boolean = config.MEMPOOL.OFFICIAL && config.MEMPOOL_SERVICES.ACCELERATIONS;
+  private startedWebsocketLoop: boolean = false;
+  private websocketConnected: boolean = false;
   private onDemandPollingEnabled = !config.MEMPOOL_SERVICES.ACCELERATIONS;
   private apiPath = config.MEMPOOL.OFFICIAL ? (config.MEMPOOL_SERVICES.API + '/accelerator/accelerations') : (config.EXTERNAL_DATA_SERVER.MEMPOOL_API + '/accelerations');
-  private _accelerations: Acceleration[] | null = null;
+  private _accelerations: Record<string, Acceleration> = {};
   private lastPoll = 0;
   private forcePoll = false;
   private myAccelerations: Record<string, { status: MyAccelerationStatus, added: number, acceleration?: Acceleration }> = {};
 
-  public get accelerations(): Acceleration[] | null {
+  public constructor() {}
+
+  public getAccelerations(): Record<string, Acceleration> {
     return this._accelerations;
   }
 
@@ -72,11 +81,18 @@ class AccelerationApi {
     }
   }
 
-  public async $updateAccelerations(): Promise<Acceleration[] | null> {
+  public async $updateAccelerations(): Promise<Record<string, Acceleration> | null> {
+    if (this.useWebsocket && this.websocketConnected) {
+      return this._accelerations;
+    }
     if (!this.onDemandPollingEnabled) {
       const accelerations = await this.$fetchAccelerations();
       if (accelerations) {
-        this._accelerations = accelerations;
+        const latestAccelerations = {};
+        for (const acc of accelerations) {
+          latestAccelerations[acc.txid] = acc;
+        }
+        this._accelerations = latestAccelerations;
         return this._accelerations;
       }
     } else {
@@ -85,7 +101,7 @@ class AccelerationApi {
     return null;
   }
 
-  private async $updateAccelerationsOnDemand(): Promise<Acceleration[] | null> {
+  private async $updateAccelerationsOnDemand(): Promise<Record<string, Acceleration> | null> {
     const shouldUpdate = this.forcePoll
       || this.countMyAccelerationsWithStatus('requested') > 0
       || (this.countMyAccelerationsWithStatus('accelerating') > 0 && this.lastPoll < (Date.now() - (10 * 60 * 1000)));
@@ -120,7 +136,11 @@ class AccelerationApi {
       }
     }
 
-    this._accelerations = Object.values(this.myAccelerations).map(({ acceleration }) => acceleration).filter(acc => acc) as Acceleration[];
+    const latestAccelerations = {};
+    for (const acc of Object.values(this.myAccelerations).map(({ acceleration }) => acceleration).filter(acc => acc) as Acceleration[]) {
+      latestAccelerations[acc.txid] = acc;
+    }
+    this._accelerations = latestAccelerations;
     return this._accelerations;
   }
 
@@ -152,6 +172,110 @@ class AccelerationApi {
     }
     return anyAccelerated;
   }
+
+  // get a list of accelerations that have changed between two sets of accelerations
+  public getAccelerationDelta(oldAccelerationMap: Record<string, Acceleration>, newAccelerationMap: Record<string, Acceleration>): string[] {
+    const changed: string[] = [];
+    const mempoolCache = mempool.getMempool();
+
+    for (const acceleration of Object.values(newAccelerationMap)) {
+      // skip transactions we don't know about
+      if (!mempoolCache[acceleration.txid]) {
+        continue;
+      }
+      if (oldAccelerationMap[acceleration.txid] == null) {
+        // new acceleration
+        changed.push(acceleration.txid);
+      } else {
+        if (oldAccelerationMap[acceleration.txid].feeDelta !== acceleration.feeDelta) {
+          // feeDelta changed
+          changed.push(acceleration.txid);
+        } else if (oldAccelerationMap[acceleration.txid].pools?.length) {
+          let poolsChanged = false;
+          const pools = new Set();
+            oldAccelerationMap[acceleration.txid].pools.forEach(pool => {
+            pools.add(pool);
+          });
+          acceleration.pools.forEach(pool => {
+            if (!pools.has(pool)) {
+              poolsChanged = true;
+            } else {
+              pools.delete(pool);
+            }
+          });
+          if (pools.size > 0) {
+            poolsChanged = true;
+          }
+          if (poolsChanged) {
+            // pools changed
+            changed.push(acceleration.txid);
+          }
+        }
+      }
+    }
+
+    for (const oldTxid of Object.keys(oldAccelerationMap)) {
+      if (!newAccelerationMap[oldTxid]) {
+        // removed
+        changed.push(oldTxid);
+      }
+    }
+
+    return changed;
+  }
+
+  private handleWebsocketMessage(msg: any): void {
+    if (msg?.accelerations !== null) {
+      const latestAccelerations = {};
+      for (const acc of msg?.accelerations || []) {
+        latestAccelerations[acc.txid] = acc;
+      }
+      this._accelerations = latestAccelerations;
+      websocketHandler.handleAccelerationsChanged(this._accelerations);
+    }
+  }
+
+  public async connectWebsocket(): Promise<void> {
+    if (this.startedWebsocketLoop) {
+      return;
+    }
+    while (this.useWebsocket) {
+      this.startedWebsocketLoop = true;
+      if (!this.ws) {
+        this.ws = new WebSocket(`${config.MEMPOOL_SERVICES.API.replace('https://', 'ws://').replace('http://', 'ws://')}/accelerator/ws`);
+        this.websocketConnected = true;
+
+        this.ws.on('open', () => {
+          logger.info('Acceleration websocket opened');
+          this.ws?.send(JSON.stringify({
+            'watch-accelerations': true
+          }));
+        });
+
+        this.ws.on('error', (error) => {
+          logger.err('Acceleration websocket error: ' + error);
+          this.ws = null;
+          this.websocketConnected = false;
+        });
+
+        this.ws.on('close', () => {
+          logger.info('Acceleration websocket closed');
+          this.ws = null;
+          this.websocketConnected = false;
+        });
+
+        this.ws.on('message', (data, isBinary) => {
+          try {
+            const parsedMsg = JSON.parse((isBinary ? data : data.toString()) as string);
+            this.handleWebsocketMessage(parsedMsg);
+          } catch (e) {
+            logger.warn('Failed to parse acceleration websocket message: ' + (e instanceof Error ? e.message : e));
+          }
+        });
+      }
+      await new Promise(resolve => setTimeout(resolve, 5000));
+    }
+  }
 }
 
 export default new AccelerationApi();

backend/src/api/websocket-handler.ts

@@ -22,6 +22,7 @@ import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository
 import Audit from './audit';
 import priceUpdater from '../tasks/price-updater';
 import { ApiPrice } from '../repositories/PricesRepository';
+import { Acceleration } from './services/acceleration';
 import accelerationApi from './services/acceleration';
 import mempool from './mempool';
 import statistics from './statistics/statistics';
@@ -60,6 +61,8 @@ class WebsocketHandler {
   private lastRbfSummary: ReplacementInfo[] | null = null;
   private mempoolSequence: number = 0;
 
+  private accelerations: Record<string, Acceleration> = {};
+
   constructor() { }
 
   addWebsocketServer(wss: WebSocket.Server) {
@@ -495,6 +498,42 @@ class WebsocketHandler {
     }
   }
 
+  handleAccelerationsChanged(accelerations: Record<string, Acceleration>): void {
+    if (!this.webSocketServers.length) {
+      throw new Error('No WebSocket.Server has been set');
+    }
+
+    const websocketAccelerationDelta = accelerationApi.getAccelerationDelta(this.accelerations, accelerations);
+    this.accelerations = accelerations;
+
+    if (!websocketAccelerationDelta.length) {
+      return;
+    }
+
+    // pre-compute acceleration delta
+    const accelerationUpdate = {
+      added: websocketAccelerationDelta.map(txid => accelerations[txid]).filter(acc => acc != null),
+      removed: websocketAccelerationDelta.filter(txid => !accelerations[txid]),
+    };
+
+    try {
+      const response = JSON.stringify({
+        accelerations: accelerationUpdate,
+      });
+
+      for (const server of this.webSocketServers) {
+        server.clients.forEach((client) => {
+          if (client.readyState !== WebSocket.OPEN) {
+            return;
+          }
+          client.send(response);
+        });
+      }
+    } catch (e) {
+      logger.debug(`Error sending acceleration update to websocket clients: ${e}`);
+    }
+  }
+
   handleReorg(): void {
     if (!this.webSocketServers.length) {
       throw new Error('No WebSocket.Server have been set');
@@ -571,7 +610,7 @@ class WebsocketHandler {
     const vBytesPerSecond = memPool.getVBytesPerSecond();
     const rbfTransactions = Common.findRbfTransactions(newTransactions, recentlyDeletedTransactions.flat());
     const da = difficultyAdjustment.getDifficultyAdjustment();
-    const accelerations = memPool.getAccelerations();
+    const accelerations = accelerationApi.getAccelerations();
     memPool.handleRbfTransactions(rbfTransactions);
     const rbfChanges = rbfCache.getRbfChanges();
     let rbfReplacements;
@@ -679,10 +718,13 @@ class WebsocketHandler {
     const addressCache = this.makeAddressCache(newTransactions);
     const removedAddressCache = this.makeAddressCache(deletedTransactions);
 
+    const websocketAccelerationDelta = accelerationApi.getAccelerationDelta(this.accelerations, accelerations);
+    this.accelerations = accelerations;
+
     // pre-compute acceleration delta
     const accelerationUpdate = {
-      added: accelerationDelta.map(txid => accelerations[txid]).filter(acc => acc != null),
-      removed: accelerationDelta.filter(txid => !accelerations[txid]),
+      added: websocketAccelerationDelta.map(txid => accelerations[txid]).filter(acc => acc != null),
+      removed: websocketAccelerationDelta.filter(txid => !accelerations[txid]),
     };
 
     // TODO - Fix indentation after PR is merged

backend/src/index.ts

@@ -233,11 +233,11 @@ class Server {
       const newMempool = await bitcoinApi.$getRawMempool();
       const minFeeMempool = memPool.limitGBT ? await bitcoinSecondClient.getRawMemPool() : null;
       const minFeeTip = memPool.limitGBT ? await bitcoinSecondClient.getBlockCount() : -1;
-      const newAccelerations = await accelerationApi.$updateAccelerations();
+      const latestAccelerations = await accelerationApi.$updateAccelerations();
       const numHandledBlocks = await blocks.$updateBlocks();
       const pollRate = config.MEMPOOL.POLL_RATE_MS * (indexer.indexerIsRunning() ? 10 : 1);
       if (numHandledBlocks === 0) {
-        await memPool.$updateMempool(newMempool, newAccelerations, minFeeMempool, minFeeTip, pollRate);
+        await memPool.$updateMempool(newMempool, latestAccelerations, minFeeMempool, minFeeTip, pollRate);
       }
       indexer.$run();
       if (config.WALLETS.ENABLED) {
@@ -318,8 +318,10 @@ class Server {
       priceUpdater.setRatesChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler));
     }
     loadingIndicators.setProgressChangedCallback(websocketHandler.handleLoadingChanged.bind(websocketHandler));
+
+    accelerationApi.connectWebsocket();
   }
-  
+
   setUpHttpApiRoutes(): void {
     bitcoinRoutes.initRoutes(this.app);
     bitcoinCoreRoutes.initRoutes(this.app);

backend/src/repositories/BlocksRepository.ts

@@ -391,7 +391,7 @@ class BlocksRepository {
   /**
    * Get blocks count for a period
    */
-   public async $blockCountBetweenHeight(startHeight: number, endHeight: number): Promise<number> {
+  public async $blockCountBetweenHeight(startHeight: number, endHeight: number): Promise<number> {
     const params: any[] = [];
     let query = `SELECT count(height) as blockCount
       FROM blocks
@@ -501,7 +501,7 @@ class BlocksRepository {
     }
 
     query += ` ORDER BY height DESC
-      LIMIT 10`;
+      LIMIT 100`;
 
     try {
       const [rows]: any[] = await DB.query(query, params);
@@ -729,7 +729,7 @@ class BlocksRepository {
   /**
    * Get the historical averaged block fee rate percentiles
    */
-   public async $getHistoricalBlockFeeRates(div: number, interval: string | null): Promise<any> {
+  public async $getHistoricalBlockFeeRates(div: number, interval: string | null): Promise<any> {
     try {
       let query = `SELECT
         CAST(AVG(height) as INT) as avgHeight,
@@ -760,7 +760,7 @@ class BlocksRepository {
   /**
    * Get the historical averaged block sizes
    */
-   public async $getHistoricalBlockSizes(div: number, interval: string | null): Promise<any> {
+  public async $getHistoricalBlockSizes(div: number, interval: string | null): Promise<any> {
     try {
       let query = `SELECT
         CAST(AVG(height) as INT) as avgHeight,
@@ -785,7 +785,7 @@ class BlocksRepository {
   /**
    * Get the historical averaged block weights
    */
-   public async $getHistoricalBlockWeights(div: number, interval: string | null): Promise<any> {
+  public async $getHistoricalBlockWeights(div: number, interval: string | null): Promise<any> {
     try {
       let query = `SELECT
         CAST(AVG(height) as INT) as avgHeight,
@@ -823,7 +823,7 @@ class BlocksRepository {
   /**
    * Get a list of blocks that have not had CPFP data indexed
    */
-   public async $getCPFPUnindexedBlocks(): Promise<number[]> {
+   public async $getCPFPUnindexedBlocks(): Promise<number[]> { 
     try {
       const blockchainInfo = await bitcoinClient.getBlockchainInfo();
       const currentBlockHeight = blockchainInfo.blocks;
@@ -893,7 +893,7 @@ class BlocksRepository {
   /**
    * Save block price by batch
    */
-   public async $saveBlockPrices(blockPrices: BlockPrice[]): Promise<void> {
+  public async $saveBlockPrices(blockPrices: BlockPrice[]): Promise<void> {
     try {
       let query = `INSERT INTO blocks_prices(height, price_id) VALUES`;
       for (const price of blockPrices) {
@@ -1153,6 +1153,57 @@ class BlocksRepository {
     blk.extras = <BlockExtension>extras;
     return <BlockExtended>blk;
   }
+
+  /**
+   * Count how many blocks are indexed
+   */
+  public async $getIndexedBlockCount(): Promise<number> {
+    try {
+      const [res]: any[] = await DB.query(`SELECT COUNT(hash) as count FROM blocks`);
+      if (!res || !res.length) {
+        logger.err(`Unable to count indexed blocks in our db`);
+        return -1;
+      }
+      return res[0].count;
+    } catch (e) {
+      logger.err(`Unable to count indexed blocks in our db. Exception: ${JSON.stringify(e)}`);
+      return -1;
+    }
+  }
+
+  /**
+   * Count how many blocks are indexed with CPFP data
+   */
+  public async $getIndexedCpfpBlockCount(): Promise<number> {
+    try {
+      const [res]: any[] = await DB.query(`SELECT COUNT(DISTINCT height) as count FROM compact_cpfp_clusters`);
+      if (!res || !res.length) {
+        logger.err(`Unable to count indexed blocks with CPFP data in our db`);
+        return -1;
+      }
+      return res[0].count;
+    } catch (e) {
+      logger.err(`Unable to count indexed blocks with CPFP data in our db. Exception: ${JSON.stringify(e)}`);
+      return -1;
+    }
+  }
+
+  /**
+   * Count how many blocks are indexed with coin stats data
+   */
+  public async $getIndexedCoinStatsBlockCount(): Promise<number> {
+    try {
+      const [res]: any[] = await DB.query(`SELECT COUNT(hash) as count FROM blocks WHERE utxoset_size IS NOT NULL && total_input_amt IS NOT NULL`);
+      if (!res || !res.length) {
+        logger.err(`Unable to count indexed blocks with coin stats data in our db`);
+        return -1;
+      }
+      return res[0].count;
+    } catch (e) {
+      logger.err(`Unable to count indexed blocks with coin stats data in our db. Exception: ${JSON.stringify(e)}`);
+      return -1;
+    }
+  }
 }
 
 export default new BlocksRepository();

frontend/src/app/app.module.server.ts

@@ -3,7 +3,7 @@ import { NgModule } from '@angular/core';
 import { ServerModule } from '@angular/platform-server';
 
 import { ZONE_SERVICE } from '@app/injection-tokens';
-import { AppModule } from '@app/app.module';
+import { AppModule } from './app.module';
 import { AppComponent } from '@components/app/app.component';
 import { HttpCacheInterceptor } from '@app/services/http-cache.interceptor';
 import { ZoneService } from '@app/services/zone.service';

frontend/src/app/app.module.ts

@@ -3,7 +3,7 @@ import { ModuleWithProviders, NgModule } from '@angular/core';
 import { HttpClientModule, HTTP_INTERCEPTORS } from '@angular/common/http';
 import { BrowserAnimationsModule } from '@angular/platform-browser/animations';
 import { ZONE_SERVICE } from '@app/injection-tokens';
-import { AppRoutingModule } from '@app/app-routing.module';
+import { AppRoutingModule } from './app-routing.module';
 import { AppComponent } from '@components/app/app.component';
 import { ElectrsApiService } from '@app/services/electrs-api.service';
 import { OrdApiService } from '@app/services/ord-api.service';

frontend/src/app/components/asset/asset.component.ts

@@ -9,7 +9,7 @@ import { AudioService } from '@app/services/audio.service';
 import { ApiService } from '@app/services/api.service';
 import { of, merge, Subscription, combineLatest } from 'rxjs';
 import { SeoService } from '@app/services/seo.service';
-import { environment } from '@app/../environments/environment';
+import { environment } from '@environments/environment';
 import { AssetsService } from '@app/services/assets.service';
 import { moveDec } from '@app/bitcoin.utils';
 

frontend/src/app/components/custom-dashboard/custom-dashboard.component.ts

@@ -73,7 +73,7 @@ export class CustomDashboardComponent implements OnInit, OnDestroy, AfterViewIni
     { index: 0, name: $localize`:@@dfc3c34e182ea73c5d784ff7c8135f087992dac1:All`, mode: 'and', filters: [], gradient: 'age' },
     { index: 1, name: $localize`Consolidation`, mode: 'and', filters: ['consolidation'], gradient: 'fee' },
     { index: 2, name: $localize`Coinjoin`, mode: 'and', filters: ['coinjoin'], gradient: 'fee' },
-    { index: 3, name: $localize`Data`, mode: 'or', filters: ['inscription', 'fake_pubkey', 'op_return'], gradient: 'fee' },
+    { index: 3, name: $localize`Data`, mode: 'or', filters: ['inscription', 'fake_pubkey', 'fake_scripthash', 'op_return'], gradient: 'fee' },
   ];
   goggleFlags = 0n;
   goggleMode: FilterMode = 'and';

frontend/src/app/components/loading-indicator/loading-indicator.component.scss

@@ -1,7 +1,7 @@
 .sticky-loading {
   position: absolute;
   right: 10px;
-  z-index: 99;
+  z-index: 1000;
   font-size: 14px;
   @media (width >= 992px) {
     left: 32px;

frontend/src/app/components/pool-ranking/pool-ranking.component.html

@@ -90,9 +90,9 @@
           <th class="d-none d-md-table-cell" i18n="mining.rank">Rank</th>
           <th class=""></th>
           <th class="" i18n="mining.pool-name">Pool</th>
-          <th class="" *ngIf="this.miningWindowPreference === '24h'" i18n="mining.hashrate">Hashrate</th>
+          <th class="" *ngIf="['24h', '3d', '1w'].includes(this.miningWindowPreference)" i18n="mining.hashrate">Hashrate</th>
           <th class="" i18n="master-page.blocks">Blocks</th>
-          <th *ngIf="auditAvailable" class="health text-right widget" [ngClass]="{'health-column': this.miningWindowPreference === '24h'}" i18n="latest-blocks.avg_health"
+          <th *ngIf="auditAvailable" class="health text-right widget" [ngClass]="{'health-column': ['24h', '3d', '1w'].includes(this.miningWindowPreference)}" i18n="latest-blocks.avg_health"
             i18n-ngbTooltip="latest-blocks.avg_health" ngbTooltip="Avg Health" placement="bottom" #health [disableTooltip]="!isEllipsisActive(health)">Avg Health</th>
           <th *ngIf="auditAvailable" class="d-none d-sm-table-cell" i18n="mining.fees-per-block">Avg Block Fees</th>
           <th class="d-none d-lg-table-cell" i18n="mining.empty-blocks">Empty Blocks</th>
@@ -105,12 +105,13 @@
             <img width="25" height="25" src="{{ pool.logo }}" [alt]="pool.name + ' mining pool logo'" onError="this.onerror=null; this.src = '/resources/mining-pools/default.svg'">
           </td>
           <td class="pool-name"><a [routerLink]="[('/mining/pool/' + pool.slug) | relativeUrl]">{{ pool.name }}</a></td>
-          <td class="" *ngIf="this.miningWindowPreference === '24h'">{{ pool.lastEstimatedHashrate | number: '1.2-2' }} {{
-            miningStats.miningUnits.hashrateUnit }}</td>
+          <td class="" *ngIf="'24h' === this.miningWindowPreference">{{ pool.lastEstimatedHashrate | number: '1.2-2' }} {{ miningStats.miningUnits.hashrateUnit }}</td>
+          <td class="" *ngIf="'3d' === this.miningWindowPreference">{{ pool.lastEstimatedHashrate3d | number: '1.2-2' }} {{ miningStats.miningUnits.hashrateUnit }}</td>
+          <td class="" *ngIf="'1w' === this.miningWindowPreference">{{ pool.lastEstimatedHashrate1w | number: '1.2-2' }} {{ miningStats.miningUnits.hashrateUnit }}</td>
           <td class="d-flex justify-content-center">
             {{ pool.blockCount }}<span class="d-none d-md-table-cell">&nbsp;({{ pool.share }}%)</span>
           </td>
-          <td *ngIf="auditAvailable" class="health text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable, 'health-column': this.miningWindowPreference === '24h'}">
+          <td *ngIf="auditAvailable" class="health text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable, 'health-column': ['24h', '3d', '1w'].includes(this.miningWindowPreference)}">
             <a
               class="health-badge badge"
               [class.badge-success]="pool.avgMatchRate >= 99"
@@ -136,8 +137,9 @@
           <td class="d-none d-md-table-cell"></td>
           <td class="text-right"></td>
           <td class=""><b i18n="mining.all-miners">All miners</b></td>
-          <td class="" *ngIf="this.miningWindowPreference === '24h'"><b>{{ miningStats.lastEstimatedHashrate | number: '1.2-2' }} {{
-              miningStats.miningUnits.hashrateUnit }}</b></td>
+          <td class="" *ngIf="'24h' === this.miningWindowPreference">{{ miningStats.lastEstimatedHashrate| number: '1.2-2' }} {{ miningStats.miningUnits.hashrateUnit }}</td>
+          <td class="" *ngIf="'3d' === this.miningWindowPreference">{{ miningStats.lastEstimatedHashrate3d | number: '1.2-2' }} {{ miningStats.miningUnits.hashrateUnit }}</td>
+          <td class="" *ngIf="'1w' === this.miningWindowPreference">{{ miningStats.lastEstimatedHashrate1w | number: '1.2-2' }} {{ miningStats.miningUnits.hashrateUnit }}</td>
           <td class=""><b>{{ miningStats.blockCount }}</b></td>
           <td *ngIf="auditAvailable"></td>
           <td *ngIf="auditAvailable"></td>

frontend/src/app/components/pool-ranking/pool-ranking.component.ts

@@ -161,9 +161,12 @@ export class PoolRankingComponent implements OnInit {
           borderColor: '#000',
           formatter: () => {
             const i = pool.blockCount.toString();
-            if (this.miningWindowPreference === '24h') {
+            if (['24h', '3d', '1w'].includes(this.miningWindowPreference)) {
+              let hashrate = pool.lastEstimatedHashrate;
+              if ('3d' === this.miningWindowPreference) { hashrate = pool.lastEstimatedHashrate3d; }
+              if ('1w' === this.miningWindowPreference) { hashrate = pool.lastEstimatedHashrate1w; }
               return `<b style="color: white">${pool.name} (${pool.share}%)</b><br>` +
-                pool.lastEstimatedHashrate.toFixed(2) + ' ' + miningStats.miningUnits.hashrateUnit +
+                hashrate.toFixed(2) + ' ' + miningStats.miningUnits.hashrateUnit +
                 `<br>` + $localize`${ i }:INTERPOLATION: blocks`;
             } else {
               return `<b style="color: white">${pool.name} (${pool.share}%)</b><br>` +
@@ -200,13 +203,10 @@ export class PoolRankingComponent implements OnInit {
         borderColor: '#000',
         formatter: () => {
           const i = totalBlockOther.toString();
-          if (this.miningWindowPreference === '24h') {
-            return `<b style="color: white">` + $localize`Other (${percentage})` + `</b><br>` +
-              totalEstimatedHashrateOther.toString() + ' ' + miningStats.miningUnits.hashrateUnit +
-              `<br>` + $localize`${ i }:INTERPOLATION: blocks`;
+          if (['24h', '3d', '1w'].includes(this.miningWindowPreference)) {
+            return `<b style="color: white">` + $localize`Other (${percentage})` + `</b><br>` + totalEstimatedHashrateOther.toFixed(2) + ' ' + miningStats.miningUnits.hashrateUnit + `<br>` + $localize`${ i }:INTERPOLATION: blocks`;
           } else {
-            return `<b style="color: white">` + $localize`Other (${percentage})` + `</b><br>` +
-              $localize`${ i }:INTERPOLATION: blocks`;
+            return `<b style="color: white">` + $localize`Other (${percentage})` + `</b><br>` + $localize`${ i }:INTERPOLATION: blocks`;
           }
         }
       },
@@ -292,6 +292,8 @@ export class PoolRankingComponent implements OnInit {
   getEmptyMiningStat(): MiningStats {
     return {
       lastEstimatedHashrate: 0,
+      lastEstimatedHashrate3d: 0,
+      lastEstimatedHashrate1w: 0,
       blockCount: 0,
       totalEmptyBlock: 0,
       totalEmptyBlockRatio: '',

frontend/src/app/components/server-health/server-health.component.html

@@ -9,7 +9,7 @@
 
   <ng-container *ngIf="(hosts$ | async) as hosts">
     <div class="status-panel">
-      <table class="status-table table table-borderless table-striped" *ngIf="(tip$ | async) as tip">
+      <table class="status-table table table-borderless table-striped">
         <tbody>
           <tr>
             <th class="rank"></th>
@@ -27,7 +27,7 @@
             <td class="updated">{{ getLastUpdateSeconds(host) }}</td>
             <td class="rtt only-small">{{ (host.rtt / 1000) | number : '1.1-1' }} {{ host.rtt == null ? '' : 's'}} {{ !host.checked ? '⏳' : (host.unreachable ? '🔥' : '✅') }}</td>
             <td class="rtt only-large">{{ host.rtt | number : '1.0-0' }} {{ host.rtt == null ? '' : 'ms'}} {{ !host.checked ? '⏳' : (host.unreachable ? '🔥' : '✅') }}</td>
-            <td class="height">{{ host.latestHeight }} {{ !host.checked ? '⏳' : (host.outOfSync ? '🚫' : (host.latestHeight && host.latestHeight < tip ? '🟧' : '✅')) }}</td>
+            <td class="height">{{ host.latestHeight }} {{ !host.checked ? '⏳' : (host.outOfSync ? '🚫' : (host.latestHeight && host.latestHeight < maxHeight ? '🟧' : '✅')) }}</td>
           </tr>
         </tbody>
       </table>

frontend/src/app/components/server-health/server-health.component.ts

@@ -1,6 +1,6 @@
 import { Component, OnInit, ChangeDetectionStrategy, SecurityContext, ChangeDetectorRef } from '@angular/core';
 import { WebsocketService } from '@app/services/websocket.service';
-import { Observable, Subject, map } from 'rxjs';
+import { Observable, Subject, map, tap } from 'rxjs';
 import { StateService } from '@app/services/state.service';
 import { HealthCheckHost } from '@interfaces/websocket.interface';
 import { DomSanitizer } from '@angular/platform-browser';
@@ -13,7 +13,7 @@ import { DomSanitizer } from '@angular/platform-browser';
 })
 export class ServerHealthComponent implements OnInit {
   hosts$: Observable<HealthCheckHost[]>;
-  tip$: Subject<number>;
+  maxHeight: number;
   interval: number;
   now: number = Date.now();
 
@@ -44,9 +44,14 @@ export class ServerHealthComponent implements OnInit {
           host.flag = this.parseFlag(host.host);
         }
         return hosts;
+      }),
+      tap(hosts => {
+        let newMaxHeight = 0;
+        for (const host of hosts) {
+          newMaxHeight = Math.max(newMaxHeight, host.latestHeight);
+        }
       })
     );
-    this.tip$ = this.stateService.chainTip$;
     this.websocketService.want(['mempool-blocks', 'stats', 'blocks', 'tomahawk']);
 
     this.interval = window.setInterval(() => {

frontend/src/app/dashboard/dashboard.component.ts

@@ -78,7 +78,7 @@ export class DashboardComponent implements OnInit, OnDestroy, AfterViewInit {
     { index: 0, name: $localize`:@@dfc3c34e182ea73c5d784ff7c8135f087992dac1:All`, mode: 'and', filters: [], gradient: 'age' },
     { index: 1, name: $localize`Consolidation`, mode: 'and', filters: ['consolidation'], gradient: 'fee' },
     { index: 2, name: $localize`Coinjoin`, mode: 'and', filters: ['coinjoin'], gradient: 'fee' },
-    { index: 3, name: $localize`Data`, mode: 'or', filters: ['inscription', 'fake_pubkey', 'op_return'], gradient: 'fee' },
+    { index: 3, name: $localize`Data`, mode: 'or', filters: ['inscription', 'fake_pubkey', 'fake_scripthash', 'op_return'], gradient: 'fee' },
   ];
   goggleFlags = 0n;
   goggleMode: FilterMode = 'and';

frontend/src/app/interfaces/node-api.interface.ts

@@ -143,6 +143,8 @@ export interface SinglePoolStats {
   rank: number;
   share: number;
   lastEstimatedHashrate: number;
+  lastEstimatedHashrate3d: number;
+  lastEstimatedHashrate1w: number;
   emptyBlockRatio: string;
   logo: string;
   slug: string;
@@ -152,6 +154,8 @@ export interface SinglePoolStats {
 export interface PoolsStats {
   blockCount: number;
   lastEstimatedHashrate: number;
+  lastEstimatedHashrate3d: number;
+  lastEstimatedHashrate1w: number;
   pools: SinglePoolStats[];
 }
 

frontend/src/app/previews.module.ts

@@ -3,7 +3,7 @@ import { CommonModule } from '@angular/common';
 import { SharedModule } from '@app/shared/shared.module';
 import { RouterModule } from '@angular/router';
 import { GraphsModule } from '@app/graphs/graphs.module';
-import { PreviewsRoutingModule } from '@app/previews.routing.module';
+import { PreviewsRoutingModule } from './previews.routing.module';
 import { TransactionPreviewComponent } from '@components/transaction/transaction-preview.component';
 import { BlockPreviewComponent } from '@components/block/block-preview.component';
 import { AddressPreviewComponent } from '@components/address/address-preview.component';

frontend/src/app/services/mining.service.ts

@@ -13,6 +13,8 @@ export interface MiningUnits {
 
 export interface MiningStats {
   lastEstimatedHashrate: number;
+  lastEstimatedHashrate3d: number;
+  lastEstimatedHashrate1w: number;
   blockCount: number;
   totalEmptyBlock: number;
   totalEmptyBlockRatio: string;
@@ -129,6 +131,8 @@ export class MiningService {
       return {
         share: parseFloat((poolStat.blockCount / stats.blockCount * 100).toFixed(2)),
         lastEstimatedHashrate: poolStat.blockCount / stats.blockCount * stats.lastEstimatedHashrate / hashrateDivider,
+        lastEstimatedHashrate3d: poolStat.blockCount / stats.blockCount * stats.lastEstimatedHashrate3d / hashrateDivider,
+        lastEstimatedHashrate1w: poolStat.blockCount / stats.blockCount * stats.lastEstimatedHashrate1w / hashrateDivider,
         emptyBlockRatio: (poolStat.emptyBlocks / poolStat.blockCount * 100).toFixed(2),
         logo: `/resources/mining-pools/` + poolStat.slug + '.svg',
         ...poolStat
@@ -137,6 +141,8 @@ export class MiningService {
 
     return {
       lastEstimatedHashrate: stats.lastEstimatedHashrate / hashrateDivider,
+      lastEstimatedHashrate3d: stats.lastEstimatedHashrate3d / hashrateDivider,
+      lastEstimatedHashrate1w: stats.lastEstimatedHashrate1w / hashrateDivider,
       blockCount: stats.blockCount,
       totalEmptyBlock: totalEmptyBlock,
       totalEmptyBlockRatio: totalEmptyBlockRatio,