Merge branch 'master' into fix_electrum_api

backend/src/api/database-migration.ts

@@ -4,7 +4,7 @@ import logger from '../logger';
 import { Common } from './common';
 
 class DatabaseMigration {
-  private static currentVersion = 23;
+  private static currentVersion = 24;
   private queryTimeout = 120000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -243,6 +243,11 @@ class DatabaseMigration {
         await this.$executeQuery('ALTER TABLE `prices` ADD `AUD` float DEFAULT "0"');
         await this.$executeQuery('ALTER TABLE `prices` ADD `JPY` float DEFAULT "0"');
       }
+
+      if (databaseSchemaVersion < 24 && isBitcoin == true) {
+        await this.$executeQuery('DROP TABLE IF EXISTS `blocks_audits`');
+        await this.$executeQuery(this.getCreateBlocksAuditsTableQuery(), await this.$checkIfTableExists('blocks_audits'));
+      }
     } catch (e) {
       throw e;
     }
@@ -567,6 +572,19 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getCreateBlocksAuditsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS blocks_audits (
+      time timestamp NOT NULL,
+      hash varchar(65) NOT NULL,
+      height int(10) unsigned NOT NULL,
+      missing_txs JSON NOT NULL,
+      added_txs JSON NOT NULL,
+      match_rate float unsigned NOT NULL,
+      PRIMARY KEY (hash),
+      INDEX (height)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
   public async $truncateIndexedData(tables: string[]) {
     const allowedTables = ['blocks', 'hashrates', 'prices'];
 

backend/src/api/mining.ts

@@ -10,11 +10,22 @@ import { escape } from 'mysql2';
 import indexer from '../indexer';
 import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
 import config from '../config';
+import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
 
 class Mining {
   constructor() {
   }
 
+  /**
+   * Get historical block predictions match rate
+   */
+   public async $getBlockPredictionsHistory(interval: string | null = null): Promise<any> {
+    return await BlocksAuditsRepository.$getBlockPredictionsHistory(
+      this.getTimeRange(interval),
+      Common.getSqlInterval(interval)
+    );
+  }
+
   /**
    * Get historical block total fee
    */

backend/src/api/websocket-handler.ts

@@ -16,6 +16,7 @@ import transactionUtils from './transaction-utils';
 import rbfCache from './rbf-cache';
 import difficultyAdjustment from './difficulty-adjustment';
 import feeApi from './fee-api';
+import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
 
 class WebsocketHandler {
   private wss: WebSocket.Server | undefined;
@@ -416,17 +417,40 @@ class WebsocketHandler {
 
     if (_mempoolBlocks[0]) {
       const matches: string[] = [];
+      const added: string[] = [];
+      const missing: string[] = [];
+
       for (const txId of txIds) {
         if (_mempoolBlocks[0].transactionIds.indexOf(txId) > -1) {
           matches.push(txId);
+        } else {
+          added.push(txId);
         }
         delete _memPool[txId];
       }
 
-      matchRate = Math.round((matches.length / (txIds.length - 1)) * 100);
+      for (const txId of _mempoolBlocks[0].transactionIds) {
+        if (matches.includes(txId) || added.includes(txId)) {
+          continue;
+        }
+        missing.push(txId);
+      }
+
+      matchRate = Math.round((Math.max(0, matches.length - missing.length - added.length) / txIds.length * 100) * 100) / 100;
       mempoolBlocks.updateMempoolBlocks(_memPool);
       mBlocks = mempoolBlocks.getMempoolBlocks();
       mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas();
+
+      if (Common.indexingEnabled()) {
+        BlocksAuditsRepository.$saveAudit({
+          time: block.timestamp,
+          height: block.height,
+          hash: block.id,
+          addedTxs: added,
+          missingTxs: missing,
+          matchRate: matchRate,
+        });
+      }
     }
 
     if (block.extras) {

backend/src/index.ts

@@ -28,6 +28,7 @@ import { Common } from './api/common';
 import poolsUpdater from './tasks/pools-updater';
 import indexer from './indexer';
 import priceUpdater from './tasks/price-updater';
+import BlocksAuditsRepository from './repositories/BlocksAuditsRepository';
 
 class Server {
   private wss: WebSocket.Server | undefined;
@@ -292,6 +293,7 @@ class Server {
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fee-rates/:interval', routes.$getHistoricalBlockFeeRates)
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', routes.$getHistoricalBlockSizeAndWeight)
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments/:interval', routes.$getDifficultyAdjustments)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/predictions/:interval', routes.$getHistoricalBlockPrediction)
         ;
     }
 

backend/src/mempool.interfaces.ts

@@ -22,6 +22,15 @@ export interface PoolStats extends PoolInfo {
   emptyBlocks: number;
 }
 
+export interface BlockAudit {
+  time: number,
+  height: number,
+  hash: string,
+  missingTxs: string[],
+  addedTxs: string[],
+  matchRate: number,
+}
+
 export interface MempoolBlock {
   blockSize: number;
   blockVSize: number;

backend/src/repositories/BlocksAuditsRepository.ts

@@ -0,0 +1,51 @@
+import DB from '../database';
+import logger from '../logger';
+import { BlockAudit } from '../mempool.interfaces';
+
+class BlocksAuditRepositories {
+  public async $saveAudit(audit: BlockAudit): Promise<void> {
+    try {
+      await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, match_rate)
+        VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
+          JSON.stringify(audit.addedTxs), audit.matchRate]);
+    } catch (e: any) {
+      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
+        logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`);
+      } else {
+        logger.err(`Cannot save block audit into db. Reason: ` + (e instanceof Error ? e.message : e));
+        throw e;
+      }
+    }
+  }
+
+  public async $getBlockPredictionsHistory(div: number, interval: string | null): Promise<any> {
+    try {
+      let query = `SELECT UNIX_TIMESTAMP(time) as time, height, match_rate FROM blocks_audits`;
+
+      if (interval !== null) {
+        query += ` WHERE time BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+      }
+
+      query += ` GROUP BY UNIX_TIMESTAMP(time) DIV ${div} ORDER BY height`;
+
+      const [rows] = await DB.query(query);
+      return rows;
+    } catch (e: any) {
+      logger.err(`Cannot fetch block prediction history. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getPredictionsCount(): Promise<number> {
+    try {
+      const [rows] = await DB.query(`SELECT count(hash) as count FROM blocks_audits`);
+      return rows[0].count;
+    } catch (e: any) {
+      logger.err(`Cannot fetch block prediction history. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+}
+
+export default new BlocksAuditRepositories();
+

backend/src/routes.ts

@@ -27,6 +27,7 @@ import BlocksRepository from './repositories/BlocksRepository';
 import HashratesRepository from './repositories/HashratesRepository';
 import difficultyAdjustment from './api/difficulty-adjustment';
 import DifficultyAdjustmentsRepository from './repositories/DifficultyAdjustmentsRepository';
+import BlocksAuditsRepository from './repositories/BlocksAuditsRepository';
 
 class Routes {
   constructor() {}
@@ -743,6 +744,20 @@ class Routes {
     }
   }
 
+  public async $getHistoricalBlockPrediction(req: Request, res: Response) {
+    try {
+      const blockPredictions = await mining.$getBlockPredictionsHistory(req.params.interval);
+      const blockCount = await BlocksAuditsRepository.$getPredictionsCount();
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.header('X-total-count', blockCount.toString());
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(blockPredictions.map(prediction => [prediction.time, prediction.height, prediction.match_rate]));
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   public async getBlock(req: Request, res: Response) {
     try {
       const block = await blocks.$getBlock(req.params.hash);