Merge branch 'master' into nymkappa/feature/rename-mining-pool

backend/src/api/audit.ts

@@ -1,5 +1,10 @@
-import logger from '../logger';
-import { BlockExtended, TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
+import config from '../config';
+import bitcoinApi from './bitcoin/bitcoin-api-factory';
+import { Common } from './common';
+import { TransactionExtended, MempoolBlockWithTransactions, AuditScore } from '../mempool.interfaces';
+import blocksRepository from '../repositories/BlocksRepository';
+import blocksAuditsRepository from '../repositories/BlocksAuditsRepository';
+import blocks from '../api/blocks';
 
 const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
 
@@ -44,8 +49,6 @@ class Audit {
 
     displacedWeight += (4000 - transactions[0].weight);
 
-    logger.warn(`${fresh.length} fresh, ${Object.keys(isCensored).length} possibly censored, ${displacedWeight} displaced weight`);
-
     // we can expect an honest miner to include 'displaced' transactions in place of recent arrivals and censored txs
     // these displaced transactions should occupy the first N weight units of the next projected block
     let displacedWeightRemaining = displacedWeight;
@@ -73,6 +76,7 @@ class Audit {
 
     // mark unexpected transactions in the mined block as 'added'
     let overflowWeight = 0;
+    let totalWeight = 0;
     for (const tx of transactions) {
       if (inTemplate[tx.txid]) {
         matches.push(tx.txid);
@@ -82,11 +86,13 @@ class Audit {
         }
         overflowWeight += tx.weight;
       }
+      totalWeight += tx.weight;
     }
 
     // transactions missing from near the end of our template are probably not being censored
-    let overflowWeightRemaining = overflowWeight;
-    let lastOverflowRate = 1.00;
+    let overflowWeightRemaining = overflowWeight - (config.MEMPOOL.BLOCK_WEIGHT_UNITS - totalWeight);
+    let maxOverflowRate = 0;
+    let rateThreshold = 0;
     index = projectedBlocks[0].transactionIds.length - 1;
     while (index >= 0) {
       const txid = projectedBlocks[0].transactionIds[index];
@@ -94,8 +100,11 @@ class Audit {
         if (isCensored[txid]) {
           delete isCensored[txid];
         }
-        lastOverflowRate = mempool[txid].effectiveFeePerVsize;
-      } else if (Math.floor(mempool[txid].effectiveFeePerVsize * 100) <= Math.ceil(lastOverflowRate * 100)) { // tolerance of 0.01 sat/vb
+        if (mempool[txid].effectiveFeePerVsize > maxOverflowRate) {
+          maxOverflowRate = mempool[txid].effectiveFeePerVsize;
+          rateThreshold = (Math.ceil(maxOverflowRate * 100) / 100) + 0.005;
+        }
+      } else if (mempool[txid].effectiveFeePerVsize <= rateThreshold) { // tolerance of 0.01 sat/vb + rounding
         if (isCensored[txid]) {
           delete isCensored[txid];
         }
@@ -113,6 +122,45 @@ class Audit {
       score
     };
   }
+
+  public async $getBlockAuditScores(fromHeight?: number, limit: number = 15): Promise<AuditScore[]> {
+    let currentHeight = fromHeight !== undefined ? fromHeight : await blocksRepository.$mostRecentBlockHeight();
+    const returnScores: AuditScore[] = [];
+
+    if (currentHeight < 0) {
+      return returnScores;
+    }
+
+    for (let i = 0; i < limit && currentHeight >= 0; i++) {
+      const block = blocks.getBlocks().find((b) => b.height === currentHeight);
+      if (block?.extras?.matchRate != null) {
+        returnScores.push({
+          hash: block.id,
+          matchRate: block.extras.matchRate
+        });
+      } else {
+        let currentHash;
+        if (!currentHash && Common.indexingEnabled()) {
+          const dbBlock = await blocksRepository.$getBlockByHeight(currentHeight);
+          if (dbBlock && dbBlock['id']) {
+            currentHash = dbBlock['id'];
+          }
+        }
+        if (!currentHash) {
+          currentHash = await bitcoinApi.$getBlockHash(currentHeight);
+        }
+        if (currentHash) {
+          const auditScore = await blocksAuditsRepository.$getBlockAuditScore(currentHash);
+          returnScores.push({
+            hash: currentHash,
+            matchRate: auditScore?.matchRate
+          });
+        }
+      }
+      currentHeight--;
+    }
+    return returnScores;
+  }
 }
 
 export default new Audit();

backend/src/api/blocks.ts

@@ -130,7 +130,7 @@ class Blocks {
     const stripped = block.tx.map((tx) => {
       return {
         txid: tx.txid,
-        vsize: tx.vsize,
+        vsize: tx.weight / 4,
         fee: tx.fee ? Math.round(tx.fee * 100000000) : 0,
         value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0) * 100000000)
       };
@@ -195,9 +195,9 @@ class Blocks {
         };
       }
 
-      const auditSummary = await BlocksAuditsRepository.$getShortBlockAudit(block.id);
-      if (auditSummary) {
-        blockExtended.extras.matchRate = auditSummary.matchRate;
+      const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
+      if (auditScore != null) {
+        blockExtended.extras.matchRate = auditScore.matchRate;
       }
     }
 

backend/src/api/database-migration.ts

@@ -4,7 +4,7 @@ import logger from '../logger';
 import { Common } from './common';
 
 class DatabaseMigration {
-  private static currentVersion = 41;
+  private static currentVersion = 43;
   private queryTimeout = 120000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -352,6 +352,14 @@ class DatabaseMigration {
     if (databaseSchemaVersion < 41 && isBitcoin === true) {
       await this.$executeQuery('UPDATE channels SET closing_reason = NULL WHERE closing_reason = 1');
     }
+
+    if (databaseSchemaVersion < 42 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `channels` ADD closing_resolved tinyint(1) DEFAULT 0');
+    }
+
+    if (databaseSchemaVersion < 43 && isBitcoin === true) {
+      await this.$executeQuery(this.getCreateLNNodeRecordsTableQuery(), await this.$checkIfTableExists('nodes_records'));
+    }
   }
 
   /**
@@ -787,6 +795,19 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getCreateLNNodeRecordsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS nodes_records (
+      public_key varchar(66) NOT NULL,
+      type int(10) unsigned NOT NULL,
+      payload blob NOT NULL,
+      UNIQUE KEY public_key_type (public_key, type),
+      INDEX (public_key),
+      FOREIGN KEY (public_key)
+        REFERENCES nodes (public_key)
+        ON DELETE CASCADE
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
   public async $truncateIndexedData(tables: string[]) {
     const allowedTables = ['blocks', 'hashrates', 'prices'];
 

backend/src/api/explorer/channels.api.ts

@@ -117,6 +117,17 @@ class ChannelsApi {
     }
   }
 
+  public async $getUnresolvedClosedChannels(): Promise<any[]> {
+    try {
+      const query = `SELECT * FROM channels WHERE status = 2 AND closing_reason = 2 AND closing_resolved = 0 AND closing_transaction_id != ''`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getUnresolvedClosedChannels error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   public async $getChannelsWithoutCreatedDate(): Promise<any[]> {
     try {
       const query = `SELECT * FROM channels WHERE created IS NULL`;

backend/src/api/explorer/nodes.api.ts

@@ -105,6 +105,18 @@ class NodesApi {
         node.closed_channel_count = rows[0].closed_channel_count;
       }
 
+      // Custom records
+      query = `
+        SELECT type, payload
+        FROM nodes_records
+        WHERE public_key = ?
+      `;
+      [rows] = await DB.query(query, [public_key]);
+      node.custom_records = {};
+      for (const record of rows) {
+        node.custom_records[record.type] = Buffer.from(record.payload, 'binary').toString('hex');
+      }
+
       return node;
     } catch (e) {
       logger.err(`Cannot get node information for ${public_key}. Reason: ${(e instanceof Error ? e.message : e)}`);

backend/src/api/lightning/clightning/clightning-convert.ts

@@ -7,6 +7,15 @@ import { Common } from '../../common';
  * Convert a clightning "listnode" entry to a lnd node entry
  */
 export function convertNode(clNode: any): ILightningApi.Node {
+  let custom_records: { [type: number]: string } | undefined = undefined;
+  if (clNode.option_will_fund) {
+    try {
+      custom_records = { '1': Buffer.from(clNode.option_will_fund.compact_lease || '', 'hex').toString('base64') };
+    } catch (e) {
+      logger.err(`Cannot decode option_will_fund compact_lease for ${clNode.nodeid}). Reason: ` + (e instanceof Error ? e.message : e));
+      custom_records = undefined;
+    }
+  }
   return {
     alias: clNode.alias ?? '',
     color: `#${clNode.color ?? ''}`,
@@ -23,6 +32,7 @@ export function convertNode(clNode: any): ILightningApi.Node {
       };
     }) ?? [],
     last_update: clNode?.last_timestamp ?? 0,
+    custom_records
   };
 }
 

backend/src/api/lightning/lightning-api.interface.ts

@@ -49,6 +49,7 @@ export namespace ILightningApi {
     }[];
     color: string;
     features: { [key: number]: Feature };
+    custom_records?: { [type: number]: string };
   }
 
   export interface Info {

backend/src/api/mempool.ts

@@ -103,12 +103,11 @@ class Mempool {
     return txTimes;
   }
 
-  public async $updateMempool() {
-    logger.debug('Updating mempool');
+  public async $updateMempool(): Promise<void> {
+    logger.debug(`Updating mempool...`);
     const start = new Date().getTime();
     let hasChange: boolean = false;
     const currentMempoolSize = Object.keys(this.mempoolCache).length;
-    let txCount = 0;
     const transactions = await bitcoinApi.$getRawMempool();
     const diff = transactions.length - currentMempoolSize;
     const newTransactions: TransactionExtended[] = [];
@@ -124,7 +123,6 @@ class Mempool {
         try {
           const transaction = await transactionUtils.$getTransactionExtended(txid);
           this.mempoolCache[txid] = transaction;
-          txCount++;
           if (this.inSync) {
             this.txPerSecondArray.push(new Date().getTime());
             this.vBytesPerSecondArray.push({
@@ -133,14 +131,9 @@ class Mempool {
             });
           }
           hasChange = true;
-          if (diff > 0) {
-            logger.debug('Fetched transaction ' + txCount + ' / ' + diff);
-          } else {
-            logger.debug('Fetched transaction ' + txCount);
-          }
           newTransactions.push(transaction);
         } catch (e) {
-          logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
+          logger.debug(`Error finding transaction '${txid}' in the mempool: ` + (e instanceof Error ? e.message : e));
         }
       }
 
@@ -197,8 +190,7 @@ class Mempool {
 
     const end = new Date().getTime();
     const time = end - start;
-    logger.debug(`New mempool size: ${Object.keys(this.mempoolCache).length} Change: ${diff}`);
-    logger.debug('Mempool updated in ' + time / 1000 + ' seconds');
+    logger.debug(`Mempool updated in ${time / 1000} seconds. New size: ${Object.keys(this.mempoolCache).length} (${diff > 0 ? '+' + diff : diff})`);
   }
 
   public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended; }) {

backend/src/api/mining/mining-routes.ts

@@ -1,6 +1,7 @@
 import { Application, Request, Response } from 'express';
 import config from "../../config";
 import logger from '../../logger';
+import audits from '../audit';
 import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
 import BlocksRepository from '../../repositories/BlocksRepository';
 import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository';
@@ -26,7 +27,11 @@ class MiningRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', this.$getHistoricalBlockSizeAndWeight)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments/:interval', this.$getDifficultyAdjustments)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/predictions/:interval', this.$getHistoricalBlockPrediction)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/scores', this.$getBlockAuditScores)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/scores/:height', this.$getBlockAuditScores)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/score/:hash', this.$getBlockAuditScore)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/:hash', this.$getBlockAudit)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/timestamp/:timestamp', this.$getHeightFromTimestamp)
     ;
   }
 
@@ -252,6 +257,52 @@ class MiningRoutes {
       res.status(500).send(e instanceof Error ? e.message : e);
     }
   }
+
+  private async $getHeightFromTimestamp(req: Request, res: Response) {
+    try {
+      const timestamp = parseInt(req.params.timestamp, 10);
+      // This will prevent people from entering milliseconds etc.
+      // Block timestamps are allowed to be up to 2 hours off, so 24 hours
+      // will never put the maximum value before the most recent block
+      const nowPlus1day = Math.floor(Date.now() / 1000) + 60 * 60 * 24;
+      // Prevent non-integers that are not seconds
+      if (!/^[1-9][0-9]*$/.test(req.params.timestamp) || timestamp > nowPlus1day) {
+        throw new Error(`Invalid timestamp, value must be Unix seconds`);
+      }
+      const result = await BlocksRepository.$getBlockHeightFromTimestamp(
+        timestamp,
+      );
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      res.json(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getBlockAuditScores(req: Request, res: Response) {
+    try {
+      const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(await audits.$getBlockAuditScores(height, 15));
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  public async $getBlockAuditScore(req: Request, res: Response) {
+    try {
+      const audit = await BlocksAuditsRepository.$getBlockAuditScore(req.params.hash);
+
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());
+      res.json(audit || 'null');
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
 }
 
 export default new MiningRoutes();