Merge branch 'master' into nymkappa/feature/align-dashboards

backend/src/api/audit.ts

@@ -119,7 +119,8 @@ class Audit {
     }
 
     const numCensored = Object.keys(isCensored).length;
-    const score = matches.length > 0 ? (matches.length / (matches.length + numCensored)) : 0;
+    const numMatches = matches.length - 1; // adjust for coinbase tx
+    const score = numMatches > 0 ? (numMatches / (numMatches + numCensored)) : 0;
 
     return {
       censored: Object.keys(isCensored),

backend/src/api/common.ts

@@ -237,14 +237,21 @@ export class Common {
     ].join('x');
   }
 
-  static utcDateToMysql(date?: number): string {
+  static utcDateToMysql(date?: number | null): string | null {
+    if (date === null) {
+      return null;
+    }
     const d = new Date((date || 0) * 1000);
     return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
   }
 
   static findSocketNetwork(addr: string): {network: string | null, url: string} {
     let network: string | null = null;
-    let url = addr.split('://')[1];
+    let url: string = addr;
+
+    if (config.LIGHTNING.BACKEND === 'cln') {
+      url = addr.split('://')[1];
+    }
 
     if (!url) {
       return {
@@ -261,7 +268,7 @@ export class Common {
       }
     } else if (addr.indexOf('i2p') !== -1) {
       network = 'i2p';
-    } else if (addr.indexOf('ipv4') !== -1) {
+    } else if (addr.indexOf('ipv4') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && isIP(url.split(':')[0]) === 4)) {
       const ipv = isIP(url.split(':')[0]);
       if (ipv === 4) {
         network = 'ipv4';
@@ -271,7 +278,7 @@ export class Common {
           url: addr,
         };
       }
-    } else if (addr.indexOf('ipv6') !== -1) {
+    } else if (addr.indexOf('ipv6') !== -1 || (config.LIGHTNING.BACKEND === 'lnd' && url.indexOf(']:'))) {
       url = url.split('[')[1].split(']')[0];
       const ipv = isIP(url);
       if (ipv === 6) {

backend/src/api/database-migration.ts

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 56;
+  private static currentVersion = 57;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -500,6 +500,11 @@ class DatabaseMigration {
       this.uniqueLog(logger.notice, '`pools` table has been truncated`');
       await this.updateToSchemaVersion(56);
     }
+
+    if (databaseSchemaVersion < 57) {
+      await this.$executeQuery(`ALTER TABLE nodes MODIFY updated_at datetime NULL`);
+      await this.updateToSchemaVersion(57);
+    }
   }
 
   /**
@@ -1012,26 +1017,16 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
-  public async $truncateIndexedData(tables: string[]) {
-    const allowedTables = ['blocks', 'hashrates', 'prices'];
+  public async $blocksReindexingTruncate(): Promise<void> {
+    logger.warn(`Truncating pools, blocks and hashrates for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`);
+    await Common.sleep$(5000);
 
-    try {
-      for (const table of tables) {
-        if (!allowedTables.includes(table)) {
-          logger.debug(`Table ${table} cannot to be re-indexed (not allowed)`);
-          continue;
-        }
-
-        await this.$executeQuery(`TRUNCATE ${table}`, true);
-        if (table === 'hashrates') {
-          await this.$executeQuery('UPDATE state set number = 0 where name = "last_hashrates_indexing"', true);
-        }
-        logger.notice(`Table ${table} has been truncated`);
-      }
-    } catch (e) {
-      logger.warn(`Unable to erase indexed data`);
-    }
-  }
+    await this.$executeQuery(`TRUNCATE blocks`);
+    await this.$executeQuery(`TRUNCATE hashrates`);
+    await this.$executeQuery('DELETE FROM `pools`');
+    await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
+    await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`);
+}
 
   private async $convertCompactCpfpTables(): Promise<void> {
     try {

backend/src/api/explorer/channels.api.ts

@@ -559,6 +559,17 @@ class ChannelsApi {
     const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
     const policy2: Partial<ILightningApi.RoutingPolicy> = channel.node2_policy || {};
 
+    // https://github.com/mempool/mempool/issues/3006
+    if ((channel.last_update ?? 0) < 1514736061) { // January 1st 2018
+      channel.last_update = null;
+    }
+    if ((policy1.last_update ?? 0) < 1514736061) { // January 1st 2018
+      policy1.last_update = null;
+    }
+    if ((policy2.last_update ?? 0) < 1514736061) { // January 1st 2018
+      policy2.last_update = null;
+    }
+
     const query = `INSERT INTO channels
       (
         id,

backend/src/api/explorer/nodes.api.ts

@@ -642,6 +642,11 @@ class NodesApi {
    */
   public async $saveNode(node: ILightningApi.Node): Promise<void> {
     try {
+      // https://github.com/mempool/mempool/issues/3006
+      if ((node.last_update ?? 0) < 1514736061) { // January 1st 2018
+        node.last_update = null;
+      }
+  
       const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? '';
       const query = `INSERT INTO nodes(
           public_key,

backend/src/api/lightning/lightning-api.interface.ts

@@ -21,7 +21,7 @@ export namespace ILightningApi {
   export interface Channel {
     channel_id: string;
     chan_point: string;
-    last_update: number;
+    last_update: number | null;
     node1_pub: string;
     node2_pub: string;
     capacity: string;
@@ -36,11 +36,11 @@ export namespace ILightningApi {
     fee_rate_milli_msat: string;
     disabled: boolean;
     max_htlc_msat: string;
-    last_update: number;
+    last_update: number | null;
   }
 
   export interface Node {
-    last_update: number;
+    last_update: number | null;
     pub_key: string;
     alias: string;
     addresses: {

backend/src/api/pools-parser.ts

@@ -73,7 +73,7 @@ class PoolsParser {
       }
     }
 
-    logger.info('Mining pools.json import completed');
+    logger.info('Mining pools-v2.json import completed');
   }
 
   /**
@@ -115,7 +115,7 @@ class PoolsParser {
       return;
     }
 
-    // Get oldest blocks mined by the pool and assume pools.json updates only concern most recent years
+    // Get oldest blocks mined by the pool and assume pools-v2.json updates only concern most recent years
     // Ignore early days of Bitcoin as there were no mining pool yet
     const [oldestPoolBlock]: any[] = await DB.query(`
       SELECT height