Node stats updates

frontend/src/app/components/time-since/time-since.component.ts

@@ -13,6 +13,7 @@ export class TimeSinceComponent implements OnInit, OnChanges, OnDestroy {
   intervals = {};
 
   @Input() time: number;
+  @Input() dateString: number;
   @Input() fastRender = false;
 
   constructor(
@@ -52,7 +53,13 @@ export class TimeSinceComponent implements OnInit, OnChanges, OnDestroy {
   }
 
   calculate() {
-    const seconds = Math.floor((+new Date() - +new Date(this.time * 1000)) / 1000);
+    let date: Date;
+    if (this.dateString) {
+      date = new Date(this.dateString)
+    } else {
+      date = new Date(this.time * 1000);
+    }
+    const seconds = Math.floor((+new Date() - +date) / 1000);
     if (seconds < 60) {
       return $localize`:@@date-base.just-now:Just now`;
     }

frontend/src/app/lightning/lightning-api.service.ts

@@ -32,6 +32,10 @@ export class LightningApiService {
     return this.httpClient.get<any>(API_BASE_URL + '/statistics/latest');
   }
 
+  listNodeStats$(publicKey: string): Observable<any> {
+    return this.httpClient.get<any>(API_BASE_URL + '/nodes/' + publicKey + '/statistics');
+  }
+
   listTopNodes$(): Observable<any> {
     return this.httpClient.get<any>(API_BASE_URL + '/nodes/top');
   }

frontend/src/app/lightning/node/node.component.html

@@ -18,12 +18,16 @@
           <table class="table table-borderless table-striped">
             <tbody>
               <tr>
-                <td i18n="address.total-received">First Seen</td>
+                <td i18n="address.total-received">First seen</td>
-                <td>{{ node.first_seen | date:'yyyy-MM-dd HH:mm' }}</td>
+                <td>
+                  <app-time-since [dateString]="node.first_seen"></app-time-since>
+                </td>
               </tr>
               <tr>
-                <td i18n="address.total-sent">Updated At</td>
+                <td i18n="address.total-sent">Last update</td>
-                <td>{{ node.updated_at | date:'yyyy-MM-dd HH:mm' }}</td>
+                <td>
+                  <app-time-since [dateString]="node.updated_at"></app-time-since>
+                </td>
               </tr>
               <tr>
                 <td i18n="address.balance">Color</td>

frontend/src/app/lightning/node/node.component.ts

@@ -12,6 +12,7 @@ import { LightningApiService } from '../lightning-api.service';
 })
 export class NodeComponent implements OnInit {
   node$: Observable<any>;
+  statistics$: Observable<any>;
   publicKey$: Observable<string>;
 
   constructor(
@@ -26,6 +27,13 @@ export class NodeComponent implements OnInit {
           return this.lightningApiService.getNode$(params.get('public_key'));
         })
       );
+
+    this.statistics$ = this.activatedRoute.paramMap
+      .pipe(
+        switchMap((params: ParamMap) => {
+          return this.lightningApiService.listNodeStats$(params.get('public_key'));
+        })
+      );
   }
 
 }

frontend/src/app/lightning/nodes-list/nodes-list.component.html

@@ -12,10 +12,10 @@
           <a [routerLink]="['/lightning/node' | relativeUrl, node.public_key]">{{ node.alias }}</a>
         </td>
         <td class="capacity text-right">
-          <app-amount [satoshis]="node.capacity_left + node.capacity_right" digitsInfo="1.2-2"></app-amount>
+          <app-amount [satoshis]="node.capacity" digitsInfo="1.2-2"></app-amount>
         </td>
         <td class="channels text-right">
-          {{ node.channels_left + node.channels_right | number }}
+          {{ node.channels | number }}
         </td>
       </tr>
     </tbody>

lightning-backend/src/api/explorer/channels.api.ts

@@ -24,6 +24,17 @@ class ChannelsApi {
     }
   }
 
+  public async $getChannelsWithoutCreatedDate(): Promise<any[]> {
+    try {
+      const query = `SELECT * FROM channels WHERE created IS NULL`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getChannelsWithoutCreatedDate error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   public async $getChannel(shortId: string): Promise<any> {
     try {
       const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.* FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key WHERE channels.id = ?`;

lightning-backend/src/api/explorer/nodes.api.ts

@@ -4,8 +4,8 @@ import DB from '../../database';
 class NodesApi {
   public async $getNode(public_key: string): Promise<any> {
     try {
-      const query = `SELECT * FROM nodes WHERE public_key = ?`;
+      const query = `SELECT nodes.*, (SELECT COUNT(*) FROM channels WHERE channels.status < 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)) AS channel_count, (SELECT SUM(capacity) FROM channels WHERE channels.status < 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)) AS capacity FROM nodes WHERE public_key = ?`;
-      const [rows]: any = await DB.query(query, [public_key]);
+      const [rows]: any = await DB.query(query, [public_key, public_key, public_key, public_key, public_key]);
       return rows[0];
     } catch (e) {
       logger.err('$getNode error: ' + (e instanceof Error ? e.message : e));
@@ -13,9 +13,20 @@ class NodesApi {
     }
   }
 
+  public async $getNodeStats(public_key: string): Promise<any> {
+    try {
+      const query = `SELECT * FROM nodes_stats WHERE public_key = ? ORDER BY added DESC`;
+      const [rows]: any = await DB.query(query, [public_key]);
+      return rows;
+    } catch (e) {
+      logger.err('$getNodeStats error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   public async $getTopCapacityNodes(): Promise<any> {
     try {
-      const query = `SELECT nodes.*, nodes_stats.capacity_left, nodes_stats.capacity_right, nodes_stats.channels_left, nodes_stats.channels_right FROM nodes LEFT JOIN nodes_stats ON nodes_stats.public_key = nodes.public_key ORDER BY nodes_stats.added DESC, nodes_stats.capacity_left + nodes_stats.capacity_right DESC LIMIT 10`;
+      const query = `SELECT nodes.*, nodes_stats.capacity, nodes_stats.channels FROM nodes LEFT JOIN nodes_stats ON nodes_stats.public_key = nodes.public_key ORDER BY nodes_stats.added DESC, nodes_stats.capacity DESC LIMIT 10`;
       const [rows]: any = await DB.query(query);
       return rows;
     } catch (e) {
@@ -26,7 +37,7 @@ class NodesApi {
 
   public async $getTopChannelsNodes(): Promise<any> {
     try {
-      const query = `SELECT nodes.*, nodes_stats.capacity_left, nodes_stats.capacity_right, nodes_stats.channels_left, nodes_stats.channels_right FROM nodes LEFT JOIN nodes_stats ON nodes_stats.public_key = nodes.public_key ORDER BY nodes_stats.added DESC, nodes_stats.channels_left + nodes_stats.channels_right DESC LIMIT 10`;
+      const query = `SELECT nodes.*, nodes_stats.capacity, nodes_stats.channels FROM nodes LEFT JOIN nodes_stats ON nodes_stats.public_key = nodes.public_key ORDER BY nodes_stats.added DESC, nodes_stats.channels DESC LIMIT 10`;
       const [rows]: any = await DB.query(query);
       return rows;
     } catch (e) {

lightning-backend/src/api/explorer/nodes.routes.ts

@@ -8,6 +8,7 @@ class NodesRoutes {
     app
     .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/latest', this.$getGeneralStats)
     .get(config.MEMPOOL.API_URL_PREFIX + 'nodes/top', this.$getTopNodes)
+    .get(config.MEMPOOL.API_URL_PREFIX + 'nodes/:public_key/statistics', this.$getHistoricalNodeStats)
     .get(config.MEMPOOL.API_URL_PREFIX + 'nodes/:public_key', this.$getNode)
   ;
   }
@@ -25,6 +26,15 @@ class NodesRoutes {
     }
   }
 
+  private async $getHistoricalNodeStats(req: Request, res: Response) {
+    try {
+      const statistics = await nodesApi.$getNodeStats(req.params.public_key);
+      res.json(statistics);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async $getGeneralStats(req: Request, res: Response) {
     try {
       const statistics = await nodesApi.$getLatestStatistics();

lightning-backend/src/database-migration.ts

@@ -238,10 +238,8 @@ class DatabaseMigration {
       id int(11) unsigned NOT NULL AUTO_INCREMENT,
       public_key varchar(66) NOT NULL DEFAULT '',
       added date NOT NULL,
-      capacity_left bigint(11) unsigned DEFAULT NULL,
+      capacity bigint(11) unsigned DEFAULT NULL,
-      capacity_right bigint(11) unsigned DEFAULT NULL,
+      channels int(11) unsigned DEFAULT NULL,
-      channels_left int(11) unsigned DEFAULT NULL,
-      channels_right int(11) unsigned DEFAULT NULL,
       PRIMARY KEY (id),
       KEY public_key (public_key)
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;

lightning-backend/src/tasks/node-sync.service.ts

@@ -26,21 +26,71 @@ class NodeSyncService {
       for (const node of networkGraph.nodes) {
         await this.$saveNode(node);
       }
+      logger.debug(`Nodes updated`);
 
       await this.$setChannelsInactive();
 
       for (const channel of networkGraph.channels) {
         await this.$saveChannel(channel);
       }
+      logger.debug(`Channels updated`);
 
       await this.$findInactiveNodesAndChannels();
+      logger.debug(`Inactive channels scan complete`);
+
       await this.$scanForClosedChannels();
+      logger.debug(`Closed channels scan complete`);
+
+      await this.$lookUpCreationDateFromChain();
+      logger.debug(`Channel creation dates scan complete`);
+
+      await this.$updateNodeFirstSeen();
+      logger.debug(`Node first seen dates scan complete`);
 
     } catch (e) {
       logger.err('$updateNodes() error: ' + (e instanceof Error ? e.message : e));
     }
   }
 
+  // This method look up the creation date of the earliest channel of the node
+  // and update the node to that date in order to get the earliest first seen date
+  private async $updateNodeFirstSeen() {
+    try {
+      const [nodes]: any[] = await DB.query(`SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node1_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created1, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node2_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created2 FROM nodes`);
+      for (const node of nodes) {
+        let lowest = 0;
+        if (node.created1) {
+          if (node.created2 && node.created2 < node.created1) {
+            lowest = node.created2;
+          } else {
+            lowest = node.created1;
+          }
+        } else if (node.created2) {
+          lowest = node.created2;
+        }
+        if (lowest && lowest < node.first_seen) {
+          const query = `UPDATE nodes SET first_seen = FROM_UNIXTIME(?) WHERE public_key = ?`;
+          const params = [lowest, node.public_key];
+          await DB.query(query, params);
+        }
+      }
+    } catch (e) {
+      logger.err('$updateNodeFirstSeen() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private async $lookUpCreationDateFromChain() {
+    try {
+      const channels = await channelsApi.$getChannelsWithoutCreatedDate();
+      for (const channel of channels) {
+        const transaction = await bitcoinClient.getRawTransaction(channel.transaction_id, 1);
+        await DB.query(`UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`, [transaction.blocktime, channel.id]);
+      }
+    } catch (e) {
+      logger.err('$setCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
   // Looking for channels whos nodes are inactive
   private async $findInactiveNodesAndChannels(): Promise<void> {
     try {
@@ -190,23 +240,27 @@ class NodeSyncService {
   private async $saveNode(node: ILightningApi.Node): Promise<void> {
     try {
       const updatedAt = this.utcDateToMysql(node.updated_at);
+      const sockets = node.sockets.join(', ');
       const query = `INSERT INTO nodes(
           public_key,
           first_seen,
           updated_at,
           alias,
-          color
+          color,
+          sockets
         )
-        VALUES (?, NOW(), ?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = ?, alias = ?, color = ?;`;
+        VALUES (?, NOW(), ?, ?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = ?, alias = ?, color = ?, sockets = ?;`;
 
       await DB.query(query, [
         node.public_key,
         updatedAt,
         node.alias,
         node.color,
+        sockets,
         updatedAt,
         node.alias,
         node.color,
+        sockets,
       ]);
     } catch (e) {
       logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));

lightning-backend/src/tasks/stats-updater.service.ts

@@ -38,9 +38,9 @@ class LightningStatsUpdater {
 
       for (const node of nodes) {
         await DB.query(
-          `INSERT INTO nodes_stats(public_key, added, capacity_left, capacity_right, channels_left, channels_right) VALUES (?, NOW(), ?, ?, ?, ?)`,
+          `INSERT INTO nodes_stats(public_key, added, capacity, channels) VALUES (?, NOW(), ?, ?)`,
-          [node.public_key, node.channels_capacity_left, node.channels_capacity_right,
+          [node.public_key, (parseInt(node.channels_capacity_left || 0, 10)) + (parseInt(node.channels_capacity_right || 0, 10)),
-            node.channels_count_left, node.channels_count_right]);
+            node.channels_count_left + node.channels_count_right]);
       }
       await DB.query(`UPDATE state SET string = ? WHERE name = 'last_node_stats'`, [currentDate]);
       logger.debug('Daily node stats has updated.');