Store and display stats and node top lists

lightning-backend/src/api/lightning/lightning-api-factory.ts

@@ -1,4 +1,4 @@
-import config from '../config';
+import config from '../../config';
 import { AbstractLightningApi } from './lightning-api-abstract-factory';
 import LndApi from './lnd/lnd-api';
 

lightning-backend/src/api/lightning/lnd/lnd-api.ts

@@ -2,8 +2,8 @@ import { AbstractLightningApi } from '../lightning-api-abstract-factory';
 import { ILightningApi } from '../lightning-api.interface';
 import * as fs from 'fs';
 import * as lnService from 'ln-service';
-import config from '../../config';
-import logger from '../../logger';
+import config from '../../../config';
+import logger from '../../../logger';
 
 class LndApi implements AbstractLightningApi {
   private lnd: any;

lightning-backend/src/api/nodes/nodes.api.ts

@@ -0,0 +1,42 @@
+import logger from '../../logger';
+import DB from '../../database';
+
+class NodesApi {
+  public async $getTopCapacityNodes(): Promise<any> {
+    try {
+      const query = `SELECT nodes.*, nodes_stats.capacity_left, nodes_stats.capacity_right, nodes_stats.channels_left, nodes_stats.channels_right FROM nodes LEFT JOIN nodes_stats ON nodes_stats.public_key = nodes.public_key ORDER BY nodes_stats.capacity_left + nodes_stats.capacity_right DESC LIMIT 10`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getTopCapacityNodes error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getTopChannelsNodes(): Promise<any> {
+    try {
+      const query = `SELECT nodes.*, nodes_stats.capacity_left, nodes_stats.capacity_right, nodes_stats.channels_left, nodes_stats.channels_right FROM nodes LEFT JOIN nodes_stats ON nodes_stats.public_key = nodes.public_key ORDER BY nodes_stats.channels_left + nodes_stats.channels_right DESC LIMIT 10`;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getTopChannelsNodes error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getLatestStatistics(): Promise<any> {
+    try {
+      const [rows]: any = await DB.query(`SELECT * FROM statistics ORDER BY id DESC LIMIT 1`);
+      const [rows2]: any = await DB.query(`SELECT * FROM statistics ORDER BY id DESC LIMIT 1 OFFSET 24`);
+      return {
+        latest: rows[0],
+        previous: rows2[0],
+      };
+    } catch (e) {
+      logger.err('$getTopChannelsNodes error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+}
+
+export default new NodesApi();

lightning-backend/src/api/nodes/nodes.routes.ts

@@ -0,0 +1,35 @@
+import config from '../../config';
+import { Express, Request, Response } from 'express';
+import nodesApi from './nodes.api';
+class NodesRoutes {
+  constructor(app: Express) {
+    app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/latest', this.$getGeneralStats)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'nodes/top', this.$getTopNodes)
+    ;
+  }
+
+  private async $getGeneralStats(req: Request, res: Response) {
+    try {
+      const statistics = await nodesApi.$getLatestStatistics();
+      res.json(statistics);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  private async $getTopNodes(req: Request, res: Response) {
+    try {
+      const topCapacityNodes = await nodesApi.$getTopCapacityNodes();
+      const topChannelsNodes = await nodesApi.$getTopChannelsNodes();
+      res.json({
+        topByCapacity: topCapacityNodes,
+        topByChannels: topChannelsNodes,
+      });
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+}
+
+export default NodesRoutes;

lightning-backend/src/config.ts

@@ -5,6 +5,7 @@ interface IConfig {
     NETWORK: 'mainnet' | 'testnet' | 'signet';
     BACKEND: 'lnd' | 'cln' | 'ldk';
     HTTP_PORT: number;
+    API_URL_PREFIX: string;
     STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
   };
   SYSLOG: {
@@ -33,6 +34,7 @@ const defaults: IConfig = {
     'NETWORK': 'mainnet',
     'BACKEND': 'lnd',
     'HTTP_PORT': 8999,
+    'API_URL_PREFIX': '/api/v1/',
     'STDOUT_LOG_MIN_PRIORITY': 'debug',
   },
   'SYSLOG': {

lightning-backend/src/database-migration.ts

@@ -76,6 +76,7 @@ class DatabaseMigration {
       await this.$executeQuery(this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
       await this.$executeQuery(this.getCreateNodesQuery(), await this.$checkIfTableExists('nodes'));
       await this.$executeQuery(this.getCreateChannelsQuery(), await this.$checkIfTableExists('channels'));
+      await this.$executeQuery(this.getCreateNodesStatsQuery(), await this.$checkIfTableExists('nodes_stats'));
     } catch (e) {
       throw e;
     }
@@ -204,28 +205,45 @@ class DatabaseMigration {
   private getCreateChannelsQuery(): string {
     return `CREATE TABLE IF NOT EXISTS channels (
       id varchar(15) NOT NULL,
-      capacity double unsigned NOT NULL,
+      capacity bigint(20) unsigned NOT NULL,
       transaction_id varchar(64) NOT NULL,
       transaction_vout int(11) NOT NULL,
-      updated_at datetime NOT NULL,
+      updated_at datetime DEFAULT NULL,
       node1_public_key varchar(66) NOT NULL,
-      node1_base_fee_mtokens double unsigned NULL,
-      node1_cltv_delta int(11) NULL,
-      node1_fee_rate int(11) NULL,
-      node1_is_disabled boolean NULL,
-      node1_max_htlc_mtokens double unsigned NULL,
-      node1_min_htlc_mtokens double unsigned NULL,
-      node1_updated_at datetime NULL,
+      node1_base_fee_mtokens bigint(20) unsigned DEFAULT NULL,
+      node1_cltv_delta int(11) DEFAULT NULL,
+      node1_fee_rate bigint(11) DEFAULT NULL,
+      node1_is_disabled tinyint(1) DEFAULT NULL,
+      node1_max_htlc_mtokens bigint(20) unsigned DEFAULT NULL,
+      node1_min_htlc_mtokens bigint(20) unsigned DEFAULT NULL,
+      node1_updated_at datetime DEFAULT NULL,
       node2_public_key varchar(66) NOT NULL,
-      node2_base_fee_mtokens double unsigned NULL,
-      node2_cltv_delta int(11) NULL,
-      node2_fee_rate int(11) NULL,
-      node2_is_disabled boolean NULL,
-      node2_max_htlc_mtokens double unsigned NULL,
-      node2_min_htlc_mtokens double unsigned NULL,
-      node2_updated_at datetime NULL,
-      CONSTRAINT PRIMARY KEY (id)
-      ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+      node2_base_fee_mtokens bigint(20) unsigned DEFAULT NULL,
+      node2_cltv_delta int(11) DEFAULT NULL,
+      node2_fee_rate bigint(11) DEFAULT NULL,
+      node2_is_disabled tinyint(1) DEFAULT NULL,
+      node2_max_htlc_mtokens bigint(20) unsigned DEFAULT NULL,
+      node2_min_htlc_mtokens bigint(20) unsigned DEFAULT NULL,
+      node2_updated_at datetime DEFAULT NULL,
+      PRIMARY KEY (id),
+      KEY node1_public_key (node1_public_key),
+      KEY node2_public_key (node2_public_key),
+      KEY node1_public_key_2 (node1_public_key,node2_public_key)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
+  private getCreateNodesStatsQuery(): string {
+    return `CREATE TABLE nodes_stats (
+      id int(11) unsigned NOT NULL AUTO_INCREMENT,
+      public_key varchar(66) NOT NULL DEFAULT '',
+      added date NOT NULL,
+      capacity_left bigint(11) unsigned DEFAULT NULL,
+      capacity_right bigint(11) unsigned DEFAULT NULL,
+      channels_left int(11) unsigned DEFAULT NULL,
+      channels_right int(11) unsigned DEFAULT NULL,
+      PRIMARY KEY (id),
+      KEY public_key (public_key)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 }
 

lightning-backend/src/index.ts

@@ -1,13 +1,20 @@
 import config from './config';
+import * as express from 'express';
+import * as http from 'http';
 import logger from './logger';
 import DB from './database';
+import { Express, Request, Response, NextFunction } from 'express';
 import databaseMigration from './database-migration';
 import statsUpdater from './tasks/stats-updater.service';
 import nodeSyncService from './tasks/node-sync.service';
+import NodesRoutes from './api/nodes/nodes.routes';
 
 logger.notice(`Mempool Server is running on port ${config.MEMPOOL.HTTP_PORT}`);
 
 class LightningServer {
+  private server: http.Server | undefined;
+  private app: Express = express();
+
   constructor() {
     this.init();
   }
@@ -18,6 +25,27 @@ class LightningServer {
 
     statsUpdater.startService();
     nodeSyncService.startService();
+
+    this.startServer();
+  }
+
+  startServer() {
+    this.app
+      .use((req: Request, res: Response, next: NextFunction) => {
+        res.setHeader('Access-Control-Allow-Origin', '*');
+        next();
+      })
+      .use(express.urlencoded({ extended: true }))
+      .use(express.text())
+    ;
+
+    this.server = http.createServer(this.app);
+
+    this.server.listen(config.MEMPOOL.HTTP_PORT, () => {
+      logger.notice(`Mempool Lightning is running on port ${config.MEMPOOL.HTTP_PORT}`);
+    });
+
+    const nodeRoutes = new NodesRoutes(this.app);
   }
 }
 

lightning-backend/src/tasks/node-sync.service.ts

@@ -1,8 +1,8 @@
 
 import DB from '../database';
 import logger from '../logger';
-import lightningApi from '../api/lightning-api-factory';
-import { ILightningApi } from '../api/lightning-api.interface';
+import lightningApi from '../api/lightning/lightning-api-factory';
+import { ILightningApi } from '../api/lightning/lightning-api.interface';
 
 class NodeSyncService {
   constructor() {}

lightning-backend/src/tasks/stats-updater.service.ts

@@ -1,7 +1,7 @@
 
 import DB from '../database';
 import logger from '../logger';
-import lightningApi from '../api/lightning-api-factory';
+import lightningApi from '../api/lightning/lightning-api-factory';
 
 class LightningStatsUpdater {
   constructor() {}
@@ -19,12 +19,25 @@ class LightningStatsUpdater {
         this.$logLightningStats();
       }, 1000 * 60 * 60);
     }, difference);
+
+    // this.$logNodeStatsDaily();
+  }
+
+  private async $logNodeStatsDaily() {
+    const query = `SELECT nodes.public_key, COUNT(DISTINCT c1.id) AS channels_count_left, COUNT(DISTINCT c2.id) AS channels_count_right, SUM(DISTINCT c1.capacity) AS channels_capacity_left, SUM(DISTINCT c2.capacity) AS channels_capacity_right FROM nodes LEFT JOIN channels AS c1 ON c1.node1_public_key = nodes.public_key LEFT JOIN channels AS c2 ON c2.node2_public_key = nodes.public_key GROUP BY nodes.public_key`;
+    const [nodes]: any = await DB.query(query);
+
+    for (const node of nodes) {
+      await DB.query(
+        `INSERT INTO nodes_stats(public_key, added, capacity_left, capacity_right, channels_left, channels_right) VALUES (?, NOW(), ?, ?, ?, ?)`,
+        [node.public_key, node.channels_capacity_left, node.channels_capacity_right, node.channels_count_left, node.channels_count_right]);
+    }
   }
 
   private async $logLightningStats() {
-    const networkInfo = await lightningApi.$getNetworkInfo();
-
     try {
+      const networkInfo = await lightningApi.$getNetworkInfo();
+
       const query = `INSERT INTO statistics(
           added,
           channel_count,