Integrate LN stats importer into the main process

backend/src/index.ts

@@ -29,11 +29,11 @@ import channelsRoutes from './api/explorer/channels.routes';
 import generalLightningRoutes from './api/explorer/general.routes';
 import lightningStatsUpdater from './tasks/lightning/stats-updater.service';
 import nodeSyncService from './tasks/lightning/node-sync.service';
-import statisticsRoutes from "./api/statistics/statistics.routes";
+import statisticsRoutes from './api/statistics/statistics.routes';
-import miningRoutes from "./api/mining/mining-routes";
+import miningRoutes from './api/mining/mining-routes';
-import bisqRoutes from "./api/bisq/bisq.routes";
+import bisqRoutes from './api/bisq/bisq.routes';
-import liquidRoutes from "./api/liquid/liquid.routes";
+import liquidRoutes from './api/liquid/liquid.routes';
-import bitcoinRoutes from "./api/bitcoin/bitcoin.routes";
+import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
 
 class Server {
   private wss: WebSocket.Server | undefined;

backend/src/tasks/lightning/stats-updater.service.ts

@@ -4,11 +4,12 @@ import logger from '../../logger';
 import lightningApi from '../../api/lightning/lightning-api-factory';
 import channelsApi from '../../api/explorer/channels.api';
 import { isIP } from 'net';
+import LightningStatsImporter from './sync-tasks/stats-importer';
 
 class LightningStatsUpdater {
   hardCodedStartTime = '2018-01-12';
 
-  public async $startService() {
+  public async $startService(): Promise<void> {
     logger.info('Starting Lightning Stats service');
     let isInSync = false;
     let error: any;
@@ -28,6 +29,8 @@ class LightningStatsUpdater {
       return;
     }
 
+    LightningStatsImporter.$run();
+
     setTimeout(() => {
       this.$runTasks();
     }, this.timeUntilMidnight());

backend/src/tasks/lightning/sync-tasks/stats-importer.ts

@@ -29,24 +29,25 @@ interface Channel {
   htlc_maximum_msat: number;
 }
 
-const topologiesFolder = config.LIGHTNING.TOPOLOGY_FOLDER;
+class LightningStatsImporter {
-const parser = new XMLParser();
+  topologiesFolder = config.LIGHTNING.TOPOLOGY_FOLDER;
+  parser = new XMLParser();
 
-let latestNodeCount = 1; // Ignore gap in the data
+  latestNodeCount = 1; // Ignore gap in the data
 
-async function $run(): Promise<void> {
+  async $run(): Promise<void> {
     // const [channels]: any[] = await DB.query('SELECT short_id from channels;');
     // logger.info('Caching funding txs for currently existing channels');
     // await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
     
-  await $importHistoricalLightningStats();
+    await this.$importHistoricalLightningStats();
   }
 
   /**
    * Parse the file content into XML, and return a list of nodes and channels
    */
-function parseFile(fileContent): any {
+  parseFile(fileContent): any {
-  const graph = parser.parse(fileContent);
+    const graph = this.parser.parse(fileContent);
     if (Object.keys(graph).length === 0) {
       return null;
     }
@@ -105,7 +106,7 @@ function parseFile(fileContent): any {
   /**
    * Generate LN network stats for one day
    */
-async function computeNetworkStats(timestamp: number, networkGraph): Promise<void> {
+  async computeNetworkStats(timestamp: number, networkGraph): Promise<void> {
     // Node counts and network shares
     let clearnetNodes = 0;
     let torNodes = 0;
@@ -237,8 +238,8 @@ async function computeNetworkStats(timestamp: number, networkGraph): Promise<voi
     }
   }
 
-export async function $importHistoricalLightningStats(): Promise<void> {
+  async $importHistoricalLightningStats(): Promise<void> {
-  const fileList = readdirSync(topologiesFolder);
+    const fileList = readdirSync(this.topologiesFolder);
     fileList.sort().reverse();
 
     const [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(added) as added FROM lightning_stats');
@@ -249,26 +250,26 @@ export async function $importHistoricalLightningStats(): Promise<void> {
 
     for (const filename of fileList) {
       const timestamp = parseInt(filename.split('_')[1], 10);
-    const fileContent = readFileSync(`${topologiesFolder}/${filename}`, 'utf8');
+      const fileContent = readFileSync(`${this.topologiesFolder}/${filename}`, 'utf8');
 
-    const graph = parseFile(fileContent);
+      const graph = this.parseFile(fileContent);
       if (!graph) {
         continue;
       }
 
       // Ignore drop of more than 90% of the node count as it's probably a missing data point
-    const diffRatio = graph.nodes.length / latestNodeCount;
+      const diffRatio = graph.nodes.length / this.latestNodeCount;
       if (diffRatio < 0.90) {
         continue;
       }
-    latestNodeCount = graph.nodes.length;
+      this.latestNodeCount = graph.nodes.length;
 
       // Stats exist already, don't calculate/insert them
       if (existingStatsTimestamps[timestamp] === true) {
         continue;
       }
 
-    logger.debug(`Processing ${topologiesFolder}/${filename}`);
+      logger.debug(`Processing ${this.topologiesFolder}/${filename}`);
 
       const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`;
       logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.channels.length} channels`);
@@ -278,10 +279,11 @@ export async function $importHistoricalLightningStats(): Promise<void> {
       await fundingTxFetcher.$fetchChannelsFundingTxs(graph.channels.map(channel => channel.scid.slice(0, -2)));
 
       logger.debug(`Generating LN network stats for ${datestr}`);
-    await computeNetworkStats(timestamp, graph);
+      await this.computeNetworkStats(timestamp, graph);
     }
 
     logger.info(`Lightning network stats historical import completed`);
   }
+}
 
-$run().then(() => process.exit(0));
+export default new LightningStatsImporter;