From faafa6db3b19eb48e93a82e13e1159d109b4cc96 Mon Sep 17 00:00:00 2001
From: softsimon <softsimon@protonmail.com>
Date: Mon, 4 Jul 2022 12:00:16 +0200
Subject: [PATCH] Backfill node and capacity

---
 lightning-backend/src/index.ts                |   8 +-
 .../src/tasks/node-sync.service.ts            |   5 +-
 .../src/tasks/stats-updater.service.ts        | 105 ++++++++++++++++--
 3 files changed, 100 insertions(+), 18 deletions(-)

diff --git a/lightning-backend/src/index.ts b/lightning-backend/src/index.ts
index 614ac8499..fc7a95f2d 100644
--- a/lightning-backend/src/index.ts
+++ b/lightning-backend/src/index.ts
@@ -1,13 +1,9 @@
-import config from './config';
-import logger from './logger';
 import DB from './database';
 import databaseMigration from './database-migration';
 import statsUpdater from './tasks/stats-updater.service';
 import nodeSyncService from './tasks/node-sync.service';
 import server from './server';
 
-logger.notice(`Mempool Server is running on port ${config.MEMPOOL.HTTP_PORT}`);
-
 class LightningServer {
   constructor() {
     this.init();
@@ -17,8 +13,8 @@ class LightningServer {
     await DB.checkDbConnection();
     await databaseMigration.$initializeOrMigrateDatabase();
 
-    statsUpdater.startService();
-    nodeSyncService.startService();
+    nodeSyncService.$startService();
+    statsUpdater.$startService();
 
     server.startServer();
   }
diff --git a/lightning-backend/src/tasks/node-sync.service.ts b/lightning-backend/src/tasks/node-sync.service.ts
index fcf3da093..095e9eeac 100644
--- a/lightning-backend/src/tasks/node-sync.service.ts
+++ b/lightning-backend/src/tasks/node-sync.service.ts
@@ -8,15 +8,14 @@ import bitcoinClient from '../api/bitcoin/bitcoin-client';
 import bitcoinApi from '../api/bitcoin/bitcoin-api-factory';
 import config from '../config';
 import { IEsploraApi } from '../api/bitcoin/esplora-api.interface';
-import e from 'express';
 
 class NodeSyncService {
   constructor() {}
 
-  public async startService() {
+  public async $startService() {
     logger.info('Starting node sync service');
 
-    this.$updateNodes();
+    await this.$updateNodes();
 
     setInterval(async () => {
       await this.$updateNodes();
diff --git a/lightning-backend/src/tasks/stats-updater.service.ts b/lightning-backend/src/tasks/stats-updater.service.ts
index ced094a66..150851a81 100644
--- a/lightning-backend/src/tasks/stats-updater.service.ts
+++ b/lightning-backend/src/tasks/stats-updater.service.ts
@@ -6,22 +6,26 @@ import lightningApi from '../api/lightning/lightning-api-factory';
 class LightningStatsUpdater {
   constructor() {}
 
-  public async startService() {
+  public async $startService() {
     logger.info('Starting Stats service');
 
     const now = new Date();
     const nextHourInterval = new Date(now.getFullYear(), now.getMonth(), now.getDate(), Math.floor(now.getHours() / 1) + 1, 0, 0, 0);
     const difference = nextHourInterval.getTime() - now.getTime();
 
-    setTimeout(() => {
-      this.$logLightningStats();
-      setInterval(() => {
-        this.$logLightningStats();
-        this.$logNodeStatsDaily();
+    // setTimeout(() => {
+      setInterval(async () => {
+        await this.$runTasks();
       }, 1000 * 60 * 60);
-    }, difference);
+    //}, difference);
 
-    this.$logNodeStatsDaily();
+    await this.$runTasks();
+  }
+
+  private async $runTasks() {
+    await this.$populateHistoricalData();
+    await this.$logLightningStatsDaily();
+    await this.$logNodeStatsDaily();
   }
 
   private async $logNodeStatsDaily() {
@@ -54,8 +58,91 @@ class LightningStatsUpdater {
     }
   }
 
-  private async $logLightningStats() {
+  // We only run this on first launch
+  private async $populateHistoricalData() {
+    const startTime = '2018-01-13';
     try {
+      const [rows]: any = await DB.query(`SELECT COUNT(*) FROM statistics`);
+      // Only store once per day
+      if (rows[0]['COUNT(*)'] > 0) {
+        return;
+      }
+      const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels ORDER BY created ASC`);
+
+      let date: Date = new Date(startTime);
+      const currentDate = new Date();
+
+      while (date < currentDate) {
+        let totalCapacity = 0;
+        let channelsCount = 0;
+        for (const channel of channels) {
+          if (new Date(channel.created) > date) {
+            break;
+          }
+          if (channel.closing_date !== null && new Date(channel.closing_date) < date) {
+            continue;
+          }
+          totalCapacity += channel.capacity;
+          channelsCount++;
+        }
+
+        const query = `INSERT INTO statistics(
+          added,
+          channel_count,
+          node_count,
+          total_capacity
+        )
+        VALUES (FROM_UNIXTIME(?), ?, ?, ?)`;
+
+      await DB.query(query, [
+        date.getTime() / 1000,
+        channelsCount,
+        0,
+        totalCapacity,
+      ]);
+
+        // Add one day and continue
+        date.setDate(date.getDate() + 1);
+      }
+
+      const [nodes]: any = await DB.query(`SELECT first_seen FROM nodes ORDER BY first_seen ASC`);
+      date = new Date(startTime);
+
+      while (date < currentDate) {
+        let nodeCount = 0;
+        for (const node of nodes) {
+          if (new Date(node.first_seen) > date) {
+            break;
+          }
+          nodeCount++;
+        }
+
+        const query = `UPDATE statistics SET node_count = ? WHERE added = FROM_UNIXTIME(?)`;
+
+        await DB.query(query, [
+          nodeCount,
+          date.getTime() / 1000,
+        ]);
+
+        // Add one day and continue
+        date.setDate(date.getDate() + 1);
+      }
+
+      logger.debug('Historical stats populated.');
+    } catch (e) {
+      logger.err('$populateHistoricalData() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  private async $logLightningStatsDaily() {
+    const currentDate = new Date().toISOString().split('T')[0];
+    try {
+      const [state]: any = await DB.query(`SELECT string FROM state WHERE name = 'last_node_stats'`);
+      // Only store once per day
+      if (state[0].string === currentDate) {
+        return;
+      }
+
       const networkGraph = await lightningApi.$getNetworkGraph();
       let total_capacity = 0;
       for (const channel of networkGraph.channels) {