From 5d7e42195f4f666bb112b5f3deba02e90cafbf2e Mon Sep 17 00:00:00 2001
From: nymkappa <vialjoris@hotmail.com>
Date: Tue, 2 Aug 2022 15:02:24 +0200
Subject: [PATCH] Reduce massive gaps in the imported historical LN data

---
 .../tasks/lightning/sync-tasks/stats-importer.ts    | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts
index 8482b558c..4f7c5ca04 100644
--- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts
+++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts
@@ -248,6 +248,8 @@ class LightningStatsImporter {
   }
 
   async $importHistoricalLightningStats(): Promise<void> {
+    let latestNodeCount = 1;
+
     const fileList = await fsPromises.readdir(this.topologiesFolder);
     fileList.sort().reverse();
 
@@ -284,6 +286,17 @@ class LightningStatsImporter {
         await fsPromises.writeFile(`${this.topologiesFolder}/${filename}.json`, JSON.stringify(graph));
       }
 
+      if (timestamp > 1556316000) {
+        // "No, the reason most likely is just that I started collection in 2019,
+        // so what I had before that is just the survivors from before, which weren't that many"
+        const diffRatio = graph.nodes.length / latestNodeCount;
+        if (diffRatio < 0.9) {
+          // Ignore drop of more than 90% of the node count as it's probably a missing data point
+          continue;
+        }
+      }
+      latestNodeCount = graph.nodes.length;
+      
       const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`;
       logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.channels.length} channels`);