From a71262f538d1083a70f2a39cd014240e1b8e927f Mon Sep 17 00:00:00 2001
From: nymkappa <vialjoris@hotmail.com>
Date: Tue, 16 Aug 2022 19:29:00 +0200
Subject: [PATCH] Assume topology file are in .json - trim log

---
 .../lightning/sync-tasks/stats-importer.ts    | 42 ++++++++++---------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts
index a75e83142..f73ab4b47 100644
--- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts
+++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts
@@ -38,8 +38,6 @@ class LightningStatsImporter {
   parser = new XMLParser();
 
   async $run(): Promise<void> {
-    logger.info(`Importing historical lightning stats`);
-
     const [channels]: any[] = await DB.query('SELECT short_id from channels;');
     logger.info('Caching funding txs for currently existing channels');
     await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
@@ -283,11 +281,11 @@ class LightningStatsImporter {
 
     // For logging purpose
     let processed = 10;
-    let totalProcessed = -1;
+    let totalProcessed = 0;
+    let logStarted = false;
 
     for (const filename of fileList) {
       processed++;
-      totalProcessed++;
 
       const timestamp = parseInt(filename.split('_')[1], 10);
 
@@ -296,6 +294,10 @@ class LightningStatsImporter {
         continue;
       }
 
+      if (filename.indexOf('.json') === -1) {
+        continue;
+      }
+
       logger.debug(`Reading ${this.topologiesFolder}/${filename}`);
       let fileContent = '';
       try {
@@ -307,25 +309,23 @@ class LightningStatsImporter {
       }
 
       let graph;
-      if (filename.indexOf('.json') !== -1) {
-        try {
-          graph = JSON.parse(fileContent);
-        } catch (e) {
-          logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`);
-          continue;
-        }
-      } else {
-        graph = this.parseFile(fileContent);
-        if (!graph) {
-          logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`);
-          continue;
-        }
-        await fsPromises.writeFile(`${this.topologiesFolder}/${filename}.json`, JSON.stringify(graph));
+      try {
+        graph = JSON.parse(fileContent);
+      } catch (e) {
+        logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`);
+        continue;
       }
-
+  
+      if (!logStarted) {
+        logger.info(`Founds a topology file that we did not import. Importing historical lightning stats now.`);
+        logStarted = true;
+      }
+      
       const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`;
       logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`);
 
+      totalProcessed++;
+
       if (processed > 10) {
         logger.info(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`);
         processed = 0;
@@ -338,7 +338,9 @@ class LightningStatsImporter {
       existingStatsTimestamps[timestamp] = stat;
     }
 
-    logger.info(`Lightning network stats historical import completed`);
+    if (totalProcessed > 0) {
+      logger.info(`Lightning network stats historical import completed`);
+    }
   }
 
   /**