machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Assume topology file are in .json - trim log

Browse Source
This commit is contained in:
nymkappa 2022-08-16 19:29:00 +02:00
parent 264ce1222a
commit a71262f538
No known key found for this signature in database
GPG Key ID: E155910B16E8BD04
1 changed files with 22 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
backend/src/tasks/lightning/sync-tasks/stats-importer.ts
View File

@ -38,8 +38,6 @@ class LightningStatsImporter {
parser = new XMLParser(); parser = new XMLParser();
async $run(): Promise<void> { async $run(): Promise<void> {
logger.info(`Importing historical lightning stats`);
const [channels]: any[] = await DB.query('SELECT short_id from channels;'); const [channels]: any[] = await DB.query('SELECT short_id from channels;');
logger.info('Caching funding txs for currently existing channels'); logger.info('Caching funding txs for currently existing channels');
await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id)); await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
@ -283,11 +281,11 @@ class LightningStatsImporter {
// For logging purpose // For logging purpose
let processed = 10; let processed = 10;
let totalProcessed = -1; let totalProcessed = 0;
let logStarted = false;
for (const filename of fileList) { for (const filename of fileList) {
processed++; processed++;
totalProcessed++;
const timestamp = parseInt(filename.split('_')[1], 10); const timestamp = parseInt(filename.split('_')[1], 10);
@ -296,6 +294,10 @@ class LightningStatsImporter {
continue; continue;
} }
if (filename.indexOf('.json') === -1) {
continue;
}
logger.debug(`Reading ${this.topologiesFolder}/${filename}`); logger.debug(`Reading ${this.topologiesFolder}/${filename}`);
let fileContent = ''; let fileContent = '';
try { try {
@ -307,25 +309,23 @@ class LightningStatsImporter {
} }
let graph; let graph;
if (filename.indexOf('.json') !== -1) { try {
try { graph = JSON.parse(fileContent);
graph = JSON.parse(fileContent); } catch (e) {
} catch (e) { logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`);
logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`); continue;
continue;
}
} else {
graph = this.parseFile(fileContent);
if (!graph) {
logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`);
continue;
}
await fsPromises.writeFile(`${this.topologiesFolder}/${filename}.json`, JSON.stringify(graph));
} }
if (!logStarted) {
logger.info(`Founds a topology file that we did not import. Importing historical lightning stats now.`);
logStarted = true;
}
const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`; const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`;
logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`); logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`);
totalProcessed++;
if (processed > 10) { if (processed > 10) {
logger.info(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`); logger.info(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`);
processed = 0; processed = 0;
@ -338,7 +338,9 @@ class LightningStatsImporter {
existingStatsTimestamps[timestamp] = stat; existingStatsTimestamps[timestamp] = stat;
} }
logger.info(`Lightning network stats historical import completed`); if (totalProcessed > 0) {
logger.info(`Lightning network stats historical import completed`);
}
} }
/** /**