machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Integrate LN stats importer into the main process

Browse Source
This commit is contained in:
nymkappa 2022-08-01 17:48:04 +02:00
parent 4ea1e98547
commit 91ada9ce75
No known key found for this signature in database
GPG Key ID: E155910B16E8BD04
3 changed files with 246 additions and 241 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
backend/src/index.ts
View File

@ -29,11 +29,11 @@ import channelsRoutes from './api/explorer/channels.routes';
import generalLightningRoutes from './api/explorer/general.routes';
import lightningStatsUpdater from './tasks/lightning/stats-updater.service';
import nodeSyncService from './tasks/lightning/node-sync.service';
import statisticsRoutes from "./api/statistics/statistics.routes";
import miningRoutes from "./api/mining/mining-routes";
import bisqRoutes from "./api/bisq/bisq.routes";
import liquidRoutes from "./api/liquid/liquid.routes";
import bitcoinRoutes from "./api/bitcoin/bitcoin.routes";
import statisticsRoutes from './api/statistics/statistics.routes';
import miningRoutes from './api/mining/mining-routes';
import bisqRoutes from './api/bisq/bisq.routes';
import liquidRoutes from './api/liquid/liquid.routes';
import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
class Server {
private wss: WebSocket.Server | undefined;

5
backend/src/tasks/lightning/stats-updater.service.ts
View File

@ -4,11 +4,12 @@ import logger from '../../logger';
import lightningApi from '../../api/lightning/lightning-api-factory';
import channelsApi from '../../api/explorer/channels.api';
import { isIP } from 'net';
import LightningStatsImporter from './sync-tasks/stats-importer';
class LightningStatsUpdater {
hardCodedStartTime = '2018-01-12';
public async $startService() {
public async $startService(): Promise<void> {
logger.info('Starting Lightning Stats service');
let isInSync = false;
let error: any;
@ -28,6 +29,8 @@ class LightningStatsUpdater {
return;
}
LightningStatsImporter.$run();
setTimeout(() => {
this.$runTasks();
}, this.timeUntilMidnight());

46
backend/src/tasks/lightning/sync-tasks/stats-importer.ts
View File

@ -29,24 +29,25 @@ interface Channel {
htlc_maximum_msat: number;
}
const topologiesFolder = config.LIGHTNING.TOPOLOGY_FOLDER;
const parser = new XMLParser();
class LightningStatsImporter {
topologiesFolder = config.LIGHTNING.TOPOLOGY_FOLDER;
parser = new XMLParser();
let latestNodeCount = 1; // Ignore gap in the data
latestNodeCount = 1; // Ignore gap in the data
async function $run(): Promise<void> {
async $run(): Promise<void> {
// const [channels]: any[] = await DB.query('SELECT short_id from channels;');
// logger.info('Caching funding txs for currently existing channels');
// await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
await $importHistoricalLightningStats();
}
await this.$importHistoricalLightningStats();
}
/**
/**
* Parse the file content into XML, and return a list of nodes and channels
*/
function parseFile(fileContent): any {
const graph = parser.parse(fileContent);
parseFile(fileContent): any {
const graph = this.parser.parse(fileContent);
if (Object.keys(graph).length === 0) {
return null;
}
@ -100,12 +101,12 @@ function parseFile(fileContent): any {
nodes: nodes,
channels: channels,
};
}
}
/**
/**
* Generate LN network stats for one day
*/
async function computeNetworkStats(timestamp: number, networkGraph): Promise<void> {
async computeNetworkStats(timestamp: number, networkGraph): Promise<void> {
// Node counts and network shares
let clearnetNodes = 0;
let torNodes = 0;
@ -235,10 +236,10 @@ async function computeNetworkStats(timestamp: number, networkGraph): Promise<voi
nodeStats[public_key].channels,
]);
}
}
}
export async function $importHistoricalLightningStats(): Promise<void> {
const fileList = readdirSync(topologiesFolder);
async $importHistoricalLightningStats(): Promise<void> {
const fileList = readdirSync(this.topologiesFolder);
fileList.sort().reverse();
const [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(added) as added FROM lightning_stats');
@ -249,26 +250,26 @@ export async function $importHistoricalLightningStats(): Promise<void> {
for (const filename of fileList) {
const timestamp = parseInt(filename.split('_')[1], 10);
const fileContent = readFileSync(`${topologiesFolder}/${filename}`, 'utf8');
const fileContent = readFileSync(`${this.topologiesFolder}/${filename}`, 'utf8');
const graph = parseFile(fileContent);
const graph = this.parseFile(fileContent);
if (!graph) {
continue;
}
// Ignore drop of more than 90% of the node count as it's probably a missing data point
const diffRatio = graph.nodes.length / latestNodeCount;
const diffRatio = graph.nodes.length / this.latestNodeCount;
if (diffRatio < 0.90) {
continue;
}
latestNodeCount = graph.nodes.length;
this.latestNodeCount = graph.nodes.length;
// Stats exist already, don't calculate/insert them
if (existingStatsTimestamps[timestamp] === true) {
continue;
}
logger.debug(`Processing ${topologiesFolder}/${filename}`);
logger.debug(`Processing ${this.topologiesFolder}/${filename}`);
const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`;
logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.channels.length} channels`);
@ -278,10 +279,11 @@ export async function $importHistoricalLightningStats(): Promise<void> {
await fundingTxFetcher.$fetchChannelsFundingTxs(graph.channels.map(channel => channel.scid.slice(0, -2)));
logger.debug(`Generating LN network stats for ${datestr}`);
await computeNetworkStats(timestamp, graph);
await this.computeNetworkStats(timestamp, graph);
}
logger.info(`Lightning network stats historical import completed`);
}
}
$run().then(() => process.exit(0));
export default new LightningStatsImporter;