From 755ac276f74b85096a210958297d758f09d5accc Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 23 Aug 2022 18:00:40 +0200 Subject: [PATCH 1/4] Node history chart full width if we can't display channel map --- frontend/src/app/lightning/node/node.component.html | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/lightning/node/node.component.html b/frontend/src/app/lightning/node/node.component.html index 12ca8eaa1..c81978bbc 100644 --- a/frontend/src/app/lightning/node/node.component.html +++ b/frontend/src/app/lightning/node/node.component.html @@ -119,7 +119,7 @@
-
+
@@ -127,6 +127,9 @@
+
+ +

Active channels map

From 35512bef8d1906982a28dc8e0b9d30750b0fa929 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 23 Aug 2022 22:47:18 +0200 Subject: [PATCH 2/4] Do not fetch node stats for channel tree graph --- backend/src/api/explorer/channels.api.ts | 45 ++++++++++++++++-------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/backend/src/api/explorer/channels.api.ts b/backend/src/api/explorer/channels.api.ts index a0a617e43..b5eac7499 100644 --- a/backend/src/api/explorer/channels.api.ts +++ b/backend/src/api/explorer/channels.api.ts @@ -288,21 +288,36 @@ class ChannelsApi { const channels: any[] = [] for (const row of allChannels) { - const activeChannelsStats: any = await nodesApi.$getActiveChannelsStats(row.public_key); - channels.push({ - status: row.status, - closing_reason: row.closing_reason, - capacity: row.capacity ?? 0, - short_id: row.short_id, - id: row.id, - fee_rate: row.node1_fee_rate ?? row.node2_fee_rate ?? 0, - node: { - alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20), - public_key: row.public_key, - channels: activeChannelsStats.active_channel_count ?? 0, - capacity: activeChannelsStats.capacity ?? 0, - } - }); + let channel; + if (index >= 0) { + const activeChannelsStats: any = await nodesApi.$getActiveChannelsStats(row.public_key); + channel = { + status: row.status, + closing_reason: row.closing_reason, + capacity: row.capacity ?? 0, + short_id: row.short_id, + id: row.id, + fee_rate: row.node1_fee_rate ?? row.node2_fee_rate ?? 0, + node: { + alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20), + public_key: row.public_key, + channels: activeChannelsStats.active_channel_count ?? 0, + capacity: activeChannelsStats.capacity ?? 0, + } + }; + } else if (index === -1) { + channel = { + capacity: row.capacity ?? 0, + short_id: row.short_id, + id: row.id, + node: { + alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20), + public_key: row.public_key, + } + }; + } + + channels.push(channel); } return channels; From 43cc9499b1dbfdd18ca0c12a1e4e07aa2e9462ef Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 24 Aug 2022 08:35:02 +0200 Subject: [PATCH 3/4] Check query input before running the mysql query --- backend/src/api/explorer/channels.routes.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/backend/src/api/explorer/channels.routes.ts b/backend/src/api/explorer/channels.routes.ts index 0fa91db92..eda3a6168 100644 --- a/backend/src/api/explorer/channels.routes.ts +++ b/backend/src/api/explorer/channels.routes.ts @@ -47,8 +47,17 @@ class ChannelsRoutes { res.status(400).send('Missing parameter: public_key'); return; } + const index = parseInt(typeof req.query.index === 'string' ? req.query.index : '0', 10) || 0; const status: string = typeof req.query.status === 'string' ? req.query.status : ''; + + if (index < -1) { + res.status(400).send('Invalid index'); + } + if (['open', 'active', 'closed'].includes(status) === false) { + res.status(400).send('Invalid status'); + } + const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, 10, status); const channelsCount = await channelsApi.$getChannelsCountForNode(req.query.public_key, status); res.header('Pragma', 'public'); From a151a90d2fd429edc3f522319e7c6defa9addcd8 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 24 Aug 2022 12:12:16 +0200 Subject: [PATCH 4/4] Import historical nodes --- .../lightning/sync-tasks/stats-importer.ts | 69 +++++++++++-------- 1 file changed, 42 insertions(+), 27 deletions(-) diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index e05ba4ab3..30cfcc62b 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -7,6 +7,7 @@ import { ILightningApi } from '../../../api/lightning/lightning-api.interface'; import { isIP } from 'net'; import { Common } from '../../../api/common'; import channelsApi from '../../../api/explorer/channels.api'; +import nodesApi from '../../../api/explorer/nodes.api'; const fsPromises = promises; @@ -32,7 +33,26 @@ class LightningStatsImporter { let clearnetTorNodes = 0; let unannouncedNodes = 0; + const [nodesInDbRaw]: any[] = await DB.query(`SELECT public_key FROM nodes`); + const nodesInDb = {}; + for (const node of nodesInDbRaw) { + nodesInDb[node.public_key] = node; + } + for (const node of networkGraph.nodes) { + // If we don't know about this node, insert it in db + if (isHistorical === true && !nodesInDb[node.pub_key]) { + await nodesApi.$saveNode({ + last_update: node.last_update, + pub_key: node.pub_key, + alias: node.alias, + addresses: node.addresses, + color: node.color, + features: node.features, + }); + nodesInDb[node.pub_key] = node; + } + let hasOnion = false; let hasClearnet = false; let isUnnanounced = true; @@ -69,7 +89,7 @@ class LightningStatsImporter { const baseFees: number[] = []; const alreadyCountedChannels = {}; - const [channelsInDbRaw]: any[] = await DB.query(`SELECT short_id, created FROM channels`); + const [channelsInDbRaw]: any[] = await DB.query(`SELECT short_id FROM channels`); const channelsInDb = {}; for (const channel of channelsInDbRaw) { channelsInDb[channel.short_id] = channel; @@ -84,29 +104,19 @@ class LightningStatsImporter { continue; } - // Channel is already in db, check if we need to update 'created' field - if (isHistorical === true) { - //@ts-ignore - if (channelsInDb[short_id] && channel.timestamp < channel.created) { - await DB.query(` - UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.short_id = ?`, - //@ts-ignore - [channel.timestamp, short_id] - ); - } else if (!channelsInDb[short_id]) { - await channelsApi.$saveChannel({ - channel_id: short_id, - chan_point: `${tx.txid}:${short_id.split('x')[2]}`, - //@ts-ignore - last_update: channel.timestamp, - node1_pub: channel.node1_pub, - node2_pub: channel.node2_pub, - capacity: (tx.value * 100000000).toString(), - node1_policy: null, - node2_policy: null, - }, 0); - channelsInDb[channel.channel_id] = channel; - } + // If we don't know about this channel, insert it in db + if (isHistorical === true && !channelsInDb[short_id]) { + await channelsApi.$saveChannel({ + channel_id: short_id, + chan_point: `${tx.txid}:${short_id.split('x')[2]}`, + last_update: channel.last_update, + node1_pub: channel.node1_pub, + node2_pub: channel.node2_pub, + capacity: (tx.value * 100000000).toString(), + node1_policy: null, + node2_policy: null, + }, 0); + channelsInDb[channel.channel_id] = channel; } if (!nodeStats[channel.node1_pub]) { @@ -281,6 +291,7 @@ class LightningStatsImporter { * Import topology files LN historical data into the database */ async $importHistoricalLightningStats(): Promise { + logger.debug('Run the historical importer'); try { let fileList: string[] = []; try { @@ -294,7 +305,7 @@ class LightningStatsImporter { fileList.sort().reverse(); const [rows]: any[] = await DB.query(` - SELECT UNIX_TIMESTAMP(added) AS added, node_count + SELECT UNIX_TIMESTAMP(added) AS added FROM lightning_stats ORDER BY added DESC `); @@ -391,12 +402,16 @@ class LightningStatsImporter { }); } + let rgb = node.rgb_color ?? '#000000'; + if (rgb.indexOf('#') === -1) { + rgb = `#${rgb}`; + } newGraph.nodes.push({ last_update: node.timestamp ?? 0, pub_key: node.id ?? null, - alias: node.alias ?? null, + alias: node.alias ?? node.id.slice(0, 20), addresses: addresses, - color: node.rgb_color ?? null, + color: rgb, features: {}, }); }