From bd1d9573d625ef5c0aec5c6ec8119bdb1627f783 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 22 Aug 2022 17:55:19 +0200 Subject: [PATCH 1/3] Reduce api size for channel world map in ln dashboard - added spinner - update cache warmer --- backend/src/api/explorer/channels.api.ts | 58 +++++++++++----- backend/src/api/explorer/channels.routes.ts | 6 +- .../nodes-channels-map.component.html | 26 ++++--- .../nodes-channels-map.component.scss | 17 +++++ .../nodes-channels-map.component.ts | 67 +++++++++++++------ frontend/src/app/services/api.service.ts | 5 +- production/nginx-cache-warmer | 2 + 7 files changed, 133 insertions(+), 48 deletions(-) diff --git a/backend/src/api/explorer/channels.api.ts b/backend/src/api/explorer/channels.api.ts index 408356639..a2db61f78 100644 --- a/backend/src/api/explorer/channels.api.ts +++ b/backend/src/api/explorer/channels.api.ts @@ -17,32 +17,60 @@ class ChannelsApi { } } - public async $getAllChannelsGeo(publicKey?: string): Promise { + public async $getAllChannelsGeo(publicKey?: string, style?: string): Promise { try { + let select: string; + if (style === 'widget') { + select = ` + nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude, + nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude + `; + } else { + select = ` + nodes_1.public_key as node1_public_key, nodes_1.alias AS node1_alias, + nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude, + nodes_2.public_key as node2_public_key, nodes_2.alias AS node2_alias, + nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude + `; + } + const params: string[] = []; - let query = `SELECT nodes_1.public_key as node1_public_key, nodes_1.alias AS node1_alias, - nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude, - nodes_2.public_key as node2_public_key, nodes_2.alias AS node2_alias, - nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude, - channels.capacity - FROM channels - JOIN nodes AS nodes_1 on nodes_1.public_key = channels.node1_public_key - JOIN nodes AS nodes_2 on nodes_2.public_key = channels.node2_public_key - WHERE nodes_1.latitude IS NOT NULL AND nodes_1.longitude IS NOT NULL - AND nodes_2.latitude IS NOT NULL AND nodes_2.longitude IS NOT NULL + let query = `SELECT ${select} + FROM channels + JOIN nodes AS nodes_1 on nodes_1.public_key = channels.node1_public_key + JOIN nodes AS nodes_2 on nodes_2.public_key = channels.node2_public_key + WHERE nodes_1.latitude IS NOT NULL AND nodes_1.longitude IS NOT NULL + AND nodes_2.latitude IS NOT NULL AND nodes_2.longitude IS NOT NULL `; if (publicKey !== undefined) { query += ' AND (nodes_1.public_key = ? OR nodes_2.public_key = ?)'; params.push(publicKey); params.push(publicKey); + } else { + query += ` AND channels.capacity > 1000000 + GROUP BY nodes_1.public_key, nodes_2.public_key + ORDER BY channels.capacity DESC + LIMIT 10000 + `; } const [rows]: any = await DB.query(query, params); - return rows.map((row) => [ - row.node1_public_key, row.node1_alias, row.node1_longitude, row.node1_latitude, - row.node2_public_key, row.node2_alias, row.node2_longitude, row.node2_latitude, - row.capacity]); + return rows.map((row) => { + if (style === 'widget') { + return [ + row.node1_longitude, row.node1_latitude, + row.node2_longitude, row.node2_latitude, + ]; + } else { + return [ + row.node1_public_key, row.node1_alias, + row.node1_longitude, row.node1_latitude, + row.node2_public_key, row.node2_alias, + row.node2_longitude, row.node2_latitude, + ]; + } + }); } catch (e) { logger.err('$getAllChannelsGeo error: ' + (e instanceof Error ? e.message : e)); throw e; diff --git a/backend/src/api/explorer/channels.routes.ts b/backend/src/api/explorer/channels.routes.ts index 09c3da668..0fa91db92 100644 --- a/backend/src/api/explorer/channels.routes.ts +++ b/backend/src/api/explorer/channels.routes.ts @@ -102,7 +102,11 @@ class ChannelsRoutes { private async $getAllChannelsGeo(req: Request, res: Response) { try { - const channels = await channelsApi.$getAllChannelsGeo(req.params?.publicKey); + const style: string = typeof req.query.style === 'string' ? req.query.style : ''; + const channels = await channelsApi.$getAllChannelsGeo(req.params?.publicKey, style); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); res.json(channels); } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); diff --git a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.html b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.html index 1d865d4b7..cc66fb158 100644 --- a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.html +++ b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.html @@ -1,16 +1,22 @@ -
-
-
-
- Lightning nodes channels world map +
+
+
+
+
+ Lightning nodes channels world map +
+ (Tor nodes excluded) +
+ +
- (Tor nodes excluded)
-
-
+
-
+
diff --git a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.scss b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.scss index ca887ad13..bf2b9b79e 100644 --- a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.scss +++ b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.scss @@ -79,4 +79,21 @@ @media (max-width: 567px) { padding-bottom: 55px; } +} + +.loading-spinner { + position: absolute; + top: 50%; + left: calc(50% - 15px); + z-index: 100; +} +.loading-spinner.widget { + position: absolute; + top: 200px; + z-index: 100; + width: 100%; + left: 0; + @media (max-width: 767.98px) { + top: 250px; + } } \ No newline at end of file diff --git a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts index 182b98a56..09e6a17bc 100644 --- a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts +++ b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts @@ -32,6 +32,7 @@ export class NodesChannelsMap implements OnInit { channelColor = '#466d9d'; channelCurve = 0; nodeSize = 4; + isLoading = true; chartInstance = undefined; chartOptions: EChartsOption = {}; @@ -74,7 +75,7 @@ export class NodesChannelsMap implements OnInit { switchMap((params: ParamMap) => { return zip( this.assetsService.getWorldMapJson$, - this.style !== 'channelpage' ? this.apiService.getChannelsGeo$(params.get('public_key') ?? undefined) : [''], + this.style !== 'channelpage' ? this.apiService.getChannelsGeo$(params.get('public_key') ?? undefined, this.style) : [''], [params.get('public_key') ?? undefined] ).pipe(tap((data) => { registerMap('world', data[0]); @@ -93,10 +94,12 @@ export class NodesChannelsMap implements OnInit { } } for (const channel of geoloc) { - if (!thisNodeGPS && data[2] === channel[0]) { - thisNodeGPS = [channel[2], channel[3]]; - } else if (!thisNodeGPS && data[2] === channel[4]) { - thisNodeGPS = [channel[6], channel[7]]; + if (this.style === 'nodepage' && !thisNodeGPS) { + if (data[2] === channel[0]) { + thisNodeGPS = [channel[2], channel[3]]; + } else if (data[2] === channel[4]) { + thisNodeGPS = [channel[6], channel[7]]; + } } // 0 - node1 pubkey @@ -105,48 +108,68 @@ export class NodesChannelsMap implements OnInit { // 4 - node2 pubkey // 5 - node2 alias // 6,7 - node2 GPS + const node1PubKey = 0; + const node1Alias = 1; + let node1GpsLat = 2; + let node1GpsLgt = 3; + const node2PubKey = 4; + const node2Alias = 5; + let node2GpsLat = 6; + let node2GpsLgt = 7; + let node1UniqueId = channel[node1PubKey]; + let node2UniqueId = channel[node2PubKey]; + if (this.style === 'widget') { + node1GpsLat = 0; + node1GpsLgt = 1; + node2GpsLat = 2; + node2GpsLgt = 3; + node1UniqueId = channel[node1GpsLat].toString() + channel[node1GpsLgt].toString(); + node2UniqueId = channel[node2GpsLat].toString() + channel[node2GpsLgt].toString(); + } // We add a bit of noise so nodes at the same location are not all // on top of each other let random = Math.random() * 2 * Math.PI; let random2 = Math.random() * 0.01; - if (!nodesPubkeys[channel[0]]) { + if (!nodesPubkeys[node1UniqueId]) { nodes.push([ - channel[2] + random2 * Math.cos(random), - channel[3] + random2 * Math.sin(random), + channel[node1GpsLat] + random2 * Math.cos(random), + channel[node1GpsLgt] + random2 * Math.sin(random), 1, - channel[0], - channel[1] + channel[node1PubKey], + channel[node1Alias] ]); - nodesPubkeys[channel[0]] = nodes[nodes.length - 1]; + nodesPubkeys[node1UniqueId] = nodes[nodes.length - 1]; } random = Math.random() * 2 * Math.PI; random2 = Math.random() * 0.01; - if (!nodesPubkeys[channel[4]]) { + if (!nodesPubkeys[node2UniqueId]) { nodes.push([ - channel[6] + random2 * Math.cos(random), - channel[7] + random2 * Math.sin(random), + channel[node2GpsLat] + random2 * Math.cos(random), + channel[node2GpsLgt] + random2 * Math.sin(random), 1, - channel[4], - channel[5] + channel[node2PubKey], + channel[node2Alias] ]); - nodesPubkeys[channel[4]] = nodes[nodes.length - 1]; + nodesPubkeys[node2UniqueId] = nodes[nodes.length - 1]; } const channelLoc = []; - channelLoc.push(nodesPubkeys[channel[0]].slice(0, 2)); - channelLoc.push(nodesPubkeys[channel[4]].slice(0, 2)); + channelLoc.push(nodesPubkeys[node1UniqueId].slice(0, 2)); + channelLoc.push(nodesPubkeys[node2UniqueId].slice(0, 2)); channelsLoc.push(channelLoc); } + if (this.style === 'nodepage' && thisNodeGPS) { this.center = [thisNodeGPS[0], thisNodeGPS[1]]; this.zoom = 10; this.channelWidth = 1; this.channelOpacity = 1; } + if (this.style === 'channelpage' && this.channel.length > 0) { this.channelWidth = 2; this.channelOpacity = 1; @@ -238,7 +261,7 @@ export class NodesChannelsMap implements OnInit { }, { large: false, - progressive: 200, + progressive: this.style === 'widget' ? 500 : 200, silent: true, type: 'lines', coordinateSystem: 'geo', @@ -266,6 +289,10 @@ export class NodesChannelsMap implements OnInit { this.chartInstance = ec; + this.chartInstance.on('finished', () => { + this.isLoading = false; + }); + if (this.style === 'widget') { this.chartInstance.getZr().on('click', (e) => { this.zone.run(() => { diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index f7f4cb5b6..5d89a168f 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -271,10 +271,11 @@ export class ApiService { return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/nodes/countries'); } - getChannelsGeo$(publicKey?: string): Observable { + getChannelsGeo$(publicKey?: string, style?: 'graph' | 'nodepage' | 'widget' | 'channelpage'): Observable { return this.httpClient.get( this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/channels-geo' + - (publicKey !== undefined ? `/${publicKey}` : '') + (publicKey !== undefined ? `/${publicKey}` : '') + + (style !== undefined ? `?style=${style}` : '') ); } } diff --git a/production/nginx-cache-warmer b/production/nginx-cache-warmer index f6ec8473d..a4ece6e0b 100755 --- a/production/nginx-cache-warmer +++ b/production/nginx-cache-warmer @@ -77,6 +77,8 @@ do for url in / \ '/api/v1/mining/difficulty-adjustments/2y' \ '/api/v1/mining/difficulty-adjustments/3y' \ '/api/v1/mining/difficulty-adjustments/all' \ + '/api/v1/lightning/channels-geo?style=widget' \ + '/api/v1/lightning/channels-geo?style=graph' \ do curl -s "https://${hostname}${url}" >/dev/null From fd46ea82bf783966f7b08c54bd400de41caba328 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 22 Aug 2022 22:06:31 +0200 Subject: [PATCH 2/3] Fix channel map size --- .../nodes-channels-map.component.html | 2 +- .../nodes-channels-map.component.scss | 59 +++++++++++-------- 2 files changed, 36 insertions(+), 25 deletions(-) diff --git a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.html b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.html index cc66fb158..7eda48b2b 100644 --- a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.html +++ b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.html @@ -8,7 +8,7 @@ (Tor nodes excluded)
-
diff --git a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.scss b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.scss index bf2b9b79e..fd93b09c5 100644 --- a/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.scss +++ b/frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.scss @@ -10,7 +10,7 @@ .full-container { padding: 0px 15px; width: 100%; - min-height: 500px; + min-height: 600px; height: calc(100% - 150px); @media (max-width: 992px) { @@ -18,17 +18,20 @@ padding-bottom: 100px; } } - .full-container.nodepage { + min-height: 400px; + margin-top: 25px; + margin-bottom: 25px; +} +.full-container.channelpage { + min-height: 400px; margin-top: 25px; margin-bottom: 25px; } - .full-container.widget { height: 250px; min-height: 250px; } - .full-container.fit-container { margin: 0; padding: 0; @@ -41,25 +44,6 @@ } } -.widget { - width: 90vw; - margin-left: auto; - margin-right: auto; - height: 250px; - -webkit-mask: linear-gradient(0deg, #11131f00 5%, #11131fff 25%); - @media (max-width: 767.98px) { - width: 100vw; - } -} - -.widget > .chart { - min-height: 250px; - -webkit-mask: linear-gradient(180deg, #11131f00 0%, #11131fff 20%); - @media (max-width: 767.98px) { - padding-bottom: 0px; - } -} - .chart { min-height: 500px; width: 100%; @@ -80,6 +64,33 @@ padding-bottom: 55px; } } +.chart.graph { + min-height: 600px; +} +.chart.nodepage { + min-height: 400px; +} +.chart.channelpage { + min-height: 400px; +} + +.widget { + width: 90vw; + margin-left: auto; + margin-right: auto; + height: 250px; + -webkit-mask: linear-gradient(0deg, #11131f00 5%, #11131fff 25%); + @media (max-width: 767.98px) { + width: 100vw; + } +} +.widget > .chart { + min-height: 250px; + -webkit-mask: linear-gradient(180deg, #11131f00 0%, #11131fff 20%); + @media (max-width: 767.98px) { + padding-bottom: 0px; + } +} .loading-spinner { position: absolute; @@ -96,4 +107,4 @@ @media (max-width: 767.98px) { top: 250px; } -} \ No newline at end of file +} From 73d29302307a23ef78fa6dfc82a014f67be28353 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 22 Aug 2022 22:15:15 +0200 Subject: [PATCH 3/3] Fix "cannot update channel list" error --- backend/src/tasks/lightning/network-sync.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/tasks/lightning/network-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts index d704012f7..f15aa0d15 100644 --- a/backend/src/tasks/lightning/network-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -98,7 +98,7 @@ class NetworkSyncService { const [closedChannelsRaw]: any[] = await DB.query(`SELECT id FROM channels WHERE status = 2`); const closedChannels = {}; for (const closedChannel of closedChannelsRaw) { - closedChannels[Common.channelShortIdToIntegerId(closedChannel.id)] = true; + closedChannels[closedChannel.id] = true; } let progress = 0;