From 54451c9a8c754ecd880363c8d8184645aa78f255 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 16 Aug 2022 16:15:34 +0000 Subject: [PATCH 01/23] Fix sticky error state on block page --- .../app/components/block/block.component.ts | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 621956d20..4862e4e5c 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -4,7 +4,7 @@ import { ActivatedRoute, ParamMap, Router } from '@angular/router'; import { ElectrsApiService } from '../../services/electrs-api.service'; import { switchMap, tap, throttleTime, catchError, map, shareReplay, startWith, pairwise } from 'rxjs/operators'; import { Transaction, Vout } from '../../interfaces/electrs.interface'; -import { Observable, of, Subscription, asyncScheduler } from 'rxjs'; +import { Observable, of, Subscription, asyncScheduler, EMPTY } from 'rxjs'; import { StateService } from '../../services/state.service'; import { SeoService } from 'src/app/services/seo.service'; import { WebsocketService } from 'src/app/services/websocket.service'; @@ -142,8 +142,21 @@ export class BlockComponent implements OnInit, OnDestroy { this.location.replaceState( this.router.createUrlTree([(this.network ? '/' + this.network : '') + '/block/', hash]).toString() ); - return this.apiService.getBlock$(hash); - }) + return this.apiService.getBlock$(hash).pipe( + catchError((err) => { + this.error = err; + this.isLoadingBlock = false; + this.isLoadingOverview = false; + return EMPTY; + }) + ); + }), + catchError((err) => { + this.error = err; + this.isLoadingBlock = false; + this.isLoadingOverview = false; + return EMPTY; + }), ); } @@ -152,7 +165,14 @@ export class BlockComponent implements OnInit, OnDestroy { return of(blockInCache); } - return this.apiService.getBlock$(blockHash); + return this.apiService.getBlock$(blockHash).pipe( + catchError((err) => { + this.error = err; + this.isLoadingBlock = false; + this.isLoadingOverview = false; + return EMPTY; + }) + ); } }), tap((block: BlockExtended) => { @@ -168,7 +188,6 @@ export class BlockComponent implements OnInit, OnDestroy { this.block = block; this.blockHeight = block.height; - const direction = (this.lastBlockHeight < this.blockHeight) ? 'right' : 'left'; this.lastBlockHeight = this.blockHeight; this.nextBlockHeight = block.height + 1; this.setNextAndPreviousBlockLink(); From 82f8bf6bb489aa2c8c9715ea779c52b61a136b26 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 16 Aug 2022 18:47:45 +0200 Subject: [PATCH 02/23] Refactor ISP pie chart to make it more consitent --- backend/src/api/explorer/nodes.api.ts | 147 ++++++++++++------ backend/src/api/explorer/nodes.routes.ts | 10 +- .../nodes-per-isp-chart.component.html | 45 +++--- .../nodes-per-isp-chart.component.scss | 7 +- .../nodes-per-isp-chart.component.ts | 110 +++++++------ frontend/src/app/services/api.service.ts | 5 +- .../components/toggle/toggle.component.html | 2 +- .../components/toggle/toggle.component.ts | 1 + 8 files changed, 192 insertions(+), 135 deletions(-) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 2d838524e..e59770d50 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -168,64 +168,115 @@ class NodesApi { } } - public async $getNodesISPRanking(groupBy: string, showTor: boolean) { + public async $getNodesISPRanking() { try { - const orderBy = groupBy === 'capacity' ? `CAST(SUM(capacity) as INT)` : `COUNT(DISTINCT nodes.public_key)`; - - // Clearnet - let query = `SELECT GROUP_CONCAT(DISTINCT(nodes.as_number)) as ispId, geo_names.names as names, - COUNT(DISTINCT nodes.public_key) as nodesCount, CAST(SUM(capacity) as INT) as capacity - FROM nodes - JOIN geo_names ON geo_names.id = nodes.as_number - JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key - GROUP BY geo_names.names - ORDER BY ${orderBy} DESC - `; - const [nodesCountPerAS]: any = await DB.query(query); + let query = ''; - let total = 0; - const nodesPerAs: any[] = []; + // List all channels and the two linked ISP + query = ` + SELECT short_id, capacity, + channels.node1_public_key AS node1PublicKey, isp1.names AS isp1, isp1.id as isp1ID, + channels.node2_public_key AS node2PublicKey, isp2.names AS isp2, isp2.id as isp2ID + FROM channels + JOIN nodes node1 ON node1.public_key = channels.node1_public_key + JOIN nodes node2 ON node2.public_key = channels.node2_public_key + JOIN geo_names isp1 ON isp1.id = node1.as_number + JOIN geo_names isp2 ON isp2.id = node2.as_number + WHERE channels.status = 1 + ORDER BY short_id DESC + `; + const [channelsIsp]: any = await DB.query(query); - for (const asGroup of nodesCountPerAS) { - if (groupBy === 'capacity') { - total += asGroup.capacity; - } else { - total += asGroup.nodesCount; + // Sum channels capacity and node count per ISP + const ispList = {}; + for (const channel of channelsIsp) { + const isp1 = JSON.parse(channel.isp1); + const isp2 = JSON.parse(channel.isp2); + + if (!ispList[isp1]) { + ispList[isp1] = { + id: channel.isp1ID, + capacity: 0, + channels: 0, + nodes: {}, + }; } + if (!ispList[isp2]) { + ispList[isp2] = { + id: channel.isp2ID, + capacity: 0, + channels: 0, + nodes: {}, + }; + } + + ispList[isp1].capacity += channel.capacity; + ispList[isp1].channels += 1; + ispList[isp1].nodes[channel.node1PublicKey] = true; + ispList[isp2].capacity += channel.capacity; + ispList[isp2].channels += 1; + ispList[isp2].nodes[channel.node2PublicKey] = true; } - // Tor - if (showTor) { - query = `SELECT COUNT(DISTINCT nodes.public_key) as nodesCount, CAST(SUM(capacity) as INT) as capacity - FROM nodes - JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key - ORDER BY ${orderBy} DESC - `; - const [nodesCountTor]: any = await DB.query(query); - - total += groupBy === 'capacity' ? nodesCountTor[0].capacity : nodesCountTor[0].nodesCount; - nodesPerAs.push({ - ispId: null, - name: 'Tor', - count: nodesCountTor[0].nodesCount, - share: Math.floor((groupBy === 'capacity' ? nodesCountTor[0].capacity : nodesCountTor[0].nodesCount) / total * 10000) / 100, - capacity: nodesCountTor[0].capacity, - }); + const ispRanking: any[] = []; + for (const isp of Object.keys(ispList)) { + ispRanking.push([ + ispList[isp].id, + isp, + ispList[isp].capacity, + ispList[isp].channels, + Object.keys(ispList[isp].nodes).length, + ]); } - for (const as of nodesCountPerAS) { - nodesPerAs.push({ - ispId: as.ispId, - name: JSON.parse(as.names), - count: as.nodesCount, - share: Math.floor((groupBy === 'capacity' ? as.capacity : as.nodesCount) / total * 10000) / 100, - capacity: as.capacity, - }); - } + // Total active channels capacity + query = `SELECT SUM(capacity) AS capacity FROM channels WHERE status = 1`; + const [totalCapacity]: any = await DB.query(query); - return nodesPerAs; + // Get the total capacity of all channels which have at least one node on clearnet + query = ` + SELECT SUM(capacity) as capacity + FROM ( + SELECT capacity, GROUP_CONCAT(socket1.type, socket2.type) as networks + FROM channels + JOIN nodes_sockets socket1 ON node1_public_key = socket1.public_key + JOIN nodes_sockets socket2 ON node2_public_key = socket2.public_key + AND channels.status = 1 + GROUP BY short_id + ) channels_tmp + WHERE channels_tmp.networks LIKE '%ipv%' + `; + const [clearnetCapacity]: any = await DB.query(query); + + // Get the total capacity of all channels which have both nodes on Tor + query = ` + SELECT SUM(capacity) as capacity + FROM ( + SELECT capacity, GROUP_CONCAT(socket1.type, socket2.type) as networks + FROM channels + JOIN nodes_sockets socket1 ON node1_public_key = socket1.public_key + JOIN nodes_sockets socket2 ON node2_public_key = socket2.public_key + AND channels.status = 1 + GROUP BY short_id + ) channels_tmp + WHERE channels_tmp.networks NOT LIKE '%ipv%' AND + channels_tmp.networks NOT LIKE '%dns%' AND + channels_tmp.networks NOT LIKE '%websocket%' + `; + const [torCapacity]: any = await DB.query(query); + + const clearnetCapacityValue = parseInt(clearnetCapacity[0].capacity, 10); + const torCapacityValue = parseInt(torCapacity[0].capacity, 10); + const unknownCapacityValue = parseInt(totalCapacity[0].capacity) - clearnetCapacityValue - torCapacityValue; + + return { + clearnetCapacity: clearnetCapacityValue, + torCapacity: torCapacityValue, + unknownCapacity: unknownCapacityValue, + ispRanking: ispRanking, + }; } catch (e) { - logger.err(`Cannot get nodes grouped by AS. Reason: ${e instanceof Error ? e.message : e}`); + logger.err(`Cannot get LN ISP ranking. Reason: ${e instanceof Error ? e.message : e}`); throw e; } } diff --git a/backend/src/api/explorer/nodes.routes.ts b/backend/src/api/explorer/nodes.routes.ts index 5e0f95acb..a07001c8d 100644 --- a/backend/src/api/explorer/nodes.routes.ts +++ b/backend/src/api/explorer/nodes.routes.ts @@ -71,15 +71,7 @@ class NodesRoutes { private async $getISPRanking(req: Request, res: Response): Promise { try { - const groupBy = req.query.groupBy as string; - const showTor = req.query.showTor as string === 'true' ? true : false; - - if (!['capacity', 'node-count'].includes(groupBy)) { - res.status(400).send(`groupBy must be one of 'capacity' or 'node-count'`); - return; - } - - const nodesPerAs = await nodesApi.$getNodesISPRanking(groupBy, showTor); + const nodesPerAs = await nodesApi.$getNodesISPRanking(); res.header('Pragma', 'public'); res.header('Cache-control', 'public'); diff --git a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html index 01be4f036..25773a06e 100644 --- a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html +++ b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html @@ -3,21 +3,24 @@
-
Tagged ISPs
-

- {{ stats.taggedISP }} +

Clearnet capacity
+

+

-
Tagged nodes
-

- {{ stats.taggedNodeCount }} +

Unknown capacity
+

+

-
Tagged capacity
-

- +

Tor capacity
+

+

@@ -25,13 +28,13 @@
- Lightning nodes per ISP + Top 100 ISP hosting LN nodes
- (Tor nodes excluded) + (Tor nodes excluded)
@@ -44,9 +47,8 @@
-
- - +
+
@@ -59,16 +61,15 @@ - - - + + + - - - + + +
Capacity
{{ asEntry.rank }}
{{ isp[5] }} - {{ asEntry.name }} - {{ asEntry.name }} + {{ isp[1] }} {{ asEntry.count }}{{ isp[4] }}
diff --git a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss index 874d901b2..c6897cda9 100644 --- a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss +++ b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss @@ -149,7 +149,8 @@ } .name { - width: 25%; + width: 35%; + max-width: 300px; @media (max-width: 576px) { width: 70%; max-width: 150px; @@ -159,14 +160,14 @@ } .share { - width: 20%; + width: 15%; @media (max-width: 576px) { display: none } } .nodes { - width: 20%; + width: 15%; @media (max-width: 576px) { width: 10%; } diff --git a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.ts b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.ts index cd8a72884..116c3215c 100644 --- a/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.ts +++ b/frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.ts @@ -26,14 +26,15 @@ export class NodesPerISPChartComponent implements OnInit { renderer: 'svg', }; timespan = ''; + sortBy = 'capacity'; + showUnknown = false; chartInstance = undefined; @HostBinding('attr.dir') dir = 'ltr'; nodesPerAsObservable$: Observable; - showTorObservable$: Observable; - groupBySubject = new Subject(); - showTorSubject = new Subject(); + sortBySubject = new Subject(); + showUnknownSubject = new Subject(); constructor( private apiService: ApiService, @@ -48,32 +49,49 @@ export class NodesPerISPChartComponent implements OnInit { ngOnInit(): void { this.seoService.setTitle($localize`Lightning nodes per ISP`); - this.showTorObservable$ = this.showTorSubject.asObservable(); - this.nodesPerAsObservable$ = combineLatest([ - this.groupBySubject.pipe(startWith(false)), - this.showTorSubject.pipe(startWith(false)), + this.sortBySubject.pipe(startWith(true)), ]) .pipe( switchMap((selectedFilters) => { - return this.apiService.getNodesPerAs( - selectedFilters[0] ? 'capacity' : 'node-count', - selectedFilters[1] // Show Tor nodes - ) + this.sortBy = selectedFilters[0] ? 'capacity' : 'node-count'; + return this.apiService.getNodesPerIsp() .pipe( - tap(data => { + tap(() => { this.isLoading = false; - this.prepareChartOptions(data); }), map(data => { - for (let i = 0; i < data.length; ++i) { - data[i].rank = i + 1; + let nodeCount = 0; + let totalCapacity = 0; + + for (let i = 0; i < data.ispRanking.length; ++i) { + nodeCount += data.ispRanking[i][4]; + totalCapacity += data.ispRanking[i][2]; + data.ispRanking[i][5] = i; } + for (let i = 0; i < data.ispRanking.length; ++i) { + data.ispRanking[i][6] = Math.round(data.ispRanking[i][4] / nodeCount * 10000) / 100; + data.ispRanking[i][7] = Math.round(data.ispRanking[i][2] / totalCapacity * 10000) / 100; + } + + if (selectedFilters[0] === true) { + data.ispRanking.sort((a, b) => b[7] - a[7]); + } else { + data.ispRanking.sort((a, b) => b[6] - a[6]); + } + + for (let i = 0; i < data.ispRanking.length; ++i) { + data.ispRanking[i][5] = i + 1; + } + + this.prepareChartOptions(data.ispRanking); + return { - taggedISP: data.length, - taggedCapacity: data.reduce((partialSum, isp) => partialSum + isp.capacity, 0), - taggedNodeCount: data.reduce((partialSum, isp) => partialSum + isp.count, 0), - data: data.slice(0, 100), + taggedISP: data.ispRanking.length, + clearnetCapacity: data.clearnetCapacity, + unknownCapacity: data.unknownCapacity, + torCapacity: data.torCapacity, + ispRanking: data.ispRanking.slice(0, 100), }; }) ); @@ -82,22 +100,22 @@ export class NodesPerISPChartComponent implements OnInit { ); if (this.widget) { - this.showTorSubject.next(false); - this.groupBySubject.next(false); + this.sortBySubject.next(false); } } - generateChartSerieData(as): PieSeriesOption[] { + generateChartSerieData(ispRanking): PieSeriesOption[] { let shareThreshold = 0.5; if (this.widget && isMobile() || isMobile()) { shareThreshold = 1; } else if (this.widget) { shareThreshold = 0.75; } - + const data: object[] = []; let totalShareOther = 0; - let totalNodeOther = 0; + let nodeCountOther = 0; + let capacityOther = 0; let edgeDistance: string | number = '10%'; if (isMobile() && this.widget) { @@ -106,18 +124,19 @@ export class NodesPerISPChartComponent implements OnInit { edgeDistance = 10; } - as.forEach((as) => { - if (as.share < shareThreshold) { - totalShareOther += as.share; - totalNodeOther += as.count; + ispRanking.forEach((isp) => { + if ((this.sortBy === 'capacity' ? isp[7] : isp[6]) < shareThreshold) { + totalShareOther += this.sortBy === 'capacity' ? isp[7] : isp[6]; + nodeCountOther += isp[4]; + capacityOther += isp[2]; return; } data.push({ itemStyle: { - color: as.ispId === null ? '#7D4698' : undefined, + color: isp[0] === null ? '#7D4698' : undefined, }, - value: as.share, - name: as.name + (isMobile() || this.widget ? `` : ` (${as.share}%)`), + value: this.sortBy === 'capacity' ? isp[7] : isp[6], + name: isp[1].replace('&', '') + (isMobile() || this.widget ? `` : ` (${this.sortBy === 'capacity' ? isp[7] : isp[6]}%)`), label: { overflow: 'truncate', width: isMobile() ? 75 : this.widget ? 125 : 250, @@ -135,13 +154,13 @@ export class NodesPerISPChartComponent implements OnInit { }, borderColor: '#000', formatter: () => { - return `${as.name} (${as.share}%)
` + - $localize`${as.count.toString()} nodes
` + - $localize`${this.amountShortenerPipe.transform(as.capacity / 100000000, 2)} BTC capacity` + return `${isp[1]} (${isp[6]}%)
` + + $localize`${isp[4].toString()} nodes
` + + $localize`${this.amountShortenerPipe.transform(isp[2] / 100000000, 2)} BTC` ; } }, - data: as.ispId, + data: isp[0], } as PieSeriesOption); }); @@ -167,8 +186,9 @@ export class NodesPerISPChartComponent implements OnInit { }, borderColor: '#000', formatter: () => { - return `${'Other'} (${totalShareOther.toFixed(2)}%)
` + - totalNodeOther.toString() + ` nodes`; + return `Other (${totalShareOther.toFixed(2)}%)
` + + $localize`${nodeCountOther.toString()} nodes
` + + $localize`${this.amountShortenerPipe.transform(capacityOther / 100000000, 2)} BTC`; } }, data: 9999 as any, @@ -177,7 +197,7 @@ export class NodesPerISPChartComponent implements OnInit { return data; } - prepareChartOptions(as): void { + prepareChartOptions(ispRanking): void { let pieSize = ['20%', '80%']; // Desktop if (isMobile() && !this.widget) { pieSize = ['15%', '60%']; @@ -194,11 +214,11 @@ export class NodesPerISPChartComponent implements OnInit { series: [ { zlevel: 0, - minShowLabelAngle: 1.8, + minShowLabelAngle: 0.9, name: 'Lightning nodes', type: 'pie', radius: pieSize, - data: this.generateChartSerieData(as), + data: this.generateChartSerieData(ispRanking), labelLine: { lineStyle: { width: 2, @@ -259,16 +279,8 @@ export class NodesPerISPChartComponent implements OnInit { this.chartInstance.setOption(this.chartOptions); } - onTorToggleStatusChanged(e): void { - this.showTorSubject.next(e); - } - onGroupToggleStatusChanged(e): void { - this.groupBySubject.next(e); - } - - isEllipsisActive(e) { - return (e.offsetWidth < e.scrollWidth); + this.sortBySubject.next(e); } } diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index 844451574..f7f4cb5b6 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -255,9 +255,8 @@ export class ApiService { return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/search', { params }); } - getNodesPerAs(groupBy: 'capacity' | 'node-count', showTorNodes: boolean): Observable { - return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/nodes/isp-ranking' - + `?groupBy=${groupBy}&showTor=${showTorNodes}`); + getNodesPerIsp(): Observable { + return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/nodes/isp-ranking'); } getNodeForCountry$(country: string): Observable { diff --git a/frontend/src/app/shared/components/toggle/toggle.component.html b/frontend/src/app/shared/components/toggle/toggle.component.html index dac33c9d8..ea67f5416 100644 --- a/frontend/src/app/shared/components/toggle/toggle.component.html +++ b/frontend/src/app/shared/components/toggle/toggle.component.html @@ -1,7 +1,7 @@
{{ textLeft }}   {{ textRight }} diff --git a/frontend/src/app/shared/components/toggle/toggle.component.ts b/frontend/src/app/shared/components/toggle/toggle.component.ts index 4bd31ffbd..f389989d9 100644 --- a/frontend/src/app/shared/components/toggle/toggle.component.ts +++ b/frontend/src/app/shared/components/toggle/toggle.component.ts @@ -10,6 +10,7 @@ export class ToggleComponent implements AfterViewInit { @Output() toggleStatusChanged = new EventEmitter(); @Input() textLeft: string; @Input() textRight: string; + @Input() checked: boolean = false; ngAfterViewInit(): void { this.toggleStatusChanged.emit(false); From 264ce1222ae0a35be897567e747e84fbe557812c Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 16 Aug 2022 19:00:08 +0200 Subject: [PATCH 03/23] Remove "invalid data skipping fix" from stats importer --- .../lightning/sync-tasks/stats-importer.ts | 27 ++++++++----------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index 7b618e66e..a75e83142 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -63,6 +63,9 @@ class LightningStatsImporter { let isUnnanounced = true; for (const socket of (node.addresses ?? [])) { + if (!socket.network?.length || !socket.addr?.length) { + continue; + } hasOnion = hasOnion || ['torv2', 'torv3'].includes(socket.network) || socket.addr.indexOf('onion') !== -1; hasClearnet = hasClearnet || ['ipv4', 'ipv6'].includes(socket.network) || [4, 6].includes(isIP(socket.addr.split(':')[0])); } @@ -263,8 +266,6 @@ class LightningStatsImporter { * Import topology files LN historical data into the database */ async $importHistoricalLightningStats(): Promise { - let latestNodeCount = 1; - const fileList = await fsPromises.readdir(this.topologiesFolder); // Insert history from the most recent to the oldest // This also put the .json cached files first @@ -292,12 +293,18 @@ class LightningStatsImporter { // Stats exist already, don't calculate/insert them if (existingStatsTimestamps[timestamp] !== undefined) { - latestNodeCount = existingStatsTimestamps[timestamp].node_count; continue; } logger.debug(`Reading ${this.topologiesFolder}/${filename}`); - const fileContent = await fsPromises.readFile(`${this.topologiesFolder}/${filename}`, 'utf8'); + let fileContent = ''; + try { + fileContent = await fsPromises.readFile(`${this.topologiesFolder}/${filename}`, 'utf8'); + } catch (e: any) { + if (e.errno == -1) { // EISDIR - Ignore directorie + continue; + } + } let graph; if (filename.indexOf('.json') !== -1) { @@ -316,18 +323,6 @@ class LightningStatsImporter { await fsPromises.writeFile(`${this.topologiesFolder}/${filename}.json`, JSON.stringify(graph)); } - if (timestamp > 1556316000) { - // "No, the reason most likely is just that I started collection in 2019, - // so what I had before that is just the survivors from before, which weren't that many" - const diffRatio = graph.nodes.length / latestNodeCount; - if (diffRatio < 0.9) { - // Ignore drop of more than 90% of the node count as it's probably a missing data point - logger.debug(`Nodes count diff ratio threshold reached, ignore the data for this day ${graph.nodes.length} nodes vs ${latestNodeCount}`); - continue; - } - } - latestNodeCount = graph.nodes.length; - const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`; logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`); From a71262f538d1083a70f2a39cd014240e1b8e927f Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 16 Aug 2022 19:29:00 +0200 Subject: [PATCH 04/23] Assume topology file are in .json - trim log --- .../lightning/sync-tasks/stats-importer.ts | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index a75e83142..f73ab4b47 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -38,8 +38,6 @@ class LightningStatsImporter { parser = new XMLParser(); async $run(): Promise { - logger.info(`Importing historical lightning stats`); - const [channels]: any[] = await DB.query('SELECT short_id from channels;'); logger.info('Caching funding txs for currently existing channels'); await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id)); @@ -283,11 +281,11 @@ class LightningStatsImporter { // For logging purpose let processed = 10; - let totalProcessed = -1; + let totalProcessed = 0; + let logStarted = false; for (const filename of fileList) { processed++; - totalProcessed++; const timestamp = parseInt(filename.split('_')[1], 10); @@ -296,6 +294,10 @@ class LightningStatsImporter { continue; } + if (filename.indexOf('.json') === -1) { + continue; + } + logger.debug(`Reading ${this.topologiesFolder}/${filename}`); let fileContent = ''; try { @@ -307,25 +309,23 @@ class LightningStatsImporter { } let graph; - if (filename.indexOf('.json') !== -1) { - try { - graph = JSON.parse(fileContent); - } catch (e) { - logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`); - continue; - } - } else { - graph = this.parseFile(fileContent); - if (!graph) { - logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`); - continue; - } - await fsPromises.writeFile(`${this.topologiesFolder}/${filename}.json`, JSON.stringify(graph)); + try { + graph = JSON.parse(fileContent); + } catch (e) { + logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`); + continue; } - + + if (!logStarted) { + logger.info(`Founds a topology file that we did not import. Importing historical lightning stats now.`); + logStarted = true; + } + const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`; logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`); + totalProcessed++; + if (processed > 10) { logger.info(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`); processed = 0; @@ -338,7 +338,9 @@ class LightningStatsImporter { existingStatsTimestamps[timestamp] = stat; } - logger.info(`Lightning network stats historical import completed`); + if (totalProcessed > 0) { + logger.info(`Lightning network stats historical import completed`); + } } /** From 8dc41257ce9b9b1622e2fd3ed03e006c0dbd7a8b Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 16 Aug 2022 21:47:52 +0200 Subject: [PATCH 05/23] Remove xml parser - Read only .topology file and assume json format --- backend/package-lock.json | 34 --------- backend/package.json | 1 - .../lightning/sync-tasks/stats-importer.ts | 73 +------------------ 3 files changed, 1 insertion(+), 107 deletions(-) diff --git a/backend/package-lock.json b/backend/package-lock.json index 4e43dc309..e83ddf252 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -16,7 +16,6 @@ "bitcoinjs-lib": "6.0.2", "crypto-js": "^4.0.0", "express": "^4.18.0", - "fast-xml-parser": "^4.0.9", "maxmind": "^4.3.6", "mysql2": "2.3.3", "node-worker-threads-pool": "^1.5.1", @@ -3136,21 +3135,6 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, - "node_modules/fast-xml-parser": { - "version": "4.0.9", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.0.9.tgz", - "integrity": "sha512-4G8EzDg2Nb1Qurs3f7BpFV4+jpMVsdgLVuG1Uv8O2OHJfVCg7gcA53obuKbmVqzd4Y7YXVBK05oJG7hzGIdyzg==", - "dependencies": { - "strnum": "^1.0.5" - }, - "bin": { - "fxparser": "src/cli/cli.js" - }, - "funding": { - "type": "paypal", - "url": "https://paypal.me/naturalintelligence" - } - }, "node_modules/fastq": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", @@ -5636,11 +5620,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/strnum": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", - "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" - }, "node_modules/supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", @@ -8556,14 +8535,6 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, - "fast-xml-parser": { - "version": "4.0.9", - "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.0.9.tgz", - "integrity": "sha512-4G8EzDg2Nb1Qurs3f7BpFV4+jpMVsdgLVuG1Uv8O2OHJfVCg7gcA53obuKbmVqzd4Y7YXVBK05oJG7hzGIdyzg==", - "requires": { - "strnum": "^1.0.5" - } - }, "fastq": { "version": "1.13.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.13.0.tgz", @@ -10398,11 +10369,6 @@ "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", "dev": true }, - "strnum": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", - "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" - }, "supports-color": { "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", diff --git a/backend/package.json b/backend/package.json index 084b57731..082449dac 100644 --- a/backend/package.json +++ b/backend/package.json @@ -38,7 +38,6 @@ "bitcoinjs-lib": "6.0.2", "crypto-js": "^4.0.0", "express": "^4.18.0", - "fast-xml-parser": "^4.0.9", "maxmind": "^4.3.6", "mysql2": "2.3.3", "node-worker-threads-pool": "^1.5.1", diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index f73ab4b47..5878f898a 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -1,6 +1,5 @@ import DB from '../../../database'; import { promises } from 'fs'; -import { XMLParser } from 'fast-xml-parser'; import logger from '../../../logger'; import fundingTxFetcher from './funding-tx-fetcher'; import config from '../../../config'; @@ -35,7 +34,6 @@ interface Channel { class LightningStatsImporter { topologiesFolder = config.LIGHTNING.TOPOLOGY_FOLDER; - parser = new XMLParser(); async $run(): Promise { const [channels]: any[] = await DB.query('SELECT short_id from channels;'); @@ -294,7 +292,7 @@ class LightningStatsImporter { continue; } - if (filename.indexOf('.json') === -1) { + if (filename.indexOf('.topology') === -1) { continue; } @@ -342,75 +340,6 @@ class LightningStatsImporter { logger.info(`Lightning network stats historical import completed`); } } - - /** - * Parse the file content into XML, and return a list of nodes and channels - */ - private parseFile(fileContent): any { - const graph = this.parser.parse(fileContent); - if (Object.keys(graph).length === 0) { - return null; - } - - const nodes: Node[] = []; - const channels: Channel[] = []; - - // If there is only one entry, the parser does not return an array, so we override this - if (!Array.isArray(graph.graphml.graph.node)) { - graph.graphml.graph.node = [graph.graphml.graph.node]; - } - if (!Array.isArray(graph.graphml.graph.edge)) { - graph.graphml.graph.edge = [graph.graphml.graph.edge]; - } - - for (const node of graph.graphml.graph.node) { - if (!node.data) { - continue; - } - const addresses: unknown[] = []; - const sockets = node.data[5].split(','); - for (const socket of sockets) { - const parts = socket.split('://'); - addresses.push({ - network: parts[0], - addr: parts[1], - }); - } - nodes.push({ - id: node.data[0], - timestamp: node.data[1], - features: node.data[2], - rgb_color: node.data[3], - alias: node.data[4], - addresses: addresses, - out_degree: node.data[6], - in_degree: node.data[7], - }); - } - - for (const channel of graph.graphml.graph.edge) { - if (!channel.data) { - continue; - } - channels.push({ - channel_id: channel.data[0], - node1_pub: channel.data[1], - node2_pub: channel.data[2], - timestamp: channel.data[3], - features: channel.data[4], - fee_base_msat: channel.data[5], - fee_rate_milli_msat: channel.data[6], - htlc_minimim_msat: channel.data[7], - cltv_expiry_delta: channel.data[8], - htlc_maximum_msat: channel.data[9], - }); - } - - return { - nodes: nodes, - edges: channels, - }; - } } export default new LightningStatsImporter; From 7520e3beba9442ec6cbb6853507fc11f4b538eb5 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 17 Aug 2022 12:53:26 +0200 Subject: [PATCH 06/23] Refactor top nodes widgets --- backend/src/api/explorer/nodes.api.ts | 15 +++++--- backend/src/api/explorer/nodes.routes.ts | 7 ++-- backend/src/mempool.interfaces.ts | 17 +++++++++ .../src/app/interfaces/node-api.interface.ts | 17 +++++++++ .../app/lightning/lightning-api.service.ts | 5 ++- .../lightning-dashboard.component.html | 19 +++++++--- .../lightning-dashboard.component.ts | 19 ++-------- .../src/app/lightning/lightning.module.ts | 6 +++ .../app/lightning/lightning.routing.module.ts | 5 +++ .../nodes-ranking.component.html | 29 +++++++++++++++ .../nodes-ranking.component.scss | 0 .../nodes-ranking/nodes-ranking.component.ts | 15 ++++++++ .../top-nodes-per-capacity.component.html | 37 +++++++++++++++++++ .../top-nodes-per-capacity.component.scss | 30 +++++++++++++++ .../top-nodes-per-capacity.component.ts | 34 +++++++++++++++++ .../top-nodes-per-channels.component.html | 37 +++++++++++++++++++ .../top-nodes-per-channels.component.scss | 30 +++++++++++++++ .../top-nodes-per-channels.component.ts | 34 +++++++++++++++++ 18 files changed, 324 insertions(+), 32 deletions(-) create mode 100644 frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.html create mode 100644 frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.scss create mode 100644 frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.ts create mode 100644 frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.html create mode 100644 frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.scss create mode 100644 frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.ts create mode 100644 frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.html create mode 100644 frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.scss create mode 100644 frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.ts diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 2d838524e..33de5ae8e 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -2,6 +2,7 @@ import logger from '../../logger'; import DB from '../../database'; import { ResultSetHeader } from 'mysql2'; import { ILightningApi } from '../lightning/lightning-api.interface'; +import { TopNodesPerCapacity, TopNodesPerChannels } from '../../mempool.interfaces'; class NodesApi { public async $getNode(public_key: string): Promise { @@ -112,18 +113,19 @@ class NodesApi { } } - public async $getTopCapacityNodes(): Promise { + public async $getTopCapacityNodes(): Promise { try { let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats'); const latestDate = rows[0].maxAdded; const query = ` - SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, node_stats.capacity, node_stats.channels + SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, + node_stats.capacity FROM node_stats JOIN nodes ON nodes.public_key = node_stats.public_key WHERE added = FROM_UNIXTIME(${latestDate}) ORDER BY capacity DESC - LIMIT 10; + LIMIT 100; `; [rows] = await DB.query(query); @@ -134,18 +136,19 @@ class NodesApi { } } - public async $getTopChannelsNodes(): Promise { + public async $getTopChannelsNodes(): Promise { try { let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats'); const latestDate = rows[0].maxAdded; const query = ` - SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, node_stats.capacity, node_stats.channels + SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, + node_stats.channels FROM node_stats JOIN nodes ON nodes.public_key = node_stats.public_key WHERE added = FROM_UNIXTIME(${latestDate}) ORDER BY channels DESC - LIMIT 10; + LIMIT 100; `; [rows] = await DB.query(query); diff --git a/backend/src/api/explorer/nodes.routes.ts b/backend/src/api/explorer/nodes.routes.ts index 5e0f95acb..e0b145bf1 100644 --- a/backend/src/api/explorer/nodes.routes.ts +++ b/backend/src/api/explorer/nodes.routes.ts @@ -2,6 +2,7 @@ import config from '../../config'; import { Application, Request, Response } from 'express'; import nodesApi from './nodes.api'; import DB from '../../database'; +import { INodesRanking } from '../../mempool.interfaces'; class NodesRoutes { constructor() { } @@ -10,7 +11,7 @@ class NodesRoutes { app .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/country/:country', this.$getNodesPerCountry) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/search/:search', this.$searchNode) - .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/top', this.$getTopNodes) + .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings', this.$getNodesRanking) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp-ranking', this.$getISPRanking) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp/:isp', this.$getNodesPerISP) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/countries', this.$getNodesCountries) @@ -56,11 +57,11 @@ class NodesRoutes { } } - private async $getTopNodes(req: Request, res: Response) { + private async $getNodesRanking(req: Request, res: Response): Promise { try { const topCapacityNodes = await nodesApi.$getTopCapacityNodes(); const topChannelsNodes = await nodesApi.$getTopChannelsNodes(); - res.json({ + res.json({ topByCapacity: topCapacityNodes, topByChannels: topChannelsNodes, }); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index c2d2ee747..5c6abf276 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -251,3 +251,20 @@ export interface RewardStats { totalFee: number; totalTx: number; } + +export interface TopNodesPerChannels { + public_key: string, + alias: string, + channels: number, +} + +export interface TopNodesPerCapacity { + public_key: string, + alias: string, + capacity: number, +} + +export interface INodesRanking { + topByCapacity: TopNodesPerCapacity[]; + topByChannels: TopNodesPerChannels[]; +} \ No newline at end of file diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 5f3506db5..905cd37a2 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -151,3 +151,20 @@ export interface RewardStats { totalFee: number; totalTx: number; } + +export interface ITopNodesPerChannels { + public_key: string, + alias: string, + channels: number, +} + +export interface ITopNodesPerCapacity { + public_key: string, + alias: string, + capacity: number, +} + +export interface INodesRanking { + topByCapacity: ITopNodesPerCapacity[]; + topByChannels: ITopNodesPerChannels[]; +} \ No newline at end of file diff --git a/frontend/src/app/lightning/lightning-api.service.ts b/frontend/src/app/lightning/lightning-api.service.ts index 2a6634558..294a2fb60 100644 --- a/frontend/src/app/lightning/lightning-api.service.ts +++ b/frontend/src/app/lightning/lightning-api.service.ts @@ -2,6 +2,7 @@ import { Injectable } from '@angular/core'; import { HttpClient, HttpParams } from '@angular/common/http'; import { Observable } from 'rxjs'; import { StateService } from '../services/state.service'; +import { INodesRanking } from '../interfaces/node-api.interface'; @Injectable({ providedIn: 'root' @@ -48,8 +49,8 @@ export class LightningApiService { return this.httpClient.get(this.apiBasePath + '/api/v1/lightning/nodes/' + publicKey + '/statistics'); } - listTopNodes$(): Observable { - return this.httpClient.get(this.apiBasePath + '/api/v1/lightning/nodes/top'); + getNodesRanking$(): Observable { + return this.httpClient.get(this.apiBasePath + '/api/v1/lightning/nodes/rankings'); } listChannelStats$(publicKey: string): Observable { diff --git a/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html b/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html index ff00f5b15..e2e2731d3 100644 --- a/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html +++ b/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html @@ -56,9 +56,13 @@
-
Top Capacity Nodes
- - + +
Top Capacity Nodes
+   + + +
+
@@ -66,9 +70,12 @@
-
Most Connected Nodes
- - + +
Most connected nodes
+   + + +
diff --git a/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.ts b/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.ts index 5d4685fb8..d601606fd 100644 --- a/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.ts +++ b/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.ts @@ -1,6 +1,7 @@ import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; import { Observable } from 'rxjs'; -import { map, share } from 'rxjs/operators'; +import { share } from 'rxjs/operators'; +import { INodesRanking } from 'src/app/interfaces/node-api.interface'; import { SeoService } from 'src/app/services/seo.service'; import { LightningApiService } from '../lightning-api.service'; @@ -11,9 +12,8 @@ import { LightningApiService } from '../lightning-api.service'; changeDetection: ChangeDetectionStrategy.OnPush, }) export class LightningDashboardComponent implements OnInit { - nodesByCapacity$: Observable; - nodesByChannels$: Observable; statistics$: Observable; + nodesRanking$: Observable; constructor( private lightningApiService: LightningApiService, @@ -23,18 +23,7 @@ export class LightningDashboardComponent implements OnInit { ngOnInit(): void { this.seoService.setTitle($localize`Lightning Dashboard`); - const sharedObservable = this.lightningApiService.listTopNodes$().pipe(share()); - - this.nodesByCapacity$ = sharedObservable - .pipe( - map((object) => object.topByCapacity), - ); - - this.nodesByChannels$ = sharedObservable - .pipe( - map((object) => object.topByChannels), - ); - + this.nodesRanking$ = this.lightningApiService.getNodesRanking$().pipe(share()); this.statistics$ = this.lightningApiService.getLatestStatistics$().pipe(share()); } diff --git a/frontend/src/app/lightning/lightning.module.ts b/frontend/src/app/lightning/lightning.module.ts index c01792815..85961edc2 100644 --- a/frontend/src/app/lightning/lightning.module.ts +++ b/frontend/src/app/lightning/lightning.module.ts @@ -24,6 +24,8 @@ import { NodesPerISP } from './nodes-per-isp/nodes-per-isp.component'; import { NodesPerCountryChartComponent } from '../lightning/nodes-per-country-chart/nodes-per-country-chart.component'; import { NodesMap } from '../lightning/nodes-map/nodes-map.component'; import { NodesChannelsMap } from '../lightning/nodes-channels-map/nodes-channels-map.component'; +import { TopNodesPerChannels } from '../lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component'; +import { TopNodesPerCapacity } from '../lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component'; @NgModule({ declarations: [ LightningDashboardComponent, @@ -45,6 +47,8 @@ import { NodesChannelsMap } from '../lightning/nodes-channels-map/nodes-channels NodesPerCountryChartComponent, NodesMap, NodesChannelsMap, + TopNodesPerChannels, + TopNodesPerCapacity, ], imports: [ CommonModule, @@ -73,6 +77,8 @@ import { NodesChannelsMap } from '../lightning/nodes-channels-map/nodes-channels NodesPerCountryChartComponent, NodesMap, NodesChannelsMap, + TopNodesPerChannels, + TopNodesPerCapacity, ], providers: [ LightningApiService, diff --git a/frontend/src/app/lightning/lightning.routing.module.ts b/frontend/src/app/lightning/lightning.routing.module.ts index 8bfb467af..cae5c1380 100644 --- a/frontend/src/app/lightning/lightning.routing.module.ts +++ b/frontend/src/app/lightning/lightning.routing.module.ts @@ -6,6 +6,7 @@ import { NodeComponent } from './node/node.component'; import { ChannelComponent } from './channel/channel.component'; import { NodesPerCountry } from './nodes-per-country/nodes-per-country.component'; import { NodesPerISP } from './nodes-per-isp/nodes-per-isp.component'; +import { NodesRanking } from './nodes-ranking/nodes-ranking.component'; const routes: Routes = [ { @@ -32,6 +33,10 @@ const routes: Routes = [ path: 'nodes/isp/:isp', component: NodesPerISP, }, + { + path: 'nodes/ranking', + component: NodesRanking, + }, { path: '**', redirectTo: '' diff --git a/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.html b/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.html new file mode 100644 index 000000000..b0b01d16a --- /dev/null +++ b/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.html @@ -0,0 +1,29 @@ +
+ + + + + + + + + +
RankAliasPlaceholder
+ + + + + + + + + + + + + + + + + +
\ No newline at end of file diff --git a/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.scss b/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.scss new file mode 100644 index 000000000..e69de29bb diff --git a/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.ts b/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.ts new file mode 100644 index 000000000..b16dc3ce4 --- /dev/null +++ b/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.ts @@ -0,0 +1,15 @@ +import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; + +@Component({ + selector: 'app-nodes-ranking', + templateUrl: './nodes-ranking.component.html', + styleUrls: ['./nodes-ranking.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class NodesRanking implements OnInit { + + ngOnInit(): void { + console.log('hi'); + } + +} diff --git a/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.html b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.html new file mode 100644 index 000000000..bb1864ba1 --- /dev/null +++ b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.html @@ -0,0 +1,37 @@ +
+ + + + + + + + + + + + + + + + + + + + + + +
AliasCapacity
+ {{ i + 1 }} + + {{ node.alias }} + + +
+ + + + + +
+
\ No newline at end of file diff --git a/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.scss b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.scss new file mode 100644 index 000000000..4662b72ff --- /dev/null +++ b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.scss @@ -0,0 +1,30 @@ +.table td, .table th { + padding: 0.5rem; +} + +.rank { + @media (min-width: 767.98px) { + width: 13%; + } + @media (max-width: 767.98px) { + padding-left: 0px; + padding-right: 0px; + } +} + +.alias { + width: 55%; + overflow: hidden; + text-overflow: ellipsis; + max-width: 350px; + @media (max-width: 767.98px) { + max-width: 175px; + } +} +.capacity { + width: 32%; + @media (max-width: 767.98px) { + padding-left: 0px; + padding-right: 0px; + } +} diff --git a/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.ts b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.ts new file mode 100644 index 000000000..b3c9171c0 --- /dev/null +++ b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.ts @@ -0,0 +1,34 @@ +import { ChangeDetectionStrategy, Component, Input, OnInit } from '@angular/core'; +import { map, Observable } from 'rxjs'; +import { INodesRanking, ITopNodesPerCapacity } from 'src/app/interfaces/node-api.interface'; + +@Component({ + selector: 'app-top-nodes-per-capacity', + templateUrl: './top-nodes-per-capacity.component.html', + styleUrls: ['./top-nodes-per-capacity.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class TopNodesPerCapacity implements OnInit { + @Input() nodes$: Observable; + @Input() widget: boolean = false; + + topNodesPerCapacity$: Observable; + skeletonRows: number[] = []; + + ngOnInit(): void { + for (let i = 1; i <= (this.widget ? 10 : 100); ++i) { + this.skeletonRows.push(i); + } + + this.topNodesPerCapacity$ = this.nodes$.pipe( + map((ranking) => { + if (this.widget === true) { + return ranking.topByCapacity.slice(0, 10); + } else { + return ranking.topByCapacity; + } + }) + ) + } + +} diff --git a/frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.html b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.html new file mode 100644 index 000000000..f581dacff --- /dev/null +++ b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.html @@ -0,0 +1,37 @@ +
+ + + + + + + + + + + + + + + + + + + + + + +
AliasChannels
+ {{ i + 1 }} + + {{ node.alias }} + + {{ node.channels | number }} +
+ + + + + +
+
\ No newline at end of file diff --git a/frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.scss b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.scss new file mode 100644 index 000000000..5335991e3 --- /dev/null +++ b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.scss @@ -0,0 +1,30 @@ +.table td, .table th { + padding: 0.5rem; +} + +.rank { + @media (min-width: 767.98px) { + width: 13%; + } + @media (max-width: 767.98px) { + padding-left: 0px; + padding-right: 0px; + } +} + +.alias { + width: 60%; + overflow: hidden; + text-overflow: ellipsis; + max-width: 350px; + @media (max-width: 767.98px) { + max-width: 175px; + } +} +.channels { + width: 27%; + @media (max-width: 767.98px) { + padding-left: 0px; + padding-right: 0px; + } +} diff --git a/frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.ts b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.ts new file mode 100644 index 000000000..46cf21eaa --- /dev/null +++ b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.ts @@ -0,0 +1,34 @@ +import { ChangeDetectionStrategy, Component, Input, OnInit } from '@angular/core'; +import { map, Observable } from 'rxjs'; +import { INodesRanking, ITopNodesPerChannels } from 'src/app/interfaces/node-api.interface'; + +@Component({ + selector: 'app-top-nodes-per-channels', + templateUrl: './top-nodes-per-channels.component.html', + styleUrls: ['./top-nodes-per-channels.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class TopNodesPerChannels implements OnInit { + @Input() nodes$: Observable; + @Input() widget: boolean = false; + + topNodesPerChannels$: Observable; + skeletonRows: number[] = []; + + ngOnInit(): void { + for (let i = 1; i <= (this.widget ? 10 : 100); ++i) { + this.skeletonRows.push(i); + } + + this.topNodesPerChannels$ = this.nodes$.pipe( + map((ranking) => { + if (this.widget === true) { + return ranking.topByChannels.slice(0, 10); + } else { + return ranking.topByChannels; + } + }) + ) + } + +} From 2359e44b1606b2a2ea2f870fab9fa9d60c3faf29 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 17 Aug 2022 16:00:30 +0200 Subject: [PATCH 07/23] Create top 100 node per capacity component --- backend/src/api/explorer/nodes.api.ts | 55 +++++++++---- backend/src/api/explorer/nodes.routes.ts | 20 ++++- backend/src/mempool.interfaces.ts | 7 +- .../src/app/interfaces/node-api.interface.ts | 7 +- .../app/lightning/lightning-api.service.ts | 8 +- .../lightning-dashboard.component.html | 6 +- .../src/app/lightning/lightning.module.ts | 3 + .../app/lightning/lightning.routing.module.ts | 5 +- .../nodes-ranking.component.html | 31 +------- .../nodes-ranking/nodes-ranking.component.ts | 9 ++- .../top-nodes-per-capacity.component.html | 78 ++++++++++++------- .../top-nodes-per-capacity.component.scss | 60 +++++++++++++- .../top-nodes-per-capacity.component.ts | 21 +++-- 13 files changed, 216 insertions(+), 94 deletions(-) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 33de5ae8e..c7b98f3a8 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -10,10 +10,10 @@ class NodesApi { // General info let query = ` SELECT public_key, alias, UNIX_TIMESTAMP(first_seen) AS first_seen, - UNIX_TIMESTAMP(updated_at) AS updated_at, color, sockets as sockets, - as_number, city_id, country_id, subdivision_id, longitude, latitude, - geo_names_iso.names as iso_code, geo_names_as.names as as_organization, geo_names_city.names as city, - geo_names_country.names as country, geo_names_subdivision.names as subdivision + UNIX_TIMESTAMP(updated_at) AS updated_at, color, sockets as sockets, + as_number, city_id, country_id, subdivision_id, longitude, latitude, + geo_names_iso.names as iso_code, geo_names_as.names as as_organization, geo_names_city.names as city, + geo_names_country.names as country, geo_names_subdivision.names as subdivision FROM nodes LEFT JOIN geo_names geo_names_as on geo_names_as.id = as_number LEFT JOIN geo_names geo_names_city on geo_names_city.id = city_id @@ -113,21 +113,46 @@ class NodesApi { } } - public async $getTopCapacityNodes(): Promise { + public async $getTopCapacityNodes(full: boolean): Promise { try { let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats'); const latestDate = rows[0].maxAdded; - const query = ` - SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, - node_stats.capacity - FROM node_stats - JOIN nodes ON nodes.public_key = node_stats.public_key - WHERE added = FROM_UNIXTIME(${latestDate}) - ORDER BY capacity DESC - LIMIT 100; - `; - [rows] = await DB.query(query); + let query: string; + if (full === false) { + query = ` + SELECT nodes.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, + node_stats.capacity + FROM node_stats + JOIN nodes ON nodes.public_key = node_stats.public_key + WHERE added = FROM_UNIXTIME(${latestDate}) + ORDER BY capacity DESC + LIMIT 100 + `; + + [rows] = await DB.query(query); + } else { + query = ` + SELECT node_stats.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(node_stats.public_key, 1, 20), alias) as alias, + CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, + CAST(COALESCE(node_stats.channels, 0) as INT) as channels, + UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt, + geo_names_city.names as city, geo_names_country.names as country + FROM node_stats + RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key + LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country' + LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city' + WHERE added = FROM_UNIXTIME(${latestDate}) + ORDER BY capacity DESC + LIMIT 100 + `; + + [rows] = await DB.query(query); + for (let i = 0; i < rows.length; ++i) { + rows[i].country = JSON.parse(rows[i].country); + rows[i].city = JSON.parse(rows[i].city); + } + } return rows; } catch (e) { diff --git a/backend/src/api/explorer/nodes.routes.ts b/backend/src/api/explorer/nodes.routes.ts index e0b145bf1..a490d946b 100644 --- a/backend/src/api/explorer/nodes.routes.ts +++ b/backend/src/api/explorer/nodes.routes.ts @@ -11,10 +11,11 @@ class NodesRoutes { app .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/country/:country', this.$getNodesPerCountry) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/search/:search', this.$searchNode) - .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings', this.$getNodesRanking) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp-ranking', this.$getISPRanking) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp/:isp', this.$getNodesPerISP) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/countries', this.$getNodesCountries) + .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings', this.$getNodesRanking) + .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/capacity', this.$getTopNodesByCapacity) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/statistics', this.$getHistoricalNodeStats) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key', this.$getNode) ; @@ -59,8 +60,11 @@ class NodesRoutes { private async $getNodesRanking(req: Request, res: Response): Promise { try { - const topCapacityNodes = await nodesApi.$getTopCapacityNodes(); + const topCapacityNodes = await nodesApi.$getTopCapacityNodes(false); const topChannelsNodes = await nodesApi.$getTopChannelsNodes(); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); res.json({ topByCapacity: topCapacityNodes, topByChannels: topChannelsNodes, @@ -70,6 +74,18 @@ class NodesRoutes { } } + private async $getTopNodesByCapacity(req: Request, res: Response): Promise { + try { + const topCapacityNodes = await nodesApi.$getTopCapacityNodes(true); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + res.json(topCapacityNodes); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + private async $getISPRanking(req: Request, res: Response): Promise { try { const groupBy = req.query.groupBy as string; diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 5c6abf276..1067f9b6d 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -259,9 +259,14 @@ export interface TopNodesPerChannels { } export interface TopNodesPerCapacity { - public_key: string, + publicKey: string, alias: string, capacity: number, + channels?: number, + firstSeen?: number, + updatedAt?: number, + city?: any, + country?: any, } export interface INodesRanking { diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 905cd37a2..4238be7bb 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -159,9 +159,14 @@ export interface ITopNodesPerChannels { } export interface ITopNodesPerCapacity { - public_key: string, + publicKey: string, alias: string, capacity: number, + channels?: number, + firstSeen?: number, + updatedAt?: number, + city?: any, + country?: any, } export interface INodesRanking { diff --git a/frontend/src/app/lightning/lightning-api.service.ts b/frontend/src/app/lightning/lightning-api.service.ts index 294a2fb60..49cc7b61e 100644 --- a/frontend/src/app/lightning/lightning-api.service.ts +++ b/frontend/src/app/lightning/lightning-api.service.ts @@ -2,7 +2,7 @@ import { Injectable } from '@angular/core'; import { HttpClient, HttpParams } from '@angular/common/http'; import { Observable } from 'rxjs'; import { StateService } from '../services/state.service'; -import { INodesRanking } from '../interfaces/node-api.interface'; +import { INodesRanking, ITopNodesPerCapacity } from '../interfaces/node-api.interface'; @Injectable({ providedIn: 'root' @@ -63,4 +63,10 @@ export class LightningApiService { (interval !== undefined ? `/${interval}` : ''), { observe: 'response' } ); } + + getTopNodesByCapacity$(): Observable { + return this.httpClient.get( + this.apiBasePath + '/api/v1/lightning/nodes/rankings/capacity' + ); + } } diff --git a/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html b/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html index e2e2731d3..3aaeed3dc 100644 --- a/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html +++ b/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html @@ -56,8 +56,8 @@
- -
Top Capacity Nodes
+ +
Top capacity nodes
  @@ -70,7 +70,7 @@
- +
Most connected nodes
  diff --git a/frontend/src/app/lightning/lightning.module.ts b/frontend/src/app/lightning/lightning.module.ts index 85961edc2..48d54c49c 100644 --- a/frontend/src/app/lightning/lightning.module.ts +++ b/frontend/src/app/lightning/lightning.module.ts @@ -24,6 +24,7 @@ import { NodesPerISP } from './nodes-per-isp/nodes-per-isp.component'; import { NodesPerCountryChartComponent } from '../lightning/nodes-per-country-chart/nodes-per-country-chart.component'; import { NodesMap } from '../lightning/nodes-map/nodes-map.component'; import { NodesChannelsMap } from '../lightning/nodes-channels-map/nodes-channels-map.component'; +import { NodesRanking } from '../lightning/nodes-ranking/nodes-ranking.component'; import { TopNodesPerChannels } from '../lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component'; import { TopNodesPerCapacity } from '../lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component'; @NgModule({ @@ -47,6 +48,7 @@ import { TopNodesPerCapacity } from '../lightning/nodes-ranking/top-nodes-per-ca NodesPerCountryChartComponent, NodesMap, NodesChannelsMap, + NodesRanking, TopNodesPerChannels, TopNodesPerCapacity, ], @@ -77,6 +79,7 @@ import { TopNodesPerCapacity } from '../lightning/nodes-ranking/top-nodes-per-ca NodesPerCountryChartComponent, NodesMap, NodesChannelsMap, + NodesRanking, TopNodesPerChannels, TopNodesPerCapacity, ], diff --git a/frontend/src/app/lightning/lightning.routing.module.ts b/frontend/src/app/lightning/lightning.routing.module.ts index cae5c1380..a3885ae7c 100644 --- a/frontend/src/app/lightning/lightning.routing.module.ts +++ b/frontend/src/app/lightning/lightning.routing.module.ts @@ -34,8 +34,11 @@ const routes: Routes = [ component: NodesPerISP, }, { - path: 'nodes/ranking', + path: 'nodes/top-capacity', component: NodesRanking, + data: { + type: 'capacity' + }, }, { path: '**', diff --git a/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.html b/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.html index b0b01d16a..9ca4ed6dc 100644 --- a/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.html +++ b/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.html @@ -1,29 +1,2 @@ -
- - - - - - - - - -
RankAliasPlaceholder
- - - - - - - - - - - - - - - - - -
\ No newline at end of file + + \ No newline at end of file diff --git a/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.ts b/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.ts index b16dc3ce4..373751be9 100644 --- a/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.ts +++ b/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.ts @@ -1,4 +1,5 @@ import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; +import { ActivatedRoute } from '@angular/router'; @Component({ selector: 'app-nodes-ranking', @@ -7,9 +8,13 @@ import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; changeDetection: ChangeDetectionStrategy.OnPush, }) export class NodesRanking implements OnInit { + type: string; + + constructor(private route: ActivatedRoute) {} ngOnInit(): void { - console.log('hi'); + this.route.data.subscribe(data => { + this.type = data.type; + }); } - } diff --git a/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.html b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.html index bb1864ba1..4f84b8134 100644 --- a/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.html +++ b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.html @@ -1,37 +1,59 @@ -
- - - - - - - - - - - - - - - - +
+

+ Top 100 nodes by capacity +

+ +
+
AliasCapacity
- {{ i + 1 }} - - {{ node.alias }} - - -
+ + + + + + + + + + + + + + + - -
AliasCapacityChannelsFirst seenLast updateLocation
- + {{ i + 1 }} - + {{ node.alias }} - + + + {{ node.channels | number }} + + + + + + {{ node?.city?.en ?? '-' }}
+ + + + + + + + + + + + + + + + +
\ No newline at end of file diff --git a/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.scss b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.scss index 4662b72ff..2b927db7f 100644 --- a/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.scss +++ b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.scss @@ -1,8 +1,20 @@ +.container-xl { + max-width: 1400px; + padding-bottom: 100px; + @media (min-width: 767.98px) { + padding-left: 50px; + padding-right: 50px; + } +} + .table td, .table th { padding: 0.5rem; } -.rank { +.full .rank { + width: 5%; +} +.widget .rank { @media (min-width: 767.98px) { width: 13%; } @@ -12,7 +24,16 @@ } } -.alias { +.full .alias { + width: 10%; + overflow: hidden; + text-overflow: ellipsis; + max-width: 350px; + @media (max-width: 767.98px) { + max-width: 175px; + } +} +.widget .alias { width: 55%; overflow: hidden; text-overflow: ellipsis; @@ -21,10 +42,43 @@ max-width: 175px; } } -.capacity { + +.full .capacity { + width: 10%; +} +.widget .capacity { width: 32%; @media (max-width: 767.98px) { padding-left: 0px; padding-right: 0px; } } + +.full .channels { + width: 15%; + padding-right: 50px; + @media (max-width: 767.98px) { + display: none; + } +} + +.full .timestamp-first { + width: 15%; + @media (max-width: 767.98px) { + display: none; + } +} + +.full .timestamp-update { + width: 15%; + @media (max-width: 767.98px) { + display: none; + } +} + +.full .location { + width: 10%; + @media (max-width: 767.98px) { + display: none; + } +} \ No newline at end of file diff --git a/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.ts b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.ts index b3c9171c0..1f72b6974 100644 --- a/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.ts +++ b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.ts @@ -1,6 +1,7 @@ import { ChangeDetectionStrategy, Component, Input, OnInit } from '@angular/core'; import { map, Observable } from 'rxjs'; import { INodesRanking, ITopNodesPerCapacity } from 'src/app/interfaces/node-api.interface'; +import { LightningApiService } from '../../lightning-api.service'; @Component({ selector: 'app-top-nodes-per-capacity', @@ -15,20 +16,24 @@ export class TopNodesPerCapacity implements OnInit { topNodesPerCapacity$: Observable; skeletonRows: number[] = []; + constructor(private apiService: LightningApiService) {} + ngOnInit(): void { for (let i = 1; i <= (this.widget ? 10 : 100); ++i) { this.skeletonRows.push(i); } - this.topNodesPerCapacity$ = this.nodes$.pipe( - map((ranking) => { - if (this.widget === true) { + console.log(this.widget); + + if (this.widget === false) { + this.topNodesPerCapacity$ = this.apiService.getTopNodesByCapacity$(); + } else { + this.topNodesPerCapacity$ = this.nodes$.pipe( + map((ranking) => { return ranking.topByCapacity.slice(0, 10); - } else { - return ranking.topByCapacity; - } - }) - ) + }) + ); + } } } From 6421bc82f2eeb7c44c9c6beb702b46920ae18457 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 17 Aug 2022 16:19:01 +0200 Subject: [PATCH 08/23] Create top 100 node per channel count component --- backend/src/api/explorer/nodes.api.ts | 51 ++++++++--- backend/src/api/explorer/nodes.routes.ts | 15 +++- backend/src/mempool.interfaces.ts | 17 ++-- .../src/app/interfaces/node-api.interface.ts | 9 +- .../app/lightning/lightning-api.service.ts | 8 +- .../app/lightning/lightning.routing.module.ts | 7 ++ .../nodes-ranking.component.html | 5 +- .../top-nodes-per-capacity.component.html | 12 +++ .../top-nodes-per-capacity.component.ts | 2 - .../top-nodes-per-channels.component.html | 90 +++++++++++++------ .../top-nodes-per-channels.component.scss | 64 +++++++++++-- .../top-nodes-per-channels.component.ts | 19 ++-- 12 files changed, 232 insertions(+), 67 deletions(-) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index c7b98f3a8..c0a891a0d 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -2,7 +2,7 @@ import logger from '../../logger'; import DB from '../../database'; import { ResultSetHeader } from 'mysql2'; import { ILightningApi } from '../lightning/lightning-api.interface'; -import { TopNodesPerCapacity, TopNodesPerChannels } from '../../mempool.interfaces'; +import { ITopNodesPerCapacity, ITopNodesPerChannels } from '../../mempool.interfaces'; class NodesApi { public async $getNode(public_key: string): Promise { @@ -113,7 +113,7 @@ class NodesApi { } } - public async $getTopCapacityNodes(full: boolean): Promise { + public async $getTopCapacityNodes(full: boolean): Promise { try { let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats'); const latestDate = rows[0].maxAdded; @@ -161,21 +161,46 @@ class NodesApi { } } - public async $getTopChannelsNodes(): Promise { + public async $getTopChannelsNodes(full: boolean): Promise { try { let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats'); const latestDate = rows[0].maxAdded; - const query = ` - SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, - node_stats.channels - FROM node_stats - JOIN nodes ON nodes.public_key = node_stats.public_key - WHERE added = FROM_UNIXTIME(${latestDate}) - ORDER BY channels DESC - LIMIT 100; - `; - [rows] = await DB.query(query); + let query: string; + if (full === false) { + query = ` + SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, + node_stats.channels + FROM node_stats + JOIN nodes ON nodes.public_key = node_stats.public_key + WHERE added = FROM_UNIXTIME(${latestDate}) + ORDER BY channels DESC + LIMIT 100; + `; + + [rows] = await DB.query(query); + } else { + query = ` + SELECT node_stats.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(node_stats.public_key, 1, 20), alias) as alias, + CAST(COALESCE(node_stats.channels, 0) as INT) as channels, + CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, + UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt, + geo_names_city.names as city, geo_names_country.names as country + FROM node_stats + RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key + LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country' + LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city' + WHERE added = FROM_UNIXTIME(${latestDate}) + ORDER BY channels DESC + LIMIT 100 + `; + + [rows] = await DB.query(query); + for (let i = 0; i < rows.length; ++i) { + rows[i].country = JSON.parse(rows[i].country); + rows[i].city = JSON.parse(rows[i].city); + } + } return rows; } catch (e) { diff --git a/backend/src/api/explorer/nodes.routes.ts b/backend/src/api/explorer/nodes.routes.ts index a490d946b..aaafe679a 100644 --- a/backend/src/api/explorer/nodes.routes.ts +++ b/backend/src/api/explorer/nodes.routes.ts @@ -16,6 +16,7 @@ class NodesRoutes { .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/countries', this.$getNodesCountries) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings', this.$getNodesRanking) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/capacity', this.$getTopNodesByCapacity) + .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/channels', this.$getTopNodesByChannels) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/statistics', this.$getHistoricalNodeStats) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key', this.$getNode) ; @@ -61,7 +62,7 @@ class NodesRoutes { private async $getNodesRanking(req: Request, res: Response): Promise { try { const topCapacityNodes = await nodesApi.$getTopCapacityNodes(false); - const topChannelsNodes = await nodesApi.$getTopChannelsNodes(); + const topChannelsNodes = await nodesApi.$getTopChannelsNodes(false); res.header('Pragma', 'public'); res.header('Cache-control', 'public'); res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); @@ -86,6 +87,18 @@ class NodesRoutes { } } + private async $getTopNodesByChannels(req: Request, res: Response): Promise { + try { + const topCapacityNodes = await nodesApi.$getTopChannelsNodes(true); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + res.json(topCapacityNodes); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + private async $getISPRanking(req: Request, res: Response): Promise { try { const groupBy = req.query.groupBy as string; diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 1067f9b6d..d4f904772 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -252,13 +252,18 @@ export interface RewardStats { totalTx: number; } -export interface TopNodesPerChannels { - public_key: string, +export interface ITopNodesPerChannels { + publicKey: string, alias: string, - channels: number, + channels?: number, + capacity: number, + firstSeen?: number, + updatedAt?: number, + city?: any, + country?: any, } -export interface TopNodesPerCapacity { +export interface ITopNodesPerCapacity { publicKey: string, alias: string, capacity: number, @@ -270,6 +275,6 @@ export interface TopNodesPerCapacity { } export interface INodesRanking { - topByCapacity: TopNodesPerCapacity[]; - topByChannels: TopNodesPerChannels[]; + topByCapacity: ITopNodesPerCapacity[]; + topByChannels: ITopNodesPerChannels[]; } \ No newline at end of file diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 4238be7bb..2d1ff8f43 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -153,9 +153,14 @@ export interface RewardStats { } export interface ITopNodesPerChannels { - public_key: string, + publicKey: string, alias: string, - channels: number, + channels?: number, + capacity: number, + firstSeen?: number, + updatedAt?: number, + city?: any, + country?: any, } export interface ITopNodesPerCapacity { diff --git a/frontend/src/app/lightning/lightning-api.service.ts b/frontend/src/app/lightning/lightning-api.service.ts index 49cc7b61e..1aacf7b89 100644 --- a/frontend/src/app/lightning/lightning-api.service.ts +++ b/frontend/src/app/lightning/lightning-api.service.ts @@ -2,7 +2,7 @@ import { Injectable } from '@angular/core'; import { HttpClient, HttpParams } from '@angular/common/http'; import { Observable } from 'rxjs'; import { StateService } from '../services/state.service'; -import { INodesRanking, ITopNodesPerCapacity } from '../interfaces/node-api.interface'; +import { INodesRanking, ITopNodesPerCapacity, ITopNodesPerChannels } from '../interfaces/node-api.interface'; @Injectable({ providedIn: 'root' @@ -69,4 +69,10 @@ export class LightningApiService { this.apiBasePath + '/api/v1/lightning/nodes/rankings/capacity' ); } + + getTopNodesByChannels$(): Observable { + return this.httpClient.get( + this.apiBasePath + '/api/v1/lightning/nodes/rankings/channels' + ); + } } diff --git a/frontend/src/app/lightning/lightning.routing.module.ts b/frontend/src/app/lightning/lightning.routing.module.ts index a3885ae7c..9e9405234 100644 --- a/frontend/src/app/lightning/lightning.routing.module.ts +++ b/frontend/src/app/lightning/lightning.routing.module.ts @@ -40,6 +40,13 @@ const routes: Routes = [ type: 'capacity' }, }, + { + path: 'nodes/top-channels', + component: NodesRanking, + data: { + type: 'channels' + }, + }, { path: '**', redirectTo: '' diff --git a/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.html b/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.html index 9ca4ed6dc..1720e9be6 100644 --- a/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.html +++ b/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.html @@ -1,2 +1,5 @@ - \ No newline at end of file + + + + \ No newline at end of file diff --git a/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.html b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.html index 4f84b8134..a9043f9d1 100644 --- a/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.html +++ b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.html @@ -51,6 +51,18 @@ + + + + + + + + + + + + diff --git a/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.ts b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.ts index 1f72b6974..2d90817f9 100644 --- a/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.ts +++ b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component.ts @@ -23,8 +23,6 @@ export class TopNodesPerCapacity implements OnInit { this.skeletonRows.push(i); } - console.log(this.widget); - if (this.widget === false) { this.topNodesPerCapacity$ = this.apiService.getTopNodesByCapacity$(); } else { diff --git a/frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.html b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.html index f581dacff..72c2aa5b1 100644 --- a/frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.html +++ b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.html @@ -1,37 +1,71 @@ -
- - - - - - - - - - - - - - - - +
+

+ Top 100 nodes by channel count +

+ +
+
AliasChannels
- {{ i + 1 }} - - {{ node.alias }} - - {{ node.channels | number }} -
+ + + + + + + + + + + + + + + - -
AliasChannelsCapacityFirst seenLast updateLocation
- + {{ i + 1 }} - + {{ node.alias }} - + {{ node.channels | number }} + + + + + + + + {{ node?.city?.en ?? '-' }}
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
\ No newline at end of file diff --git a/frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.scss b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.scss index 5335991e3..5af0e8339 100644 --- a/frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.scss +++ b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.scss @@ -1,8 +1,20 @@ +.container-xl { + max-width: 1400px; + padding-bottom: 100px; + @media (min-width: 767.98px) { + padding-left: 50px; + padding-right: 50px; + } +} + .table td, .table th { padding: 0.5rem; } -.rank { +.full .rank { + width: 5%; +} +.widget .rank { @media (min-width: 767.98px) { width: 13%; } @@ -12,8 +24,8 @@ } } -.alias { - width: 60%; +.full .alias { + width: 10%; overflow: hidden; text-overflow: ellipsis; max-width: 350px; @@ -21,10 +33,52 @@ max-width: 175px; } } -.channels { - width: 27%; +.widget .alias { + width: 55%; + overflow: hidden; + text-overflow: ellipsis; + max-width: 350px; + @media (max-width: 767.98px) { + max-width: 175px; + } +} + +.full .channels { + width: 10%; +} +.widget .channels { + width: 32%; @media (max-width: 767.98px) { padding-left: 0px; padding-right: 0px; } } + +.full .capacity { + width: 15%; + padding-right: 50px; + @media (max-width: 767.98px) { + display: none; + } +} + +.full .timestamp-first { + width: 15%; + @media (max-width: 767.98px) { + display: none; + } +} + +.full .timestamp-update { + width: 15%; + @media (max-width: 767.98px) { + display: none; + } +} + +.full .location { + width: 10%; + @media (max-width: 767.98px) { + display: none; + } +} \ No newline at end of file diff --git a/frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.ts b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.ts index 46cf21eaa..c2821c596 100644 --- a/frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.ts +++ b/frontend/src/app/lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component.ts @@ -1,6 +1,7 @@ import { ChangeDetectionStrategy, Component, Input, OnInit } from '@angular/core'; import { map, Observable } from 'rxjs'; import { INodesRanking, ITopNodesPerChannels } from 'src/app/interfaces/node-api.interface'; +import { LightningApiService } from '../../lightning-api.service'; @Component({ selector: 'app-top-nodes-per-channels', @@ -15,20 +16,22 @@ export class TopNodesPerChannels implements OnInit { topNodesPerChannels$: Observable; skeletonRows: number[] = []; + constructor(private apiService: LightningApiService) {} + ngOnInit(): void { for (let i = 1; i <= (this.widget ? 10 : 100); ++i) { this.skeletonRows.push(i); } - this.topNodesPerChannels$ = this.nodes$.pipe( - map((ranking) => { - if (this.widget === true) { + if (this.widget === false) { + this.topNodesPerChannels$ = this.apiService.getTopNodesByChannels$(); + } else { + this.topNodesPerChannels$ = this.nodes$.pipe( + map((ranking) => { return ranking.topByChannels.slice(0, 10); - } else { - return ranking.topByChannels; - } - }) - ) + }) + ); + } } } From 9c8fd6431e7de902cb7944c94f92fe54299dab73 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 17 Aug 2022 21:20:11 +0200 Subject: [PATCH 09/23] Create node rankings page with 3 different rankings --- backend/src/api/explorer/nodes.api.ts | 48 +++++++++++ backend/src/api/explorer/nodes.routes.ts | 13 +++ backend/src/mempool.interfaces.ts | 11 +++ .../src/app/interfaces/node-api.interface.ts | 11 +++ .../app/lightning/lightning-api.service.ts | 8 +- .../lightning-dashboard.component.html | 4 +- .../src/app/lightning/lightning.module.ts | 7 ++ .../app/lightning/lightning.routing.module.ts | 5 ++ .../oldest-nodes/oldest-nodes.component.html | 71 ++++++++++++++++ .../oldest-nodes/oldest-nodes.component.scss | 84 +++++++++++++++++++ .../oldest-nodes/oldest-nodes.component.ts | 36 ++++++++ .../nodes-rankings-dashboard.component.html | 47 +++++++++++ .../nodes-rankings-dashboard.component.scss | 33 ++++++++ .../nodes-rankings-dashboard.component.ts | 25 ++++++ 14 files changed, 401 insertions(+), 2 deletions(-) create mode 100644 frontend/src/app/lightning/nodes-ranking/oldest-nodes/oldest-nodes.component.html create mode 100644 frontend/src/app/lightning/nodes-ranking/oldest-nodes/oldest-nodes.component.scss create mode 100644 frontend/src/app/lightning/nodes-ranking/oldest-nodes/oldest-nodes.component.ts create mode 100644 frontend/src/app/lightning/nodes-rankings-dashboard/nodes-rankings-dashboard.component.html create mode 100644 frontend/src/app/lightning/nodes-rankings-dashboard/nodes-rankings-dashboard.component.scss create mode 100644 frontend/src/app/lightning/nodes-rankings-dashboard/nodes-rankings-dashboard.component.ts diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index c0a891a0d..339364374 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -209,6 +209,54 @@ class NodesApi { } } + public async $getOldestNodes(full: boolean): Promise { + try { + let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats'); + const latestDate = rows[0].maxAdded; + + let query: string; + if (full === false) { + query = ` + SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, + node_stats.channels + FROM node_stats + JOIN nodes ON nodes.public_key = node_stats.public_key + WHERE added = FROM_UNIXTIME(${latestDate}) + ORDER BY first_seen + LIMIT 100; + `; + + [rows] = await DB.query(query); + } else { + query = ` + SELECT node_stats.public_key AS publicKey, IF(nodes.alias = '', SUBSTRING(node_stats.public_key, 1, 20), alias) as alias, + CAST(COALESCE(node_stats.channels, 0) as INT) as channels, + CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, + UNIX_TIMESTAMP(nodes.first_seen) as firstSeen, UNIX_TIMESTAMP(nodes.updated_at) as updatedAt, + geo_names_city.names as city, geo_names_country.names as country + FROM node_stats + RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key + LEFT JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country' + LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city' + WHERE added = FROM_UNIXTIME(${latestDate}) + ORDER BY first_seen + LIMIT 100 + `; + + [rows] = await DB.query(query); + for (let i = 0; i < rows.length; ++i) { + rows[i].country = JSON.parse(rows[i].country); + rows[i].city = JSON.parse(rows[i].city); + } + } + + return rows; + } catch (e) { + logger.err('$getTopChannelsNodes error: ' + (e instanceof Error ? e.message : e)); + throw e; + } + } + public async $searchNodeByPublicKeyOrAlias(search: string) { try { const searchStripped = search.replace('%', '') + '%'; diff --git a/backend/src/api/explorer/nodes.routes.ts b/backend/src/api/explorer/nodes.routes.ts index aaafe679a..482f01f0f 100644 --- a/backend/src/api/explorer/nodes.routes.ts +++ b/backend/src/api/explorer/nodes.routes.ts @@ -17,6 +17,7 @@ class NodesRoutes { .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings', this.$getNodesRanking) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/capacity', this.$getTopNodesByCapacity) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/channels', this.$getTopNodesByChannels) + .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/rankings/age', this.$getOldestNodes) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/statistics', this.$getHistoricalNodeStats) .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key', this.$getNode) ; @@ -99,6 +100,18 @@ class NodesRoutes { } } + private async $getOldestNodes(req: Request, res: Response): Promise { + try { + const topCapacityNodes = await nodesApi.$getOldestNodes(true); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + res.json(topCapacityNodes); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + private async $getISPRanking(req: Request, res: Response): Promise { try { const groupBy = req.query.groupBy as string; diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index d4f904772..d72b13576 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -277,4 +277,15 @@ export interface ITopNodesPerCapacity { export interface INodesRanking { topByCapacity: ITopNodesPerCapacity[]; topByChannels: ITopNodesPerChannels[]; +} + +export interface IOldestNodes { + publicKey: string, + alias: string, + firstSeen: number, + channels?: number, + capacity: number, + updatedAt?: number, + city?: any, + country?: any, } \ No newline at end of file diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 2d1ff8f43..838208cc3 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -177,4 +177,15 @@ export interface ITopNodesPerCapacity { export interface INodesRanking { topByCapacity: ITopNodesPerCapacity[]; topByChannels: ITopNodesPerChannels[]; +} + +export interface IOldestNodes { + publicKey: string, + alias: string, + firstSeen: number, + channels?: number, + capacity: number, + updatedAt?: number, + city?: any, + country?: any, } \ No newline at end of file diff --git a/frontend/src/app/lightning/lightning-api.service.ts b/frontend/src/app/lightning/lightning-api.service.ts index 1aacf7b89..f14b0b382 100644 --- a/frontend/src/app/lightning/lightning-api.service.ts +++ b/frontend/src/app/lightning/lightning-api.service.ts @@ -2,7 +2,7 @@ import { Injectable } from '@angular/core'; import { HttpClient, HttpParams } from '@angular/common/http'; import { Observable } from 'rxjs'; import { StateService } from '../services/state.service'; -import { INodesRanking, ITopNodesPerCapacity, ITopNodesPerChannels } from '../interfaces/node-api.interface'; +import { INodesRanking, IOldestNodes, ITopNodesPerCapacity, ITopNodesPerChannels } from '../interfaces/node-api.interface'; @Injectable({ providedIn: 'root' @@ -75,4 +75,10 @@ export class LightningApiService { this.apiBasePath + '/api/v1/lightning/nodes/rankings/channels' ); } + + getOldestNodes$(): Observable { + return this.httpClient.get( + this.apiBasePath + '/api/v1/lightning/nodes/rankings/age' + ); + } } diff --git a/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html b/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html index 3aaeed3dc..fe678a2fc 100644 --- a/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html +++ b/frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html @@ -42,6 +42,7 @@
+
@@ -53,6 +54,7 @@
+
@@ -61,12 +63,12 @@   -
+
diff --git a/frontend/src/app/lightning/lightning.module.ts b/frontend/src/app/lightning/lightning.module.ts index 48d54c49c..7ca02b2ba 100644 --- a/frontend/src/app/lightning/lightning.module.ts +++ b/frontend/src/app/lightning/lightning.module.ts @@ -27,6 +27,9 @@ import { NodesChannelsMap } from '../lightning/nodes-channels-map/nodes-channels import { NodesRanking } from '../lightning/nodes-ranking/nodes-ranking.component'; import { TopNodesPerChannels } from '../lightning/nodes-ranking/top-nodes-per-channels/top-nodes-per-channels.component'; import { TopNodesPerCapacity } from '../lightning/nodes-ranking/top-nodes-per-capacity/top-nodes-per-capacity.component'; +import { OldestNodes } from '../lightning/nodes-ranking/oldest-nodes/oldest-nodes.component'; +import { NodesRankingsDashboard } from '../lightning/nodes-rankings-dashboard/nodes-rankings-dashboard.component'; + @NgModule({ declarations: [ LightningDashboardComponent, @@ -51,6 +54,8 @@ import { TopNodesPerCapacity } from '../lightning/nodes-ranking/top-nodes-per-ca NodesRanking, TopNodesPerChannels, TopNodesPerCapacity, + OldestNodes, + NodesRankingsDashboard, ], imports: [ CommonModule, @@ -82,6 +87,8 @@ import { TopNodesPerCapacity } from '../lightning/nodes-ranking/top-nodes-per-ca NodesRanking, TopNodesPerChannels, TopNodesPerCapacity, + OldestNodes, + NodesRankingsDashboard, ], providers: [ LightningApiService, diff --git a/frontend/src/app/lightning/lightning.routing.module.ts b/frontend/src/app/lightning/lightning.routing.module.ts index 9e9405234..56734ea7b 100644 --- a/frontend/src/app/lightning/lightning.routing.module.ts +++ b/frontend/src/app/lightning/lightning.routing.module.ts @@ -7,6 +7,7 @@ import { ChannelComponent } from './channel/channel.component'; import { NodesPerCountry } from './nodes-per-country/nodes-per-country.component'; import { NodesPerISP } from './nodes-per-isp/nodes-per-isp.component'; import { NodesRanking } from './nodes-ranking/nodes-ranking.component'; +import { NodesRankingsDashboard } from './nodes-rankings-dashboard/nodes-rankings-dashboard.component'; const routes: Routes = [ { @@ -33,6 +34,10 @@ const routes: Routes = [ path: 'nodes/isp/:isp', component: NodesPerISP, }, + { + path: 'nodes/rankings', + component: NodesRankingsDashboard, + }, { path: 'nodes/top-capacity', component: NodesRanking, diff --git a/frontend/src/app/lightning/nodes-ranking/oldest-nodes/oldest-nodes.component.html b/frontend/src/app/lightning/nodes-ranking/oldest-nodes/oldest-nodes.component.html new file mode 100644 index 000000000..ef18f9b72 --- /dev/null +++ b/frontend/src/app/lightning/nodes-ranking/oldest-nodes/oldest-nodes.component.html @@ -0,0 +1,71 @@ +
+

+ Top 100 nodes by capacity +

+ +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
AliasFirst seenCapacityChannelsLast updateLocation
+ {{ i + 1 }} + + {{ node.alias }} + + ‎{{ node.firstSeen * 1000 | date: 'yyyy-MM-dd' }} + + + + {{ node.channels | number }} + + + + {{ node?.city?.en ?? '-' }} +
+ + + + + + + + + + + + + +
+
+
\ No newline at end of file diff --git a/frontend/src/app/lightning/nodes-ranking/oldest-nodes/oldest-nodes.component.scss b/frontend/src/app/lightning/nodes-ranking/oldest-nodes/oldest-nodes.component.scss new file mode 100644 index 000000000..f69a3ba67 --- /dev/null +++ b/frontend/src/app/lightning/nodes-ranking/oldest-nodes/oldest-nodes.component.scss @@ -0,0 +1,84 @@ +.container-xl { + max-width: 1400px; + padding-bottom: 100px; + @media (min-width: 767.98px) { + padding-left: 50px; + padding-right: 50px; + } +} + +.table td, .table th { + padding: 0.5rem; +} + +.full .rank { + width: 5%; +} +.widget .rank { + @media (min-width: 767.98px) { + width: 13%; + } + @media (max-width: 767.98px) { + padding-left: 0px; + padding-right: 0px; + } +} + +.full .alias { + width: 10%; + overflow: hidden; + text-overflow: ellipsis; + max-width: 350px; + @media (max-width: 767.98px) { + max-width: 175px; + } +} +.widget .alias { + width: 50%; + overflow: hidden; + text-overflow: ellipsis; + max-width: 300px; + @media (max-width: 767.98px) { + max-width: 170px; + } +} + +.full .capacity { + width: 10%; +} +.widget .capacity { + width: 32%; + @media (max-width: 767.98px) { + padding-left: 0px; + padding-right: 0px; + } +} + +.full .channels { + width: 15%; + padding-right: 50px; + @media (max-width: 767.98px) { + display: none; + } +} + +.full .timestamp-first { + width: 50%; + @media (max-width: 767.98px) { + display: none; + } +} + +.full .timestamp-update { + width: 15%; + @media (max-width: 767.98px) { + display: none; + } +} + +.full .location { + width: 10%; + @media (max-width: 767.98px) { + display: none; + } +} \ No newline at end of file diff --git a/frontend/src/app/lightning/nodes-ranking/oldest-nodes/oldest-nodes.component.ts b/frontend/src/app/lightning/nodes-ranking/oldest-nodes/oldest-nodes.component.ts new file mode 100644 index 000000000..23f248b0e --- /dev/null +++ b/frontend/src/app/lightning/nodes-ranking/oldest-nodes/oldest-nodes.component.ts @@ -0,0 +1,36 @@ +import { ChangeDetectionStrategy, Component, Input, OnInit } from '@angular/core'; +import { map, Observable } from 'rxjs'; +import { IOldestNodes } from '../../../interfaces/node-api.interface'; +import { LightningApiService } from '../../lightning-api.service'; + +@Component({ + selector: 'app-oldest-nodes', + templateUrl: './oldest-nodes.component.html', + styleUrls: ['./oldest-nodes.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class OldestNodes implements OnInit { + @Input() widget: boolean = false; + + oldestNodes$: Observable; + skeletonRows: number[] = []; + + constructor(private apiService: LightningApiService) {} + + ngOnInit(): void { + for (let i = 1; i <= (this.widget ? 10 : 100); ++i) { + this.skeletonRows.push(i); + } + + if (this.widget === false) { + this.oldestNodes$ = this.apiService.getOldestNodes$(); + } else { + this.oldestNodes$ = this.apiService.getOldestNodes$().pipe( + map((nodes: IOldestNodes[]) => { + return nodes.slice(0, 10); + }) + ); + } + } + +} diff --git a/frontend/src/app/lightning/nodes-rankings-dashboard/nodes-rankings-dashboard.component.html b/frontend/src/app/lightning/nodes-rankings-dashboard/nodes-rankings-dashboard.component.html new file mode 100644 index 000000000..0359244f3 --- /dev/null +++ b/frontend/src/app/lightning/nodes-rankings-dashboard/nodes-rankings-dashboard.component.html @@ -0,0 +1,47 @@ +
+
+ +
+
+
+ +
Top capacity nodes
+   + + + +
+
+
+ +
+
+
+ +
Most connected nodes
+   + + + +
+
+
+ +
+
+
+ +
Oldest nodes
+   + + + +
+
+
+ +
+
\ No newline at end of file diff --git a/frontend/src/app/lightning/nodes-rankings-dashboard/nodes-rankings-dashboard.component.scss b/frontend/src/app/lightning/nodes-rankings-dashboard/nodes-rankings-dashboard.component.scss new file mode 100644 index 000000000..28e80d451 --- /dev/null +++ b/frontend/src/app/lightning/nodes-rankings-dashboard/nodes-rankings-dashboard.component.scss @@ -0,0 +1,33 @@ +.main { + max-width: 90%; +} + +.col { + padding-bottom: 20px; + padding-left: 10px; + padding-right: 10px; +} + +.card { + background-color: #1d1f31; +} + +.card-title { + font-size: 1rem; + color: #4a68b9; +} +.card-title > a { + color: #4a68b9; +} + +.card-text { + font-size: 22px; +} + +.title-link, .title-link:hover, .title-link:focus, .title-link:active { + text-align: center; + display: block; + margin-bottom: 10px; + text-decoration: none; + color: inherit; +} \ No newline at end of file diff --git a/frontend/src/app/lightning/nodes-rankings-dashboard/nodes-rankings-dashboard.component.ts b/frontend/src/app/lightning/nodes-rankings-dashboard/nodes-rankings-dashboard.component.ts new file mode 100644 index 000000000..4b39d8467 --- /dev/null +++ b/frontend/src/app/lightning/nodes-rankings-dashboard/nodes-rankings-dashboard.component.ts @@ -0,0 +1,25 @@ +import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; +import { Observable, share } from 'rxjs'; +import { INodesRanking } from 'src/app/interfaces/node-api.interface'; +import { SeoService } from 'src/app/services/seo.service'; +import { LightningApiService } from '../lightning-api.service'; + +@Component({ + selector: 'app-nodes-rankings-dashboard', + templateUrl: './nodes-rankings-dashboard.component.html', + styleUrls: ['./nodes-rankings-dashboard.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class NodesRankingsDashboard implements OnInit { + nodesRanking$: Observable; + + constructor( + private lightningApiService: LightningApiService, + private seoService: SeoService, + ) {} + + ngOnInit(): void { + this.seoService.setTitle($localize`Top lightning nodes`); + this.nodesRanking$ = this.lightningApiService.getNodesRanking$().pipe(share()); + } +} From 350aedd934b95e498acf077543b52424218510d3 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 17 Aug 2022 21:29:04 +0200 Subject: [PATCH 10/23] Create top 100 oldest nodes full page --- .../src/app/lightning/lightning.routing.module.ts | 7 +++++++ .../nodes-ranking/nodes-ranking.component.html | 4 +++- .../oldest-nodes/oldest-nodes.component.html | 8 ++++---- .../oldest-nodes/oldest-nodes.component.scss | 12 ++++++------ .../nodes-rankings-dashboard.component.html | 2 +- 5 files changed, 21 insertions(+), 12 deletions(-) diff --git a/frontend/src/app/lightning/lightning.routing.module.ts b/frontend/src/app/lightning/lightning.routing.module.ts index 56734ea7b..13fd20b6a 100644 --- a/frontend/src/app/lightning/lightning.routing.module.ts +++ b/frontend/src/app/lightning/lightning.routing.module.ts @@ -52,6 +52,13 @@ const routes: Routes = [ type: 'channels' }, }, + { + path: 'nodes/oldest', + component: NodesRanking, + data: { + type: 'oldest' + }, + }, { path: '**', redirectTo: '' diff --git a/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.html b/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.html index 1720e9be6..5bd03941e 100644 --- a/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.html +++ b/frontend/src/app/lightning/nodes-ranking/nodes-ranking.component.html @@ -2,4 +2,6 @@ - \ No newline at end of file + + + diff --git a/frontend/src/app/lightning/nodes-ranking/oldest-nodes/oldest-nodes.component.html b/frontend/src/app/lightning/nodes-ranking/oldest-nodes/oldest-nodes.component.html index ef18f9b72..5b96400c2 100644 --- a/frontend/src/app/lightning/nodes-ranking/oldest-nodes/oldest-nodes.component.html +++ b/frontend/src/app/lightning/nodes-ranking/oldest-nodes/oldest-nodes.component.html @@ -1,6 +1,6 @@
-

- Top 100 nodes by capacity +

+ Top 100 oldest lightning nodes

@@ -8,7 +8,7 @@ Alias - First seen + First seen Capacity Channels Last update @@ -22,7 +22,7 @@ {{ node.alias }} - + ‎{{ node.firstSeen * 1000 | date: 'yyyy-MM-dd' }} diff --git a/frontend/src/app/lightning/nodes-ranking/oldest-nodes/oldest-nodes.component.scss b/frontend/src/app/lightning/nodes-ranking/oldest-nodes/oldest-nodes.component.scss index f69a3ba67..5f77ab41b 100644 --- a/frontend/src/app/lightning/nodes-ranking/oldest-nodes/oldest-nodes.component.scss +++ b/frontend/src/app/lightning/nodes-ranking/oldest-nodes/oldest-nodes.component.scss @@ -45,9 +45,12 @@ .full .capacity { width: 10%; + @media (max-width: 767.98px) { + display: none; + } } .widget .capacity { - width: 32%; + width: 10%; @media (max-width: 767.98px) { padding-left: 0px; padding-right: 0px; @@ -63,14 +66,11 @@ } .full .timestamp-first { - width: 50%; - @media (max-width: 767.98px) { - display: none; - } + width: 10%; } .full .timestamp-update { - width: 15%; + width: 20%; @media (max-width: 767.98px) { display: none; } diff --git a/frontend/src/app/lightning/nodes-rankings-dashboard/nodes-rankings-dashboard.component.html b/frontend/src/app/lightning/nodes-rankings-dashboard/nodes-rankings-dashboard.component.html index 0359244f3..93f7d1fc3 100644 --- a/frontend/src/app/lightning/nodes-rankings-dashboard/nodes-rankings-dashboard.component.html +++ b/frontend/src/app/lightning/nodes-rankings-dashboard/nodes-rankings-dashboard.component.html @@ -32,7 +32,7 @@
- +
Oldest nodes
  Date: Thu, 18 Aug 2022 07:48:58 +0200 Subject: [PATCH 11/23] Wrap LN importer into try/catch --- .../lightning/sync-tasks/stats-importer.ts | 138 +++++++++--------- 1 file changed, 71 insertions(+), 67 deletions(-) diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index 5878f898a..7e00f5b07 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -262,82 +262,86 @@ class LightningStatsImporter { * Import topology files LN historical data into the database */ async $importHistoricalLightningStats(): Promise { - const fileList = await fsPromises.readdir(this.topologiesFolder); - // Insert history from the most recent to the oldest - // This also put the .json cached files first - fileList.sort().reverse(); + try { + const fileList = await fsPromises.readdir(this.topologiesFolder); + // Insert history from the most recent to the oldest + // This also put the .json cached files first + fileList.sort().reverse(); - const [rows]: any[] = await DB.query(` - SELECT UNIX_TIMESTAMP(added) AS added, node_count - FROM lightning_stats - ORDER BY added DESC - `); - const existingStatsTimestamps = {}; - for (const row of rows) { - existingStatsTimestamps[row.added] = row; - } - - // For logging purpose - let processed = 10; - let totalProcessed = 0; - let logStarted = false; - - for (const filename of fileList) { - processed++; - - const timestamp = parseInt(filename.split('_')[1], 10); - - // Stats exist already, don't calculate/insert them - if (existingStatsTimestamps[timestamp] !== undefined) { - continue; + const [rows]: any[] = await DB.query(` + SELECT UNIX_TIMESTAMP(added) AS added, node_count + FROM lightning_stats + ORDER BY added DESC + `); + const existingStatsTimestamps = {}; + for (const row of rows) { + existingStatsTimestamps[row.added] = row; } - if (filename.indexOf('.topology') === -1) { - continue; - } + // For logging purpose + let processed = 10; + let totalProcessed = 0; + let logStarted = false; - logger.debug(`Reading ${this.topologiesFolder}/${filename}`); - let fileContent = ''; - try { - fileContent = await fsPromises.readFile(`${this.topologiesFolder}/${filename}`, 'utf8'); - } catch (e: any) { - if (e.errno == -1) { // EISDIR - Ignore directorie + for (const filename of fileList) { + processed++; + + const timestamp = parseInt(filename.split('_')[1], 10); + + // Stats exist already, don't calculate/insert them + if (existingStatsTimestamps[timestamp] !== undefined) { continue; } + + if (filename.indexOf('.topology') === -1) { + continue; + } + + logger.debug(`Reading ${this.topologiesFolder}/${filename}`); + let fileContent = ''; + try { + fileContent = await fsPromises.readFile(`${this.topologiesFolder}/${filename}`, 'utf8'); + } catch (e: any) { + if (e.errno == -1) { // EISDIR - Ignore directorie + continue; + } + } + + let graph; + try { + graph = JSON.parse(fileContent); + } catch (e) { + logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`); + continue; + } + + if (!logStarted) { + logger.info(`Founds a topology file that we did not import. Importing historical lightning stats now.`); + logStarted = true; + } + + const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`; + logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`); + + totalProcessed++; + + if (processed > 10) { + logger.info(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`); + processed = 0; + } else { + logger.debug(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`); + } + await fundingTxFetcher.$fetchChannelsFundingTxs(graph.edges.map(channel => channel.channel_id.slice(0, -2))); + const stat = await this.computeNetworkStats(timestamp, graph); + + existingStatsTimestamps[timestamp] = stat; } - let graph; - try { - graph = JSON.parse(fileContent); - } catch (e) { - logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`); - continue; + if (totalProcessed > 0) { + logger.info(`Lightning network stats historical import completed`); } - - if (!logStarted) { - logger.info(`Founds a topology file that we did not import. Importing historical lightning stats now.`); - logStarted = true; - } - - const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`; - logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`); - - totalProcessed++; - - if (processed > 10) { - logger.info(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`); - processed = 0; - } else { - logger.debug(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`); - } - await fundingTxFetcher.$fetchChannelsFundingTxs(graph.edges.map(channel => channel.channel_id.slice(0, -2))); - const stat = await this.computeNetworkStats(timestamp, graph); - - existingStatsTimestamps[timestamp] = stat; - } - - if (totalProcessed > 0) { - logger.info(`Lightning network stats historical import completed`); + } catch (e) { + logger.err(`Lightning network stats historical failed. Reason: ${e instanceof Error ? e.message : e}`); } } } From 57e0980134d130a20dfc3626600f3d44facae077 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 18 Aug 2022 10:59:03 +0200 Subject: [PATCH 12/23] Import json topology --- backend/src/api/common.ts | 4 + .../sync-tasks/funding-tx-fetcher.ts | 4 +- .../lightning/sync-tasks/stats-importer.ts | 89 +++++++++++++------ 3 files changed, 66 insertions(+), 31 deletions(-) diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index fe12e0f40..8635ee96f 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -207,6 +207,10 @@ export class Common { /** Decodes a channel id returned by lnd as uint64 to a short channel id */ static channelIntegerIdToShortId(id: string): string { + if (id.indexOf('/') !== -1) { + id = id.slice(0, -2); + } + if (id.indexOf('x') !== -1) { // Already a short id return id; } diff --git a/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts b/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts index 9dbc21c72..76865dc40 100644 --- a/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts +++ b/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts @@ -71,9 +71,7 @@ class FundingTxFetcher { } public async $fetchChannelOpenTx(channelId: string): Promise<{timestamp: number, txid: string, value: number}> { - if (channelId.indexOf('x') === -1) { - channelId = Common.channelIntegerIdToShortId(channelId); - } + channelId = Common.channelIntegerIdToShortId(channelId); if (this.fundingTxCache[channelId]) { return this.fundingTxCache[channelId]; diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index 7e00f5b07..7879ec676 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -8,30 +8,6 @@ import { isIP } from 'net'; const fsPromises = promises; -interface Node { - id: string; - timestamp: number; - features: string; - rgb_color: string; - alias: string; - addresses: unknown[]; - out_degree: number; - in_degree: number; -} - -interface Channel { - channel_id: string; - node1_pub: string; - node2_pub: string; - timestamp: number; - features: string; - fee_base_msat: number; - fee_rate_milli_msat: number; - htlc_minimim_msat: number; - cltv_expiry_delta: number; - htlc_maximum_msat: number; -} - class LightningStatsImporter { topologiesFolder = config.LIGHTNING.TOPOLOGY_FOLDER; @@ -59,11 +35,11 @@ class LightningStatsImporter { let isUnnanounced = true; for (const socket of (node.addresses ?? [])) { - if (!socket.network?.length || !socket.addr?.length) { + if (!socket.network?.length && !socket.addr?.length) { continue; } - hasOnion = hasOnion || ['torv2', 'torv3'].includes(socket.network) || socket.addr.indexOf('onion') !== -1; - hasClearnet = hasClearnet || ['ipv4', 'ipv6'].includes(socket.network) || [4, 6].includes(isIP(socket.addr.split(':')[0])); + hasOnion = hasOnion || ['torv2', 'torv3'].includes(socket.network) || socket.addr.indexOf('onion') !== -1 || socket.addr.indexOf('torv2') !== -1 || socket.addr.indexOf('torv3') !== -1; + hasClearnet = hasClearnet || ['ipv4', 'ipv6'].includes(socket.network) || [4, 6].includes(isIP(socket.addr.split(':')[0])) || socket.addr.indexOf('ipv4') !== -1 || socket.addr.indexOf('ipv6') !== -1;; } if (hasOnion && hasClearnet) { clearnetTorNodes++; @@ -293,7 +269,7 @@ class LightningStatsImporter { continue; } - if (filename.indexOf('.topology') === -1) { + if (filename.indexOf('topology_') === -1) { continue; } @@ -310,6 +286,7 @@ class LightningStatsImporter { let graph; try { graph = JSON.parse(fileContent); + graph = await this.cleanupTopology(graph); } catch (e) { logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`); continue; @@ -344,6 +321,62 @@ class LightningStatsImporter { logger.err(`Lightning network stats historical failed. Reason: ${e instanceof Error ? e.message : e}`); } } + + async cleanupTopology(graph) { + const newGraph = { + nodes: [], + edges: [], + }; + + for (const node of graph.nodes) { + const addressesParts = (node.addresses ?? '').split(','); + const addresses: any[] = []; + for (const address of addressesParts) { + addresses.push({ + network: '', + addr: address + }); + } + + newGraph.nodes.push({ + last_update: node.timestamp ?? 0, + pub_key: node.id ?? null, + alias: node.alias ?? null, + addresses: addresses, + color: node.rgb_color ?? null, + features: {}, + }); + } + + for (const adjacency of graph.adjacency) { + if (adjacency.length === 0) { + continue; + } else { + for (const edge of adjacency) { + newGraph.edges.push({ + channel_id: edge.scid, + chan_point: '', + last_update: edge.timestamp, + node1_pub: edge.source ?? null, + node2_pub: edge.destination ?? null, + capacity: '0', // Will be fetch later + node1_policy: { + time_lock_delta: edge.cltv_expiry_delta, + min_htlc: edge.htlc_minimim_msat, + fee_base_msat: edge.fee_base_msat, + fee_rate_milli_msat: edge.fee_proportional_millionths, + max_htlc_msat: edge.htlc_maximum_msat, + last_update: edge.timestamp, + disabled: false, + }, + node2_policy: null, + }); + } + } + } + + return newGraph; + } } export default new LightningStatsImporter; From 0243769a0294a05d17b513e25d98621e08e966b2 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Thu, 18 Aug 2022 11:14:34 +0200 Subject: [PATCH 13/23] Improve error logging in ln import --- .../src/tasks/lightning/sync-tasks/stats-importer.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index 7879ec676..141f4d1a3 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -239,7 +239,13 @@ class LightningStatsImporter { */ async $importHistoricalLightningStats(): Promise { try { - const fileList = await fsPromises.readdir(this.topologiesFolder); + let fileList: string[] = []; + try { + fileList = await fsPromises.readdir(this.topologiesFolder); + } catch (e) { + logger.err(`Unable to open topology folder at ${this.topologiesFolder}`); + throw e; + } // Insert history from the most recent to the oldest // This also put the .json cached files first fileList.sort().reverse(); @@ -281,6 +287,8 @@ class LightningStatsImporter { if (e.errno == -1) { // EISDIR - Ignore directorie continue; } + logger.err(`Unable to open ${this.topologiesFolder}/${filename}`); + continue; } let graph; From 5d22023d192b3b5b9bfcaa5186eb5ba42cd3d0c3 Mon Sep 17 00:00:00 2001 From: junderw Date: Fri, 19 Aug 2022 01:05:02 +0900 Subject: [PATCH 14/23] Fix times Co-authored-by: slaninas --- backend/src/api/difficulty-adjustment.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/api/difficulty-adjustment.ts b/backend/src/api/difficulty-adjustment.ts index 1f85fdb80..6d188266c 100644 --- a/backend/src/api/difficulty-adjustment.ts +++ b/backend/src/api/difficulty-adjustment.ts @@ -47,8 +47,8 @@ class DifficultyAdjustmentApi { } const timeAvg = timeAvgMins * 60 * 1000 ; - const remainingTime = (remainingBlocks * timeAvg) + (now * 1000); - const estimatedRetargetDate = remainingTime + now; + const remainingTime = remainingBlocks * timeAvg; + const estimatedRetargetDate = remainingTime + now * 1000; return { progressPercent, From 5ab05e4e12edd2763b59557918f2ba953890ff56 Mon Sep 17 00:00:00 2001 From: junderw Date: Fri, 19 Aug 2022 01:12:53 +0900 Subject: [PATCH 15/23] Add contributors/junderw.txt --- contributors/junderw.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 contributors/junderw.txt diff --git a/contributors/junderw.txt b/contributors/junderw.txt new file mode 100644 index 000000000..23f0e9a95 --- /dev/null +++ b/contributors/junderw.txt @@ -0,0 +1,3 @@ +I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of August 19, 2022. + +Signed: junderw From e2f60a6761a1ddfb6dcfee7392b6947fd6bf073e Mon Sep 17 00:00:00 2001 From: softsimon Date: Thu, 18 Aug 2022 22:39:58 +0400 Subject: [PATCH 16/23] Hide features before feature activated --- .../app/components/tx-features/tx-features.component.html | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frontend/src/app/components/tx-features/tx-features.component.html b/frontend/src/app/components/tx-features/tx-features.component.html index e3569de8d..2027a8437 100644 --- a/frontend/src/app/components/tx-features/tx-features.component.html +++ b/frontend/src/app/components/tx-features/tx-features.component.html @@ -1,3 +1,4 @@ + SegWit SegWit @@ -5,7 +6,9 @@ SegWit + + Taproot Taproot @@ -16,6 +19,9 @@ + + RBF RBF + From 78f28c4bc3429e6c399e726e3b66f48f858ab264 Mon Sep 17 00:00:00 2001 From: softsimon Date: Thu, 18 Aug 2022 23:09:44 +0400 Subject: [PATCH 17/23] Remove taproot privacy claims --- .../app/components/tx-features/tx-features.component.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/app/components/tx-features/tx-features.component.html b/frontend/src/app/components/tx-features/tx-features.component.html index 2027a8437..6c9345095 100644 --- a/frontend/src/app/components/tx-features/tx-features.component.html +++ b/frontend/src/app/components/tx-features/tx-features.component.html @@ -9,13 +9,13 @@ -Taproot +Taproot - Taproot + Taproot - Taproot + Taproot - Taproot + Taproot From 298edb643015ad697283ee7aeafb0c88931a01d3 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Fri, 19 Aug 2022 08:55:45 +0200 Subject: [PATCH 18/23] Reverted wrong fix in 2300 --- backend/src/api/difficulty-adjustment.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/api/difficulty-adjustment.ts b/backend/src/api/difficulty-adjustment.ts index 6d188266c..1f85fdb80 100644 --- a/backend/src/api/difficulty-adjustment.ts +++ b/backend/src/api/difficulty-adjustment.ts @@ -47,8 +47,8 @@ class DifficultyAdjustmentApi { } const timeAvg = timeAvgMins * 60 * 1000 ; - const remainingTime = remainingBlocks * timeAvg; - const estimatedRetargetDate = remainingTime + now * 1000; + const remainingTime = (remainingBlocks * timeAvg) + (now * 1000); + const estimatedRetargetDate = remainingTime + now; return { progressPercent, From 1ef4485a26d04249f3760b87bd54f6b5ca1f38d2 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Fri, 19 Aug 2022 11:57:59 +0200 Subject: [PATCH 19/23] Use timestamp instead of date in stats table --- backend/src/api/database-migration.ts | 12 +++++++++++- .../src/tasks/lightning/sync-tasks/stats-importer.ts | 2 ++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index f3512248f..46b7ded6e 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 37; + private static currentVersion = 38; private queryTimeout = 120000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -328,6 +328,16 @@ class DatabaseMigration { if (databaseSchemaVersion < 37 && isBitcoin == true) { await this.$executeQuery(this.getCreateLNNodesSocketsTableQuery(), await this.$checkIfTableExists('nodes_sockets')); } + + if (databaseSchemaVersion < 38 && isBitcoin == true) { + if (config.LIGHTNING.ENABLED) { + this.uniqueLog(logger.notice, `'lightning_stats' and 'node_stats' tables have been truncated.`); + } + await this.$executeQuery(`TRUNCATE lightning_stats`); + await this.$executeQuery(`TRUNCATE node_stats`); + await this.$executeQuery('ALTER TABLE `lightning_stats` CHANGE `added` `added` timestamp NULL'); + await this.$executeQuery('ALTER TABLE `node_stats` CHANGE `added` `added` timestamp NULL'); + } } /** diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index 141f4d1a3..20d5d65d9 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -272,10 +272,12 @@ class LightningStatsImporter { // Stats exist already, don't calculate/insert them if (existingStatsTimestamps[timestamp] !== undefined) { + totalProcessed++; continue; } if (filename.indexOf('topology_') === -1) { + totalProcessed++; continue; } From e7d99e96538da131070615805904a825d479c7fb Mon Sep 17 00:00:00 2001 From: nymkappa Date: Wed, 17 Aug 2022 08:15:27 +0200 Subject: [PATCH 20/23] Insert channels from historical data --- backend/src/api/database-migration.ts | 1 - backend/src/api/explorer/channels.api.ts | 15 ++++--- .../tasks/lightning/network-sync.service.ts | 4 +- .../lightning/sync-tasks/stats-importer.ts | 45 ++++++++++++++++--- 4 files changed, 50 insertions(+), 15 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 46b7ded6e..5cbd78d40 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -248,7 +248,6 @@ class DatabaseMigration { } if (databaseSchemaVersion < 25 && isBitcoin === true) { - await this.$executeQuery(`INSERT INTO state VALUES('last_node_stats', 0, '1970-01-01');`); await this.$executeQuery(this.getCreateLightningStatisticsQuery(), await this.$checkIfTableExists('lightning_stats')); await this.$executeQuery(this.getCreateNodesQuery(), await this.$checkIfTableExists('nodes')); await this.$executeQuery(this.getCreateChannelsQuery(), await this.$checkIfTableExists('channels')); diff --git a/backend/src/api/explorer/channels.api.ts b/backend/src/api/explorer/channels.api.ts index 67003be57..f214a7d90 100644 --- a/backend/src/api/explorer/channels.api.ts +++ b/backend/src/api/explorer/channels.api.ts @@ -61,9 +61,14 @@ class ChannelsApi { } } - public async $getChannelsByStatus(status: number): Promise { + public async $getChannelsByStatus(status: number | number[]): Promise { try { - const query = `SELECT * FROM channels WHERE status = ?`; + let query: string; + if (Array.isArray(status)) { + query = `SELECT * FROM channels WHERE status IN (${status.join(',')})`; + } else { + query = `SELECT * FROM channels WHERE status = ?`; + } const [rows]: any = await DB.query(query, [status]); return rows; } catch (e) { @@ -337,7 +342,7 @@ class ChannelsApi { /** * Save or update a channel present in the graph */ - public async $saveChannel(channel: ILightningApi.Channel): Promise { + public async $saveChannel(channel: ILightningApi.Channel, status = 1): Promise { const [ txid, vout ] = channel.chan_point.split(':'); const policy1: Partial = channel.node1_policy || {}; @@ -369,11 +374,11 @@ class ChannelsApi { node2_min_htlc_mtokens, node2_updated_at ) - VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ${status}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE capacity = ?, updated_at = ?, - status = 1, + status = ${status}, node1_public_key = ?, node1_base_fee_mtokens = ?, node1_cltv_delta = ?, diff --git a/backend/src/tasks/lightning/network-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts index f0122c5ca..ad468939d 100644 --- a/backend/src/tasks/lightning/network-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -232,8 +232,8 @@ class NetworkSyncService { let progress = 0; try { - logger.info(`Starting closed channels scan...`); - const channels = await channelsApi.$getChannelsByStatus(0); + logger.info(`Starting closed channels scan`); + const channels = await channelsApi.$getChannelsByStatus([0, 1]); for (const channel of channels) { const spendingTx = await bitcoinApi.$getOutspend(channel.transaction_id, channel.transaction_vout); if (spendingTx.spent === true && spendingTx.status?.confirmed === true) { diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index 20d5d65d9..b754415ea 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -5,6 +5,8 @@ import fundingTxFetcher from './funding-tx-fetcher'; import config from '../../../config'; import { ILightningApi } from '../../../api/lightning/lightning-api.interface'; import { isIP } from 'net'; +import { Common } from '../../../api/common'; +import channelsApi from '../../../api/explorer/channels.api'; const fsPromises = promises; @@ -22,7 +24,8 @@ class LightningStatsImporter { /** * Generate LN network stats for one day */ - public async computeNetworkStats(timestamp: number, networkGraph: ILightningApi.NetworkGraph): Promise { + public async computeNetworkStats(timestamp: number, + networkGraph: ILightningApi.NetworkGraph, isHistorical: boolean = false): Promise { // Node counts and network shares let clearnetNodes = 0; let torNodes = 0; @@ -66,11 +69,14 @@ class LightningStatsImporter { const baseFees: number[] = []; const alreadyCountedChannels = {}; + const [channelsInDbRaw]: any[] = await DB.query(`SELECT short_id, created FROM channels`); + const channelsInDb = {}; + for (const channel of channelsInDbRaw) { + channelsInDb[channel.short_id] = channel; + } + for (const channel of networkGraph.edges) { - let short_id = channel.channel_id; - if (short_id.indexOf('/') !== -1) { - short_id = short_id.slice(0, -2); - } + const short_id = Common.channelIntegerIdToShortId(channel.channel_id); const tx = await fundingTxFetcher.$fetchChannelOpenTx(short_id); if (!tx) { @@ -78,6 +84,31 @@ class LightningStatsImporter { continue; } + // Channel is already in db, check if we need to update 'created' field + if (isHistorical === true) { + //@ts-ignore + if (channelsInDb[short_id] && channel.timestamp < channel.created) { + await DB.query(` + UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.short_id = ?`, + //@ts-ignore + [channel.timestamp, short_id] + ); + } else if (!channelsInDb[short_id]) { + await channelsApi.$saveChannel({ + channel_id: short_id, + chan_point: `${tx.txid}:${short_id.split('x')[2]}`, + //@ts-ignore + last_update: channel.timestamp, + node1_pub: channel.node1_pub, + node2_pub: channel.node2_pub, + capacity: (tx.value * 100000000).toString(), + node1_policy: null, + node2_policy: null, + }, 0); + channelsInDb[channel.channel_id] = channel; + } + } + if (!nodeStats[channel.node1_pub]) { nodeStats[channel.node1_pub] = { capacity: 0, @@ -102,7 +133,7 @@ class LightningStatsImporter { nodeStats[channel.node2_pub].channels++; } - if (channel.node1_policy !== undefined) { // Coming from the node + if (isHistorical === false) { // Coming from the node for (const policy of [channel.node1_policy, channel.node2_policy]) { if (policy && parseInt(policy.fee_rate_milli_msat, 10) < 5000) { avgFeeRate += parseInt(policy.fee_rate_milli_msat, 10); @@ -113,7 +144,7 @@ class LightningStatsImporter { baseFees.push(parseInt(policy.fee_base_msat, 10)); } } - } else { // Coming from the historical import + } else { // @ts-ignore if (channel.fee_rate_milli_msat < 5000) { // @ts-ignore From 59f0e2d34541dbeaa401370859bf292611e956d5 Mon Sep 17 00:00:00 2001 From: wiz Date: Fri, 19 Aug 2022 22:38:18 +0900 Subject: [PATCH 21/23] Unpublish the Github Sponsor links --- .github/FUNDING.yml | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index 5c05c1d83..000000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1,12 +0,0 @@ -# These are supported funding model platforms - -github: ['mempool'] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] -patreon: # Replace with a single Patreon username -open_collective: # Replace with a single Open Collective username -ko_fi: # Replace with a single Ko-fi username -tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel -community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry -liberapay: # Replace with a single Liberapay username -issuehunt: # Replace with a single IssueHunt username -otechie: # Replace with a single Otechie username -custom: ['https://mempool.space/sponsor'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] From 64c07cf2d2880fe70b879bff87563e0f5b2f0ead Mon Sep 17 00:00:00 2001 From: nymkappa Date: Fri, 19 Aug 2022 16:38:16 +0200 Subject: [PATCH 22/23] Fix undefined public_key --- backend/src/api/explorer/nodes.api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 88b434711..295e69bc0 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -169,7 +169,7 @@ class NodesApi { let query: string; if (full === false) { query = ` - SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, + SELECT nodes.public_key as publicKey, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, node_stats.channels FROM node_stats JOIN nodes ON nodes.public_key = node_stats.public_key From 5b9b717a936f21f9d35551526dc03f909651efaa Mon Sep 17 00:00:00 2001 From: nymkappa Date: Fri, 19 Aug 2022 18:07:26 +0200 Subject: [PATCH 23/23] Fix LN stats importer with new data "cleanupTopology" structure --- backend/src/index.ts | 2 +- .../lightning/sync-tasks/stats-importer.ts | 34 +++++++++++++------ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index 683f964f0..d1e3cee8d 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -189,7 +189,7 @@ class Server { await networkSyncService.$startService(); await lightningStatsUpdater.$startService(); } catch(e) { - logger.err(`Lightning backend crashed. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`); + logger.err(`Nodejs lightning backend crashed. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`); await Common.sleep$(1000 * 60); this.$runLightningBackend(); }; diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index b754415ea..e05ba4ab3 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -146,28 +146,40 @@ class LightningStatsImporter { } } else { // @ts-ignore - if (channel.fee_rate_milli_msat < 5000) { + if (channel.node1_policy.fee_rate_milli_msat < 5000) { // @ts-ignore - avgFeeRate += parseInt(channel.fee_rate_milli_msat, 10); + avgFeeRate += parseInt(channel.node1_policy.fee_rate_milli_msat, 10); // @ts-ignore - feeRates.push(parseInt(channel.fee_rate_milli_msat), 10); + feeRates.push(parseInt(channel.node1_policy.fee_rate_milli_msat), 10); } // @ts-ignore - if (channel.fee_base_msat < 5000) { + if (channel.node1_policy.fee_base_msat < 5000) { // @ts-ignore - avgBaseFee += parseInt(channel.fee_base_msat, 10); + avgBaseFee += parseInt(channel.node1_policy.fee_base_msat, 10); // @ts-ignore - baseFees.push(parseInt(channel.fee_base_msat), 10); + baseFees.push(parseInt(channel.node1_policy.fee_base_msat), 10); } } } + let medCapacity = 0; + let medFeeRate = 0; + let medBaseFee = 0; + let avgCapacity = 0; + avgFeeRate /= Math.max(networkGraph.edges.length, 1); avgBaseFee /= Math.max(networkGraph.edges.length, 1); - const medCapacity = capacities.sort((a, b) => b - a)[Math.round(capacities.length / 2 - 1)]; - const medFeeRate = feeRates.sort((a, b) => b - a)[Math.round(feeRates.length / 2 - 1)]; - const medBaseFee = baseFees.sort((a, b) => b - a)[Math.round(baseFees.length / 2 - 1)]; - const avgCapacity = Math.round(capacity / Math.max(capacities.length, 1)); + + if (capacities.length > 0) { + medCapacity = capacities.sort((a, b) => b - a)[Math.round(capacities.length / 2 - 1)]; + avgCapacity = Math.round(capacity / Math.max(capacities.length, 1)); + } + if (feeRates.length > 0) { + medFeeRate = feeRates.sort((a, b) => b - a)[Math.round(feeRates.length / 2 - 1)]; + } + if (baseFees.length > 0) { + medBaseFee = baseFees.sort((a, b) => b - a)[Math.round(baseFees.length / 2 - 1)]; + } let query = `INSERT INTO lightning_stats( added, @@ -350,7 +362,7 @@ class LightningStatsImporter { logger.debug(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`); } await fundingTxFetcher.$fetchChannelsFundingTxs(graph.edges.map(channel => channel.channel_id.slice(0, -2))); - const stat = await this.computeNetworkStats(timestamp, graph); + const stat = await this.computeNetworkStats(timestamp, graph, true); existingStatsTimestamps[timestamp] = stat; }