From 07821769cd05310403e50b287fbad7ebcd6f01e8 Mon Sep 17 00:00:00 2001 From: softsimon Date: Thu, 5 May 2022 23:19:24 +0400 Subject: [PATCH] Node stats updates --- .../time-since/time-since.component.ts | 9 ++- .../app/lightning/lightning-api.service.ts | 4 ++ .../app/lightning/node/node.component.html | 12 ++-- .../src/app/lightning/node/node.component.ts | 8 +++ .../nodes-list/nodes-list.component.html | 4 +- .../src/api/explorer/channels.api.ts | 11 ++++ .../src/api/explorer/nodes.api.ts | 19 ++++-- .../src/api/explorer/nodes.routes.ts | 10 ++++ lightning-backend/src/database-migration.ts | 6 +- .../src/tasks/node-sync.service.ts | 58 ++++++++++++++++++- .../src/tasks/stats-updater.service.ts | 6 +- 11 files changed, 127 insertions(+), 20 deletions(-) diff --git a/frontend/src/app/components/time-since/time-since.component.ts b/frontend/src/app/components/time-since/time-since.component.ts index 0fbf745de..1162116ec 100644 --- a/frontend/src/app/components/time-since/time-since.component.ts +++ b/frontend/src/app/components/time-since/time-since.component.ts @@ -13,6 +13,7 @@ export class TimeSinceComponent implements OnInit, OnChanges, OnDestroy { intervals = {}; @Input() time: number; + @Input() dateString: number; @Input() fastRender = false; constructor( @@ -52,7 +53,13 @@ export class TimeSinceComponent implements OnInit, OnChanges, OnDestroy { } calculate() { - const seconds = Math.floor((+new Date() - +new Date(this.time * 1000)) / 1000); + let date: Date; + if (this.dateString) { + date = new Date(this.dateString) + } else { + date = new Date(this.time * 1000); + } + const seconds = Math.floor((+new Date() - +date) / 1000); if (seconds < 60) { return $localize`:@@date-base.just-now:Just now`; } diff --git a/frontend/src/app/lightning/lightning-api.service.ts b/frontend/src/app/lightning/lightning-api.service.ts index 326ac063f..a49923546 100644 --- a/frontend/src/app/lightning/lightning-api.service.ts +++ b/frontend/src/app/lightning/lightning-api.service.ts @@ -32,6 +32,10 @@ export class LightningApiService { return this.httpClient.get(API_BASE_URL + '/statistics/latest'); } + listNodeStats$(publicKey: string): Observable { + return this.httpClient.get(API_BASE_URL + '/nodes/' + publicKey + '/statistics'); + } + listTopNodes$(): Observable { return this.httpClient.get(API_BASE_URL + '/nodes/top'); } diff --git a/frontend/src/app/lightning/node/node.component.html b/frontend/src/app/lightning/node/node.component.html index c85f63c78..c4d618dcf 100644 --- a/frontend/src/app/lightning/node/node.component.html +++ b/frontend/src/app/lightning/node/node.component.html @@ -18,12 +18,16 @@ - - + + - - + + diff --git a/frontend/src/app/lightning/node/node.component.ts b/frontend/src/app/lightning/node/node.component.ts index b62198d0d..6949e1826 100644 --- a/frontend/src/app/lightning/node/node.component.ts +++ b/frontend/src/app/lightning/node/node.component.ts @@ -12,6 +12,7 @@ import { LightningApiService } from '../lightning-api.service'; }) export class NodeComponent implements OnInit { node$: Observable; + statistics$: Observable; publicKey$: Observable; constructor( @@ -26,6 +27,13 @@ export class NodeComponent implements OnInit { return this.lightningApiService.getNode$(params.get('public_key')); }) ); + + this.statistics$ = this.activatedRoute.paramMap + .pipe( + switchMap((params: ParamMap) => { + return this.lightningApiService.listNodeStats$(params.get('public_key')); + }) + ); } } diff --git a/frontend/src/app/lightning/nodes-list/nodes-list.component.html b/frontend/src/app/lightning/nodes-list/nodes-list.component.html index 64deb7b60..65a7a558a 100644 --- a/frontend/src/app/lightning/nodes-list/nodes-list.component.html +++ b/frontend/src/app/lightning/nodes-list/nodes-list.component.html @@ -12,10 +12,10 @@ {{ node.alias }} diff --git a/lightning-backend/src/api/explorer/channels.api.ts b/lightning-backend/src/api/explorer/channels.api.ts index 157cbc97a..c13bc9319 100644 --- a/lightning-backend/src/api/explorer/channels.api.ts +++ b/lightning-backend/src/api/explorer/channels.api.ts @@ -24,6 +24,17 @@ class ChannelsApi { } } + public async $getChannelsWithoutCreatedDate(): Promise { + try { + const query = `SELECT * FROM channels WHERE created IS NULL`; + const [rows]: any = await DB.query(query); + return rows; + } catch (e) { + logger.err('$getChannelsWithoutCreatedDate error: ' + (e instanceof Error ? e.message : e)); + throw e; + } + } + public async $getChannel(shortId: string): Promise { try { const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.* FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key WHERE channels.id = ?`; diff --git a/lightning-backend/src/api/explorer/nodes.api.ts b/lightning-backend/src/api/explorer/nodes.api.ts index 5ed30c0e4..0b034d230 100644 --- a/lightning-backend/src/api/explorer/nodes.api.ts +++ b/lightning-backend/src/api/explorer/nodes.api.ts @@ -4,8 +4,8 @@ import DB from '../../database'; class NodesApi { public async $getNode(public_key: string): Promise { try { - const query = `SELECT * FROM nodes WHERE public_key = ?`; - const [rows]: any = await DB.query(query, [public_key]); + const query = `SELECT nodes.*, (SELECT COUNT(*) FROM channels WHERE channels.status < 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)) AS channel_count, (SELECT SUM(capacity) FROM channels WHERE channels.status < 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)) AS capacity FROM nodes WHERE public_key = ?`; + const [rows]: any = await DB.query(query, [public_key, public_key, public_key, public_key, public_key]); return rows[0]; } catch (e) { logger.err('$getNode error: ' + (e instanceof Error ? e.message : e)); @@ -13,9 +13,20 @@ class NodesApi { } } + public async $getNodeStats(public_key: string): Promise { + try { + const query = `SELECT * FROM nodes_stats WHERE public_key = ? ORDER BY added DESC`; + const [rows]: any = await DB.query(query, [public_key]); + return rows; + } catch (e) { + logger.err('$getNodeStats error: ' + (e instanceof Error ? e.message : e)); + throw e; + } + } + public async $getTopCapacityNodes(): Promise { try { - const query = `SELECT nodes.*, nodes_stats.capacity_left, nodes_stats.capacity_right, nodes_stats.channels_left, nodes_stats.channels_right FROM nodes LEFT JOIN nodes_stats ON nodes_stats.public_key = nodes.public_key ORDER BY nodes_stats.added DESC, nodes_stats.capacity_left + nodes_stats.capacity_right DESC LIMIT 10`; + const query = `SELECT nodes.*, nodes_stats.capacity, nodes_stats.channels FROM nodes LEFT JOIN nodes_stats ON nodes_stats.public_key = nodes.public_key ORDER BY nodes_stats.added DESC, nodes_stats.capacity DESC LIMIT 10`; const [rows]: any = await DB.query(query); return rows; } catch (e) { @@ -26,7 +37,7 @@ class NodesApi { public async $getTopChannelsNodes(): Promise { try { - const query = `SELECT nodes.*, nodes_stats.capacity_left, nodes_stats.capacity_right, nodes_stats.channels_left, nodes_stats.channels_right FROM nodes LEFT JOIN nodes_stats ON nodes_stats.public_key = nodes.public_key ORDER BY nodes_stats.added DESC, nodes_stats.channels_left + nodes_stats.channels_right DESC LIMIT 10`; + const query = `SELECT nodes.*, nodes_stats.capacity, nodes_stats.channels FROM nodes LEFT JOIN nodes_stats ON nodes_stats.public_key = nodes.public_key ORDER BY nodes_stats.added DESC, nodes_stats.channels DESC LIMIT 10`; const [rows]: any = await DB.query(query); return rows; } catch (e) { diff --git a/lightning-backend/src/api/explorer/nodes.routes.ts b/lightning-backend/src/api/explorer/nodes.routes.ts index 73bef9f26..1b86abb69 100644 --- a/lightning-backend/src/api/explorer/nodes.routes.ts +++ b/lightning-backend/src/api/explorer/nodes.routes.ts @@ -8,6 +8,7 @@ class NodesRoutes { app .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/latest', this.$getGeneralStats) .get(config.MEMPOOL.API_URL_PREFIX + 'nodes/top', this.$getTopNodes) + .get(config.MEMPOOL.API_URL_PREFIX + 'nodes/:public_key/statistics', this.$getHistoricalNodeStats) .get(config.MEMPOOL.API_URL_PREFIX + 'nodes/:public_key', this.$getNode) ; } @@ -25,6 +26,15 @@ class NodesRoutes { } } + private async $getHistoricalNodeStats(req: Request, res: Response) { + try { + const statistics = await nodesApi.$getNodeStats(req.params.public_key); + res.json(statistics); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + private async $getGeneralStats(req: Request, res: Response) { try { const statistics = await nodesApi.$getLatestStatistics(); diff --git a/lightning-backend/src/database-migration.ts b/lightning-backend/src/database-migration.ts index 769783460..8c16da676 100644 --- a/lightning-backend/src/database-migration.ts +++ b/lightning-backend/src/database-migration.ts @@ -238,10 +238,8 @@ class DatabaseMigration { id int(11) unsigned NOT NULL AUTO_INCREMENT, public_key varchar(66) NOT NULL DEFAULT '', added date NOT NULL, - capacity_left bigint(11) unsigned DEFAULT NULL, - capacity_right bigint(11) unsigned DEFAULT NULL, - channels_left int(11) unsigned DEFAULT NULL, - channels_right int(11) unsigned DEFAULT NULL, + capacity bigint(11) unsigned DEFAULT NULL, + channels int(11) unsigned DEFAULT NULL, PRIMARY KEY (id), KEY public_key (public_key) ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; diff --git a/lightning-backend/src/tasks/node-sync.service.ts b/lightning-backend/src/tasks/node-sync.service.ts index 952090f31..1479ce13a 100644 --- a/lightning-backend/src/tasks/node-sync.service.ts +++ b/lightning-backend/src/tasks/node-sync.service.ts @@ -26,21 +26,71 @@ class NodeSyncService { for (const node of networkGraph.nodes) { await this.$saveNode(node); } + logger.debug(`Nodes updated`); await this.$setChannelsInactive(); for (const channel of networkGraph.channels) { await this.$saveChannel(channel); } + logger.debug(`Channels updated`); await this.$findInactiveNodesAndChannels(); + logger.debug(`Inactive channels scan complete`); + await this.$scanForClosedChannels(); + logger.debug(`Closed channels scan complete`); + + await this.$lookUpCreationDateFromChain(); + logger.debug(`Channel creation dates scan complete`); + + await this.$updateNodeFirstSeen(); + logger.debug(`Node first seen dates scan complete`); } catch (e) { logger.err('$updateNodes() error: ' + (e instanceof Error ? e.message : e)); } } + // This method look up the creation date of the earliest channel of the node + // and update the node to that date in order to get the earliest first seen date + private async $updateNodeFirstSeen() { + try { + const [nodes]: any[] = await DB.query(`SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node1_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created1, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node2_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created2 FROM nodes`); + for (const node of nodes) { + let lowest = 0; + if (node.created1) { + if (node.created2 && node.created2 < node.created1) { + lowest = node.created2; + } else { + lowest = node.created1; + } + } else if (node.created2) { + lowest = node.created2; + } + if (lowest && lowest < node.first_seen) { + const query = `UPDATE nodes SET first_seen = FROM_UNIXTIME(?) WHERE public_key = ?`; + const params = [lowest, node.public_key]; + await DB.query(query, params); + } + } + } catch (e) { + logger.err('$updateNodeFirstSeen() error: ' + (e instanceof Error ? e.message : e)); + } + } + + private async $lookUpCreationDateFromChain() { + try { + const channels = await channelsApi.$getChannelsWithoutCreatedDate(); + for (const channel of channels) { + const transaction = await bitcoinClient.getRawTransaction(channel.transaction_id, 1); + await DB.query(`UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`, [transaction.blocktime, channel.id]); + } + } catch (e) { + logger.err('$setCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e)); + } + } + // Looking for channels whos nodes are inactive private async $findInactiveNodesAndChannels(): Promise { try { @@ -190,23 +240,27 @@ class NodeSyncService { private async $saveNode(node: ILightningApi.Node): Promise { try { const updatedAt = this.utcDateToMysql(node.updated_at); + const sockets = node.sockets.join(', '); const query = `INSERT INTO nodes( public_key, first_seen, updated_at, alias, - color + color, + sockets ) - VALUES (?, NOW(), ?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = ?, alias = ?, color = ?;`; + VALUES (?, NOW(), ?, ?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = ?, alias = ?, color = ?, sockets = ?;`; await DB.query(query, [ node.public_key, updatedAt, node.alias, node.color, + sockets, updatedAt, node.alias, node.color, + sockets, ]); } catch (e) { logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e)); diff --git a/lightning-backend/src/tasks/stats-updater.service.ts b/lightning-backend/src/tasks/stats-updater.service.ts index 0c61922e9..c96273ff1 100644 --- a/lightning-backend/src/tasks/stats-updater.service.ts +++ b/lightning-backend/src/tasks/stats-updater.service.ts @@ -38,9 +38,9 @@ class LightningStatsUpdater { for (const node of nodes) { await DB.query( - `INSERT INTO nodes_stats(public_key, added, capacity_left, capacity_right, channels_left, channels_right) VALUES (?, NOW(), ?, ?, ?, ?)`, - [node.public_key, node.channels_capacity_left, node.channels_capacity_right, - node.channels_count_left, node.channels_count_right]); + `INSERT INTO nodes_stats(public_key, added, capacity, channels) VALUES (?, NOW(), ?, ?)`, + [node.public_key, (parseInt(node.channels_capacity_left || 0, 10)) + (parseInt(node.channels_capacity_right || 0, 10)), + node.channels_count_left + node.channels_count_right]); } await DB.query(`UPDATE state SET string = ? WHERE name = 'last_node_stats'`, [currentDate]); logger.debug('Daily node stats has updated.');
First Seen{{ node.first_seen | date:'yyyy-MM-dd HH:mm' }}First seen + +
Updated At{{ node.updated_at | date:'yyyy-MM-dd HH:mm' }}Last update + +
Color - + - {{ node.channels_left + node.channels_right | number }} + {{ node.channels | number }}