From 665d85204b1d56830c60df24c118d4c36b5aee85 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 9 Jul 2022 17:45:34 +0200 Subject: [PATCH 1/7] Backfill node_stats --- .../src/tasks/lightning/node-sync.service.ts | 4 +- .../tasks/lightning/stats-updater.service.ts | 84 ++++++++++++++++--- 2 files changed, 77 insertions(+), 11 deletions(-) diff --git a/backend/src/tasks/lightning/node-sync.service.ts b/backend/src/tasks/lightning/node-sync.service.ts index c5a6c8a9d..b7e23a7fc 100644 --- a/backend/src/tasks/lightning/node-sync.service.ts +++ b/backend/src/tasks/lightning/node-sync.service.ts @@ -44,7 +44,9 @@ class NodeSyncService { await this.$lookUpCreationDateFromChain(); await this.$updateNodeFirstSeen(); await this.$scanForClosedChannels(); - await this.$runClosedChannelsForensics(); + if (config.MEMPOOL.BACKEND === 'esplora') { + await this.$runClosedChannelsForensics(); + } } catch (e) { logger.err('$updateNodes() error: ' + (e instanceof Error ? e.message : e)); diff --git a/backend/src/tasks/lightning/stats-updater.service.ts b/backend/src/tasks/lightning/stats-updater.service.ts index 01de7ede1..5e8c0dcbb 100644 --- a/backend/src/tasks/lightning/stats-updater.service.ts +++ b/backend/src/tasks/lightning/stats-updater.service.ts @@ -6,7 +6,7 @@ import channelsApi from '../../api/explorer/channels.api'; import * as net from 'net'; class LightningStatsUpdater { - constructor() {} + hardCodedStartTime = '2018-01-12'; public async $startService() { logger.info('Starting Lightning Stats service'); @@ -47,7 +47,8 @@ class LightningStatsUpdater { } private async $runTasks() { - await this.$populateHistoricalData(); + await this.$populateHistoricalStatistics(); + await this.$populateHistoricalNodeStatistics(); await this.$logLightningStatsDaily(); await this.$logNodeStatsDaily(); } @@ -85,11 +86,10 @@ class LightningStatsUpdater { } // We only run this on first launch - private async $populateHistoricalData() { - const startTime = '2018-01-13'; + private async $populateHistoricalStatistics() { try { const [rows]: any = await DB.query(`SELECT COUNT(*) FROM lightning_stats`); - // Only store once per day + // Only run if table is empty if (rows[0]['COUNT(*)'] > 0) { return; } @@ -97,10 +97,11 @@ class LightningStatsUpdater { const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels ORDER BY created ASC`); - let date: Date = new Date(startTime); + let date: Date = new Date(this.hardCodedStartTime); const currentDate = new Date(); while (date < currentDate) { + date.setUTCDate(date.getUTCDate() + 1); let totalCapacity = 0; let channelsCount = 0; for (const channel of channels) { @@ -140,9 +141,10 @@ class LightningStatsUpdater { } const [nodes]: any = await DB.query(`SELECT first_seen, sockets FROM nodes ORDER BY first_seen ASC`); - date = new Date(startTime); + date = new Date(this.hardCodedStartTime); while (date < currentDate) { + date.setUTCDate(date.getUTCDate() + 1); let nodeCount = 0; let clearnetNodes = 0; let torNodes = 0; @@ -181,9 +183,6 @@ class LightningStatsUpdater { unannouncedNodes, date.getTime() / 1000, ]); - - // Add one day and continue - date.setDate(date.getDate() + 1); } logger.info('Historical stats populated.'); @@ -192,6 +191,71 @@ class LightningStatsUpdater { } } + private async $populateHistoricalNodeStatistics() { + try { + const [rows]: any = await DB.query(`SELECT COUNT(*) FROM node_stats`); + // Only run if table is empty + if (rows[0]['COUNT(*)'] > 0) { + return; + } + logger.info(`Running historical node stats population...`); + + const [nodes]: any = await DB.query(`SELECT public_key, first_seen, alias FROM nodes ORDER BY first_seen ASC`); + + for (const node of nodes) { + const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels WHERE node1_public_key = ? OR node2_public_key = ? ORDER BY created ASC`, [node.public_key, node.public_key]); + + let date: Date = new Date(this.hardCodedStartTime); + const currentDate = new Date(); + + let lastTotalCapacity = 0; + let lastChannelsCount = 0; + + while (date < currentDate) { + date.setUTCDate(date.getUTCDate() + 1); + let totalCapacity = 0; + let channelsCount = 0; + for (const channel of channels) { + if (new Date(channel.created) > date) { + break; + } + if (channel.closing_date !== null && new Date(channel.closing_date) < date) { + continue; + } + totalCapacity += channel.capacity; + channelsCount++; + } + + if (lastTotalCapacity === totalCapacity && lastChannelsCount === channelsCount) { + continue; + } + + lastTotalCapacity = totalCapacity; + lastChannelsCount = channelsCount; + + const query = `INSERT INTO node_stats( + public_key, + added, + capacity, + channels + ) + VALUES (?, FROM_UNIXTIME(?), ?, ?)`; + + await DB.query(query, [ + node.public_key, + date.getTime() / 1000, + totalCapacity, + channelsCount, + ]); + } + logger.debug('Updated node_stats for: ' + node.alias); + } + logger.info('Historical stats populated.'); + } catch (e) { + logger.err('$populateHistoricalNodeData() error: ' + (e instanceof Error ? e.message : e)); + } + } + private async $logLightningStatsDaily() { const currentDate = new Date().toISOString().split('T')[0]; try { From 90b9c5fe8ae69561d52df034baabfab51a898283 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Mon, 11 Jul 2022 13:32:23 +0200 Subject: [PATCH 2/7] frontend URLs: `*../resources` -> `/resources` --- frontend/src/app/services/audio.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/services/audio.service.ts b/frontend/src/app/services/audio.service.ts index bb765aa38..a0b27d168 100644 --- a/frontend/src/app/services/audio.service.ts +++ b/frontend/src/app/services/audio.service.ts @@ -18,7 +18,7 @@ export class AudioService { return; } this.isPlaying = true; - this.audio.src = '../../../resources/sounds/' + name + '.mp3'; + this.audio.src = '/resources/sounds/' + name + '.mp3'; this.audio.load(); this.audio.volume = 0.65; // 65% volume this.audio.play().catch((e) => { From 355e89ce55e80b4f4e050101f5270a1f0728fa01 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Mon, 11 Jul 2022 13:32:24 +0200 Subject: [PATCH 3/7] frontend URLs: `./resources` -> `/resources` This patch was created by: find ./frontend -type f -exec sed -i 's|\./resources|/resources|g' {} \; --- .../app/components/about/about.component.html | 2 +- .../bisq-master-page.component.html | 16 ++++++++-------- .../blocks-list/blocks-list.component.html | 2 +- .../blocks-list/blocks-list.component.ts | 4 ++-- .../clipboard/clipboard.component.html | 2 +- .../liquid-master-page.component.html | 16 ++++++++-------- .../master-page/master-page.component.html | 16 ++++++++-------- .../pool-ranking/pool-ranking.component.html | 2 +- .../src/app/components/pool/pool.component.html | 2 +- .../src/app/components/pool/pool.component.ts | 2 +- .../privacy-policy/privacy-policy.component.html | 2 +- .../components/sponsor/sponsor.component.html | 6 +++--- .../terms-of-service.component.html | 2 +- .../trademark-policy.component.html | 10 +++++----- .../src/app/components/transaction/libwally.js | 2 +- .../src/app/dashboard/dashboard.component.html | 2 +- .../src/app/dashboard/dashboard.component.ts | 2 +- frontend/src/app/services/mining.service.ts | 2 +- 18 files changed, 46 insertions(+), 46 deletions(-) diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index 7fb796939..e09eddc78 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -2,7 +2,7 @@
™ - +
v{{ packetJsonVersion }} [{{ frontendGitCommitHash }}]
diff --git a/frontend/src/app/components/bisq-master-page/bisq-master-page.component.html b/frontend/src/app/components/bisq-master-page/bisq-master-page.component.html index b1735adee..dfd0f0695 100644 --- a/frontend/src/app/components/bisq-master-page/bisq-master-page.component.html +++ b/frontend/src/app/components/bisq-master-page/bisq-master-page.component.html @@ -2,7 +2,7 @@