From 665d85204b1d56830c60df24c118d4c36b5aee85 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 9 Jul 2022 17:45:34 +0200 Subject: [PATCH 01/25] Backfill node_stats --- .../src/tasks/lightning/node-sync.service.ts | 4 +- .../tasks/lightning/stats-updater.service.ts | 84 ++++++++++++++++--- 2 files changed, 77 insertions(+), 11 deletions(-) diff --git a/backend/src/tasks/lightning/node-sync.service.ts b/backend/src/tasks/lightning/node-sync.service.ts index c5a6c8a9d..b7e23a7fc 100644 --- a/backend/src/tasks/lightning/node-sync.service.ts +++ b/backend/src/tasks/lightning/node-sync.service.ts @@ -44,7 +44,9 @@ class NodeSyncService { await this.$lookUpCreationDateFromChain(); await this.$updateNodeFirstSeen(); await this.$scanForClosedChannels(); - await this.$runClosedChannelsForensics(); + if (config.MEMPOOL.BACKEND === 'esplora') { + await this.$runClosedChannelsForensics(); + } } catch (e) { logger.err('$updateNodes() error: ' + (e instanceof Error ? e.message : e)); diff --git a/backend/src/tasks/lightning/stats-updater.service.ts b/backend/src/tasks/lightning/stats-updater.service.ts index 01de7ede1..5e8c0dcbb 100644 --- a/backend/src/tasks/lightning/stats-updater.service.ts +++ b/backend/src/tasks/lightning/stats-updater.service.ts @@ -6,7 +6,7 @@ import channelsApi from '../../api/explorer/channels.api'; import * as net from 'net'; class LightningStatsUpdater { - constructor() {} + hardCodedStartTime = '2018-01-12'; public async $startService() { logger.info('Starting Lightning Stats service'); @@ -47,7 +47,8 @@ class LightningStatsUpdater { } private async $runTasks() { - await this.$populateHistoricalData(); + await this.$populateHistoricalStatistics(); + await this.$populateHistoricalNodeStatistics(); await this.$logLightningStatsDaily(); await this.$logNodeStatsDaily(); } @@ -85,11 +86,10 @@ class LightningStatsUpdater { } // We only run this on first launch - private async $populateHistoricalData() { - const startTime = '2018-01-13'; + private async $populateHistoricalStatistics() { try { const [rows]: any = await DB.query(`SELECT COUNT(*) FROM lightning_stats`); - // Only store once per day + // Only run if table is empty if (rows[0]['COUNT(*)'] > 0) { return; } @@ -97,10 +97,11 @@ class LightningStatsUpdater { const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels ORDER BY created ASC`); - let date: Date = new Date(startTime); + let date: Date = new Date(this.hardCodedStartTime); const currentDate = new Date(); while (date < currentDate) { + date.setUTCDate(date.getUTCDate() + 1); let totalCapacity = 0; let channelsCount = 0; for (const channel of channels) { @@ -140,9 +141,10 @@ class LightningStatsUpdater { } const [nodes]: any = await DB.query(`SELECT first_seen, sockets FROM nodes ORDER BY first_seen ASC`); - date = new Date(startTime); + date = new Date(this.hardCodedStartTime); while (date < currentDate) { + date.setUTCDate(date.getUTCDate() + 1); let nodeCount = 0; let clearnetNodes = 0; let torNodes = 0; @@ -181,9 +183,6 @@ class LightningStatsUpdater { unannouncedNodes, date.getTime() / 1000, ]); - - // Add one day and continue - date.setDate(date.getDate() + 1); } logger.info('Historical stats populated.'); @@ -192,6 +191,71 @@ class LightningStatsUpdater { } } + private async $populateHistoricalNodeStatistics() { + try { + const [rows]: any = await DB.query(`SELECT COUNT(*) FROM node_stats`); + // Only run if table is empty + if (rows[0]['COUNT(*)'] > 0) { + return; + } + logger.info(`Running historical node stats population...`); + + const [nodes]: any = await DB.query(`SELECT public_key, first_seen, alias FROM nodes ORDER BY first_seen ASC`); + + for (const node of nodes) { + const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels WHERE node1_public_key = ? OR node2_public_key = ? ORDER BY created ASC`, [node.public_key, node.public_key]); + + let date: Date = new Date(this.hardCodedStartTime); + const currentDate = new Date(); + + let lastTotalCapacity = 0; + let lastChannelsCount = 0; + + while (date < currentDate) { + date.setUTCDate(date.getUTCDate() + 1); + let totalCapacity = 0; + let channelsCount = 0; + for (const channel of channels) { + if (new Date(channel.created) > date) { + break; + } + if (channel.closing_date !== null && new Date(channel.closing_date) < date) { + continue; + } + totalCapacity += channel.capacity; + channelsCount++; + } + + if (lastTotalCapacity === totalCapacity && lastChannelsCount === channelsCount) { + continue; + } + + lastTotalCapacity = totalCapacity; + lastChannelsCount = channelsCount; + + const query = `INSERT INTO node_stats( + public_key, + added, + capacity, + channels + ) + VALUES (?, FROM_UNIXTIME(?), ?, ?)`; + + await DB.query(query, [ + node.public_key, + date.getTime() / 1000, + totalCapacity, + channelsCount, + ]); + } + logger.debug('Updated node_stats for: ' + node.alias); + } + logger.info('Historical stats populated.'); + } catch (e) { + logger.err('$populateHistoricalNodeData() error: ' + (e instanceof Error ? e.message : e)); + } + } + private async $logLightningStatsDaily() { const currentDate = new Date().toISOString().split('T')[0]; try { From 0093eab26963880c22b58d3b86874e2de719760e Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 11 Jul 2022 08:41:28 +0200 Subject: [PATCH 02/25] Liquid always uses esplora (regression of #2039) --- backend/src/api/blocks.ts | 9 ++++----- backend/src/api/common.ts | 2 +- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 62f3efa88..89b3d767f 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -527,13 +527,12 @@ class Blocks { } } - let block = await bitcoinClient.getBlock(hash); - // Not Bitcoin network, return the block as it if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { - return block; + return await bitcoinApi.$getBlock(hash); } + let block = await bitcoinClient.getBlock(hash); block = prepareBlock(block); // Bitcoin network, add our custom data on top @@ -547,8 +546,8 @@ class Blocks { return blockExtended; } - public async $getStrippedBlockTransactions(hash: string, skipMemoryCache: boolean = false, - skipDBLookup: boolean = false): Promise + public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false, + skipDBLookup = false): Promise { if (skipMemoryCache === false) { // Check the memory cache diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 1dc9f66ea..fe6b858e0 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -172,7 +172,7 @@ export class Common { static indexingEnabled(): boolean { return ( - ['mainnet', 'testnet', 'signet', 'regtest'].includes(config.MEMPOOL.NETWORK) && + ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) && config.DATABASE.ENABLED === true && config.MEMPOOL.INDEXING_BLOCKS_AMOUNT !== 0 ); From f2e703e9281145a4ce409684e39837a73ff9071a Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 11 Jul 2022 09:36:42 +0200 Subject: [PATCH 03/25] Fix graphs button layout --- frontend/src/app/components/graphs/graphs.component.html | 6 +++--- frontend/src/app/components/graphs/graphs.component.ts | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/components/graphs/graphs.component.html b/frontend/src/app/components/graphs/graphs.component.html index 8914adf0a..6f93676f6 100644 --- a/frontend/src/app/components/graphs/graphs.component.html +++ b/frontend/src/app/components/graphs/graphs.component.html @@ -1,10 +1,10 @@
- Mempool -
+
-
+
Date: Mon, 11 Jul 2022 09:53:32 +0200 Subject: [PATCH 04/25] Set /block API cache duration according to block age --- backend/src/routes.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/backend/src/routes.ts b/backend/src/routes.ts index aeda734e1..0bd3aa7cf 100644 --- a/backend/src/routes.ts +++ b/backend/src/routes.ts @@ -761,7 +761,19 @@ class Routes { public async getBlock(req: Request, res: Response) { try { const block = await blocks.$getBlock(req.params.hash); - res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString()); + + const blockAge = new Date().getTime() / 1000 - block.timestamp; + const day = 24 * 3600; + let cacheDuration; + if (blockAge > 365 * day) { + cacheDuration = 30 * day; + } else if (blockAge > 30 * day) { + cacheDuration = 10 * day; + } else { + cacheDuration = 600 + } + + res.setHeader('Expires', new Date(Date.now() + 1000 * cacheDuration).toUTCString()); res.json(block); } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); From 38ac38849ef934913cf35ca74f696c8117a65c0e Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 11 Jul 2022 10:55:25 +0200 Subject: [PATCH 05/25] [Indexer] Set log level accordingly - Remove indexing ETAs --- backend/src/api/blocks.ts | 18 ++++++++++++------ backend/src/api/mining.ts | 12 ++++++++---- backend/src/repositories/BlocksRepository.ts | 2 +- backend/src/tasks/price-updater.ts | 4 ++++ 4 files changed, 25 insertions(+), 11 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 89b3d767f..5d4602224 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -280,8 +280,7 @@ class Blocks { const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds); const progress = Math.round(totalIndexed / indexedBlocks.length * 10000) / 100; - const timeLeft = Math.round((indexedBlocks.length - totalIndexed) / blockPerSeconds); - logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`); + logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`); timer = new Date().getTime() / 1000; indexedThisRun = 0; } @@ -293,7 +292,11 @@ class Blocks { totalIndexed++; newlyIndexed++; } - logger.notice(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`); + if (newlyIndexed > 0) { + logger.notice(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`); + } else { + logger.debug(`Blocks summaries indexing completed: indexed ${newlyIndexed} blocks`); + } } catch (e) { logger.err(`Blocks summaries indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`); throw e; @@ -348,8 +351,7 @@ class Blocks { const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds); const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100; - const timeLeft = Math.round((indexingBlockAmount - totalIndexed) / blockPerSeconds); - logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`); + logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds`); timer = new Date().getTime() / 1000; indexedThisRun = 0; loadingIndicators.setProgress('block-indexing', progress, false); @@ -365,7 +367,11 @@ class Blocks { currentBlockHeight -= chunkSize; } - logger.notice(`Block indexing completed: indexed ${newlyIndexed} blocks`); + if (newlyIndexed > 0) { + logger.notice(`Block indexing completed: indexed ${newlyIndexed} blocks`); + } else { + logger.debug(`Block indexing completed: indexed ${newlyIndexed} blocks`); + } loadingIndicators.setProgress('block-indexing', 100); } catch (e) { logger.err('Block indexing failed. Trying again in 10 seconds. Reason: ' + (e instanceof Error ? e.message : e)); diff --git a/backend/src/api/mining.ts b/backend/src/api/mining.ts index 50a86e73e..80726d07b 100644 --- a/backend/src/api/mining.ts +++ b/backend/src/api/mining.ts @@ -250,9 +250,8 @@ class Mining { const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); const weeksPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds)); const progress = Math.round(totalIndexed / totalWeekIndexed * 10000) / 100; - const timeLeft = Math.round((totalWeekIndexed - totalIndexed) / weeksPerSeconds); const formattedDate = new Date(fromTimestamp).toUTCString(); - logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`); + logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`); timer = new Date().getTime() / 1000; indexedThisRun = 0; loadingIndicators.setProgress('weekly-hashrate-indexing', progress, false); @@ -265,6 +264,8 @@ class Mining { await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', new Date().getUTCDate()); if (newlyIndexed > 0) { logger.notice(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`); + } else { + logger.debug(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`); } loadingIndicators.setProgress('weekly-hashrate-indexing', 100); } catch (e) { @@ -339,9 +340,8 @@ class Mining { const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); const daysPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds)); const progress = Math.round(totalIndexed / totalDayIndexed * 10000) / 100; - const timeLeft = Math.round((totalDayIndexed - totalIndexed) / daysPerSeconds); const formattedDate = new Date(fromTimestamp).toUTCString(); - logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds | left: ~${timeLeft} seconds`); + logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`); timer = new Date().getTime() / 1000; indexedThisRun = 0; loadingIndicators.setProgress('daily-hashrate-indexing', progress); @@ -369,6 +369,8 @@ class Mining { await HashratesRepository.$setLatestRun('last_hashrates_indexing', new Date().getUTCDate()); if (newlyIndexed > 0) { logger.notice(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`); + } else { + logger.debug(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`); } loadingIndicators.setProgress('daily-hashrate-indexing', 100); } catch (e) { @@ -446,6 +448,8 @@ class Mining { if (totalIndexed > 0) { logger.notice(`Indexed ${totalIndexed} difficulty adjustments`); + } else { + logger.debug(`Indexed ${totalIndexed} difficulty adjustments`); } } diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 4388c46f6..83f368248 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -446,7 +446,7 @@ class BlocksRepository { ++idx; } - logger.info(`${idx} blocks hash validated in ${new Date().getTime() - start} ms`); + logger.debug(`${idx} blocks hash validated in ${new Date().getTime() - start} ms`); return true; } catch (e) { logger.err('Cannot validate chain of block hash. Reason: ' + (e instanceof Error ? e.message : e)); diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index caad6c54b..2d4087d8b 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -177,6 +177,8 @@ class PriceUpdater { } if (insertedCount > 0) { logger.notice(`Inserted ${insertedCount} MtGox USD weekly price history into db`); + } else { + logger.debug(`Inserted ${insertedCount} MtGox USD weekly price history into db`); } // Insert Kraken weekly prices @@ -251,6 +253,8 @@ class PriceUpdater { if (totalInserted > 0) { logger.notice(`Inserted ${totalInserted} hourly historical prices into the db`); + } else { + logger.debug(`Inserted ${totalInserted} hourly historical prices into the db`); } } } From 90b9c5fe8ae69561d52df034baabfab51a898283 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Mon, 11 Jul 2022 13:32:23 +0200 Subject: [PATCH 06/25] frontend URLs: `*../resources` -> `/resources` --- frontend/src/app/services/audio.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/services/audio.service.ts b/frontend/src/app/services/audio.service.ts index bb765aa38..a0b27d168 100644 --- a/frontend/src/app/services/audio.service.ts +++ b/frontend/src/app/services/audio.service.ts @@ -18,7 +18,7 @@ export class AudioService { return; } this.isPlaying = true; - this.audio.src = '../../../resources/sounds/' + name + '.mp3'; + this.audio.src = '/resources/sounds/' + name + '.mp3'; this.audio.load(); this.audio.volume = 0.65; // 65% volume this.audio.play().catch((e) => { From 355e89ce55e80b4f4e050101f5270a1f0728fa01 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Mon, 11 Jul 2022 13:32:24 +0200 Subject: [PATCH 07/25] frontend URLs: `./resources` -> `/resources` This patch was created by: find ./frontend -type f -exec sed -i 's|\./resources|/resources|g' {} \; --- .../app/components/about/about.component.html | 2 +- .../bisq-master-page.component.html | 16 ++++++++-------- .../blocks-list/blocks-list.component.html | 2 +- .../blocks-list/blocks-list.component.ts | 4 ++-- .../clipboard/clipboard.component.html | 2 +- .../liquid-master-page.component.html | 16 ++++++++-------- .../master-page/master-page.component.html | 16 ++++++++-------- .../pool-ranking/pool-ranking.component.html | 2 +- .../src/app/components/pool/pool.component.html | 2 +- .../src/app/components/pool/pool.component.ts | 2 +- .../privacy-policy/privacy-policy.component.html | 2 +- .../components/sponsor/sponsor.component.html | 6 +++--- .../terms-of-service.component.html | 2 +- .../trademark-policy.component.html | 10 +++++----- .../src/app/components/transaction/libwally.js | 2 +- .../src/app/dashboard/dashboard.component.html | 2 +- .../src/app/dashboard/dashboard.component.ts | 2 +- frontend/src/app/services/mining.service.ts | 2 +- 18 files changed, 46 insertions(+), 46 deletions(-) diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index 7fb796939..e09eddc78 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -2,7 +2,7 @@
™ - +
v{{ packetJsonVersion }} [{{ frontendGitCommitHash }}]
diff --git a/frontend/src/app/components/bisq-master-page/bisq-master-page.component.html b/frontend/src/app/components/bisq-master-page/bisq-master-page.component.html index b1735adee..dfd0f0695 100644 --- a/frontend/src/app/components/bisq-master-page/bisq-master-page.component.html +++ b/frontend/src/app/components/bisq-master-page/bisq-master-page.component.html @@ -2,7 +2,7 @@