From 0b4615cbf09e718e5d1b581fe07b0d52a303c32b Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Tue, 30 May 2023 10:05:10 -0700 Subject: [PATCH 1/5] [indexer] reindex diff adjustments and hashrates upon mining pool update --- backend/src/api/mining/mining.ts | 19 +++++++++++++++++++ backend/src/api/pools-parser.ts | 11 ++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index 20da92de3..4fb5aeda0 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -13,11 +13,15 @@ import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository'; import PricesRepository from '../../repositories/PricesRepository'; import { bitcoinCoreApi } from '../bitcoin/bitcoin-api-factory'; import { IEsploraApi } from '../bitcoin/esplora-api.interface'; +import database from '../../database'; class Mining { private blocksPriceIndexingRunning = false; public lastHashrateIndexingDate: number | null = null; public lastWeeklyHashrateIndexingDate: number | null = null; + + public reindexHashrateRequested = false; + public reindexDifficultyAdjustmentRequested = false; /** * Get historical blocks health @@ -289,6 +293,14 @@ class Mining { * Generate daily hashrate data */ public async $generateNetworkHashrateHistory(): Promise { + // If a re-index was requested, truncate first + if (this.reindexHashrateRequested === true) { + logger.notice(`hashrates will now be re-indexed`); + await database.query(`TRUNCATE hashrates`); + this.lastHashrateIndexingDate = 0; + this.reindexHashrateRequested = false; + } + // We only run this once a day around midnight const today = new Date().getUTCDate(); if (today === this.lastHashrateIndexingDate) { @@ -394,6 +406,13 @@ class Mining { * Index difficulty adjustments */ public async $indexDifficultyAdjustments(): Promise { + // If a re-index was requested, truncate first + if (this.reindexDifficultyAdjustmentRequested === true) { + logger.notice(`difficulty_adjustments will now be re-indexed`); + await database.query(`TRUNCATE difficulty_adjustments`); + this.reindexDifficultyAdjustmentRequested = false; + } + const indexedHeightsArray = await DifficultyAdjustmentsRepository.$getAdjustmentsHeights(); const indexedHeights = {}; for (const height of indexedHeightsArray) { diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index 0d34925ab..f1619cac4 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -4,6 +4,7 @@ import config from '../config'; import PoolsRepository from '../repositories/PoolsRepository'; import { PoolTag } from '../mempool.interfaces'; import diskCache from './disk-cache'; +import mining from './mining/mining'; class PoolsParser { miningPools: any[] = []; @@ -73,7 +74,7 @@ class PoolsParser { if (JSON.stringify(pool.addresses) !== poolDB.addresses || JSON.stringify(pool.regexes) !== poolDB.regexes) { // Pool addresses changed or coinbase tags changed - logger.notice(`Updating addresses and/or coinbase tags for ${pool.name} mining pool. If 'AUTOMATIC_BLOCK_REINDEXING' is enabled, we will re-index its blocks and 'unknown' blocks`); + logger.notice(`Updating addresses and/or coinbase tags for ${pool.name} mining pool.`); await PoolsRepository.$updateMiningPoolTags(poolDB.id, pool.addresses, pool.regexes); await this.$deleteBlocksForPool(poolDB); } @@ -142,6 +143,10 @@ class PoolsParser { WHERE pool_id = ?`, [pool.id] ); + + // Re-index hashrates and difficulty adjustments later + mining.reindexHashrateRequested = true; + mining.reindexDifficultyAdjustmentRequested = true; } private async $deleteUnknownBlocks(): Promise { @@ -152,6 +157,10 @@ class PoolsParser { WHERE pool_id = ? AND height >= 130635`, [unknownPool[0].id] ); + + // Re-index hashrates and difficulty adjustments later + mining.reindexHashrateRequested = true; + mining.reindexDifficultyAdjustmentRequested = true; } } From 107bdbc2094e3fd81a44fea79764dc852793c878 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Tue, 30 May 2023 10:13:07 -0700 Subject: [PATCH 2/5] [indexer] show indexer progress in /status component --- frontend/src/app/components/start/start.component.html | 2 ++ frontend/src/app/components/start/start.component.ts | 4 +++- .../src/app/components/status-view/status-view.component.html | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/frontend/src/app/components/start/start.component.html b/frontend/src/app/components/start/start.component.html index 2527c52b2..ec4bf4805 100644 --- a/frontend/src/app/components/start/start.component.html +++ b/frontend/src/app/components/start/start.component.html @@ -1,3 +1,5 @@ + +
diff --git a/frontend/src/app/components/start/start.component.ts b/frontend/src/app/components/start/start.component.ts index 7e83fb516..52fe1812c 100644 --- a/frontend/src/app/components/start/start.component.ts +++ b/frontend/src/app/components/start/start.component.ts @@ -1,4 +1,4 @@ -import { Component, ElementRef, HostListener, OnInit, OnDestroy, ViewChild } from '@angular/core'; +import { Component, ElementRef, HostListener, OnInit, OnDestroy, ViewChild, Input } from '@angular/core'; import { Subscription } from 'rxjs'; import { StateService } from '../../services/state.service'; import { specialBlocks } from '../../app.constants'; @@ -9,6 +9,8 @@ import { specialBlocks } from '../../app.constants'; styleUrls: ['./start.component.scss'], }) export class StartComponent implements OnInit, OnDestroy { + @Input() showLoadingIndicator = false; + interval = 60; colors = ['#5E35B1', '#ffffff']; diff --git a/frontend/src/app/components/status-view/status-view.component.html b/frontend/src/app/components/status-view/status-view.component.html index ec91e37a4..f766c8ea9 100644 --- a/frontend/src/app/components/status-view/status-view.component.html +++ b/frontend/src/app/components/status-view/status-view.component.html @@ -1,2 +1,2 @@ - + From e7e7b30807db3fcdc4b0b29a5913fad0548736ca Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Tue, 30 May 2023 10:16:56 -0700 Subject: [PATCH 3/5] fix log --- backend/src/api/database-migration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 21c87f9e2..2b1e5ea8e 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -1039,7 +1039,7 @@ class DatabaseMigration { } public async $blocksReindexingTruncate(): Promise { - logger.warn(`Truncating pools, blocks and hashrates for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`); + logger.warn(`Truncating pools, blocks, hashrates and difficulty_adjustments tables for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`); await Common.sleep$(5000); await this.$executeQuery(`TRUNCATE blocks`); From 62169cee3f25ef53b4f2ecf3524ec70a74887b6e Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Tue, 30 May 2023 10:25:41 -0700 Subject: [PATCH 4/5] [indexer] oldest known mining pool block per network --- backend/src/api/pools-parser.ts | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index f1619cac4..c81c549cc 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -129,7 +129,15 @@ class PoolsParser { LIMIT 1`, [pool.id] ); - const oldestBlockHeight = oldestPoolBlock.length ?? 0 > 0 ? oldestPoolBlock[0].height : 130635; + + let firstKnownBlockPool = 130635; // https://mempool.space/block/0000000000000a067d94ff753eec72830f1205ad3a4c216a08a80c832e551a52 + if (config.MEMPOOL.NETWORK === 'testnet') { + firstKnownBlockPool = 21106; // https://mempool.space/testnet/block/0000000070b701a5b6a1b965f6a38e0472e70b2bb31b973e4638dec400877581 + } else if (config.MEMPOOL.NETWORK === 'signet') { + firstKnownBlockPool = 0; + } + + const oldestBlockHeight = oldestPoolBlock.length ?? 0 > 0 ? oldestPoolBlock[0].height : firstKnownBlockPool; const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`); this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${oldestBlockHeight} for re-indexing`); await DB.query(` @@ -150,11 +158,18 @@ class PoolsParser { } private async $deleteUnknownBlocks(): Promise { + let firstKnownBlockPool = 130635; // https://mempool.space/block/0000000000000a067d94ff753eec72830f1205ad3a4c216a08a80c832e551a52 + if (config.MEMPOOL.NETWORK === 'testnet') { + firstKnownBlockPool = 21106; // https://mempool.space/testnet/block/0000000070b701a5b6a1b965f6a38e0472e70b2bb31b973e4638dec400877581 + } else if (config.MEMPOOL.NETWORK === 'signet') { + firstKnownBlockPool = 0; + } + const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`); - this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height 130635 for re-indexing`); + this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${firstKnownBlockPool} for re-indexing`); await DB.query(` DELETE FROM blocks - WHERE pool_id = ? AND height >= 130635`, + WHERE pool_id = ? AND height >= ${firstKnownBlockPool}`, [unknownPool[0].id] ); From ea51ab8d0bf96bf4b3265ec6e2c9b3a9a018e9f1 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Tue, 30 May 2023 10:42:41 -0700 Subject: [PATCH 5/5] [indexer] show github sha when successufly updated pools json --- backend/src/api/pools-parser.ts | 2 -- backend/src/tasks/pools-updater.ts | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/backend/src/api/pools-parser.ts b/backend/src/api/pools-parser.ts index c81c549cc..66f09a9f7 100644 --- a/backend/src/api/pools-parser.ts +++ b/backend/src/api/pools-parser.ts @@ -80,8 +80,6 @@ class PoolsParser { } } } - - logger.info('Mining pools-v2.json import completed'); } /** diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index b24da124e..6e8173c21 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -71,7 +71,7 @@ class PoolsUpdater { poolsParser.setMiningPools(poolsJson); if (config.DATABASE.ENABLED === false) { // Don't run db operations - logger.info('Mining pools-v2.json import completed (no database)'); + logger.info(`Mining pools-v2.json (${githubSha}) import completed (no database)`); return; } @@ -84,7 +84,7 @@ class PoolsUpdater { logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, logger.tags.mining); await DB.query('ROLLBACK;'); } - logger.info('PoolsUpdater completed'); + logger.info(`Mining pools-v2.json (${githubSha}) import completed`); } catch (e) { this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week