Merge pull request #3820 from mempool/nymkappa/reindexing-pools-update

Mining pool update / re-indexer improvment
This commit is contained in:
softsimon 2023-06-14 19:16:08 +02:00 committed by GitHub
commit 4390ffe3b6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 56 additions and 11 deletions

View File

@ -1051,7 +1051,7 @@ class DatabaseMigration {
} }
public async $blocksReindexingTruncate(): Promise<void> { public async $blocksReindexingTruncate(): Promise<void> {
logger.warn(`Truncating pools, blocks and hashrates for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`); logger.warn(`Truncating pools, blocks, hashrates and difficulty_adjustments tables for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`);
await Common.sleep$(5000); await Common.sleep$(5000);
await this.$executeQuery(`TRUNCATE blocks`); await this.$executeQuery(`TRUNCATE blocks`);

View File

@ -13,12 +13,16 @@ import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
import PricesRepository from '../../repositories/PricesRepository'; import PricesRepository from '../../repositories/PricesRepository';
import { bitcoinCoreApi } from '../bitcoin/bitcoin-api-factory'; import { bitcoinCoreApi } from '../bitcoin/bitcoin-api-factory';
import { IEsploraApi } from '../bitcoin/esplora-api.interface'; import { IEsploraApi } from '../bitcoin/esplora-api.interface';
import database from '../../database';
class Mining { class Mining {
private blocksPriceIndexingRunning = false; private blocksPriceIndexingRunning = false;
public lastHashrateIndexingDate: number | null = null; public lastHashrateIndexingDate: number | null = null;
public lastWeeklyHashrateIndexingDate: number | null = null; public lastWeeklyHashrateIndexingDate: number | null = null;
public reindexHashrateRequested = false;
public reindexDifficultyAdjustmentRequested = false;
/** /**
* Get historical blocks health * Get historical blocks health
*/ */
@ -289,6 +293,14 @@ class Mining {
* Generate daily hashrate data * Generate daily hashrate data
*/ */
public async $generateNetworkHashrateHistory(): Promise<void> { public async $generateNetworkHashrateHistory(): Promise<void> {
// If a re-index was requested, truncate first
if (this.reindexHashrateRequested === true) {
logger.notice(`hashrates will now be re-indexed`);
await database.query(`TRUNCATE hashrates`);
this.lastHashrateIndexingDate = 0;
this.reindexHashrateRequested = false;
}
// We only run this once a day around midnight // We only run this once a day around midnight
const today = new Date().getUTCDate(); const today = new Date().getUTCDate();
if (today === this.lastHashrateIndexingDate) { if (today === this.lastHashrateIndexingDate) {
@ -394,6 +406,13 @@ class Mining {
* Index difficulty adjustments * Index difficulty adjustments
*/ */
public async $indexDifficultyAdjustments(): Promise<void> { public async $indexDifficultyAdjustments(): Promise<void> {
// If a re-index was requested, truncate first
if (this.reindexDifficultyAdjustmentRequested === true) {
logger.notice(`difficulty_adjustments will now be re-indexed`);
await database.query(`TRUNCATE difficulty_adjustments`);
this.reindexDifficultyAdjustmentRequested = false;
}
const indexedHeightsArray = await DifficultyAdjustmentsRepository.$getAdjustmentsHeights(); const indexedHeightsArray = await DifficultyAdjustmentsRepository.$getAdjustmentsHeights();
const indexedHeights = {}; const indexedHeights = {};
for (const height of indexedHeightsArray) { for (const height of indexedHeightsArray) {

View File

@ -4,6 +4,7 @@ import config from '../config';
import PoolsRepository from '../repositories/PoolsRepository'; import PoolsRepository from '../repositories/PoolsRepository';
import { PoolTag } from '../mempool.interfaces'; import { PoolTag } from '../mempool.interfaces';
import diskCache from './disk-cache'; import diskCache from './disk-cache';
import mining from './mining/mining';
class PoolsParser { class PoolsParser {
miningPools: any[] = []; miningPools: any[] = [];
@ -73,14 +74,12 @@ class PoolsParser {
if (JSON.stringify(pool.addresses) !== poolDB.addresses || if (JSON.stringify(pool.addresses) !== poolDB.addresses ||
JSON.stringify(pool.regexes) !== poolDB.regexes) { JSON.stringify(pool.regexes) !== poolDB.regexes) {
// Pool addresses changed or coinbase tags changed // Pool addresses changed or coinbase tags changed
logger.notice(`Updating addresses and/or coinbase tags for ${pool.name} mining pool. If 'AUTOMATIC_BLOCK_REINDEXING' is enabled, we will re-index its blocks and 'unknown' blocks`); logger.notice(`Updating addresses and/or coinbase tags for ${pool.name} mining pool.`);
await PoolsRepository.$updateMiningPoolTags(poolDB.id, pool.addresses, pool.regexes); await PoolsRepository.$updateMiningPoolTags(poolDB.id, pool.addresses, pool.regexes);
await this.$deleteBlocksForPool(poolDB); await this.$deleteBlocksForPool(poolDB);
} }
} }
} }
logger.info('Mining pools-v2.json import completed');
} }
/** /**
@ -128,7 +127,15 @@ class PoolsParser {
LIMIT 1`, LIMIT 1`,
[pool.id] [pool.id]
); );
const oldestBlockHeight = oldestPoolBlock.length ?? 0 > 0 ? oldestPoolBlock[0].height : 130635;
let firstKnownBlockPool = 130635; // https://mempool.space/block/0000000000000a067d94ff753eec72830f1205ad3a4c216a08a80c832e551a52
if (config.MEMPOOL.NETWORK === 'testnet') {
firstKnownBlockPool = 21106; // https://mempool.space/testnet/block/0000000070b701a5b6a1b965f6a38e0472e70b2bb31b973e4638dec400877581
} else if (config.MEMPOOL.NETWORK === 'signet') {
firstKnownBlockPool = 0;
}
const oldestBlockHeight = oldestPoolBlock.length ?? 0 > 0 ? oldestPoolBlock[0].height : firstKnownBlockPool;
const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`); const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${oldestBlockHeight} for re-indexing`); this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${oldestBlockHeight} for re-indexing`);
await DB.query(` await DB.query(`
@ -142,16 +149,31 @@ class PoolsParser {
WHERE pool_id = ?`, WHERE pool_id = ?`,
[pool.id] [pool.id]
); );
// Re-index hashrates and difficulty adjustments later
mining.reindexHashrateRequested = true;
mining.reindexDifficultyAdjustmentRequested = true;
} }
private async $deleteUnknownBlocks(): Promise<void> { private async $deleteUnknownBlocks(): Promise<void> {
let firstKnownBlockPool = 130635; // https://mempool.space/block/0000000000000a067d94ff753eec72830f1205ad3a4c216a08a80c832e551a52
if (config.MEMPOOL.NETWORK === 'testnet') {
firstKnownBlockPool = 21106; // https://mempool.space/testnet/block/0000000070b701a5b6a1b965f6a38e0472e70b2bb31b973e4638dec400877581
} else if (config.MEMPOOL.NETWORK === 'signet') {
firstKnownBlockPool = 0;
}
const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`); const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height 130635 for re-indexing`); this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${firstKnownBlockPool} for re-indexing`);
await DB.query(` await DB.query(`
DELETE FROM blocks DELETE FROM blocks
WHERE pool_id = ? AND height >= 130635`, WHERE pool_id = ? AND height >= ${firstKnownBlockPool}`,
[unknownPool[0].id] [unknownPool[0].id]
); );
// Re-index hashrates and difficulty adjustments later
mining.reindexHashrateRequested = true;
mining.reindexDifficultyAdjustmentRequested = true;
} }
} }

View File

@ -71,7 +71,7 @@ class PoolsUpdater {
poolsParser.setMiningPools(poolsJson); poolsParser.setMiningPools(poolsJson);
if (config.DATABASE.ENABLED === false) { // Don't run db operations if (config.DATABASE.ENABLED === false) { // Don't run db operations
logger.info('Mining pools-v2.json import completed (no database)'); logger.info(`Mining pools-v2.json (${githubSha}) import completed (no database)`);
return; return;
} }
@ -84,7 +84,7 @@ class PoolsUpdater {
logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, logger.tags.mining); logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, logger.tags.mining);
await DB.query('ROLLBACK;'); await DB.query('ROLLBACK;');
} }
logger.info('PoolsUpdater completed'); logger.info(`Mining pools-v2.json (${githubSha}) import completed`);
} catch (e) { } catch (e) {
this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week

View File

@ -1,3 +1,5 @@
<app-indexing-progress *ngIf="showLoadingIndicator"></app-indexing-progress>
<ng-container *ngIf="specialEvent"> <ng-container *ngIf="specialEvent">
<div class="pyro"> <div class="pyro">
<div class="before"></div> <div class="before"></div>

View File

@ -1,4 +1,4 @@
import { Component, ElementRef, HostListener, OnInit, OnDestroy, ViewChild } from '@angular/core'; import { Component, ElementRef, HostListener, OnInit, OnDestroy, ViewChild, Input } from '@angular/core';
import { Subscription } from 'rxjs'; import { Subscription } from 'rxjs';
import { StateService } from '../../services/state.service'; import { StateService } from '../../services/state.service';
import { specialBlocks } from '../../app.constants'; import { specialBlocks } from '../../app.constants';
@ -9,6 +9,8 @@ import { specialBlocks } from '../../app.constants';
styleUrls: ['./start.component.scss'], styleUrls: ['./start.component.scss'],
}) })
export class StartComponent implements OnInit, OnDestroy { export class StartComponent implements OnInit, OnDestroy {
@Input() showLoadingIndicator = false;
interval = 60; interval = 60;
colors = ['#5E35B1', '#ffffff']; colors = ['#5E35B1', '#ffffff'];

View File

@ -1,2 +1,2 @@
<app-start></app-start> <app-start [showLoadingIndicator]="true"></app-start>
<app-footer></app-footer> <app-footer></app-footer>