machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3820 from mempool/nymkappa/reindexing-pools-update

Browse Source 
Mining pool update / re-indexer improvment
This commit is contained in:
softsimon 2023-06-14 19:16:08 +02:00 committed by GitHub
commit 4390ffe3b6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 56 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/api/database-migration.ts
View File

@ -1051,7 +1051,7 @@ class DatabaseMigration {
}
public async $blocksReindexingTruncate(): Promise<void> {
logger.warn(`Truncating pools, blocks and hashrates for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`);
logger.warn(`Truncating pools, blocks, hashrates and difficulty_adjustments tables for re-indexing (using '--reindex-blocks'). You can cancel this command within 5 seconds`);
await Common.sleep$(5000);
await this.$executeQuery(`TRUNCATE blocks`);

19
backend/src/api/mining/mining.ts
View File

@ -13,11 +13,15 @@ import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
import PricesRepository from '../../repositories/PricesRepository';
import { bitcoinCoreApi } from '../bitcoin/bitcoin-api-factory';
import { IEsploraApi } from '../bitcoin/esplora-api.interface';
import database from '../../database';
class Mining {
private blocksPriceIndexingRunning = false;
public lastHashrateIndexingDate: number | null = null;
public lastWeeklyHashrateIndexingDate: number | null = null;
public reindexHashrateRequested = false;
public reindexDifficultyAdjustmentRequested = false;
/**
* Get historical blocks health
@ -289,6 +293,14 @@ class Mining {
* Generate daily hashrate data
*/
public async $generateNetworkHashrateHistory(): Promise<void> {
// If a re-index was requested, truncate first
if (this.reindexHashrateRequested === true) {
logger.notice(`hashrates will now be re-indexed`);
await database.query(`TRUNCATE hashrates`);
this.lastHashrateIndexingDate = 0;
this.reindexHashrateRequested = false;
}
// We only run this once a day around midnight
const today = new Date().getUTCDate();
if (today === this.lastHashrateIndexingDate) {
@ -394,6 +406,13 @@ class Mining {
* Index difficulty adjustments
*/
public async $indexDifficultyAdjustments(): Promise<void> {
// If a re-index was requested, truncate first
if (this.reindexDifficultyAdjustmentRequested === true) {
logger.notice(`difficulty_adjustments will now be re-indexed`);
await database.query(`TRUNCATE difficulty_adjustments`);
this.reindexDifficultyAdjustmentRequested = false;
}
const indexedHeightsArray = await DifficultyAdjustmentsRepository.$getAdjustmentsHeights();
const indexedHeights = {};
for (const height of indexedHeightsArray) {

34
backend/src/api/pools-parser.ts
View File

@ -4,6 +4,7 @@ import config from '../config';
import PoolsRepository from '../repositories/PoolsRepository';
import { PoolTag } from '../mempool.interfaces';
import diskCache from './disk-cache';
import mining from './mining/mining';
class PoolsParser {
miningPools: any[] = [];
@ -73,14 +74,12 @@ class PoolsParser {
if (JSON.stringify(pool.addresses) !== poolDB.addresses ||
JSON.stringify(pool.regexes) !== poolDB.regexes) {
// Pool addresses changed or coinbase tags changed
logger.notice(`Updating addresses and/or coinbase tags for ${pool.name} mining pool. If 'AUTOMATIC_BLOCK_REINDEXING' is enabled, we will re-index its blocks and 'unknown' blocks`);
logger.notice(`Updating addresses and/or coinbase tags for ${pool.name} mining pool.`);
await PoolsRepository.$updateMiningPoolTags(poolDB.id, pool.addresses, pool.regexes);
await this.$deleteBlocksForPool(poolDB);
}
}
}
logger.info('Mining pools-v2.json import completed');
}
/**
@ -128,7 +127,15 @@ class PoolsParser {
LIMIT 1`,
[pool.id]
);
const oldestBlockHeight = oldestPoolBlock.length ?? 0 > 0 ? oldestPoolBlock[0].height : 130635;
let firstKnownBlockPool = 130635; // https://mempool.space/block/0000000000000a067d94ff753eec72830f1205ad3a4c216a08a80c832e551a52
if (config.MEMPOOL.NETWORK === 'testnet') {
firstKnownBlockPool = 21106; // https://mempool.space/testnet/block/0000000070b701a5b6a1b965f6a38e0472e70b2bb31b973e4638dec400877581
} else if (config.MEMPOOL.NETWORK === 'signet') {
firstKnownBlockPool = 0;
}
const oldestBlockHeight = oldestPoolBlock.length ?? 0 > 0 ? oldestPoolBlock[0].height : firstKnownBlockPool;
const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${oldestBlockHeight} for re-indexing`);
await DB.query(`
@ -142,16 +149,31 @@ class PoolsParser {
WHERE pool_id = ?`,
[pool.id]
);
// Re-index hashrates and difficulty adjustments later
mining.reindexHashrateRequested = true;
mining.reindexDifficultyAdjustmentRequested = true;
}
private async $deleteUnknownBlocks(): Promise<void> {
let firstKnownBlockPool = 130635; // https://mempool.space/block/0000000000000a067d94ff753eec72830f1205ad3a4c216a08a80c832e551a52
if (config.MEMPOOL.NETWORK === 'testnet') {
firstKnownBlockPool = 21106; // https://mempool.space/testnet/block/0000000070b701a5b6a1b965f6a38e0472e70b2bb31b973e4638dec400877581
} else if (config.MEMPOOL.NETWORK === 'signet') {
firstKnownBlockPool = 0;
}
const [unknownPool] = await DB.query(`SELECT id from pools where slug = "unknown"`);
this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height 130635 for re-indexing`);
this.uniqueLog(logger.notice, `Deleting blocks with unknown mining pool from height ${firstKnownBlockPool} for re-indexing`);
await DB.query(`
DELETE FROM blocks
WHERE pool_id = ? AND height >= 130635`,
WHERE pool_id = ? AND height >= ${firstKnownBlockPool}`,
[unknownPool[0].id]
);
// Re-index hashrates and difficulty adjustments later
mining.reindexHashrateRequested = true;
mining.reindexDifficultyAdjustmentRequested = true;
}
}

4
backend/src/tasks/pools-updater.ts
View File

@ -71,7 +71,7 @@ class PoolsUpdater {
poolsParser.setMiningPools(poolsJson);
if (config.DATABASE.ENABLED === false) { // Don't run db operations
logger.info('Mining pools-v2.json import completed (no database)');
logger.info(`Mining pools-v2.json (${githubSha}) import completed (no database)`);
return;
}
@ -84,7 +84,7 @@ class PoolsUpdater {
logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, logger.tags.mining);
await DB.query('ROLLBACK;');
}
logger.info('PoolsUpdater completed');
logger.info(`Mining pools-v2.json (${githubSha}) import completed`);
} catch (e) {
this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week

2
frontend/src/app/components/start/start.component.html
View File

@ -1,3 +1,5 @@
<app-indexing-progress *ngIf="showLoadingIndicator"></app-indexing-progress>
<ng-container *ngIf="specialEvent">
<div class="pyro">
<div class="before"></div>

4
frontend/src/app/components/start/start.component.ts
View File

@ -1,4 +1,4 @@
import { Component, ElementRef, HostListener, OnInit, OnDestroy, ViewChild } from '@angular/core';
import { Component, ElementRef, HostListener, OnInit, OnDestroy, ViewChild, Input } from '@angular/core';
import { Subscription } from 'rxjs';
import { StateService } from '../../services/state.service';
import { specialBlocks } from '../../app.constants';
@ -9,6 +9,8 @@ import { specialBlocks } from '../../app.constants';
styleUrls: ['./start.component.scss'],
})
export class StartComponent implements OnInit, OnDestroy {
@Input() showLoadingIndicator = false;
interval = 60;
colors = ['#5E35B1', '#ffffff'];

2
frontend/src/app/components/status-view/status-view.component.html
View File

@ -1,2 +1,2 @@
<app-start></app-start>
<app-start [showLoadingIndicator]="true"></app-start>
<app-footer></app-footer>