From e0952a4c1d7fd1fd41ccd2360c1ba34e6eb07a05 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Sat, 16 Jul 2022 09:22:45 +0200 Subject: [PATCH] Wait for the price updater to complete before saving blocks prices --- backend/src/api/blocks.ts | 15 ++++++++++ backend/src/api/mining/mining.ts | 14 +++++----- backend/src/indexer.ts | 29 +++++++++++++++++--- backend/src/repositories/PricesRepository.ts | 5 ++++ backend/src/tasks/price-updater.ts | 9 +++--- 5 files changed, 57 insertions(+), 15 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 30f9fbf78..45ffd6079 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -22,6 +22,8 @@ import poolsParser from './pools-parser'; import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; import mining from './mining/mining'; import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository'; +import PricesRepository from '../repositories/PricesRepository'; +import priceUpdater from '../tasks/price-updater'; class Blocks { private blocks: BlockExtended[] = []; @@ -457,6 +459,19 @@ class Blocks { } await blocksRepository.$saveBlockInDatabase(blockExtended); + const lastestPriceId = await PricesRepository.$getLatestPriceId(); + if (priceUpdater.historyInserted === true && lastestPriceId !== null) { + await blocksRepository.$saveBlockPrices([{ + height: blockExtended.height, + priceId: lastestPriceId, + }]); + } else { + logger.info(`Cannot save block price for ${blockExtended.height} because the price updater hasnt completed yet. Trying again in 10 seconds.`) + setTimeout(() => { + indexer.runSingleTask('blocksPrices'); + }, 10000); + } + // Save blocks summary for visualization if it's enabled if (Common.blocksSummariesIndexingEnabled() === true) { await this.$getStrippedBlockTransactions(blockExtended.id, true); diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index 55e749596..55cd33bd3 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -473,7 +473,7 @@ class Mining { for (const block of blocksWithoutPrices) { // Quick optimisation, out mtgox feed only goes back to 2010-07-19 02:00:00, so skip the first 68951 blocks - if (block.height < 68951) { + if (['mainnet', 'testnet'].includes(config.MEMPOOL.NETWORK) && block.height < 68951) { blocksPrices.push({ height: block.height, priceId: prices[0].id, @@ -492,11 +492,11 @@ class Mining { if (blocksPrices.length >= 100000) { totalInserted += blocksPrices.length; + let logStr = `Linking ${blocksPrices.length} blocks to their closest price`; if (blocksWithoutPrices.length > 200000) { - logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`); - } else { - logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`); + logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`; } + logger.debug(logStr); await BlocksRepository.$saveBlockPrices(blocksPrices); blocksPrices.length = 0; } @@ -504,11 +504,11 @@ class Mining { if (blocksPrices.length > 0) { totalInserted += blocksPrices.length; + let logStr = `Linking ${blocksPrices.length} blocks to their closest price`; if (blocksWithoutPrices.length > 200000) { - logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`); - } else { - logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`); + logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`; } + logger.debug(logStr); await BlocksRepository.$saveBlockPrices(blocksPrices); } } catch (e) { diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index e452a42f4..26a407291 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -6,13 +6,12 @@ import logger from './logger'; import HashratesRepository from './repositories/HashratesRepository'; import bitcoinClient from './api/bitcoin/bitcoin-client'; import priceUpdater from './tasks/price-updater'; +import PricesRepository from './repositories/PricesRepository'; class Indexer { runIndexer = true; indexerRunning = false; - - constructor() { - } + tasksRunning: string[] = []; public reindex() { if (Common.indexingEnabled()) { @@ -20,6 +19,28 @@ class Indexer { } } + public async runSingleTask(task: 'blocksPrices') { + if (!Common.indexingEnabled()) { + return; + } + + if (task === 'blocksPrices' && !this.tasksRunning.includes(task)) { + this.tasksRunning.push(task); + const lastestPriceId = await PricesRepository.$getLatestPriceId(); + if (priceUpdater.historyInserted === false || lastestPriceId === null) { + logger.debug(`Blocks prices indexer is waiting for the price updater to complete`) + setTimeout(() => { + this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task) + this.runSingleTask('blocksPrices'); + }, 10000); + } else { + logger.debug(`Blocks prices indexer will run now`) + await mining.$indexBlockPrices(); + this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task) + } + } + } + public async $run() { if (!Common.indexingEnabled() || this.runIndexer === false || this.indexerRunning === true || mempool.hasPriority() @@ -50,7 +71,7 @@ class Indexer { return; } - await mining.$indexBlockPrices(); + this.runSingleTask('blocksPrices'); await mining.$indexDifficultyAdjustments(); await this.$resetHashratesIndexingState(); // TODO - Remove this as it's not efficient await mining.$generateNetworkHashrateHistory(); diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index 92fb4860f..cc79ff2a6 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -27,6 +27,11 @@ class PricesRepository { return oldestRow[0] ? oldestRow[0].time : 0; } + public async $getLatestPriceId(): Promise { + const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`); + return oldestRow[0] ? oldestRow[0].id : null; + } + public async $getLatestPriceTime(): Promise { const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`); return oldestRow[0] ? oldestRow[0].time : 0; diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index a5901d7f7..81066efb2 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -1,4 +1,5 @@ import * as fs from 'fs'; +import { Common } from '../api/common'; import config from '../config'; import logger from '../logger'; import PricesRepository from '../repositories/PricesRepository'; @@ -34,10 +35,10 @@ export interface Prices { } class PriceUpdater { - historyInserted: boolean = false; - lastRun: number = 0; - lastHistoricalRun: number = 0; - running: boolean = false; + public historyInserted = false; + lastRun = 0; + lastHistoricalRun = 0; + running = false; feeds: PriceFeed[] = []; currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY']; latestPrices: Prices;