From d483362a9b60c26ec09ddfa7c9307a05f581e6aa Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 4 Mar 2023 10:51:13 +0900 Subject: [PATCH 1/4] Handle missing price (show 0) --- backend/src/api/websocket-handler.ts | 9 +- backend/src/mempool.interfaces.ts | 1 - backend/src/repositories/PricesRepository.ts | 154 ++++++++++++------ backend/src/tasks/price-feeds/bitfinex-api.ts | 3 - backend/src/tasks/price-updater.ts | 36 ++-- .../components/amount/amount.component.html | 8 +- frontend/src/app/fiat/fiat.component.html | 6 +- frontend/src/app/services/price.service.ts | 4 +- 8 files changed, 141 insertions(+), 80 deletions(-) diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index c1c3b3995..a96264825 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -1,8 +1,8 @@ import logger from '../logger'; import * as WebSocket from 'ws'; import { - BlockExtended, TransactionExtended, WebsocketResponse, MempoolBlock, MempoolBlockDelta, - OptimizedStatistic, ILoadingIndicators, IConversionRates + BlockExtended, TransactionExtended, WebsocketResponse, + OptimizedStatistic, ILoadingIndicators } from '../mempool.interfaces'; import blocks from './blocks'; import memPool from './mempool'; @@ -20,6 +20,7 @@ import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository import Audit from './audit'; import { deepClone } from '../utils/clone'; import priceUpdater from '../tasks/price-updater'; +import { ApiPrice } from '../repositories/PricesRepository'; class WebsocketHandler { private wss: WebSocket.Server | undefined; @@ -193,7 +194,7 @@ class WebsocketHandler { }); } - handleNewConversionRates(conversionRates: IConversionRates) { + handleNewConversionRates(conversionRates: ApiPrice) { if (!this.wss) { throw new Error('WebSocket.Server is not set'); } @@ -214,7 +215,7 @@ class WebsocketHandler { 'mempoolInfo': memPool.getMempoolInfo(), 'vBytesPerSecond': memPool.getVBytesPerSecond(), 'blocks': _blocks, - 'conversions': priceUpdater.latestPrices, + 'conversions': priceUpdater.getLatestPrices(), 'mempool-blocks': mempoolBlocks.getMempoolBlocks(), 'transactions': memPool.getLatestTransactions(), 'backendInfo': backendInfo.getBackendInfo(), diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index a7937e01d..8662770bc 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -293,7 +293,6 @@ interface RequiredParams { } export interface ILoadingIndicators { [name: string]: number; } -export interface IConversionRates { [currency: string]: number; } export interface IBackendInfo { hostname: string; diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index 6493735ee..4cbc06afd 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -1,6 +1,5 @@ import DB from '../database'; import logger from '../logger'; -import { IConversionRates } from '../mempool.interfaces'; import priceUpdater from '../tasks/price-updater'; export interface ApiPrice { @@ -13,6 +12,16 @@ export interface ApiPrice { AUD: number, JPY: number, } +const ApiPriceFields = ` + UNIX_TIMESTAMP(time) as time, + USD, + EUR, + GBP, + CAD, + CHF, + AUD, + JPY +`; export interface ExchangeRates { USDEUR: number, @@ -39,7 +48,7 @@ export const MAX_PRICES = { }; class PricesRepository { - public async $savePrices(time: number, prices: IConversionRates): Promise { + public async $savePrices(time: number, prices: ApiPrice): Promise { if (prices.USD === -1) { // Some historical price entries have no USD prices, so we just ignore them to avoid future UX issues // As of today there are only 4 (on 2013-09-05, 2013-0909, 2013-09-12 and 2013-09-26) so that's fine @@ -60,77 +69,115 @@ class PricesRepository { VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ? )`, [time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY] ); - } catch (e: any) { + } catch (e) { logger.err(`Cannot save exchange rate into db. Reason: ` + (e instanceof Error ? e.message : e)); throw e; } } public async $getOldestPriceTime(): Promise { - const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != 0 ORDER BY time LIMIT 1`); + const [oldestRow] = await DB.query(` + SELECT UNIX_TIMESTAMP(time) AS time + FROM prices + ORDER BY time + LIMIT 1 + `); return oldestRow[0] ? oldestRow[0].time : 0; } public async $getLatestPriceId(): Promise { - const [oldestRow] = await DB.query(`SELECT id from prices WHERE USD != 0 ORDER BY time DESC LIMIT 1`); - return oldestRow[0] ? oldestRow[0].id : null; - } - - public async $getLatestPriceTime(): Promise { - const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != 0 ORDER BY time DESC LIMIT 1`); - return oldestRow[0] ? oldestRow[0].time : 0; - } - - public async $getPricesTimes(): Promise { - const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != 0 ORDER BY time`); - return times.map(time => time.time); - } - - public async $getPricesTimesAndId(): Promise { - const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time, id, USD from prices ORDER BY time`); - return times; - } - - public async $getLatestConversionRates(): Promise { - const [rates]: any[] = await DB.query(` - SELECT USD, EUR, GBP, CAD, CHF, AUD, JPY + const [oldestRow] = await DB.query(` + SELECT id FROM prices ORDER BY time DESC LIMIT 1` ); - if (!rates || rates.length === 0) { + return oldestRow[0] ? oldestRow[0].id : null; + } + + public async $getLatestPriceTime(): Promise { + const [oldestRow] = await DB.query(` + SELECT UNIX_TIMESTAMP(time) AS time + FROM prices + ORDER BY time DESC + LIMIT 1` + ); + return oldestRow[0] ? oldestRow[0].time : 0; + } + + public async $getPricesTimes(): Promise { + const [times] = await DB.query(` + SELECT UNIX_TIMESTAMP(time) AS time + FROM prices + WHERE USD != -1 + ORDER BY time + `); + if (!Array.isArray(times)) { + return []; + } + return times.map(time => time.time); + } + + public async $getPricesTimesAndId(): Promise<{time: number, id: number, USD: number}[]> { + const [times] = await DB.query(` + SELECT + UNIX_TIMESTAMP(time) AS time, + id, + USD + FROM prices + ORDER BY time + `); + return times as {time: number, id: number, USD: number}[]; + } + + public async $getLatestConversionRates(): Promise { + const [rates] = await DB.query(` + SELECT ${ApiPriceFields} + FROM prices + ORDER BY time DESC + LIMIT 1` + ); + + if (!Array.isArray(rates) || rates.length === 0) { return priceUpdater.getEmptyPricesObj(); } - return rates[0]; + return rates[0] as ApiPrice; } public async $getNearestHistoricalPrice(timestamp: number | undefined): Promise { try { - const [rates]: any[] = await DB.query(` - SELECT *, UNIX_TIMESTAMP(time) AS time + const [rates] = await DB.query(` + SELECT ${ApiPriceFields} FROM prices WHERE UNIX_TIMESTAMP(time) < ? ORDER BY time DESC LIMIT 1`, [timestamp] ); - if (!rates) { + if (!Array.isArray(rates)) { throw Error(`Cannot get single historical price from the database`); } // Compute fiat exchange rates - const latestPrice = await this.$getLatestConversionRates(); + let latestPrice = rates[0] as ApiPrice; + if (latestPrice.USD === -1) { + latestPrice = priceUpdater.getEmptyPricesObj(); + } + + const computeFx = (usd: number, other: number): number => + Math.round(Math.max(other, 0) / Math.max(usd, 1) * 100) / 100; + const exchangeRates: ExchangeRates = { - USDEUR: Math.round(latestPrice.EUR / latestPrice.USD * 100) / 100, - USDGBP: Math.round(latestPrice.GBP / latestPrice.USD * 100) / 100, - USDCAD: Math.round(latestPrice.CAD / latestPrice.USD * 100) / 100, - USDCHF: Math.round(latestPrice.CHF / latestPrice.USD * 100) / 100, - USDAUD: Math.round(latestPrice.AUD / latestPrice.USD * 100) / 100, - USDJPY: Math.round(latestPrice.JPY / latestPrice.USD * 100) / 100, + USDEUR: computeFx(latestPrice.USD, latestPrice.EUR), + USDGBP: computeFx(latestPrice.USD, latestPrice.GBP), + USDCAD: computeFx(latestPrice.USD, latestPrice.CAD), + USDCHF: computeFx(latestPrice.USD, latestPrice.CHF), + USDAUD: computeFx(latestPrice.USD, latestPrice.AUD), + USDJPY: computeFx(latestPrice.USD, latestPrice.JPY), }; return { - prices: rates, + prices: rates as ApiPrice[], exchangeRates: exchangeRates }; } catch (e) { @@ -141,28 +188,35 @@ class PricesRepository { public async $getHistoricalPrices(): Promise { try { - const [rates]: any[] = await DB.query(` - SELECT *, UNIX_TIMESTAMP(time) AS time + const [rates] = await DB.query(` + SELECT ${ApiPriceFields} FROM prices ORDER BY time DESC `); - if (!rates) { + if (!Array.isArray(rates)) { throw Error(`Cannot get average historical price from the database`); } // Compute fiat exchange rates - const latestPrice: ApiPrice = rates[0]; + let latestPrice = rates[0] as ApiPrice; + if (latestPrice.USD === -1) { + latestPrice = priceUpdater.getEmptyPricesObj(); + } + + const computeFx = (usd: number, other: number): number => + Math.round(Math.max(other, 0) / Math.max(usd, 1) * 100) / 100; + const exchangeRates: ExchangeRates = { - USDEUR: Math.round(latestPrice.EUR / latestPrice.USD * 100) / 100, - USDGBP: Math.round(latestPrice.GBP / latestPrice.USD * 100) / 100, - USDCAD: Math.round(latestPrice.CAD / latestPrice.USD * 100) / 100, - USDCHF: Math.round(latestPrice.CHF / latestPrice.USD * 100) / 100, - USDAUD: Math.round(latestPrice.AUD / latestPrice.USD * 100) / 100, - USDJPY: Math.round(latestPrice.JPY / latestPrice.USD * 100) / 100, + USDEUR: computeFx(latestPrice.USD, latestPrice.EUR), + USDGBP: computeFx(latestPrice.USD, latestPrice.GBP), + USDCAD: computeFx(latestPrice.USD, latestPrice.CAD), + USDCHF: computeFx(latestPrice.USD, latestPrice.CHF), + USDAUD: computeFx(latestPrice.USD, latestPrice.AUD), + USDJPY: computeFx(latestPrice.USD, latestPrice.JPY), }; return { - prices: rates, + prices: rates as ApiPrice[], exchangeRates: exchangeRates }; } catch (e) { diff --git a/backend/src/tasks/price-feeds/bitfinex-api.ts b/backend/src/tasks/price-feeds/bitfinex-api.ts index 0e06c3af7..30b70e9eb 100644 --- a/backend/src/tasks/price-feeds/bitfinex-api.ts +++ b/backend/src/tasks/price-feeds/bitfinex-api.ts @@ -8,9 +8,6 @@ class BitfinexApi implements PriceFeed { public url: string = 'https://api.bitfinex.com/v1/pubticker/BTC'; public urlHist: string = 'https://api-pub.bitfinex.com/v2/candles/trade:{GRANULARITY}:tBTC{CURRENCY}/hist'; - constructor() { - } - public async $fetchPrice(currency): Promise { const response = await query(this.url + currency); if (response && response['last_price']) { diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index b39e152ae..ccb8d3e68 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -2,8 +2,7 @@ import * as fs from 'fs'; import path from 'path'; import config from '../config'; import logger from '../logger'; -import { IConversionRates } from '../mempool.interfaces'; -import PricesRepository, { MAX_PRICES } from '../repositories/PricesRepository'; +import PricesRepository, { ApiPrice, MAX_PRICES } from '../repositories/PricesRepository'; import BitfinexApi from './price-feeds/bitfinex-api'; import BitflyerApi from './price-feeds/bitflyer-api'; import CoinbaseApi from './price-feeds/coinbase-api'; @@ -21,18 +20,18 @@ export interface PriceFeed { } export interface PriceHistory { - [timestamp: number]: IConversionRates; + [timestamp: number]: ApiPrice; } class PriceUpdater { public historyInserted = false; - lastRun = 0; - lastHistoricalRun = 0; - running = false; - feeds: PriceFeed[] = []; - currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY']; - latestPrices: IConversionRates; - private ratesChangedCallback: ((rates: IConversionRates) => void) | undefined; + private lastRun = 0; + private lastHistoricalRun = 0; + private running = false; + private feeds: PriceFeed[] = []; + private currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY']; + private latestPrices: ApiPrice; + private ratesChangedCallback: ((rates: ApiPrice) => void) | undefined; constructor() { this.latestPrices = this.getEmptyPricesObj(); @@ -44,8 +43,13 @@ class PriceUpdater { this.feeds.push(new GeminiApi()); } - public getEmptyPricesObj(): IConversionRates { + public getLatestPrices(): ApiPrice { + return this.latestPrices; + } + + public getEmptyPricesObj(): ApiPrice { return { + time: 0, USD: -1, EUR: -1, GBP: -1, @@ -56,7 +60,7 @@ class PriceUpdater { }; } - public setRatesChangedCallback(fn: (rates: IConversionRates) => void) { + public setRatesChangedCallback(fn: (rates: ApiPrice) => void): void { this.ratesChangedCallback = fn; } @@ -156,6 +160,10 @@ class PriceUpdater { } this.lastRun = new Date().getTime() / 1000; + + if (this.latestPrices.USD === -1) { + this.latestPrices = await PricesRepository.$getLatestConversionRates(); + } } /** @@ -224,7 +232,7 @@ class PriceUpdater { // Group them by timestamp and currency, for example // grouped[123456789]['USD'] = [1, 2, 3, 4]; - const grouped: any = {}; + const grouped = {}; for (const historicalEntry of historicalPrices) { for (const time in historicalEntry) { if (existingPriceTimes.includes(parseInt(time, 10))) { @@ -249,7 +257,7 @@ class PriceUpdater { // Average prices and insert everything into the db let totalInserted = 0; for (const time in grouped) { - const prices: IConversionRates = this.getEmptyPricesObj(); + const prices: ApiPrice = this.getEmptyPricesObj(); for (const currency in grouped[time]) { if (grouped[time][currency].length === 0) { continue; diff --git a/frontend/src/app/components/amount/amount.component.html b/frontend/src/app/components/amount/amount.component.html index ce9c02d78..27fd59110 100644 --- a/frontend/src/app/components/amount/amount.component.html +++ b/frontend/src/app/components/amount/amount.component.html @@ -3,13 +3,15 @@ {{ addPlus && satoshis >= 0 ? '+' : '' }} {{ ( - (blockConversion.price[currency] >= 0 ? blockConversion.price[currency] : null) ?? - (blockConversion.price['USD'] * blockConversion.exchangeRates['USD' + currency]) ?? 0 + (blockConversion.price[currency] > -1 ? blockConversion.price[currency] : null) ?? + (blockConversion.price['USD'] > -1 ? blockConversion.price['USD'] * blockConversion.exchangeRates['USD' + currency] : null) ?? 0 ) * satoshis / 100000000 | fiatCurrency : digitsInfo : currency }} - {{ addPlus && satoshis >= 0 ? '+' : '' }}{{ (conversions ? conversions[currency] : 0) * satoshis / 100000000 | fiatCurrency : digitsInfo : currency }} + {{ addPlus && satoshis >= 0 ? '+' : '' }} + {{ (conversions[currency] > -1 ? conversions[currency] : 0) * satoshis / 100000000 | fiatCurrency : digitsInfo : currency }} + diff --git a/frontend/src/app/fiat/fiat.component.html b/frontend/src/app/fiat/fiat.component.html index 998153d29..00dd1250a 100644 --- a/frontend/src/app/fiat/fiat.component.html +++ b/frontend/src/app/fiat/fiat.component.html @@ -1,14 +1,14 @@ {{ ( - (blockConversion.price[currency] >= 0 ? blockConversion.price[currency] : null) ?? - (blockConversion.price['USD'] * blockConversion.exchangeRates['USD' + currency]) ?? 0 + (blockConversion.price[currency] > -1 ? blockConversion.price[currency] : null) ?? + (blockConversion.price['USD'] > -1 ? blockConversion.price['USD'] * blockConversion.exchangeRates['USD' + currency] : null) ?? 0 ) * value / 100000000 | fiatCurrency : digitsInfo : currency }} - {{ (conversions[currency] ?? conversions['USD'] ?? 0) * value / 100000000 | fiatCurrency : digitsInfo : currency }} + {{ (conversions[currency] > -1 ? conversions[currency] : 0) * value / 100000000 | fiatCurrency : digitsInfo : currency }} \ No newline at end of file diff --git a/frontend/src/app/services/price.service.ts b/frontend/src/app/services/price.service.ts index 93c4ce449..4236205ca 100644 --- a/frontend/src/app/services/price.service.ts +++ b/frontend/src/app/services/price.service.ts @@ -89,7 +89,7 @@ export class PriceService { return this.singlePriceObservable$.pipe( map((conversion) => { if (conversion.prices.length <= 0) { - return this.getEmptyPrice(); + return undefined; } return { price: { @@ -113,7 +113,7 @@ export class PriceService { return this.priceObservable$.pipe( map((conversion) => { - if (!blockTimestamp) { + if (!blockTimestamp || !conversion) { return undefined; } From 2e74d7fa4ad661e35b797ab8dd998478756ad899 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 25 Feb 2023 11:28:44 +0900 Subject: [PATCH 2/4] Remove mining db stats - replaced by runtime state variable --- backend/src/api/database-migration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index bf552889d..1ef31c90b 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -1037,7 +1037,7 @@ class DatabaseMigration { await this.$executeQuery('DELETE FROM `pools`'); await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1'); await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`); -} + } private async $convertCompactCpfpTables(): Promise { try { From 32a260473ab3845da2c3c50b949eb07b67a15dfd Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 25 Feb 2023 11:46:52 +0900 Subject: [PATCH 3/4] Update some mining indexing logs --- backend/src/api/mining/mining.ts | 28 +++++++++---------- backend/src/indexer.ts | 8 +++--- .../DifficultyAdjustmentsRepository.ts | 18 ++++++------ .../src/repositories/HashratesRepository.ts | 22 +++++++-------- 4 files changed, 38 insertions(+), 38 deletions(-) diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index 78d313b48..45b00e021 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -117,7 +117,7 @@ class Mining { poolsStatistics['lastEstimatedHashrate'] = await bitcoinClient.getNetworkHashPs(totalBlock24h); } catch (e) { poolsStatistics['lastEstimatedHashrate'] = 0; - logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate'); + logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate', logger.tags.mining); } return poolsStatistics; @@ -145,7 +145,7 @@ class Mining { try { currentEstimatedHashrate = await bitcoinClient.getNetworkHashPs(totalBlock24h); } catch (e) { - logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate'); + logger.debug('Bitcoin Core is not available, using zeroed value for current hashrate', logger.tags.mining); } return { @@ -208,7 +208,7 @@ class Mining { const startedAt = new Date().getTime() / 1000; let timer = new Date().getTime() / 1000; - logger.debug(`Indexing weekly mining pool hashrate`); + logger.debug(`Indexing weekly mining pool hashrate`, logger.tags.mining); loadingIndicators.setProgress('weekly-hashrate-indexing', 0); while (toTimestamp > genesisTimestamp && toTimestamp > oldestConsecutiveBlockTimestamp) { @@ -245,7 +245,7 @@ class Mining { }); } - newlyIndexed += hashrates.length; + newlyIndexed += hashrates.length / Math.max(1, pools.length); await HashratesRepository.$saveHashrates(hashrates); hashrates.length = 0; } @@ -256,7 +256,7 @@ class Mining { const weeksPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds)); const progress = Math.round(totalIndexed / totalWeekIndexed * 10000) / 100; const formattedDate = new Date(fromTimestamp).toUTCString(); - logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`); + logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds.toFixed(2)} weeks/sec | total: ~${totalIndexed}/${Math.round(totalWeekIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining); timer = new Date().getTime() / 1000; indexedThisRun = 0; loadingIndicators.setProgress('weekly-hashrate-indexing', progress, false); @@ -268,14 +268,14 @@ class Mining { } this.lastWeeklyHashrateIndexingDate = new Date().getUTCDate(); if (newlyIndexed > 0) { - logger.notice(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`, logger.tags.mining); + logger.notice(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed} weeks`, logger.tags.mining); } else { - logger.debug(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`, logger.tags.mining); + logger.debug(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed} weeks`, logger.tags.mining); } loadingIndicators.setProgress('weekly-hashrate-indexing', 100); } catch (e) { loadingIndicators.setProgress('weekly-hashrate-indexing', 100); - logger.err(`Weekly mining pools hashrates indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`); + logger.err(`Weekly mining pools hashrates indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining); throw e; } } @@ -308,7 +308,7 @@ class Mining { const startedAt = new Date().getTime() / 1000; let timer = new Date().getTime() / 1000; - logger.debug(`Indexing daily network hashrate`); + logger.debug(`Indexing daily network hashrate`, logger.tags.mining); loadingIndicators.setProgress('daily-hashrate-indexing', 0); while (toTimestamp > genesisTimestamp && toTimestamp > oldestConsecutiveBlockTimestamp) { @@ -346,7 +346,7 @@ class Mining { const daysPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds)); const progress = Math.round(totalIndexed / totalDayIndexed * 10000) / 100; const formattedDate = new Date(fromTimestamp).toUTCString(); - logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`); + logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds.toFixed(2)} days/sec | total: ~${totalIndexed}/${Math.round(totalDayIndexed)} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining); timer = new Date().getTime() / 1000; indexedThisRun = 0; loadingIndicators.setProgress('daily-hashrate-indexing', progress); @@ -380,7 +380,7 @@ class Mining { loadingIndicators.setProgress('daily-hashrate-indexing', 100); } catch (e) { loadingIndicators.setProgress('daily-hashrate-indexing', 100); - logger.err(`Daily network hashrate indexing failed. Trying again in 10 seconds. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining); + logger.err(`Daily network hashrate indexing failed. Trying again later. Reason: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining); throw e; } } @@ -446,7 +446,7 @@ class Mining { const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer)); if (elapsedSeconds > 5) { const progress = Math.round(totalBlockChecked / blocks.length * 100); - logger.info(`Indexing difficulty adjustment at block #${block.height} | Progress: ${progress}%`); + logger.info(`Indexing difficulty adjustment at block #${block.height} | Progress: ${progress}%`, logger.tags.mining); timer = new Date().getTime() / 1000; } } @@ -499,7 +499,7 @@ class Mining { if (blocksWithoutPrices.length > 200000) { logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`; } - logger.debug(logStr); + logger.debug(logStr, logger.tags.mining); await BlocksRepository.$saveBlockPrices(blocksPrices); blocksPrices.length = 0; } @@ -511,7 +511,7 @@ class Mining { if (blocksWithoutPrices.length > 200000) { logStr += ` | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`; } - logger.debug(logStr); + logger.debug(logStr, logger.tags.mining); await BlocksRepository.$saveBlockPrices(blocksPrices); } } catch (e) { diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 1665e443f..3b16ad155 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -76,13 +76,13 @@ class Indexer { this.tasksRunning.push(task); const lastestPriceId = await PricesRepository.$getLatestPriceId(); if (priceUpdater.historyInserted === false || lastestPriceId === null) { - logger.debug(`Blocks prices indexer is waiting for the price updater to complete`); + logger.debug(`Blocks prices indexer is waiting for the price updater to complete`, logger.tags.mining); setTimeout(() => { this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task); this.runSingleTask('blocksPrices'); }, 10000); } else { - logger.debug(`Blocks prices indexer will run now`); + logger.debug(`Blocks prices indexer will run now`, logger.tags.mining); await mining.$indexBlockPrices(); this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task); } @@ -112,7 +112,7 @@ class Indexer { this.runIndexer = false; this.indexerRunning = true; - logger.info(`Running mining indexer`); + logger.debug(`Running mining indexer`); await this.checkAvailableCoreIndexes(); @@ -122,7 +122,7 @@ class Indexer { const chainValid = await blocks.$generateBlockDatabase(); if (chainValid === false) { // Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration - logger.warn(`The chain of block hash is invalid, re-indexing invalid data in 10 seconds.`); + logger.warn(`The chain of block hash is invalid, re-indexing invalid data in 10 seconds.`, logger.tags.mining); setTimeout(() => this.reindex(), 10000); this.indexerRunning = false; return; diff --git a/backend/src/repositories/DifficultyAdjustmentsRepository.ts b/backend/src/repositories/DifficultyAdjustmentsRepository.ts index 1c101bcf2..0b19cc640 100644 --- a/backend/src/repositories/DifficultyAdjustmentsRepository.ts +++ b/backend/src/repositories/DifficultyAdjustmentsRepository.ts @@ -20,9 +20,9 @@ class DifficultyAdjustmentsRepository { await DB.query(query, params); } catch (e: any) { if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart - logger.debug(`Cannot save difficulty adjustment at block ${adjustment.height}, already indexed, ignoring`); + logger.debug(`Cannot save difficulty adjustment at block ${adjustment.height}, already indexed, ignoring`, logger.tags.mining); } else { - logger.err(`Cannot save difficulty adjustment at block ${adjustment.height}. Reason: ${e instanceof Error ? e.message : e}`); + logger.err(`Cannot save difficulty adjustment at block ${adjustment.height}. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining); throw e; } } @@ -54,7 +54,7 @@ class DifficultyAdjustmentsRepository { const [rows] = await DB.query(query); return rows as IndexedDifficultyAdjustment[]; } catch (e) { - logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e)); + logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e), logger.tags.mining); throw e; } } @@ -83,7 +83,7 @@ class DifficultyAdjustmentsRepository { const [rows] = await DB.query(query); return rows as IndexedDifficultyAdjustment[]; } catch (e) { - logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e)); + logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e), logger.tags.mining); throw e; } } @@ -93,27 +93,27 @@ class DifficultyAdjustmentsRepository { const [rows]: any[] = await DB.query(`SELECT height FROM difficulty_adjustments`); return rows.map(block => block.height); } catch (e: any) { - logger.err(`Cannot get difficulty adjustment block heights. Reason: ${e instanceof Error ? e.message : e}`); + logger.err(`Cannot get difficulty adjustment block heights. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining); throw e; } } public async $deleteAdjustementsFromHeight(height: number): Promise { try { - logger.info(`Delete newer difficulty adjustments from height ${height} from the database`); + logger.info(`Delete newer difficulty adjustments from height ${height} from the database`, logger.tags.mining); await DB.query(`DELETE FROM difficulty_adjustments WHERE height >= ?`, [height]); } catch (e: any) { - logger.err(`Cannot delete difficulty adjustments from the database. Reason: ${e instanceof Error ? e.message : e}`); + logger.err(`Cannot delete difficulty adjustments from the database. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining); throw e; } } public async $deleteLastAdjustment(): Promise { try { - logger.info(`Delete last difficulty adjustment from the database`); + logger.info(`Delete last difficulty adjustment from the database`, logger.tags.mining); await DB.query(`DELETE FROM difficulty_adjustments ORDER BY time LIMIT 1`); } catch (e: any) { - logger.err(`Cannot delete last difficulty adjustment from the database. Reason: ${e instanceof Error ? e.message : e}`); + logger.err(`Cannot delete last difficulty adjustment from the database. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining); throw e; } } diff --git a/backend/src/repositories/HashratesRepository.ts b/backend/src/repositories/HashratesRepository.ts index c380e87d9..875f77b34 100644 --- a/backend/src/repositories/HashratesRepository.ts +++ b/backend/src/repositories/HashratesRepository.ts @@ -25,7 +25,7 @@ class HashratesRepository { try { await DB.query(query); } catch (e: any) { - logger.err('Cannot save indexed hashrate into db. Reason: ' + (e instanceof Error ? e.message : e)); + logger.err('Cannot save indexed hashrate into db. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining); throw e; } } @@ -51,7 +51,7 @@ class HashratesRepository { const [rows]: any[] = await DB.query(query); return rows; } catch (e) { - logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e)); + logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining); throw e; } } @@ -78,7 +78,7 @@ class HashratesRepository { const [rows]: any[] = await DB.query(query); return rows; } catch (e) { - logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e)); + logger.err('Cannot fetch network hashrate history. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining); throw e; } } @@ -93,7 +93,7 @@ class HashratesRepository { const [rows]: any[] = await DB.query(query); return rows.map(row => row.timestamp); } catch (e) { - logger.err('Cannot retreive indexed weekly hashrate timestamps. Reason: ' + (e instanceof Error ? e.message : e)); + logger.err('Cannot retreive indexed weekly hashrate timestamps. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining); throw e; } } @@ -128,7 +128,7 @@ class HashratesRepository { const [rows]: any[] = await DB.query(query); return rows; } catch (e) { - logger.err('Cannot fetch weekly pools hashrate history. Reason: ' + (e instanceof Error ? e.message : e)); + logger.err('Cannot fetch weekly pools hashrate history. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining); throw e; } } @@ -158,7 +158,7 @@ class HashratesRepository { const [rows]: any[] = await DB.query(query, [pool.id]); boundaries = rows[0]; } catch (e) { - logger.err('Cannot fetch hashrate start/end timestamps for this pool. Reason: ' + (e instanceof Error ? e.message : e)); + logger.err('Cannot fetch hashrate start/end timestamps for this pool. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining); } // Get hashrates entries between boundaries @@ -173,7 +173,7 @@ class HashratesRepository { const [rows]: any[] = await DB.query(query, [boundaries.firstTimestamp, boundaries.lastTimestamp, pool.id]); return rows; } catch (e) { - logger.err('Cannot fetch pool hashrate history for this pool. Reason: ' + (e instanceof Error ? e.message : e)); + logger.err('Cannot fetch pool hashrate history for this pool. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining); throw e; } } @@ -192,7 +192,7 @@ class HashratesRepository { } return rows[0]['number']; } catch (e) { - logger.err(`Cannot retrieve last indexing run for ${key}. Reason: ` + (e instanceof Error ? e.message : e)); + logger.err(`Cannot retrieve last indexing run for ${key}. Reason: ` + (e instanceof Error ? e.message : e), logger.tags.mining); throw e; } } @@ -201,7 +201,7 @@ class HashratesRepository { * Delete most recent data points for re-indexing */ public async $deleteLastEntries() { - logger.info(`Delete latest hashrates data points from the database`); + logger.info(`Delete latest hashrates data points from the database`, logger.tags.mining); try { const [rows]: any[] = await DB.query(`SELECT MAX(hashrate_timestamp) as timestamp FROM hashrates GROUP BY type`); @@ -212,7 +212,7 @@ class HashratesRepository { mining.lastHashrateIndexingDate = null; mining.lastWeeklyHashrateIndexingDate = null; } catch (e) { - logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e)); + logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining); } } @@ -228,7 +228,7 @@ class HashratesRepository { mining.lastHashrateIndexingDate = null; mining.lastWeeklyHashrateIndexingDate = null; } catch (e) { - logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e)); + logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e), logger.tags.mining); } } } From 001be82f5abd836aa9d8f6cf44282edbe152d1fb Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 25 Feb 2023 14:01:48 +0900 Subject: [PATCH 4/4] Move some notice into info --- backend/src/api/mining/mining.ts | 6 +++--- backend/src/tasks/lightning/sync-tasks/stats-importer.ts | 2 +- backend/src/tasks/pools-updater.ts | 2 +- backend/src/tasks/price-feeds/kraken-api.ts | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index 45b00e021..0ab2267c3 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -268,7 +268,7 @@ class Mining { } this.lastWeeklyHashrateIndexingDate = new Date().getUTCDate(); if (newlyIndexed > 0) { - logger.notice(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed} weeks`, logger.tags.mining); + logger.info(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed} weeks`, logger.tags.mining); } else { logger.debug(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed} weeks`, logger.tags.mining); } @@ -373,7 +373,7 @@ class Mining { this.lastHashrateIndexingDate = new Date().getUTCDate(); if (newlyIndexed > 0) { - logger.notice(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`, logger.tags.mining); + logger.info(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`, logger.tags.mining); } else { logger.debug(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`, logger.tags.mining); } @@ -452,7 +452,7 @@ class Mining { } if (totalIndexed > 0) { - logger.notice(`Indexed ${totalIndexed} difficulty adjustments`, logger.tags.mining); + logger.info(`Indexed ${totalIndexed} difficulty adjustments`, logger.tags.mining); } else { logger.debug(`Indexed ${totalIndexed} difficulty adjustments`, logger.tags.mining); } diff --git a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts index 14f592a14..d009ce052 100644 --- a/backend/src/tasks/lightning/sync-tasks/stats-importer.ts +++ b/backend/src/tasks/lightning/sync-tasks/stats-importer.ts @@ -411,7 +411,7 @@ class LightningStatsImporter { } if (totalProcessed > 0) { - logger.notice(`Lightning network stats historical import completed`, logger.tags.ln); + logger.info(`Lightning network stats historical import completed`, logger.tags.ln); } } catch (e) { logger.err(`Lightning network stats historical failed. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.ln); diff --git a/backend/src/tasks/pools-updater.ts b/backend/src/tasks/pools-updater.ts index 32de85f3a..d8b77cf01 100644 --- a/backend/src/tasks/pools-updater.ts +++ b/backend/src/tasks/pools-updater.ts @@ -82,7 +82,7 @@ class PoolsUpdater { logger.err(`Could not migrate mining pools, rolling back. Exception: ${JSON.stringify(e)}`, logger.tags.mining); await DB.query('ROLLBACK;'); } - logger.notice('PoolsUpdater completed'); + logger.info('PoolsUpdater completed'); } catch (e) { this.lastRun = now - (oneWeek - oneDay); // Try again in 24h instead of waiting next week diff --git a/backend/src/tasks/price-feeds/kraken-api.ts b/backend/src/tasks/price-feeds/kraken-api.ts index c6b3c0c11..ebc784c6f 100644 --- a/backend/src/tasks/price-feeds/kraken-api.ts +++ b/backend/src/tasks/price-feeds/kraken-api.ts @@ -98,7 +98,7 @@ class KrakenApi implements PriceFeed { } if (Object.keys(priceHistory).length > 0) { - logger.notice(`Inserted ${Object.keys(priceHistory).length} Kraken EUR, USD, GBP, JPY, CAD, CHF and AUD weekly price history into db`, logger.tags.mining); + logger.info(`Inserted ${Object.keys(priceHistory).length} Kraken EUR, USD, GBP, JPY, CAD, CHF and AUD weekly price history into db`, logger.tags.mining); } } }