From bfaddfc3456a8d16a4615c3406aaa26208a44ae2 Mon Sep 17 00:00:00 2001 From: natsoni Date: Thu, 7 Mar 2024 10:36:33 +0100 Subject: [PATCH 01/11] Add CURRENCY_API_KEY option to config files --- backend/mempool-config.sample.json | 3 ++- backend/src/__fixtures__/mempool-config.template.json | 3 ++- backend/src/__tests__/config.test.ts | 1 + backend/src/config.ts | 2 ++ docker/backend/mempool-config.json | 3 ++- docker/backend/start.sh | 2 ++ frontend/src/app/services/state.service.ts | 2 ++ 7 files changed, 13 insertions(+), 3 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 5d2cf1fba..473988651 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -36,7 +36,8 @@ "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, "PRICE_UPDATES_PER_HOUR": 1, - "MAX_TRACKED_ADDRESSES": 100 + "MAX_TRACKED_ADDRESSES": 100, + "CURRENCY_API_KEY": "your-api-key-from-freecurrencyapi.com" }, "CORE_RPC": { "HOST": "127.0.0.1", diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 26ae6fb28..1bc4de045 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -37,7 +37,8 @@ "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, "PRICE_UPDATES_PER_HOUR": 1, - "MAX_TRACKED_ADDRESSES": 1 + "MAX_TRACKED_ADDRESSES": 1, + "CURRENCY_API_KEY": "__MEMPOOL_CURRENCY_API_KEY__" }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 5066e0ef7..f382dcefa 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -51,6 +51,7 @@ describe('Mempool Backend Config', () => { ALLOW_UNREACHABLE: true, PRICE_UPDATES_PER_HOUR: 1, MAX_TRACKED_ADDRESSES: 1, + CURRENCY_API_KEY: '' }); expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true }); diff --git a/backend/src/config.ts b/backend/src/config.ts index 3330adca0..fea1150f8 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -42,6 +42,7 @@ interface IConfig { ALLOW_UNREACHABLE: boolean; PRICE_UPDATES_PER_HOUR: number; MAX_TRACKED_ADDRESSES: number; + CURRENCY_API_KEY: string; }; ESPLORA: { REST_API_URL: string; @@ -200,6 +201,7 @@ const defaults: IConfig = { 'ALLOW_UNREACHABLE': true, 'PRICE_UPDATES_PER_HOUR': 1, 'MAX_TRACKED_ADDRESSES': 1, + 'CURRENCY_API_KEY': '', }, 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index eca4cf14c..185bea19e 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -37,7 +37,8 @@ "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", "PRICE_UPDATES_PER_HOUR": __MEMPOOL_PRICE_UPDATES_PER_HOUR__, - "MAX_TRACKED_ADDRESSES": __MEMPOOL_MAX_TRACKED_ADDRESSES__ + "MAX_TRACKED_ADDRESSES": __MEMPOOL_MAX_TRACKED_ADDRESSES__, + "CURRENCY_API_KEY": "__MEMPOOL_CURRENCY_API_KEY__" }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/docker/backend/start.sh b/docker/backend/start.sh index b700bba32..a71b4b813 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -39,6 +39,7 @@ __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__=${MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT:=4000000} __MEMPOOL_ALLOW_UNREACHABLE__=${MEMPOOL_ALLOW_UNREACHABLE:=true} __MEMPOOL_PRICE_UPDATES_PER_HOUR__=${MEMPOOL_PRICE_UPDATES_PER_HOUR:=1} __MEMPOOL_MAX_TRACKED_ADDRESSES__=${MEMPOOL_MAX_TRACKED_ADDRESSES:=1} +__MEMPOOL_CURRENCY_API_KEY__=${MEMPOOL_CURRENCY_API_KEY:=""} # CORE_RPC __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1} @@ -195,6 +196,7 @@ sed -i "s!__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__!${__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGH sed -i "s!__MEMPOOL_ALLOW_UNREACHABLE__!${__MEMPOOL_ALLOW_UNREACHABLE__}!g" mempool-config.json sed -i "s!__MEMPOOL_PRICE_UPDATES_PER_HOUR__!${__MEMPOOL_PRICE_UPDATES_PER_HOUR__}!g" mempool-config.json sed -i "s!__MEMPOOL_MAX_TRACKED_ADDRESSES__!${__MEMPOOL_MAX_TRACKED_ADDRESSES__}!g" mempool-config.json +sed -i "s!__MEMPOOL_CURRENCY_API_KEY__!${__MEMPOOL_CURRENCY_API_KEY__}!g" mempool-config.json sed -i "s!__CORE_RPC_HOST__!${__CORE_RPC_HOST__}!g" mempool-config.json sed -i "s!__CORE_RPC_PORT__!${__CORE_RPC_PORT__}!g" mempool-config.json diff --git a/frontend/src/app/services/state.service.ts b/frontend/src/app/services/state.service.ts index e54d89403..9daebc209 100644 --- a/frontend/src/app/services/state.service.ts +++ b/frontend/src/app/services/state.service.ts @@ -49,6 +49,7 @@ export interface Env { SIGNET_BLOCK_AUDIT_START_HEIGHT: number; HISTORICAL_PRICE: boolean; ACCELERATOR: boolean; + ADDITIONAL_CURRENCIES: boolean; GIT_COMMIT_HASH_MEMPOOL_SPACE?: string; PACKAGE_JSON_VERSION_MEMPOOL_SPACE?: string; } @@ -82,6 +83,7 @@ const defaultEnv: Env = { 'SIGNET_BLOCK_AUDIT_START_HEIGHT': 0, 'HISTORICAL_PRICE': true, 'ACCELERATOR': false, + 'ADDITIONAL_CURRENCIES': false, }; @Injectable({ From f121d165447a7dfc2ac6cd1d5589b81e900a8481 Mon Sep 17 00:00:00 2001 From: natsoni Date: Thu, 7 Mar 2024 10:48:32 +0100 Subject: [PATCH 02/11] Add more fiat currencies using fx rates from FreeCurrencyAPI --- backend/src/api/database-migration.ts | 32 +- backend/src/repositories/PricesRepository.ts | 301 ++++++++++++++++-- .../tasks/price-feeds/free-currency-api.ts | 72 +++++ backend/src/tasks/price-updater.ts | 136 ++++++++ frontend/src/app/app.constants.ts | 130 ++++++++ .../fiat-selector/fiat-selector.component.ts | 5 + frontend/src/app/services/price.service.ts | 89 +++++- 7 files changed, 727 insertions(+), 38 deletions(-) create mode 100644 backend/src/tasks/price-feeds/free-currency-api.ts diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index f493e4eb3..a8eeddb04 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 71; + private static currentVersion = 72; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -606,6 +606,36 @@ class DatabaseMigration { await this.$executeQuery('ALTER TABLE `federation_txos` ADD emergencyKey TINYINT NOT NULL DEFAULT 0'); await this.updateToSchemaVersion(71); } + + if (databaseSchemaVersion < 72) { + await this.$executeQuery('ALTER TABLE `prices` ADD `BGN` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `BRL` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `CNY` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `CZK` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `DKK` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `HKD` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `HRK` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `HUF` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `IDR` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `ILS` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `INR` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `ISK` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `KRW` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `MXN` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `MYR` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `NOK` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `NZD` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `PHP` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `PLN` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `RON` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `RUB` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `SEK` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `SGD` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `THB` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `TRY` float DEFAULT "-1"'); + await this.$executeQuery('ALTER TABLE `prices` ADD `ZAR` float DEFAULT "-1"'); + await this.updateToSchemaVersion(72); + } } /** diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index ed9d1fd72..0ff8d9d95 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -1,5 +1,6 @@ import DB from '../database'; import logger from '../logger'; +import config from '../config'; import priceUpdater from '../tasks/price-updater'; export interface ApiPrice { @@ -11,17 +12,81 @@ export interface ApiPrice { CHF: number, AUD: number, JPY: number, + BGN: number, + BRL: number, + CNY: number, + CZK: number, + DKK: number, + HKD: number, + HRK: number, + HUF: number, + IDR: number, + ILS: number, + INR: number, + ISK: number, + KRW: number, + MXN: number, + MYR: number, + NOK: number, + NZD: number, + PHP: number, + PLN: number, + RON: number, + RUB: number, + SEK: number, + SGD: number, + THB: number, + TRY: number, + ZAR: number, } -const ApiPriceFields = ` - UNIX_TIMESTAMP(time) as time, - USD, - EUR, - GBP, - CAD, - CHF, - AUD, - JPY -`; + +const ApiPriceFields = config.MEMPOOL.CURRENCY_API_KEY ? + ` + UNIX_TIMESTAMP(time) as time, + USD, + EUR, + GBP, + CAD, + CHF, + AUD, + JPY, + BGN, + BRL, + CNY, + CZK, + DKK, + HKD, + HRK, + HUF, + IDR, + ILS, + INR, + ISK, + KRW, + MXN, + MYR, + NOK, + NZD, + PHP, + PLN, + RON, + RUB, + SEK, + SGD, + THB, + TRY, + ZAR + `: + ` + UNIX_TIMESTAMP(time) as time, + USD, + EUR, + GBP, + CAD, + CHF, + AUD, + JPY + `; export interface ExchangeRates { USDEUR: number, @@ -30,6 +95,32 @@ export interface ExchangeRates { USDCHF: number, USDAUD: number, USDJPY: number, + USDBGN?: number, + USDBRL?: number, + USDCNY?: number, + USDCZK?: number, + USDDKK?: number, + USDHKD?: number, + USDHRK?: number, + USDHUF?: number, + USDIDR?: number, + USDILS?: number, + USDINR?: number, + USDISK?: number, + USDKRW?: number, + USDMXN?: number, + USDMYR?: number, + USDNOK?: number, + USDNZD?: number, + USDPHP?: number, + USDPLN?: number, + USDRON?: number, + USDRUB?: number, + USDSEK?: number, + USDSGD?: number, + USDTHB?: number, + USDTRY?: number, + USDZAR?: number, } export interface Conversion { @@ -45,6 +136,32 @@ export const MAX_PRICES = { CHF: 100000000, AUD: 100000000, JPY: 10000000000, + BGN: 1000000000, + BRL: 1000000000, + CNY: 1000000000, + CZK: 10000000000, + DKK: 1000000000, + HKD: 1000000000, + HRK: 1000000000, + HUF: 10000000000, + IDR: 100000000000, + ILS: 1000000000, + INR: 10000000000, + ISK: 10000000000, + KRW: 100000000000, + MXN: 1000000000, + MYR: 1000000000, + NOK: 1000000000, + NZD: 1000000000, + PHP: 10000000000, + PLN: 1000000000, + RON: 1000000000, + RUB: 10000000000, + SEK: 1000000000, + SGD: 100000000, + THB: 10000000000, + TRY: 10000000000, + ZAR: 10000000000, }; class PricesRepository { @@ -64,17 +181,49 @@ class PricesRepository { } try { - await DB.query(` - INSERT INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY) - VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ? )`, - [time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY] - ); + if (!config.MEMPOOL.CURRENCY_API_KEY) { // Store only the 7 main currencies + await DB.query(` + INSERT INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY) + VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ? )`, + [time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY] + ); + } else { // Store all 7 main currencies + all the currencies obtained with the external API + await DB.query(` + INSERT INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY, BGN, BRL, CNY, CZK, DKK, HKD, HRK, HUF, IDR, ILS, INR, ISK, KRW, MXN, MYR, NOK, NZD, PHP, PLN, RON, RUB, SEK, SGD, THB, TRY, ZAR) + VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ? , ?, ?, ?, ?, ?, ?, ?, ? , ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? , ? )`, + [time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY, prices.BGN, prices.BRL, prices.CNY, prices.CZK, prices.DKK, + prices.HKD, prices.HRK, prices.HUF, prices.IDR, prices.ILS, prices.INR, prices.ISK, prices.KRW, prices.MXN, prices.MYR, prices.NOK, prices.NZD, + prices.PHP, prices.PLN, prices.RON, prices.RUB, prices.SEK, prices.SGD, prices.THB, prices.TRY, prices.ZAR] + ); + } } catch (e) { logger.err(`Cannot save exchange rate into db. Reason: ` + (e instanceof Error ? e.message : e)); throw e; } } + public async $saveAdditionalCurrencyPrices(time: number, prices: ApiPrice, legacyCurrencies: string[]): Promise { + try { + await DB.query(` + UPDATE prices + SET BGN = ?, BRL = ?, CNY = ?, CZK = ?, DKK = ?, HKD = ?, HRK = ?, HUF = ?, IDR = ?, ILS = ?, INR = ?, ISK = ?, KRW = ?, MXN = ?, MYR = ?, NOK = ?, NZD = ?, PHP = ?, PLN = ?, RON = ?, RUB = ?, SEK = ?, SGD = ?, THB = ?, TRY = ?, ZAR = ? + WHERE UNIX_TIMESTAMP(time) = ?`, + [prices.BGN, prices.BRL, prices.CNY, prices.CZK, prices.DKK, prices.HKD, prices.HRK, prices.HUF, prices.IDR, prices.ILS, prices.INR, prices.ISK, prices.KRW, prices.MXN, prices.MYR, prices.NOK, prices.NZD, prices.PHP, prices.PLN, prices.RON, prices.RUB, prices.SEK, prices.SGD, prices.THB, prices.TRY, prices.ZAR, time] + ); + for (const currency of legacyCurrencies) { + await DB.query(` + UPDATE prices + SET ${currency} = ? + WHERE UNIX_TIMESTAMP(time) = ?`, + [prices[currency], time] + ); + } + } catch (e) { + logger.err(`Cannot update exchange rate into db. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } + public async $getOldestPriceTime(): Promise { const [oldestRow] = await DB.query(` SELECT UNIX_TIMESTAMP(time) AS time @@ -118,6 +267,28 @@ class PricesRepository { return times.map(time => time.time); } + public async $getPricesTimesWithMissingFields(): Promise<{time: number, USD: number, eur_missing: boolean, gbp_missing: boolean, cad_missing: boolean, chf_missing: boolean, aud_missing: boolean, jpy_missing: boolean}[]> { + const [times] = await DB.query(` + SELECT UNIX_TIMESTAMP(time) AS time, + USD, + CASE WHEN EUR = -1 THEN TRUE ELSE FALSE END AS eur_missing, + CASE WHEN GBP = -1 THEN TRUE ELSE FALSE END AS gbp_missing, + CASE WHEN CAD = -1 THEN TRUE ELSE FALSE END AS cad_missing, + CASE WHEN CHF = -1 THEN TRUE ELSE FALSE END AS chf_missing, + CASE WHEN AUD = -1 THEN TRUE ELSE FALSE END AS aud_missing, + CASE WHEN JPY = -1 THEN TRUE ELSE FALSE END AS jpy_missing + FROM prices + WHERE USD != -1 + AND -1 IN (EUR, GBP, CAD, CHF, AUD, JPY, BGN, BRL, CNY, CZK, DKK, HKD, HRK, HUF, IDR, ILS, INR, ISK, KRW, + MXN, MYR, NOK, NZD, PHP, PLN, RON, RUB, SEK, SGD, THB, TRY, ZAR) + ORDER BY time + `); + if (!Array.isArray(times)) { + return []; + } + return times as {time: number, USD: number, eur_missing: boolean, gbp_missing: boolean, cad_missing: boolean, chf_missing: boolean, aud_missing: boolean, jpy_missing: boolean}[]; + } + public async $getPricesTimesAndId(): Promise<{time: number, id: number, USD: number}[]> { const [times] = await DB.query(` SELECT @@ -167,14 +338,48 @@ class PricesRepository { const computeFx = (usd: number, other: number): number => Math.round(Math.max(other, 0) / Math.max(usd, 1) * 100) / 100; - const exchangeRates: ExchangeRates = { - USDEUR: computeFx(latestPrice.USD, latestPrice.EUR), - USDGBP: computeFx(latestPrice.USD, latestPrice.GBP), - USDCAD: computeFx(latestPrice.USD, latestPrice.CAD), - USDCHF: computeFx(latestPrice.USD, latestPrice.CHF), - USDAUD: computeFx(latestPrice.USD, latestPrice.AUD), - USDJPY: computeFx(latestPrice.USD, latestPrice.JPY), - }; + const exchangeRates: ExchangeRates = config.MEMPOOL.CURRENCY_API_KEY ? + { + USDEUR: computeFx(latestPrice.USD, latestPrice.EUR), + USDGBP: computeFx(latestPrice.USD, latestPrice.GBP), + USDCAD: computeFx(latestPrice.USD, latestPrice.CAD), + USDCHF: computeFx(latestPrice.USD, latestPrice.CHF), + USDAUD: computeFx(latestPrice.USD, latestPrice.AUD), + USDJPY: computeFx(latestPrice.USD, latestPrice.JPY), + USDBGN: computeFx(latestPrice.USD, latestPrice.BGN), + USDBRL: computeFx(latestPrice.USD, latestPrice.BRL), + USDCNY: computeFx(latestPrice.USD, latestPrice.CNY), + USDCZK: computeFx(latestPrice.USD, latestPrice.CZK), + USDDKK: computeFx(latestPrice.USD, latestPrice.DKK), + USDHKD: computeFx(latestPrice.USD, latestPrice.HKD), + USDHRK: computeFx(latestPrice.USD, latestPrice.HRK), + USDHUF: computeFx(latestPrice.USD, latestPrice.HUF), + USDIDR: computeFx(latestPrice.USD, latestPrice.IDR), + USDILS: computeFx(latestPrice.USD, latestPrice.ILS), + USDINR: computeFx(latestPrice.USD, latestPrice.INR), + USDISK: computeFx(latestPrice.USD, latestPrice.ISK), + USDKRW: computeFx(latestPrice.USD, latestPrice.KRW), + USDMXN: computeFx(latestPrice.USD, latestPrice.MXN), + USDMYR: computeFx(latestPrice.USD, latestPrice.MYR), + USDNOK: computeFx(latestPrice.USD, latestPrice.NOK), + USDNZD: computeFx(latestPrice.USD, latestPrice.NZD), + USDPHP: computeFx(latestPrice.USD, latestPrice.PHP), + USDPLN: computeFx(latestPrice.USD, latestPrice.PLN), + USDRON: computeFx(latestPrice.USD, latestPrice.RON), + USDRUB: computeFx(latestPrice.USD, latestPrice.RUB), + USDSEK: computeFx(latestPrice.USD, latestPrice.SEK), + USDSGD: computeFx(latestPrice.USD, latestPrice.SGD), + USDTHB: computeFx(latestPrice.USD, latestPrice.THB), + USDTRY: computeFx(latestPrice.USD, latestPrice.TRY), + USDZAR: computeFx(latestPrice.USD, latestPrice.ZAR), + } : { + USDEUR: computeFx(latestPrice.USD, latestPrice.EUR), + USDGBP: computeFx(latestPrice.USD, latestPrice.GBP), + USDCAD: computeFx(latestPrice.USD, latestPrice.CAD), + USDCHF: computeFx(latestPrice.USD, latestPrice.CHF), + USDAUD: computeFx(latestPrice.USD, latestPrice.AUD), + USDJPY: computeFx(latestPrice.USD, latestPrice.JPY), + }; return { prices: rates as ApiPrice[], @@ -206,14 +411,48 @@ class PricesRepository { const computeFx = (usd: number, other: number): number => Math.round(Math.max(other, 0) / Math.max(usd, 1) * 100) / 100; - const exchangeRates: ExchangeRates = { - USDEUR: computeFx(latestPrice.USD, latestPrice.EUR), - USDGBP: computeFx(latestPrice.USD, latestPrice.GBP), - USDCAD: computeFx(latestPrice.USD, latestPrice.CAD), - USDCHF: computeFx(latestPrice.USD, latestPrice.CHF), - USDAUD: computeFx(latestPrice.USD, latestPrice.AUD), - USDJPY: computeFx(latestPrice.USD, latestPrice.JPY), - }; + const exchangeRates: ExchangeRates = config.MEMPOOL.CURRENCY_API_KEY ? + { + USDEUR: computeFx(latestPrice.USD, latestPrice.EUR), + USDGBP: computeFx(latestPrice.USD, latestPrice.GBP), + USDCAD: computeFx(latestPrice.USD, latestPrice.CAD), + USDCHF: computeFx(latestPrice.USD, latestPrice.CHF), + USDAUD: computeFx(latestPrice.USD, latestPrice.AUD), + USDJPY: computeFx(latestPrice.USD, latestPrice.JPY), + USDBGN: computeFx(latestPrice.USD, latestPrice.BGN), + USDBRL: computeFx(latestPrice.USD, latestPrice.BRL), + USDCNY: computeFx(latestPrice.USD, latestPrice.CNY), + USDCZK: computeFx(latestPrice.USD, latestPrice.CZK), + USDDKK: computeFx(latestPrice.USD, latestPrice.DKK), + USDHKD: computeFx(latestPrice.USD, latestPrice.HKD), + USDHRK: computeFx(latestPrice.USD, latestPrice.HRK), + USDHUF: computeFx(latestPrice.USD, latestPrice.HUF), + USDIDR: computeFx(latestPrice.USD, latestPrice.IDR), + USDILS: computeFx(latestPrice.USD, latestPrice.ILS), + USDINR: computeFx(latestPrice.USD, latestPrice.INR), + USDISK: computeFx(latestPrice.USD, latestPrice.ISK), + USDKRW: computeFx(latestPrice.USD, latestPrice.KRW), + USDMXN: computeFx(latestPrice.USD, latestPrice.MXN), + USDMYR: computeFx(latestPrice.USD, latestPrice.MYR), + USDNOK: computeFx(latestPrice.USD, latestPrice.NOK), + USDNZD: computeFx(latestPrice.USD, latestPrice.NZD), + USDPHP: computeFx(latestPrice.USD, latestPrice.PHP), + USDPLN: computeFx(latestPrice.USD, latestPrice.PLN), + USDRON: computeFx(latestPrice.USD, latestPrice.RON), + USDRUB: computeFx(latestPrice.USD, latestPrice.RUB), + USDSEK: computeFx(latestPrice.USD, latestPrice.SEK), + USDSGD: computeFx(latestPrice.USD, latestPrice.SGD), + USDTHB: computeFx(latestPrice.USD, latestPrice.THB), + USDTRY: computeFx(latestPrice.USD, latestPrice.TRY), + USDZAR: computeFx(latestPrice.USD, latestPrice.ZAR), + } : { + USDEUR: computeFx(latestPrice.USD, latestPrice.EUR), + USDGBP: computeFx(latestPrice.USD, latestPrice.GBP), + USDCAD: computeFx(latestPrice.USD, latestPrice.CAD), + USDCHF: computeFx(latestPrice.USD, latestPrice.CHF), + USDAUD: computeFx(latestPrice.USD, latestPrice.AUD), + USDJPY: computeFx(latestPrice.USD, latestPrice.JPY), + }; return { prices: rates as ApiPrice[], diff --git a/backend/src/tasks/price-feeds/free-currency-api.ts b/backend/src/tasks/price-feeds/free-currency-api.ts new file mode 100644 index 000000000..cd9c1d498 --- /dev/null +++ b/backend/src/tasks/price-feeds/free-currency-api.ts @@ -0,0 +1,72 @@ +import { query } from '../../utils/axios-query'; +import { ConversionFeed, ConversionRates } from '../price-updater'; + +const emptyRates = { + AUD: -1, + BGN: -1, + BRL: -1, + CAD: -1, + CHF: -1, + CNY: -1, + CZK: -1, + DKK: -1, + EUR: -1, + GBP: -1, + HKD: -1, + HRK: -1, + HUF: -1, + IDR: -1, + ILS: -1, + INR: -1, + ISK: -1, + JPY: -1, + KRW: -1, + MXN: -1, + MYR: -1, + NOK: -1, + NZD: -1, + PHP: -1, + PLN: -1, + RON: -1, + RUB: -1, + SEK: -1, + SGD: -1, + THB: -1, + TRY: -1, + ZAR: -1, +}; + +class FreeCurrencyApi implements ConversionFeed { + private API_KEY: string; + + constructor(apiKey: string) { + this.API_KEY = apiKey; + } + + public async $getQuota(): Promise { + const response = await query(`https://api.freecurrencyapi.com/v1/status?apikey=${this.API_KEY}`); + if (response && response['quotas']) { + return response['quotas']; + } + return null; + } + + public async $fetchLatestConversionRates(): Promise { + const response = await query(`https://api.freecurrencyapi.com/v1/latest?apikey=${this.API_KEY}`); + if (response && response['data']) { + return response['data']; + } + return emptyRates; + } + + public async $fetchConversionRates(date: string): Promise { + const response = await query(`https://api.freecurrencyapi.com/v1/historical?date=${date}&apikey=${this.API_KEY}`); + if (response && response['data'] && response['data'][date]) { + return response['data'][date]; + } + return emptyRates; + } + +} + +export default FreeCurrencyApi; diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index 0d5ca5958..f23879831 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -8,6 +8,7 @@ import BitflyerApi from './price-feeds/bitflyer-api'; import CoinbaseApi from './price-feeds/coinbase-api'; import GeminiApi from './price-feeds/gemini-api'; import KrakenApi from './price-feeds/kraken-api'; +import FreeCurrencyApi from './price-feeds/free-currency-api'; export interface PriceFeed { name: string; @@ -23,6 +24,16 @@ export interface PriceHistory { [timestamp: number]: ApiPrice; } +export interface ConversionFeed { + $getQuota(): Promise; + $fetchLatestConversionRates(): Promise; + $fetchConversionRates(date: string): Promise; +} + +export interface ConversionRates { + [currency: string]: number +} + function getMedian(arr: number[]): number { const sortedArr = arr.slice().sort((a, b) => a - b); const mid = Math.floor(sortedArr.length / 2); @@ -33,6 +44,8 @@ function getMedian(arr: number[]): number { class PriceUpdater { public historyInserted = false; + private additionalCurrenciesHistoryInserted = false; + private additionalCurrenciesHistoryRunning = false; private timeBetweenUpdatesMs = 360_0000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR; private cyclePosition = -1; private firstRun = true; @@ -42,6 +55,10 @@ class PriceUpdater { private feeds: PriceFeed[] = []; private currencies: string[] = ['USD', 'EUR', 'GBP', 'CAD', 'CHF', 'AUD', 'JPY']; private latestPrices: ApiPrice; + private currencyConversionFeed: ConversionFeed | undefined; + private newCurrencies: string[] = ['BGN', 'BRL', 'CNY', 'CZK', 'DKK', 'HKD', 'HRK', 'HUF', 'IDR', 'ILS', 'INR', 'ISK', 'KRW', 'MXN', 'MYR', 'NOK', 'NZD', 'PHP', 'PLN', 'RON', 'RUB', 'SEK', 'SGD', 'THB', 'TRY', 'ZAR']; + private lastTimeConversionsRatesFetched: number = 0; + private latestConversionsRatesFromFeed: ConversionRates = {}; private ratesChangedCallback: ((rates: ApiPrice) => void) | undefined; constructor() { @@ -53,6 +70,7 @@ class PriceUpdater { this.feeds.push(new BitfinexApi()); this.feeds.push(new GeminiApi()); + this.currencyConversionFeed = new FreeCurrencyApi(config.MEMPOOL.CURRENCY_API_KEY); this.setCyclePosition(); } @@ -70,6 +88,32 @@ class PriceUpdater { CHF: -1, AUD: -1, JPY: -1, + BGN: -1, + BRL: -1, + CNY: -1, + CZK: -1, + DKK: -1, + HKD: -1, + HRK: -1, + HUF: -1, + IDR: -1, + ILS: -1, + INR: -1, + ISK: -1, + KRW: -1, + MXN: -1, + MYR: -1, + NOK: -1, + NZD: -1, + PHP: -1, + PLN: -1, + RON: -1, + RUB: -1, + SEK: -1, + SGD: -1, + THB: -1, + TRY: -1, + ZAR: -1, }; } @@ -99,6 +143,18 @@ class PriceUpdater { if ((Math.round(new Date().getTime() / 1000) - this.lastHistoricalRun) > 3600 * 24) { // Once a day, look for missing prices (could happen due to network connectivity issues) this.historyInserted = false; + this.additionalCurrenciesHistoryInserted = false; + } + + if (config.MEMPOOL.CURRENCY_API_KEY && this.currencyConversionFeed && (Math.round(new Date().getTime() / 1000) - this.lastTimeConversionsRatesFetched) > 3600 * 24) { + // Once a day, fetch conversion rates from api: we don't need more granularity for fiat currencies and have a limited number of requests + try { + this.latestConversionsRatesFromFeed = await this.currencyConversionFeed.$fetchLatestConversionRates(); + this.lastTimeConversionsRatesFetched = Math.round(new Date().getTime() / 1000); + logger.debug(`Fetched currencies conversion rates from external API: ${JSON.stringify(this.latestConversionsRatesFromFeed)}`); + } catch (e) { + logger.err(`Cannot fetch conversion rates from the API. Reason: ${(e instanceof Error ? e.message : e)}`); + } } try { @@ -106,6 +162,9 @@ class PriceUpdater { if (this.historyInserted === false && config.DATABASE.ENABLED === true) { await this.$insertHistoricalPrices(); } + if (this.additionalCurrenciesHistoryInserted === false && config.DATABASE.ENABLED === true && config.MEMPOOL.CURRENCY_API_KEY && !this.additionalCurrenciesHistoryRunning) { + await this.$insertMissingAdditionalPrices(); + } } catch (e: any) { logger.err(`Cannot save BTC prices in db. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining); } @@ -185,6 +244,14 @@ class PriceUpdater { } } + if (config.MEMPOOL.CURRENCY_API_KEY && this.latestPrices.USD > 0 && Object.keys(this.latestConversionsRatesFromFeed).length > 0) { + for (const conversionCurrency of this.newCurrencies) { + if (this.latestConversionsRatesFromFeed[conversionCurrency] > 0 && this.latestPrices.USD * this.latestConversionsRatesFromFeed[conversionCurrency] < MAX_PRICES[conversionCurrency]) { + this.latestPrices[conversionCurrency] = Math.round(this.latestPrices.USD * this.latestConversionsRatesFromFeed[conversionCurrency]); + } + } + } + if (config.DATABASE.ENABLED === true && this.cyclePosition === 0) { // Save everything in db try { @@ -320,6 +387,75 @@ class PriceUpdater { logger.debug(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`, logger.tags.mining); } } + + /** + * Find missing prices for additional currencies and insert them in the database + * We calculate the additional prices from the USD price and the conversion rates + */ + private async $insertMissingAdditionalPrices(): Promise { + this.additionalCurrenciesHistoryRunning = true; + const priceTimesToFill = await PricesRepository.$getPricesTimesWithMissingFields(); + if (priceTimesToFill.length === 0) { + return; + } + logger.debug(`Fetching missing conversion rates from external API to fill ${priceTimesToFill.length} rows`, logger.tags.mining); + + let conversionRates: { [timestamp: number]: ConversionRates } = {}; + let totalInserted = 0; + + let requestCounter = 0; + + for (const priceTime of priceTimesToFill) { + const missingLegacyCurrencies = this.getMissingLegacyCurrencies(priceTime); // In the case a legacy currency (EUR, GBP, CAD, CHF, AUD, JPY) + const year = new Date(priceTime.time * 1000).getFullYear(); // is missing, we use the same process as for the new currencies + const yearTimestamp = new Date(year, 0, 1).getTime() / 1000; + if (conversionRates[yearTimestamp] === undefined) { + try { + if (requestCounter >= 10) { + await new Promise(resolve => setTimeout(resolve, 60_000)); // avoid getting 429'd + requestCounter = 0; + } + conversionRates[yearTimestamp] = await this.currencyConversionFeed?.$fetchConversionRates(`${year}-01-01`); + ++requestCounter; + } catch (e) { + logger.err(`Cannot fetch conversion rates from the API for year ${year}. Reason: ${(e instanceof Error ? e.message : e)}`); + } + } + + if (conversionRates[yearTimestamp] === undefined) { + continue; + } + + const prices: ApiPrice = this.getEmptyPricesObj(); + + let willInsert = false; + for (const conversionCurrency of this.newCurrencies.concat(missingLegacyCurrencies)) { + if (conversionRates[yearTimestamp][conversionCurrency] > 0 && priceTime.USD * conversionRates[yearTimestamp][conversionCurrency] < MAX_PRICES[conversionCurrency]) { + prices[conversionCurrency] = year >= 2013 ? Math.round(priceTime.USD * conversionRates[yearTimestamp][conversionCurrency]) : Math.round(priceTime.USD * conversionRates[yearTimestamp][conversionCurrency] * 100) / 100; + willInsert = true; + } + } + + if (willInsert) { + await PricesRepository.$saveAdditionalCurrencyPrices(priceTime.time, prices, missingLegacyCurrencies); + ++totalInserted; + } + } + logger.debug(`Inserted ${totalInserted} missing additional currency prices into the db`, logger.tags.mining); + this.additionalCurrenciesHistoryInserted = true; + this.additionalCurrenciesHistoryRunning = false; + } + + // Helper function to get legacy missing currencies in a row (EUR, GBP, CAD, CHF, AUD, JPY) + private getMissingLegacyCurrencies(priceTime: any): string[] { + const missingCurrencies: string[] = []; + ['eur', 'gbp', 'cad', 'chf', 'aud', 'jpy'].forEach(currency => { + if (priceTime[`${currency}_missing`]) { + missingCurrencies.push(currency.toUpperCase()); + } + }); + return missingCurrencies; + } } export default new PriceUpdater(); diff --git a/frontend/src/app/app.constants.ts b/frontend/src/app/app.constants.ts index f510c6480..17105d97e 100644 --- a/frontend/src/app/app.constants.ts +++ b/frontend/src/app/app.constants.ts @@ -268,4 +268,134 @@ export const fiatCurrencies = { code: 'USD', indexed: true, }, + BGN: { + name: 'Bulgarian Lev', + code: 'BGN', + indexed: true, + }, + BRL: { + name: 'Brazilian Real', + code: 'BRL', + indexed: true, + }, + CNY: { + name: 'Chinese Yuan', + code: 'CNY', + indexed: true, + }, + CZK: { + name: 'Czech Koruna', + code: 'CZK', + indexed: true, + }, + DKK: { + name: 'Danish Krone', + code: 'DKK', + indexed: true, + }, + HKD: { + name: 'Hong Kong Dollar', + code: 'HKD', + indexed: true, + }, + HRK: { + name: 'Croatian Kuna', + code: 'HRK', + indexed: true, + }, + HUF: { + name: 'Hungarian Forint', + code: 'HUF', + indexed: true, + }, + IDR: { + name: 'Indonesian Rupiah', + code: 'IDR', + indexed: true, + }, + ILS: { + name: 'Israeli Shekel', + code: 'ILS', + indexed: true, + }, + INR: { + name: 'Indian Rupee', + code: 'INR', + indexed: true, + }, + ISK: { + name: 'Icelandic Krona', + code: 'ISK', + indexed: true, + }, + KRW: { + name: 'South Korean Won', + code: 'KRW', + indexed: true, + }, + MXN: { + name: 'Mexican Peso', + code: 'MXN', + indexed: true, + }, + MYR: { + name: 'Malaysian Ringgit', + code: 'MYR', + indexed: true, + }, + NOK: { + name: 'Norwegian Krone', + code: 'NOK', + indexed: true, + }, + NZD: { + name: 'New Zealand Dollar', + code: 'NZD', + indexed: true, + }, + PHP: { + name: 'Philippine Peso', + code: 'PHP', + indexed: true, + }, + PLN: { + name: 'Polish Zloty', + code: 'PLN', + indexed: true, + }, + RON: { + name: 'Romanian Leu', + code: 'RON', + indexed: true, + }, + RUB: { + name: 'Russian Ruble', + code: 'RUB', + indexed: true, + }, + SEK: { + name: 'Swedish Krona', + code: 'SEK', + indexed: true, + }, + SGD: { + name: 'Singapore Dollar', + code: 'SGD', + indexed: true, + }, + THB: { + name: 'Thai Baht', + code: 'THB', + indexed: true, + }, + TRY: { + name: 'Turkish Lira', + code: 'TRY', + indexed: true, + }, + ZAR: { + name: 'South African Rand', + code: 'ZAR', + indexed: true, + }, }; \ No newline at end of file diff --git a/frontend/src/app/components/fiat-selector/fiat-selector.component.ts b/frontend/src/app/components/fiat-selector/fiat-selector.component.ts index 337ef11f3..f2538fec9 100644 --- a/frontend/src/app/components/fiat-selector/fiat-selector.component.ts +++ b/frontend/src/app/components/fiat-selector/fiat-selector.component.ts @@ -35,6 +35,11 @@ export class FiatSelectorComponent implements OnInit { this.stateService.fiatCurrency$.subscribe((fiat) => { this.fiatForm.get('fiat')?.setValue(fiat); }); + if (!this.stateService.env.ADDITIONAL_CURRENCIES) { + this.currencies = this.currencies.filter((currency: any) => { + return ['AUD', 'CAD', 'EUR', 'JPY', 'GBP', 'CHF', 'USD'].includes(currency[0]); + }); + } } changeFiat() { diff --git a/frontend/src/app/services/price.service.ts b/frontend/src/app/services/price.service.ts index 4236205ca..41f928343 100644 --- a/frontend/src/app/services/price.service.ts +++ b/frontend/src/app/services/price.service.ts @@ -13,6 +13,32 @@ export interface ApiPrice { CHF: number, AUD: number, JPY: number, + BGN?: number, + BRL?: number, + CNY?: number, + CZK?: number, + DKK?: number, + HKD?: number, + HRK?: number, + HUF?: number, + IDR?: number, + ILS?: number, + INR?: number, + ISK?: number, + KRW?: number, + MXN?: number, + MYR?: number, + NOK?: number, + NZD?: number, + PHP?: number, + PLN?: number, + RON?: number, + RUB?: number, + SEK?: number, + SGD?: number, + THB?: number, + TRY?: number, + ZAR?: number, } export interface ExchangeRates { USDEUR: number, @@ -21,6 +47,32 @@ export interface ExchangeRates { USDCHF: number, USDAUD: number, USDJPY: number, + USDBGN?: number, + USDBRL?: number, + USDCNY?: number, + USDCZK?: number, + USDDKK?: number, + USDHKD?: number, + USDHRK?: number, + USDHUF?: number, + USDIDR?: number, + USDILS?: number, + USDINR?: number, + USDISK?: number, + USDKRW?: number, + USDMXN?: number, + USDMYR?: number, + USDNOK?: number, + USDNZD?: number, + USDPHP?: number, + USDPLN?: number, + USDRON?: number, + USDRUB?: number, + USDSEK?: number, + USDSGD?: number, + USDTHB?: number, + USDTRY?: number, + USDZAR?: number, } export interface Conversion { prices: ApiPrice[], @@ -60,10 +112,19 @@ export class PriceService { getEmptyPrice(): Price { return { - price: { + price: this.stateService.env.ADDITIONAL_CURRENCIES ? { + USD: 0, EUR: 0, GBP: 0, CAD: 0, CHF: 0, AUD: 0, JPY: 0, BGN: 0, BRL: 0, CNY: 0, CZK: 0, DKK: 0, HKD: 0, HRK: 0, HUF: 0, IDR: 0, + ILS: 0, INR: 0, ISK: 0, KRW: 0, MXN: 0, MYR: 0, NOK: 0, NZD: 0, PHP: 0, PLN: 0, RON: 0, RUB: 0, SEK: 0, SGD: 0, THB: 0, TRY: 0, + ZAR: 0 + } : + { USD: 0, EUR: 0, GBP: 0, CAD: 0, CHF: 0, AUD: 0, JPY: 0, }, - exchangeRates: { + exchangeRates: this.stateService.env.ADDITIONAL_CURRENCIES ? { + USDEUR: 0, USDGBP: 0, USDCAD: 0, USDCHF: 0, USDAUD: 0, USDJPY: 0, USDBGN: 0, USDBRL: 0, USDCNY: 0, USDCZK: 0, USDDKK: 0, USDHKD: 0, + USDHRK: 0, USDHUF: 0, USDIDR: 0, USDILS: 0, USDINR: 0, USDISK: 0, USDKRW: 0, USDMXN: 0, USDMYR: 0, USDNOK: 0, USDNZD: 0, USDPHP: 0, + USDPLN: 0, USDRON: 0, USDRUB: 0, USDSEK: 0, USDSGD: 0, USDTHB: 0, USDTRY: 0, USDZAR: 0 + } : { USDEUR: 0, USDGBP: 0, USDCAD: 0, USDCHF: 0, USDAUD: 0, USDJPY: 0, }, }; @@ -92,7 +153,17 @@ export class PriceService { return undefined; } return { - price: { + price: this.stateService.env.ADDITIONAL_CURRENCIES ? { + USD: conversion.prices[0].USD, EUR: conversion.prices[0].EUR, GBP: conversion.prices[0].GBP, CAD: conversion.prices[0].CAD, + CHF: conversion.prices[0].CHF, AUD: conversion.prices[0].AUD, JPY: conversion.prices[0].JPY, BGN: conversion.prices[0].BGN, + BRL: conversion.prices[0].BRL, CNY: conversion.prices[0].CNY, CZK: conversion.prices[0].CZK, DKK: conversion.prices[0].DKK, + HKD: conversion.prices[0].HKD, HRK: conversion.prices[0].HRK, HUF: conversion.prices[0].HUF, IDR: conversion.prices[0].IDR, + ILS: conversion.prices[0].ILS, INR: conversion.prices[0].INR, ISK: conversion.prices[0].ISK, KRW: conversion.prices[0].KRW, + MXN: conversion.prices[0].MXN, MYR: conversion.prices[0].MYR, NOK: conversion.prices[0].NOK, NZD: conversion.prices[0].NZD, + PHP: conversion.prices[0].PHP, PLN: conversion.prices[0].PLN, RON: conversion.prices[0].RON, RUB: conversion.prices[0].RUB, + SEK: conversion.prices[0].SEK, SGD: conversion.prices[0].SGD, THB: conversion.prices[0].THB, TRY: conversion.prices[0].TRY, + ZAR: conversion.prices[0].ZAR + } : { USD: conversion.prices[0].USD, EUR: conversion.prices[0].EUR, GBP: conversion.prices[0].GBP, CAD: conversion.prices[0].CAD, CHF: conversion.prices[0].CHF, AUD: conversion.prices[0].AUD, JPY: conversion.prices[0].JPY }, @@ -122,9 +193,15 @@ export class PriceService { exchangeRates: conversion.exchangeRates, }; for (const price of conversion.prices) { - historicalPrice.prices[price.time] = { - USD: price.USD, EUR: price.EUR, GBP: price.GBP, CAD: price.CAD, - CHF: price.CHF, AUD: price.AUD, JPY: price.JPY + historicalPrice.prices[price.time] = this.stateService.env.ADDITIONAL_CURRENCIES ? { + USD: price.USD, EUR: price.EUR, GBP: price.GBP, CAD: price.CAD, CHF: price.CHF, AUD: price.AUD, + JPY: price.JPY, BGN: price.BGN, BRL: price.BRL, CNY: price.CNY, CZK: price.CZK, DKK: price.DKK, + HKD: price.HKD, HRK: price.HRK, HUF: price.HUF, IDR: price.IDR, ILS: price.ILS, INR: price.INR, + ISK: price.ISK, KRW: price.KRW, MXN: price.MXN, MYR: price.MYR, NOK: price.NOK, NZD: price.NZD, + PHP: price.PHP, PLN: price.PLN, RON: price.RON, RUB: price.RUB, SEK: price.SEK, SGD: price.SGD, + THB: price.THB, TRY: price.TRY, ZAR: price.ZAR + } : { + USD: price.USD, EUR: price.EUR, GBP: price.GBP, CAD: price.CAD, CHF: price.CHF, AUD: price.AUD, JPY: price.JPY }; } From b043d698ca61d4d0cef3666ba15cf2ad7929e185 Mon Sep 17 00:00:00 2001 From: natsoni Date: Thu, 7 Mar 2024 18:33:19 +0100 Subject: [PATCH 03/11] Allow historical price api to return data of a single currency --- backend/src/api/mining/mining-routes.ts | 17 +++++--- backend/src/repositories/PricesRepository.ts | 38 ++++++++++++++++-- frontend/src/app/services/api.service.ts | 41 ++++++++++++++++++-- 3 files changed, 85 insertions(+), 11 deletions(-) diff --git a/backend/src/api/mining/mining-routes.ts b/backend/src/api/mining/mining-routes.ts index 6b9ef7b9f..abf2fd2d5 100644 --- a/backend/src/api/mining/mining-routes.ts +++ b/backend/src/api/mining/mining-routes.ts @@ -51,13 +51,20 @@ class MiningRoutes { res.status(400).send('Prices are not available on testnets.'); return; } - if (req.query.timestamp) { - res.status(200).send(await PricesRepository.$getNearestHistoricalPrice( - parseInt(req.query.timestamp ?? 0, 10) - )); + const timestamp = parseInt(req.query.timestamp as string, 10) || 0; + const currency = req.query.currency as string; + + let response; + if (timestamp && currency) { + response = await PricesRepository.$getNearestHistoricalPrice(timestamp, currency); + } else if (timestamp) { + response = await PricesRepository.$getNearestHistoricalPrice(timestamp); + } else if (currency) { + response = await PricesRepository.$getHistoricalPrices(currency); } else { - res.status(200).send(await PricesRepository.$getHistoricalPrices()); + response = await PricesRepository.$getHistoricalPrices(); } + res.status(200).send(response); } catch (e) { res.status(500).send(e instanceof Error ? e.message : e); } diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index 0ff8d9d95..d72ea45b7 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -315,7 +315,7 @@ class PricesRepository { return rates[0] as ApiPrice; } - public async $getNearestHistoricalPrice(timestamp: number | undefined): Promise { + public async $getNearestHistoricalPrice(timestamp: number | undefined, currency?: string): Promise { try { const [rates] = await DB.query(` SELECT ${ApiPriceFields} @@ -379,7 +379,23 @@ class PricesRepository { USDCHF: computeFx(latestPrice.USD, latestPrice.CHF), USDAUD: computeFx(latestPrice.USD, latestPrice.AUD), USDJPY: computeFx(latestPrice.USD, latestPrice.JPY), + }; + + if (currency) { + if (!latestPrice[currency]) { + return null; + } + const filteredRates = rates.map((rate: any) => { + return { + time: rate.time, + [currency]: rate[currency] + }; + }); + return { + prices: filteredRates as ApiPrice[], + exchangeRates: exchangeRates }; + } return { prices: rates as ApiPrice[], @@ -391,7 +407,7 @@ class PricesRepository { } } - public async $getHistoricalPrices(): Promise { + public async $getHistoricalPrices(currency?: string): Promise { try { const [rates] = await DB.query(` SELECT ${ApiPriceFields} @@ -411,7 +427,7 @@ class PricesRepository { const computeFx = (usd: number, other: number): number => Math.round(Math.max(other, 0) / Math.max(usd, 1) * 100) / 100; - const exchangeRates: ExchangeRates = config.MEMPOOL.CURRENCY_API_KEY ? + const exchangeRates: ExchangeRates = config.MEMPOOL.CURRENCY_API_KEY ? { USDEUR: computeFx(latestPrice.USD, latestPrice.EUR), USDGBP: computeFx(latestPrice.USD, latestPrice.GBP), @@ -452,7 +468,23 @@ class PricesRepository { USDCHF: computeFx(latestPrice.USD, latestPrice.CHF), USDAUD: computeFx(latestPrice.USD, latestPrice.AUD), USDJPY: computeFx(latestPrice.USD, latestPrice.JPY), + }; + + if (currency) { + if (!latestPrice[currency]) { + return null; + } + const filteredRates = rates.map((rate: any) => { + return { + time: rate.time, + [currency]: rate[currency] + }; + }); + return { + prices: filteredRates as ApiPrice[], + exchangeRates: exchangeRates }; + } return { prices: rates as ApiPrice[], diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index 00e7075aa..ea2f3a074 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -405,7 +405,7 @@ export class ApiService { ); } - getHistoricalPrice$(timestamp: number | undefined): Observable { + getHistoricalPrice$(timestamp: number | undefined, currency?: string): Observable { if (this.stateService.isAnyTestnet()) { return of({ prices: [], @@ -416,12 +416,47 @@ export class ApiService { USDCHF: 0, USDAUD: 0, USDJPY: 0, + USDBGN: 0, + USDBRL: 0, + USDCNY: 0, + USDCZK: 0, + USDDKK: 0, + USDHKD: 0, + USDHRK: 0, + USDHUF: 0, + USDIDR: 0, + USDILS: 0, + USDINR: 0, + USDISK: 0, + USDKRW: 0, + USDMXN: 0, + USDMYR: 0, + USDNOK: 0, + USDNZD: 0, + USDPHP: 0, + USDPLN: 0, + USDRON: 0, + USDRUB: 0, + USDSEK: 0, + USDSGD: 0, + USDTHB: 0, + USDTRY: 0, + USDZAR: 0, } }); } + const queryParams = []; + + if (timestamp) { + queryParams.push(`timestamp=${timestamp}`); + } + + if (currency) { + queryParams.push(`currency=${currency}`); + } return this.httpClient.get( - this.apiBaseUrl + this.apiBasePath + '/api/v1/historical-price' + - (timestamp ? `?timestamp=${timestamp}` : '') + `${this.apiBaseUrl}${this.apiBasePath}/api/v1/historical-price` + + (queryParams.length > 0 ? `?${queryParams.join('&')}` : '') ); } } From ccf1121f195180f3da195bc4fb910fe10a834f6e Mon Sep 17 00:00:00 2001 From: natsoni Date: Sat, 9 Mar 2024 16:32:21 +0100 Subject: [PATCH 04/11] Fetch historical data based on timestamp and currency --- backend/src/repositories/PricesRepository.ts | 31 +++++++++--- .../app/components/block/block.component.html | 2 +- .../app/components/block/block.component.ts | 6 +-- .../transaction/transaction.component.ts | 12 +++-- .../transactions-list.component.ts | 48 +++++++++++++++++-- .../tx-bowtie-graph-tooltip.component.ts | 16 ++++--- frontend/src/app/services/price.service.ts | 14 ++++-- 7 files changed, 100 insertions(+), 29 deletions(-) diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index d72ea45b7..e5acec861 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -329,14 +329,29 @@ class PricesRepository { throw Error(`Cannot get single historical price from the database`); } + let pricesUsedForExchangeRates; // If we don't have a fx API key, we need to use the latest prices to compute the exchange rates + if (!config.MEMPOOL.CURRENCY_API_KEY) { + const [latestPrices] = await DB.query(` + SELECT ${ApiPriceFields} + FROM prices + ORDER BY time DESC + LIMIT 1 + `); + if (!Array.isArray(latestPrices)) { + throw Error(`Cannot get single historical price from the database`); + } + pricesUsedForExchangeRates = latestPrices[0] as ApiPrice; + } else { + pricesUsedForExchangeRates = rates[0] as ApiPrice; + } + // Compute fiat exchange rates - let latestPrice = rates[0] as ApiPrice; + let latestPrice = pricesUsedForExchangeRates; if (!latestPrice || latestPrice.USD === -1) { latestPrice = priceUpdater.getEmptyPricesObj(); } - const computeFx = (usd: number, other: number): number => - Math.round(Math.max(other, 0) / Math.max(usd, 1) * 100) / 100; + const computeFx = (usd: number, other: number): number => usd <= 0.05 ? 0 : Math.round(Math.max(other, 0) / usd * 100) / 100; const exchangeRates: ExchangeRates = config.MEMPOOL.CURRENCY_API_KEY ? { @@ -388,7 +403,8 @@ class PricesRepository { const filteredRates = rates.map((rate: any) => { return { time: rate.time, - [currency]: rate[currency] + [currency]: rate[currency], + ['USD']: rate['USD'] }; }); return { @@ -424,8 +440,8 @@ class PricesRepository { latestPrice = priceUpdater.getEmptyPricesObj(); } - const computeFx = (usd: number, other: number): number => - Math.round(Math.max(other, 0) / Math.max(usd, 1) * 100) / 100; + const computeFx = (usd: number, other: number): number => + usd <= 0 ? 0 : Math.round(Math.max(other, 0) / usd * 100) / 100; const exchangeRates: ExchangeRates = config.MEMPOOL.CURRENCY_API_KEY ? { @@ -477,7 +493,8 @@ class PricesRepository { const filteredRates = rates.map((rate: any) => { return { time: rate.time, - [currency]: rate[currency] + [currency]: rate[currency], + ['USD']: rate['USD'] }; }); return { diff --git a/frontend/src/app/components/block/block.component.html b/frontend/src/app/components/block/block.component.html index 777be0907..6b5a6a846 100644 --- a/frontend/src/app/components/block/block.component.html +++ b/frontend/src/app/components/block/block.component.html @@ -335,7 +335,7 @@
- +
diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 1eb1c4798..3cb60e048 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -526,9 +526,9 @@ export class BlockComponent implements OnInit, OnDestroy { if (this.priceSubscription) { this.priceSubscription.unsubscribe(); } - this.priceSubscription = block$.pipe( - switchMap((block) => { - return this.priceService.getBlockPrice$(block.timestamp).pipe( + this.priceSubscription = combineLatest([this.stateService.fiatCurrency$, block$]).pipe( + switchMap(([currency, block]) => { + return this.priceService.getBlockPrice$(block.timestamp, true, currency).pipe( tap((price) => { this.blockConversion = price; }) diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index 0167a3d43..ea397ee90 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -76,6 +76,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { mempoolBlocksSubscription: Subscription; blocksSubscription: Subscription; miningSubscription: Subscription; + currencyChangeSubscription: Subscription; fragmentParams: URLSearchParams; rbfTransaction: undefined | Transaction; replaced: boolean = false; @@ -493,10 +494,12 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { } } this.fetchRbfHistory$.next(this.tx.txid); - - this.priceService.getBlockPrice$(tx.status?.block_time, true).pipe( - tap((price) => { - this.blockConversion = price; + this.currencyChangeSubscription?.unsubscribe(); + this.currencyChangeSubscription = this.stateService.fiatCurrency$.pipe( + switchMap((currency) => { + return tx.status.block_time ? this.priceService.getBlockPrice$(tx.status.block_time, true, currency).pipe( + tap((price) => tx['price'] = price), + ) : of(undefined); }) ).subscribe(); @@ -810,6 +813,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { this.mempoolBlocksSubscription.unsubscribe(); this.blocksSubscription.unsubscribe(); this.miningSubscription?.unsubscribe(); + this.currencyChangeSubscription?.unsubscribe(); this.leaveTransaction(); } } diff --git a/frontend/src/app/components/transactions-list/transactions-list.component.ts b/frontend/src/app/components/transactions-list/transactions-list.component.ts index da9bdfe04..8f91489c1 100644 --- a/frontend/src/app/components/transactions-list/transactions-list.component.ts +++ b/frontend/src/app/components/transactions-list/transactions-list.component.ts @@ -32,11 +32,14 @@ export class TransactionsListComponent implements OnInit, OnChanges { @Input() outputIndex: number; @Input() address: string = ''; @Input() rowLimit = 12; + @Input() blockTime: number = 0; // Used for price calculation if all the transactions are in the same block @Output() loadMore = new EventEmitter(); latestBlock$: Observable; outspendsSubscription: Subscription; + currencyChangeSubscription: Subscription; + currency: string; refreshOutspends$: ReplaySubject = new ReplaySubject(); refreshChannels$: ReplaySubject = new ReplaySubject(); showDetails$ = new BehaviorSubject(false); @@ -125,6 +128,35 @@ export class TransactionsListComponent implements OnInit, OnChanges { ) , ).subscribe(() => this.ref.markForCheck()); + + this.currencyChangeSubscription = this.stateService.fiatCurrency$ + .subscribe(currency => { + this.currency = currency; + this.refreshPrice(); + }); + } + + refreshPrice(): void { + // Loop over all transactions + if (!this.transactions || !this.transactions.length || !this.currency) { + return; + } + const confirmedTxs = this.transactions.filter((tx) => tx.status.confirmed).length; + if (!this.blockTime) { + this.transactions.forEach((tx) => { + if (!this.blockTime) { + if (tx.status.block_time) { + this.priceService.getBlockPrice$(tx.status.block_time, confirmedTxs < 10, this.currency).pipe( + tap((price) => tx['price'] = price), + ).subscribe(); + } + } + }); + } else { + this.priceService.getBlockPrice$(this.blockTime, true, this.currency).pipe( + tap((price) => this.transactions.forEach((tx) => tx['price'] = price)), + ).subscribe(); + } } ngOnChanges(changes): void { @@ -148,6 +180,7 @@ export class TransactionsListComponent implements OnInit, OnChanges { this.transactionsLength = this.transactions.length; this.cacheService.setTxCache(this.transactions); + const confirmedTxs = this.transactions.filter((tx) => tx.status.confirmed).length; this.transactions.forEach((tx) => { tx['@voutLimit'] = true; tx['@vinLimit'] = true; @@ -197,10 +230,18 @@ export class TransactionsListComponent implements OnInit, OnChanges { } } - this.priceService.getBlockPrice$(tx.status.block_time).pipe( - tap((price) => tx['price'] = price) - ).subscribe(); + if (!this.blockTime && tx.status.block_time && this.currency) { + this.priceService.getBlockPrice$(tx.status.block_time, confirmedTxs < 10, this.currency).pipe( + tap((price) => tx['price'] = price), + ).subscribe(); + } }); + + if (this.blockTime && this.transactions?.length && this.currency) { + this.priceService.getBlockPrice$(this.blockTime, true, this.currency).pipe( + tap((price) => this.transactions.forEach((tx) => tx['price'] = price)), + ).subscribe(); + } const txIds = this.transactions.filter((tx) => !tx._outspends).map((tx) => tx.txid); if (txIds.length && !this.cached) { this.refreshOutspends$.next(txIds); @@ -308,5 +349,6 @@ export class TransactionsListComponent implements OnInit, OnChanges { ngOnDestroy(): void { this.outspendsSubscription.unsubscribe(); + this.currencyChangeSubscription?.unsubscribe(); } } diff --git a/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts b/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts index 924982983..aa98779ca 100644 --- a/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts +++ b/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts @@ -1,5 +1,5 @@ import { Component, ElementRef, ViewChild, Input, OnChanges, OnInit } from '@angular/core'; -import { tap } from 'rxjs'; +import { Subscription, of, switchMap, tap } from 'rxjs'; import { Price, PriceService } from '../../services/price.service'; import { StateService } from '../../services/state.service'; import { environment } from '../../../environments/environment'; @@ -35,6 +35,7 @@ export class TxBowtieGraphTooltipComponent implements OnChanges { tooltipPosition = { x: 0, y: 0 }; blockConversion: Price; + currencyChangeSubscription: Subscription; nativeAssetId = this.stateService.network === 'liquidtestnet' ? environment.nativeTestAssetId : environment.nativeAssetId; @@ -47,11 +48,14 @@ export class TxBowtieGraphTooltipComponent implements OnChanges { ngOnChanges(changes): void { if (changes.line?.currentValue) { - this.priceService.getBlockPrice$(changes.line?.currentValue.timestamp, true).pipe( - tap((price) => { - this.blockConversion = price; - }) - ).subscribe(); + this.currencyChangeSubscription?.unsubscribe(); + this.currencyChangeSubscription = this.stateService.fiatCurrency$.pipe( + switchMap((currency) => { + return changes.line?.currentValue.timestamp ? this.priceService.getBlockPrice$(changes.line?.currentValue.timestamp, true, currency).pipe( + tap((price) => this.blockConversion = price), + ) : of(undefined); + }) + ).subscribe(); } if (changes.cursorPosition && changes.cursorPosition.currentValue) { diff --git a/frontend/src/app/services/price.service.ts b/frontend/src/app/services/price.service.ts index 41f928343..7a2f42c95 100644 --- a/frontend/src/app/services/price.service.ts +++ b/frontend/src/app/services/price.service.ts @@ -98,6 +98,8 @@ export class PriceService { lastQueriedTimestamp: number; lastPriceHistoryUpdate: number; + lastQueriedCurrency: string; + lastQueriedHistoricalCurrency: string; historicalPrice: ConversionDict = { prices: null, @@ -130,7 +132,7 @@ export class PriceService { }; } - getBlockPrice$(blockTimestamp: number, singlePrice = false): Observable { + getBlockPrice$(blockTimestamp: number, singlePrice = false, currency: string): Observable { if (this.stateService.env.BASE_MODULE !== 'mempool' || !this.stateService.env.HISTORICAL_PRICE) { return of(undefined); } @@ -142,9 +144,10 @@ export class PriceService { * query a different timestamp than the last one */ if (singlePrice) { - if (!this.singlePriceObservable$ || (this.singlePriceObservable$ && blockTimestamp !== this.lastQueriedTimestamp)) { - this.singlePriceObservable$ = this.apiService.getHistoricalPrice$(blockTimestamp).pipe(shareReplay()); + if (!this.singlePriceObservable$ || (this.singlePriceObservable$ && (blockTimestamp !== this.lastQueriedTimestamp || currency !== this.lastQueriedCurrency))) { + this.singlePriceObservable$ = this.apiService.getHistoricalPrice$(blockTimestamp, currency).pipe(shareReplay()); this.lastQueriedTimestamp = blockTimestamp; + this.lastQueriedCurrency = currency; } return this.singlePriceObservable$.pipe( @@ -177,9 +180,10 @@ export class PriceService { * Query all price history only once. The observable is invalidated after 1 hour */ else { - if (!this.priceObservable$ || (this.priceObservable$ && (now - this.lastPriceHistoryUpdate > 3600))) { - this.priceObservable$ = this.apiService.getHistoricalPrice$(undefined).pipe(shareReplay()); + if (!this.priceObservable$ || (this.priceObservable$ && (now - this.lastPriceHistoryUpdate > 3600 || currency !== this.lastQueriedHistoricalCurrency))) { + this.priceObservable$ = this.apiService.getHistoricalPrice$(undefined, currency).pipe(shareReplay()); this.lastPriceHistoryUpdate = new Date().getTime() / 1000; + this.lastQueriedHistoricalCurrency = currency; } return this.priceObservable$.pipe( From b11164005cb5d4dfd7250bae36858e1e57b16766 Mon Sep 17 00:00:00 2001 From: natsoni Date: Sun, 10 Mar 2024 16:34:43 +0100 Subject: [PATCH 05/11] Add FIAT_PRICE category to backend config --- backend/mempool-config.sample.json | 7 +++++-- .../src/__fixtures__/mempool-config.template.json | 7 +++++-- backend/src/__tests__/config.test.ts | 6 +++++- backend/src/config.ts | 12 ++++++++++-- docker/backend/mempool-config.json | 7 +++++-- docker/backend/start.sh | 10 ++++++++-- 6 files changed, 38 insertions(+), 11 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 473988651..cac5d2e61 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -36,8 +36,7 @@ "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, "PRICE_UPDATES_PER_HOUR": 1, - "MAX_TRACKED_ADDRESSES": 100, - "CURRENCY_API_KEY": "your-api-key-from-freecurrencyapi.com" + "MAX_TRACKED_ADDRESSES": 100 }, "CORE_RPC": { "HOST": "127.0.0.1", @@ -156,5 +155,9 @@ "MEMPOOL_SERVICES": { "API": "https://mempool.space/api", "ACCELERATIONS": false + }, + "FIAT_PRICE": { + "ENABLED": true, + "API_KEY": "your-api-key-from-freecurrencyapi.com" } } diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 1bc4de045..5cbff22a3 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -37,8 +37,7 @@ "MAX_PUSH_TX_SIZE_WEIGHT": 4000000, "ALLOW_UNREACHABLE": true, "PRICE_UPDATES_PER_HOUR": 1, - "MAX_TRACKED_ADDRESSES": 1, - "CURRENCY_API_KEY": "__MEMPOOL_CURRENCY_API_KEY__" + "MAX_TRACKED_ADDRESSES": 1 }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", @@ -148,5 +147,9 @@ "ENABLED": false, "UNIX_SOCKET_PATH": "/tmp/redis.sock", "BATCH_QUERY_BASE_SIZE": 5000 + }, + "FIAT_PRICE": { + "ENABLED": true, + "API_KEY": "__MEMPOOL_CURRENCY_API_KEY__" } } diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index f382dcefa..e261e2adc 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -51,7 +51,6 @@ describe('Mempool Backend Config', () => { ALLOW_UNREACHABLE: true, PRICE_UPDATES_PER_HOUR: 1, MAX_TRACKED_ADDRESSES: 1, - CURRENCY_API_KEY: '' }); expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true }); @@ -153,6 +152,11 @@ describe('Mempool Backend Config', () => { UNIX_SOCKET_PATH: '', BATCH_QUERY_BASE_SIZE: 5000, }); + + expect(config.FIAT_PRICE).toStrictEqual({ + ENABLED: true, + API_KEY: '', + }); }); }); diff --git a/backend/src/config.ts b/backend/src/config.ts index fea1150f8..ee3645e58 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -42,7 +42,6 @@ interface IConfig { ALLOW_UNREACHABLE: boolean; PRICE_UPDATES_PER_HOUR: number; MAX_TRACKED_ADDRESSES: number; - CURRENCY_API_KEY: string; }; ESPLORA: { REST_API_URL: string; @@ -159,6 +158,10 @@ interface IConfig { UNIX_SOCKET_PATH: string; BATCH_QUERY_BASE_SIZE: number; }, + FIAT_PRICE: { + ENABLED: boolean; + API_KEY: string; + }, } const defaults: IConfig = { @@ -201,7 +204,6 @@ const defaults: IConfig = { 'ALLOW_UNREACHABLE': true, 'PRICE_UPDATES_PER_HOUR': 1, 'MAX_TRACKED_ADDRESSES': 1, - 'CURRENCY_API_KEY': '', }, 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', @@ -318,6 +320,10 @@ const defaults: IConfig = { 'UNIX_SOCKET_PATH': '', 'BATCH_QUERY_BASE_SIZE': 5000, }, + 'FIAT_PRICE': { + 'ENABLED': true, + 'API_KEY': '', + }, }; class Config implements IConfig { @@ -339,6 +345,7 @@ class Config implements IConfig { REPLICATION: IConfig['REPLICATION']; MEMPOOL_SERVICES: IConfig['MEMPOOL_SERVICES']; REDIS: IConfig['REDIS']; + FIAT_PRICE: IConfig['FIAT_PRICE']; constructor() { const configs = this.merge(configFromFile, defaults); @@ -360,6 +367,7 @@ class Config implements IConfig { this.REPLICATION = configs.REPLICATION; this.MEMPOOL_SERVICES = configs.MEMPOOL_SERVICES; this.REDIS = configs.REDIS; + this.FIAT_PRICE = configs.FIAT_PRICE; } merge = (...objects: object[]): IConfig => { diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 185bea19e..f8935706c 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -37,8 +37,7 @@ "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", "PRICE_UPDATES_PER_HOUR": __MEMPOOL_PRICE_UPDATES_PER_HOUR__, - "MAX_TRACKED_ADDRESSES": __MEMPOOL_MAX_TRACKED_ADDRESSES__, - "CURRENCY_API_KEY": "__MEMPOOL_CURRENCY_API_KEY__" + "MAX_TRACKED_ADDRESSES": __MEMPOOL_MAX_TRACKED_ADDRESSES__ }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", @@ -154,5 +153,9 @@ "ENABLED": __REDIS_ENABLED__, "UNIX_SOCKET_PATH": "__REDIS_UNIX_SOCKET_PATH__", "BATCH_QUERY_BASE_SIZE": __REDIS_BATCH_QUERY_BASE_SIZE__ + }, + "FIAT_PRICE": { + "ENABLED": __FIAT_PRICE_ENABLED__, + "API_KEY": "__FIAT_PRICE_API_KEY__" } } diff --git a/docker/backend/start.sh b/docker/backend/start.sh index a71b4b813..50ecc17ab 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -39,7 +39,6 @@ __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__=${MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT:=4000000} __MEMPOOL_ALLOW_UNREACHABLE__=${MEMPOOL_ALLOW_UNREACHABLE:=true} __MEMPOOL_PRICE_UPDATES_PER_HOUR__=${MEMPOOL_PRICE_UPDATES_PER_HOUR:=1} __MEMPOOL_MAX_TRACKED_ADDRESSES__=${MEMPOOL_MAX_TRACKED_ADDRESSES:=1} -__MEMPOOL_CURRENCY_API_KEY__=${MEMPOOL_CURRENCY_API_KEY:=""} # CORE_RPC __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1} @@ -156,6 +155,10 @@ __REDIS_ENABLED__=${REDIS_ENABLED:=false} __REDIS_UNIX_SOCKET_PATH__=${REDIS_UNIX_SOCKET_PATH:=true} __REDIS_BATCH_QUERY_BASE_SIZE__=${REDIS_BATCH_QUERY_BASE_SIZE:=5000} +# FIAT_PRICE +__FIAT_PRICE_ENABLED__=${FIAT_PRICE_ENABLED:=true} +__FIAT_PRICE_API_KEY__=${FIAT_PRICE_API_KEY:=""} + mkdir -p "${__MEMPOOL_CACHE_DIR__}" sed -i "s!__MEMPOOL_NETWORK__!${__MEMPOOL_NETWORK__}!g" mempool-config.json @@ -196,7 +199,6 @@ sed -i "s!__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__!${__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGH sed -i "s!__MEMPOOL_ALLOW_UNREACHABLE__!${__MEMPOOL_ALLOW_UNREACHABLE__}!g" mempool-config.json sed -i "s!__MEMPOOL_PRICE_UPDATES_PER_HOUR__!${__MEMPOOL_PRICE_UPDATES_PER_HOUR__}!g" mempool-config.json sed -i "s!__MEMPOOL_MAX_TRACKED_ADDRESSES__!${__MEMPOOL_MAX_TRACKED_ADDRESSES__}!g" mempool-config.json -sed -i "s!__MEMPOOL_CURRENCY_API_KEY__!${__MEMPOOL_CURRENCY_API_KEY__}!g" mempool-config.json sed -i "s!__CORE_RPC_HOST__!${__CORE_RPC_HOST__}!g" mempool-config.json sed -i "s!__CORE_RPC_PORT__!${__CORE_RPC_PORT__}!g" mempool-config.json @@ -303,4 +305,8 @@ sed -i "s!__REDIS_ENABLED__!${__REDIS_ENABLED__}!g" mempool-config.json sed -i "s!__REDIS_UNIX_SOCKET_PATH__!${__REDIS_UNIX_SOCKET_PATH__}!g" mempool-config.json sed -i "s!__REDIS_BATCH_QUERY_BASE_SIZE__!${__REDIS_BATCH_QUERY_BASE_SIZE__}!g" mempool-config.json +# FIAT_PRICE +sed -i "s!__FIAT_PRICE_ENABLED__!${__FIAT_PRICE_ENABLED__}!g" mempool-config.json +sed -i "s!__FIAT_PRICE_API_KEY__!${__FIAT_PRICE_API_KEY__}!g" mempool-config.json + node /backend/package/index.js From a5099fed75477e1a342d14b9c146a7f46d5155cb Mon Sep 17 00:00:00 2001 From: natsoni Date: Sun, 10 Mar 2024 17:12:19 +0100 Subject: [PATCH 06/11] Add backend checks for enabling fiat prices and update config paths --- backend/src/index.ts | 14 ++++++++++---- backend/src/indexer.ts | 12 +++++++----- backend/src/repositories/PricesRepository.ts | 10 +++++----- backend/src/tasks/price-updater.ts | 8 ++++---- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index 213319946..1988c7c56 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -131,7 +131,7 @@ class Server { .use(express.text({ type: ['text/plain', 'application/base64'] })) ; - if (config.DATABASE.ENABLED) { + if (config.DATABASE.ENABLED && config.FIAT_PRICE.ENABLED) { await priceUpdater.$initializeLatestPriceWithDb(); } @@ -168,7 +168,9 @@ class Server { setInterval(refreshIcons, 3600_000); } - priceUpdater.$run(); + if (config.FIAT_PRICE.ENABLED) { + priceUpdater.$run(); + } await chainTips.updateOrphanedBlocks(); this.setUpHttpApiRoutes(); @@ -220,7 +222,9 @@ class Server { await memPool.$updateMempool(newMempool, newAccelerations, pollRate); } indexer.$run(); - priceUpdater.$run(); + if (config.FIAT_PRICE.ENABLED) { + priceUpdater.$run(); + } // rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS const elapsed = Date.now() - start; @@ -284,7 +288,9 @@ class Server { memPool.setAsyncMempoolChangedCallback(websocketHandler.$handleMempoolChange.bind(websocketHandler)); blocks.setNewAsyncBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler)); } - priceUpdater.setRatesChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler)); + if (config.FIAT_PRICE.ENABLED) { + priceUpdater.setRatesChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler)); + } loadingIndicators.setProgressChangedCallback(websocketHandler.handleLoadingChanged.bind(websocketHandler)); } diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index dcb91d010..663a64c6d 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -116,7 +116,7 @@ class Indexer { switch (task) { case 'blocksPrices': { - if (!['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) { + if (!['testnet', 'signet'].includes(config.MEMPOOL.NETWORK) && config.FIAT_PRICE.ENABLED) { let lastestPriceId; try { lastestPriceId = await PricesRepository.$getLatestPriceId(); @@ -148,10 +148,12 @@ class Indexer { return; } - try { - await priceUpdater.$run(); - } catch (e) { - logger.err(`Running priceUpdater failed. Reason: ` + (e instanceof Error ? e.message : e)); + if (config.FIAT_PRICE.ENABLED) { + try { + await priceUpdater.$run(); + } catch (e) { + logger.err(`Running priceUpdater failed. Reason: ` + (e instanceof Error ? e.message : e)); + } } // Do not attempt to index anything unless Bitcoin Core is fully synced diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index e5acec861..f9498f313 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -40,7 +40,7 @@ export interface ApiPrice { ZAR: number, } -const ApiPriceFields = config.MEMPOOL.CURRENCY_API_KEY ? +const ApiPriceFields = config.FIAT_PRICE.API_KEY ? ` UNIX_TIMESTAMP(time) as time, USD, @@ -181,7 +181,7 @@ class PricesRepository { } try { - if (!config.MEMPOOL.CURRENCY_API_KEY) { // Store only the 7 main currencies + if (!config.FIAT_PRICE.API_KEY) { // Store only the 7 main currencies await DB.query(` INSERT INTO prices(time, USD, EUR, GBP, CAD, CHF, AUD, JPY) VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ? )`, @@ -330,7 +330,7 @@ class PricesRepository { } let pricesUsedForExchangeRates; // If we don't have a fx API key, we need to use the latest prices to compute the exchange rates - if (!config.MEMPOOL.CURRENCY_API_KEY) { + if (!config.FIAT_PRICE.API_KEY) { const [latestPrices] = await DB.query(` SELECT ${ApiPriceFields} FROM prices @@ -353,7 +353,7 @@ class PricesRepository { const computeFx = (usd: number, other: number): number => usd <= 0.05 ? 0 : Math.round(Math.max(other, 0) / usd * 100) / 100; - const exchangeRates: ExchangeRates = config.MEMPOOL.CURRENCY_API_KEY ? + const exchangeRates: ExchangeRates = config.FIAT_PRICE.API_KEY ? { USDEUR: computeFx(latestPrice.USD, latestPrice.EUR), USDGBP: computeFx(latestPrice.USD, latestPrice.GBP), @@ -443,7 +443,7 @@ class PricesRepository { const computeFx = (usd: number, other: number): number => usd <= 0 ? 0 : Math.round(Math.max(other, 0) / usd * 100) / 100; - const exchangeRates: ExchangeRates = config.MEMPOOL.CURRENCY_API_KEY ? + const exchangeRates: ExchangeRates = config.FIAT_PRICE.API_KEY ? { USDEUR: computeFx(latestPrice.USD, latestPrice.EUR), USDGBP: computeFx(latestPrice.USD, latestPrice.GBP), diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index f23879831..bfdce5f8c 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -70,7 +70,7 @@ class PriceUpdater { this.feeds.push(new BitfinexApi()); this.feeds.push(new GeminiApi()); - this.currencyConversionFeed = new FreeCurrencyApi(config.MEMPOOL.CURRENCY_API_KEY); + this.currencyConversionFeed = new FreeCurrencyApi(config.FIAT_PRICE.API_KEY); this.setCyclePosition(); } @@ -146,7 +146,7 @@ class PriceUpdater { this.additionalCurrenciesHistoryInserted = false; } - if (config.MEMPOOL.CURRENCY_API_KEY && this.currencyConversionFeed && (Math.round(new Date().getTime() / 1000) - this.lastTimeConversionsRatesFetched) > 3600 * 24) { + if (config.FIAT_PRICE.API_KEY && this.currencyConversionFeed && (Math.round(new Date().getTime() / 1000) - this.lastTimeConversionsRatesFetched) > 3600 * 24) { // Once a day, fetch conversion rates from api: we don't need more granularity for fiat currencies and have a limited number of requests try { this.latestConversionsRatesFromFeed = await this.currencyConversionFeed.$fetchLatestConversionRates(); @@ -162,7 +162,7 @@ class PriceUpdater { if (this.historyInserted === false && config.DATABASE.ENABLED === true) { await this.$insertHistoricalPrices(); } - if (this.additionalCurrenciesHistoryInserted === false && config.DATABASE.ENABLED === true && config.MEMPOOL.CURRENCY_API_KEY && !this.additionalCurrenciesHistoryRunning) { + if (this.additionalCurrenciesHistoryInserted === false && config.DATABASE.ENABLED === true && config.FIAT_PRICE.API_KEY && !this.additionalCurrenciesHistoryRunning) { await this.$insertMissingAdditionalPrices(); } } catch (e: any) { @@ -244,7 +244,7 @@ class PriceUpdater { } } - if (config.MEMPOOL.CURRENCY_API_KEY && this.latestPrices.USD > 0 && Object.keys(this.latestConversionsRatesFromFeed).length > 0) { + if (config.FIAT_PRICE.API_KEY && this.latestPrices.USD > 0 && Object.keys(this.latestConversionsRatesFromFeed).length > 0) { for (const conversionCurrency of this.newCurrencies) { if (this.latestConversionsRatesFromFeed[conversionCurrency] > 0 && this.latestPrices.USD * this.latestConversionsRatesFromFeed[conversionCurrency] < MAX_PRICES[conversionCurrency]) { this.latestPrices[conversionCurrency] = Math.round(this.latestPrices.USD * this.latestConversionsRatesFromFeed[conversionCurrency]); From 8e158e178672904e05e9ea1b31217e153cb8ab06 Mon Sep 17 00:00:00 2001 From: natsoni Date: Mon, 11 Mar 2024 14:28:06 +0100 Subject: [PATCH 07/11] Fix lastHistoricalRun variable to be in seconds --- backend/src/tasks/price-updater.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index bfdce5f8c..5cbb68ee7 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -320,7 +320,7 @@ class PriceUpdater { await this.$insertMissingRecentPrices('hour'); this.historyInserted = true; - this.lastHistoricalRun = new Date().getTime(); + this.lastHistoricalRun = Math.round(new Date().getTime() / 1000); } /** From 669cf592699c597b6d4719bd5534282d2552431f Mon Sep 17 00:00:00 2001 From: natsoni Date: Mon, 11 Mar 2024 15:27:43 +0100 Subject: [PATCH 08/11] Set monthly granulary for fx rates --- backend/src/repositories/PricesRepository.ts | 26 +++----- .../tasks/price-feeds/free-currency-api.ts | 1 + backend/src/tasks/price-updater.ts | 61 ++++++++++++------- 3 files changed, 49 insertions(+), 39 deletions(-) diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index f9498f313..97566b2fc 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -329,24 +329,18 @@ class PricesRepository { throw Error(`Cannot get single historical price from the database`); } - let pricesUsedForExchangeRates; // If we don't have a fx API key, we need to use the latest prices to compute the exchange rates - if (!config.FIAT_PRICE.API_KEY) { - const [latestPrices] = await DB.query(` - SELECT ${ApiPriceFields} - FROM prices - ORDER BY time DESC - LIMIT 1 - `); - if (!Array.isArray(latestPrices)) { - throw Error(`Cannot get single historical price from the database`); - } - pricesUsedForExchangeRates = latestPrices[0] as ApiPrice; - } else { - pricesUsedForExchangeRates = rates[0] as ApiPrice; - } + const [latestPrices] = await DB.query(` + SELECT ${ApiPriceFields} + FROM prices + ORDER BY time DESC + LIMIT 1 + `); + if (!Array.isArray(latestPrices)) { + throw Error(`Cannot get single historical price from the database`); + } // Compute fiat exchange rates - let latestPrice = pricesUsedForExchangeRates; + let latestPrice = latestPrices[0] as ApiPrice; if (!latestPrice || latestPrice.USD === -1) { latestPrice = priceUpdater.getEmptyPricesObj(); } diff --git a/backend/src/tasks/price-feeds/free-currency-api.ts b/backend/src/tasks/price-feeds/free-currency-api.ts index cd9c1d498..8d6175b95 100644 --- a/backend/src/tasks/price-feeds/free-currency-api.ts +++ b/backend/src/tasks/price-feeds/free-currency-api.ts @@ -33,6 +33,7 @@ const emptyRates = { SGD: -1, THB: -1, TRY: -1, + USD: -1, ZAR: -1, }; diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index 5cbb68ee7..7ed4cb178 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -26,8 +26,8 @@ export interface PriceHistory { export interface ConversionFeed { $getQuota(): Promise; - $fetchLatestConversionRates(): Promise; - $fetchConversionRates(date: string): Promise; + $fetchLatestConversionRates(): Promise; + $fetchConversionRates(date: string): Promise; } export interface ConversionRates { @@ -46,6 +46,7 @@ class PriceUpdater { public historyInserted = false; private additionalCurrenciesHistoryInserted = false; private additionalCurrenciesHistoryRunning = false; + private lastFailedHistoricalRun = 0; private timeBetweenUpdatesMs = 360_0000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR; private cyclePosition = -1; private firstRun = true; @@ -146,6 +147,11 @@ class PriceUpdater { this.additionalCurrenciesHistoryInserted = false; } + if (this.lastFailedHistoricalRun > 0 && (Math.round(new Date().getTime() / 1000) - this.lastFailedHistoricalRun) > 60) { + // If the last attempt to insert missing prices failed, we try again after 60 seconds + this.additionalCurrenciesHistoryInserted = false; + } + if (config.FIAT_PRICE.API_KEY && this.currencyConversionFeed && (Math.round(new Date().getTime() / 1000) - this.lastTimeConversionsRatesFetched) > 3600 * 24) { // Once a day, fetch conversion rates from api: we don't need more granularity for fiat currencies and have a limited number of requests try { @@ -165,6 +171,7 @@ class PriceUpdater { if (this.additionalCurrenciesHistoryInserted === false && config.DATABASE.ENABLED === true && config.FIAT_PRICE.API_KEY && !this.additionalCurrenciesHistoryRunning) { await this.$insertMissingAdditionalPrices(); } + } catch (e: any) { logger.err(`Cannot save BTC prices in db. Reason: ${e instanceof Error ? e.message : e}`, logger.tags.mining); } @@ -393,46 +400,53 @@ class PriceUpdater { * We calculate the additional prices from the USD price and the conversion rates */ private async $insertMissingAdditionalPrices(): Promise { - this.additionalCurrenciesHistoryRunning = true; + this.lastFailedHistoricalRun = 0; const priceTimesToFill = await PricesRepository.$getPricesTimesWithMissingFields(); if (priceTimesToFill.length === 0) { return; } + try { + const remainingQuota = await this.currencyConversionFeed?.$getQuota(); + if (remainingQuota['month']['remaining'] < 500) { // We need some calls left for the daily updates + logger.debug(`Not enough currency API credit to insert missing prices in ${priceTimesToFill.length} rows (${remainingQuota['month']['remaining']} calls left).`, logger.tags.mining); + this.additionalCurrenciesHistoryInserted = true; // Do not try again until next day + return; + } + } catch (e) { + logger.err(`Cannot fetch currency API credit, insertion of missing prices aborted. Reason: ${(e instanceof Error ? e.message : e)}`); + return; + } + + this.additionalCurrenciesHistoryRunning = true; logger.debug(`Fetching missing conversion rates from external API to fill ${priceTimesToFill.length} rows`, logger.tags.mining); let conversionRates: { [timestamp: number]: ConversionRates } = {}; let totalInserted = 0; - let requestCounter = 0; - - for (const priceTime of priceTimesToFill) { + for (let i = 0; i < priceTimesToFill.length; i++) { + const priceTime = priceTimesToFill[i]; const missingLegacyCurrencies = this.getMissingLegacyCurrencies(priceTime); // In the case a legacy currency (EUR, GBP, CAD, CHF, AUD, JPY) const year = new Date(priceTime.time * 1000).getFullYear(); // is missing, we use the same process as for the new currencies - const yearTimestamp = new Date(year, 0, 1).getTime() / 1000; - if (conversionRates[yearTimestamp] === undefined) { - try { - if (requestCounter >= 10) { - await new Promise(resolve => setTimeout(resolve, 60_000)); // avoid getting 429'd - requestCounter = 0; - } - conversionRates[yearTimestamp] = await this.currencyConversionFeed?.$fetchConversionRates(`${year}-01-01`); - ++requestCounter; - } catch (e) { - logger.err(`Cannot fetch conversion rates from the API for year ${year}. Reason: ${(e instanceof Error ? e.message : e)}`); + const month = new Date(priceTime.time * 1000).getMonth(); + const yearMonthTimestamp = new Date(year, month, 1).getTime() / 1000; + if (conversionRates[yearMonthTimestamp] === undefined) { + conversionRates[yearMonthTimestamp] = await this.currencyConversionFeed?.$fetchConversionRates(`${year}-${month + 1 < 10 ? `0${month + 1}` : `${month + 1}`}-01`) || { USD: -1 }; + if (conversionRates[yearMonthTimestamp]['USD'] < 0) { + logger.err(`Cannot fetch conversion rates from the API for ${year}-${month + 1 < 10 ? `0${month + 1}` : `${month + 1}`}-01. Aborting insertion of missing prices.`, logger.tags.mining); + this.lastFailedHistoricalRun = Math.round(new Date().getTime() / 1000); + break; } } - if (conversionRates[yearTimestamp] === undefined) { - continue; - } - const prices: ApiPrice = this.getEmptyPricesObj(); let willInsert = false; for (const conversionCurrency of this.newCurrencies.concat(missingLegacyCurrencies)) { - if (conversionRates[yearTimestamp][conversionCurrency] > 0 && priceTime.USD * conversionRates[yearTimestamp][conversionCurrency] < MAX_PRICES[conversionCurrency]) { - prices[conversionCurrency] = year >= 2013 ? Math.round(priceTime.USD * conversionRates[yearTimestamp][conversionCurrency]) : Math.round(priceTime.USD * conversionRates[yearTimestamp][conversionCurrency] * 100) / 100; + if (conversionRates[yearMonthTimestamp][conversionCurrency] > 0 && priceTime.USD * conversionRates[yearMonthTimestamp][conversionCurrency] < MAX_PRICES[conversionCurrency]) { + prices[conversionCurrency] = year >= 2013 ? Math.round(priceTime.USD * conversionRates[yearMonthTimestamp][conversionCurrency]) : Math.round(priceTime.USD * conversionRates[yearMonthTimestamp][conversionCurrency] * 100) / 100; willInsert = true; + } else { + prices[conversionCurrency] = 0; } } @@ -441,6 +455,7 @@ class PriceUpdater { ++totalInserted; } } + logger.debug(`Inserted ${totalInserted} missing additional currency prices into the db`, logger.tags.mining); this.additionalCurrenciesHistoryInserted = true; this.additionalCurrenciesHistoryRunning = false; From b99e5c4160d9039bac7d43cb0be3a4b69088e007 Mon Sep 17 00:00:00 2001 From: natsoni Date: Mon, 11 Mar 2024 18:02:30 +0100 Subject: [PATCH 09/11] Populate historical fiat prices from latest to oldest --- backend/src/repositories/PricesRepository.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index 97566b2fc..31098193d 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -281,7 +281,7 @@ class PricesRepository { WHERE USD != -1 AND -1 IN (EUR, GBP, CAD, CHF, AUD, JPY, BGN, BRL, CNY, CZK, DKK, HKD, HRK, HUF, IDR, ILS, INR, ISK, KRW, MXN, MYR, NOK, NZD, PHP, PLN, RON, RUB, SEK, SGD, THB, TRY, ZAR) - ORDER BY time + ORDER BY time DESC `); if (!Array.isArray(times)) { return []; From a95d17950ffd02678a45d5a7e61ec315db9e41c2 Mon Sep 17 00:00:00 2001 From: natsoni Date: Tue, 12 Mar 2024 20:50:48 +0100 Subject: [PATCH 10/11] Add fake entry for missing price data before 2010-07-19 --- backend/src/repositories/PricesRepository.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/backend/src/repositories/PricesRepository.ts b/backend/src/repositories/PricesRepository.ts index 31098193d..13392f0cf 100644 --- a/backend/src/repositories/PricesRepository.ts +++ b/backend/src/repositories/PricesRepository.ts @@ -401,6 +401,13 @@ class PricesRepository { ['USD']: rate['USD'] }; }); + if (filteredRates.length === 0) { // No price data before 2010-07-19: add a fake entry + filteredRates.push({ + time: 1279497600, + [currency]: 0, + ['USD']: 0 + }); + } return { prices: filteredRates as ApiPrice[], exchangeRates: exchangeRates From 7bc6ef25168ec48c4b4bdc3c3d54edf9db023117 Mon Sep 17 00:00:00 2001 From: natsoni Date: Thu, 14 Mar 2024 13:08:15 +0100 Subject: [PATCH 11/11] Fix tx fee to display historical price --- .../src/app/components/transaction/transaction.component.html | 2 +- .../src/app/components/transaction/transaction.component.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index adff17e8f..b767d74d8 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -533,7 +533,7 @@ Fee - {{ tx.fee | number }} sat + {{ tx.fee | number }} sat Fee rate diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index ea397ee90..612bcd5ae 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -109,7 +109,6 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { hideFlow: boolean = this.stateService.hideFlow.value; overrideFlowPreference: boolean = null; flowEnabled: boolean; - blockConversion: Price; tooltipPosition: { x: number, y: number }; isMobile: boolean;