From 25c0eb62b24439283096c4e04f59ff6dcf37c9f7 Mon Sep 17 00:00:00 2001 From: natsoni Date: Tue, 4 Jun 2024 10:58:04 +0200 Subject: [PATCH] More robust price service --- .../tasks/price-feeds/free-currency-api.ts | 2 +- backend/src/tasks/price-updater.ts | 33 +++++++++++++------ backend/src/utils/axios-query.ts | 9 ++++- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/backend/src/tasks/price-feeds/free-currency-api.ts b/backend/src/tasks/price-feeds/free-currency-api.ts index 2a21a16d2..48e511aa8 100644 --- a/backend/src/tasks/price-feeds/free-currency-api.ts +++ b/backend/src/tasks/price-feeds/free-currency-api.ts @@ -76,7 +76,7 @@ class FreeCurrencyApi implements ConversionFeed { } public async $fetchConversionRates(date: string): Promise { - const response = await query(`${this.API_URL_PREFIX}historical?date=${date}&apikey=${this.API_KEY}`); + const response = await query(`${this.API_URL_PREFIX}historical?date=${date}&apikey=${this.API_KEY}`, true); if (response && response['data'] && (response['data'][date] || this.PAID)) { if (this.PAID) { response['data'] = this.convertData(response['data']); diff --git a/backend/src/tasks/price-updater.ts b/backend/src/tasks/price-updater.ts index 16fe713b7..467669a6f 100644 --- a/backend/src/tasks/price-updater.ts +++ b/backend/src/tasks/price-updater.ts @@ -59,7 +59,7 @@ class PriceUpdater { private currencyConversionFeed: ConversionFeed | undefined; private newCurrencies: string[] = ['BGN', 'BRL', 'CNY', 'CZK', 'DKK', 'HKD', 'HRK', 'HUF', 'IDR', 'ILS', 'INR', 'ISK', 'KRW', 'MXN', 'MYR', 'NOK', 'NZD', 'PHP', 'PLN', 'RON', 'RUB', 'SEK', 'SGD', 'THB', 'TRY', 'ZAR']; private lastTimeConversionsRatesFetched: number = 0; - private latestConversionsRatesFromFeed: ConversionRates = {}; + private latestConversionsRatesFromFeed: ConversionRates = { USD: -1 }; private ratesChangedCallback: ((rates: ApiPrice) => void) | undefined; constructor() { @@ -157,9 +157,9 @@ class PriceUpdater { try { this.latestConversionsRatesFromFeed = await this.currencyConversionFeed.$fetchLatestConversionRates(); this.lastTimeConversionsRatesFetched = Math.round(new Date().getTime() / 1000); - logger.debug(`Fetched currencies conversion rates from external API: ${JSON.stringify(this.latestConversionsRatesFromFeed)}`); + logger.debug(`Fetched currencies conversion rates from conversions API: ${JSON.stringify(this.latestConversionsRatesFromFeed)}`); } catch (e) { - logger.err(`Cannot fetch conversion rates from the API. Reason: ${(e instanceof Error ? e.message : e)}`); + logger.err(`Cannot fetch conversion rates from conversions API. Reason: ${(e instanceof Error ? e.message : e)}`); } } @@ -408,17 +408,17 @@ class PriceUpdater { try { const remainingQuota = await this.currencyConversionFeed?.$getQuota(); if (remainingQuota['month']['remaining'] < 500) { // We need some calls left for the daily updates - logger.debug(`Not enough currency API credit to insert missing prices in ${priceTimesToFill.length} rows (${remainingQuota['month']['remaining']} calls left).`, logger.tags.mining); + logger.debug(`Not enough conversions API credit to insert missing prices in ${priceTimesToFill.length} rows (${remainingQuota['month']['remaining']} calls left).`, logger.tags.mining); this.additionalCurrenciesHistoryInserted = true; // Do not try again until next day return; } } catch (e) { - logger.err(`Cannot fetch currency API credit, insertion of missing prices aborted. Reason: ${(e instanceof Error ? e.message : e)}`); + logger.err(`Cannot fetch conversions API credit, insertion of missing prices aborted. Reason: ${(e instanceof Error ? e.message : e)}`); return; } this.additionalCurrenciesHistoryRunning = true; - logger.debug(`Fetching missing conversion rates from external API to fill ${priceTimesToFill.length} rows`, logger.tags.mining); + logger.debug(`Inserting missing historical conversion rates using conversions API to fill ${priceTimesToFill.length} rows`, logger.tags.mining); let conversionRates: { [timestamp: number]: ConversionRates } = {}; let totalInserted = 0; @@ -430,10 +430,23 @@ class PriceUpdater { const month = new Date(priceTime.time * 1000).getMonth(); const yearMonthTimestamp = new Date(year, month, 1).getTime() / 1000; if (conversionRates[yearMonthTimestamp] === undefined) { - conversionRates[yearMonthTimestamp] = await this.currencyConversionFeed?.$fetchConversionRates(`${year}-${month + 1 < 10 ? `0${month + 1}` : `${month + 1}`}-01`) || { USD: -1 }; - if (conversionRates[yearMonthTimestamp]['USD'] < 0) { - logger.err(`Cannot fetch conversion rates from the API for ${year}-${month + 1 < 10 ? `0${month + 1}` : `${month + 1}`}-01. Aborting insertion of missing prices.`, logger.tags.mining); - this.lastFailedHistoricalRun = Math.round(new Date().getTime() / 1000); + try { + if (year === new Date().getFullYear() && month === new Date().getMonth()) { // For rows in the current month, we use the latest conversion rates + conversionRates[yearMonthTimestamp] = this.latestConversionsRatesFromFeed; + } else { + conversionRates[yearMonthTimestamp] = await this.currencyConversionFeed?.$fetchConversionRates(`${year}-${month + 1 < 10 ? `0${month + 1}` : `${month + 1}`}-15`) || { USD: -1 }; + } + + if (conversionRates[yearMonthTimestamp]['USD'] < 0) { + throw new Error('Incorrect USD conversion rate'); + } + } catch (e) { + if ((e instanceof Error ? e.message : '').includes('429')) { // Continue 60 seconds later if and only if error is 429 + this.lastFailedHistoricalRun = Math.round(new Date().getTime() / 1000); + logger.info(`Got a 429 error from conversions API. This is expected to happen a few times during the initial historical price insertion, process will resume in 60 seconds.`, logger.tags.mining); + } else { + logger.err(`Cannot fetch conversion rates from conversions API for ${year}-${month + 1 < 10 ? `0${month + 1}` : `${month + 1}`}-01, trying again next day. Error: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining); + } break; } } diff --git a/backend/src/utils/axios-query.ts b/backend/src/utils/axios-query.ts index 0a155fd55..e641d3ce9 100644 --- a/backend/src/utils/axios-query.ts +++ b/backend/src/utils/axios-query.ts @@ -5,7 +5,7 @@ import config from '../config'; import logger from '../logger'; import * as https from 'https'; -export async function query(path): Promise { +export async function query(path, throwOnFail: boolean = false): Promise { type axiosOptions = { headers: { 'User-Agent': string @@ -21,6 +21,7 @@ export async function query(path): Promise { timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000 }; let retry = 0; + let lastError: any = null; while (retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) { try { @@ -50,6 +51,7 @@ export async function query(path): Promise { } return data.data; } catch (e) { + lastError = e; logger.warn(`Could not connect to ${path} (Attempt ${retry + 1}/${config.MEMPOOL.EXTERNAL_MAX_RETRY}). Reason: ` + (e instanceof Error ? e.message : e)); retry++; } @@ -59,5 +61,10 @@ export async function query(path): Promise { } logger.err(`Could not connect to ${path}. All ${config.MEMPOOL.EXTERNAL_MAX_RETRY} attempts failed`); + + if (throwOnFail && lastError) { + throw lastError; + } + return undefined; }