More robust price service

backend/src/tasks/price-feeds/free-currency-api.ts

@@ -76,7 +76,7 @@ class FreeCurrencyApi implements ConversionFeed {
   }
 
   public async $fetchConversionRates(date: string): Promise<ConversionRates> {
-    const response = await query(`${this.API_URL_PREFIX}historical?date=${date}&apikey=${this.API_KEY}`);
+    const response = await query(`${this.API_URL_PREFIX}historical?date=${date}&apikey=${this.API_KEY}`, true);
     if (response && response['data'] && (response['data'][date] || this.PAID)) {
       if (this.PAID) {
         response['data'] = this.convertData(response['data']);

backend/src/tasks/price-updater.ts

@@ -59,7 +59,7 @@ class PriceUpdater {
   private currencyConversionFeed: ConversionFeed | undefined;
   private newCurrencies: string[] = ['BGN', 'BRL', 'CNY', 'CZK', 'DKK', 'HKD', 'HRK', 'HUF', 'IDR', 'ILS', 'INR', 'ISK', 'KRW', 'MXN', 'MYR', 'NOK', 'NZD', 'PHP', 'PLN', 'RON', 'RUB', 'SEK', 'SGD', 'THB', 'TRY', 'ZAR'];
   private lastTimeConversionsRatesFetched: number = 0;
-  private latestConversionsRatesFromFeed: ConversionRates = {};
+  private latestConversionsRatesFromFeed: ConversionRates = { USD: -1 };
   private ratesChangedCallback: ((rates: ApiPrice) => void) | undefined;
 
   constructor() {
@@ -157,9 +157,9 @@ class PriceUpdater {
       try {
         this.latestConversionsRatesFromFeed = await this.currencyConversionFeed.$fetchLatestConversionRates();
         this.lastTimeConversionsRatesFetched = Math.round(new Date().getTime() / 1000);
-        logger.debug(`Fetched currencies conversion rates from external API: ${JSON.stringify(this.latestConversionsRatesFromFeed)}`);
+        logger.debug(`Fetched currencies conversion rates from conversions API: ${JSON.stringify(this.latestConversionsRatesFromFeed)}`);
       } catch (e) {
-        logger.err(`Cannot fetch conversion rates from the API. Reason: ${(e instanceof Error ? e.message : e)}`);
+        logger.err(`Cannot fetch conversion rates from conversions API. Reason: ${(e instanceof Error ? e.message : e)}`);
       }
     }
 
@@ -408,17 +408,17 @@ class PriceUpdater {
     try {
       const remainingQuota = await this.currencyConversionFeed?.$getQuota();
       if (remainingQuota['month']['remaining'] < 500) { // We need some calls left for the daily updates
-        logger.debug(`Not enough currency API credit to insert missing prices in ${priceTimesToFill.length} rows (${remainingQuota['month']['remaining']} calls left).`, logger.tags.mining);
+        logger.debug(`Not enough conversions API credit to insert missing prices in ${priceTimesToFill.length} rows (${remainingQuota['month']['remaining']} calls left).`, logger.tags.mining);
         this.additionalCurrenciesHistoryInserted = true; // Do not try again until next day
         return;
       }
     } catch (e) {
-      logger.err(`Cannot fetch currency API credit, insertion of missing prices aborted. Reason: ${(e instanceof Error ? e.message : e)}`);
+      logger.err(`Cannot fetch conversions API credit, insertion of missing prices aborted. Reason: ${(e instanceof Error ? e.message : e)}`);
       return;
     }
 
     this.additionalCurrenciesHistoryRunning = true;
-    logger.debug(`Fetching missing conversion rates from external API to fill ${priceTimesToFill.length} rows`, logger.tags.mining);
+    logger.debug(`Inserting missing historical conversion rates using conversions API to fill ${priceTimesToFill.length} rows`, logger.tags.mining);
 
     let conversionRates: { [timestamp: number]: ConversionRates } = {};
     let totalInserted = 0;
@@ -430,10 +430,23 @@ class PriceUpdater {
       const month = new Date(priceTime.time * 1000).getMonth();
       const yearMonthTimestamp = new Date(year, month, 1).getTime() / 1000;
       if (conversionRates[yearMonthTimestamp] === undefined) {
-        conversionRates[yearMonthTimestamp] = await this.currencyConversionFeed?.$fetchConversionRates(`${year}-${month + 1 < 10 ? `0${month + 1}` : `${month + 1}`}-01`) || { USD: -1 };
+        try {
+          if (year === new Date().getFullYear() && month === new Date().getMonth()) { // For rows in the current month, we use the latest conversion rates
+            conversionRates[yearMonthTimestamp] = this.latestConversionsRatesFromFeed;
+          } else {
+            conversionRates[yearMonthTimestamp] = await this.currencyConversionFeed?.$fetchConversionRates(`${year}-${month + 1 < 10 ? `0${month + 1}` : `${month + 1}`}-15`) || { USD: -1 };
+          }
+
           if (conversionRates[yearMonthTimestamp]['USD'] < 0) {
-          logger.err(`Cannot fetch conversion rates from the API for ${year}-${month + 1 < 10 ? `0${month + 1}` : `${month + 1}`}-01. Aborting insertion of missing prices.`, logger.tags.mining);
+            throw new Error('Incorrect USD conversion rate');
+          }
+        } catch (e) {
+          if ((e instanceof Error ? e.message : '').includes('429')) { // Continue 60 seconds later if and only if error is 429
             this.lastFailedHistoricalRun = Math.round(new Date().getTime() / 1000);
+            logger.info(`Got a 429 error from conversions API. This is expected to happen a few times during the initial historical price insertion, process will resume in 60 seconds.`, logger.tags.mining);
+          } else {
+            logger.err(`Cannot fetch conversion rates from conversions API for ${year}-${month + 1 < 10 ? `0${month + 1}` : `${month + 1}`}-01, trying again next day. Error: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
+          }
           break;
         }
       }

backend/src/utils/axios-query.ts

@@ -5,7 +5,7 @@ import config from '../config';
 import logger from '../logger';
 import * as https from 'https';
 
-export async function query(path): Promise<object | undefined> {
+export async function query(path, throwOnFail: boolean = false): Promise<object | undefined> {
  type axiosOptions = {
    headers: {
      'User-Agent': string
@@ -21,6 +21,7 @@ export async function query(path): Promise<object | undefined> {
    timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
  };
  let retry = 0;
+ let lastError: any = null;
 
  while (retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
    try {
@@ -50,6 +51,7 @@ export async function query(path): Promise<object | undefined> {
      }
      return data.data;
    } catch (e) {
+     lastError = e;
      logger.warn(`Could not connect to ${path} (Attempt ${retry + 1}/${config.MEMPOOL.EXTERNAL_MAX_RETRY}). Reason: ` + (e instanceof Error ? e.message : e));
      retry++;
    }
@@ -59,5 +61,10 @@ export async function query(path): Promise<object | undefined> {
  }
 
  logger.err(`Could not connect to ${path}. All ${config.MEMPOOL.EXTERNAL_MAX_RETRY} attempts failed`);
+
+ if (throwOnFail && lastError) {
+    throw lastError;
+  }
+
  return undefined;
 }