Merge branch 'master' into mononaut/core-gettxsforblock

backend/README.md

@@ -181,7 +181,7 @@ Create a new wallet, if needed:
    bitcoin-cli -regtest createwallet test
    ```
 
-Load wallet (this command may take a while if you have lot of UTXOs):
+Load wallet (this command may take a while if you have a lot of UTXOs):
    ```
    bitcoin-cli -regtest loadwallet test
    ```
@@ -233,9 +233,9 @@ By default, mining pools will be not automatically updated regularly (`config.ME
 
 To manually update your mining pools, you can use the `--update-pools` command line flag when you run the nodejs backend. For example `npm run start --update-pools`. This will trigger the mining pools update and automatically re-index appropriate blocks.
 
-You can enabled the automatic mining pools update by settings `config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` to `true` in your `mempool-config.json`.
+You can enable the automatic mining pools update by settings `config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING` to `true` in your `mempool-config.json`.
 
-When a `coinbase tag` or `coinbase address` change is detected, all blocks tagged to the `unknown` mining pools (starting from height 130635) will be deleted from the `blocks` table. Additionaly, all blocks which were tagged to the pool which has been updated will also be deleted from the `blocks` table. Of course, those blocks will be automatically reindexed.
+When a `coinbase tag` or `coinbase address` change is detected, all blocks tagged to the `unknown` mining pools (starting from height 130635) will be deleted from the `blocks` table. Additionally, all blocks which were tagged to the pool which has been updated will also be deleted from the `blocks` table. Of course, those blocks will be automatically reindexed.
 
 ### Re-index tables
 

backend/package-lock.json

@@ -23,7 +23,7 @@
         "rust-gbt": "file:./rust-gbt",
         "socks-proxy-agent": "~7.0.0",
         "typescript": "~4.9.3",
-        "ws": "~8.17.0"
+        "ws": "~8.17.1"
       },
       "devDependencies": {
         "@babel/code-frame": "^7.18.6",
@@ -7690,9 +7690,9 @@
       }
     },
     "node_modules/ws": {
-      "version": "8.17.0",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.0.tgz",
-      "integrity": "sha512-uJq6108EgZMAl20KagGkzCKfMEjxmKvZHG7Tlq0Z6nOky7YF7aq4mOx6xK8TJ/i1LeK4Qus7INktacctDgY8Ow==",
+      "version": "8.17.1",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz",
+      "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==",
       "engines": {
         "node": ">=10.0.0"
       },
@@ -13424,9 +13424,9 @@
       }
     },
     "ws": {
-      "version": "8.17.0",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.0.tgz",
-      "integrity": "sha512-uJq6108EgZMAl20KagGkzCKfMEjxmKvZHG7Tlq0Z6nOky7YF7aq4mOx6xK8TJ/i1LeK4Qus7INktacctDgY8Ow==",
+      "version": "8.17.1",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.17.1.tgz",
+      "integrity": "sha512-6XQFvXTkbfUOZOKKILFG1PDK2NDQs4azKQl26T0YS5CxqWLgXajbPZ+h4gZekJyRqFU8pvnbAbbs/3TgRPy+GQ==",
       "requires": {}
     },
     "y18n": {

backend/package.json

@@ -52,7 +52,7 @@
     "redis": "^4.6.6",
     "socks-proxy-agent": "~7.0.0",
     "typescript": "~4.9.3",
-    "ws": "~8.17.0"
+    "ws": "~8.17.1"
   },
   "devDependencies": {
     "@babel/code-frame": "^7.18.6",

backend/src/api/services/acceleration.ts

@@ -31,10 +31,7 @@ export interface AccelerationHistory {
   feeDelta: number,
   blockHash: string,
   blockHeight: number,
-  pools: {
-    pool_unique_id: number,
-    username: string,
-  }[],
+  pools: number[];
 };
 
 class AccelerationApi {

backend/src/repositories/AccelerationRepository.ts

@@ -308,10 +308,10 @@ class AccelerationRepository {
         }
         const accelerationSummaries = accelerations.map(acc => ({
           ...acc,
-          pools: acc.pools.map(pool => pool.pool_unique_id),
+          pools: acc.pools,
         }))
         for (const acc of accelerations) {
-          if (blockTxs[acc.txid] && acc.pools.some(pool => pool.pool_unique_id === block.extras.pool.id)) {
+          if (blockTxs[acc.txid] && acc.pools.includes(block.extras.pool.id)) {
             const tx = blockTxs[acc.txid];
             const accelerationInfo = accelerationCosts.getAccelerationInfo(tx, boostRate, transactions);
             accelerationInfo.cost = Math.max(0, Math.min(acc.feeDelta, accelerationInfo.cost));

backend/src/tasks/price-feeds/free-currency-api.ts

@@ -76,7 +76,7 @@ class FreeCurrencyApi implements ConversionFeed {
   }
 
   public async $fetchConversionRates(date: string): Promise<ConversionRates> {
-    const response = await query(`${this.API_URL_PREFIX}historical?date=${date}&apikey=${this.API_KEY}`);
+    const response = await query(`${this.API_URL_PREFIX}historical?date=${date}&apikey=${this.API_KEY}`, true);
     if (response && response['data'] && (response['data'][date] || this.PAID)) {
       if (this.PAID) {
         response['data'] = this.convertData(response['data']);

backend/src/tasks/price-updater.ts

@@ -59,7 +59,7 @@ class PriceUpdater {
   private currencyConversionFeed: ConversionFeed | undefined;
   private newCurrencies: string[] = ['BGN', 'BRL', 'CNY', 'CZK', 'DKK', 'HKD', 'HRK', 'HUF', 'IDR', 'ILS', 'INR', 'ISK', 'KRW', 'MXN', 'MYR', 'NOK', 'NZD', 'PHP', 'PLN', 'RON', 'RUB', 'SEK', 'SGD', 'THB', 'TRY', 'ZAR'];
   private lastTimeConversionsRatesFetched: number = 0;
-  private latestConversionsRatesFromFeed: ConversionRates = {};
+  private latestConversionsRatesFromFeed: ConversionRates = { USD: -1 };
   private ratesChangedCallback: ((rates: ApiPrice) => void) | undefined;
 
   constructor() {
@@ -157,9 +157,9 @@ class PriceUpdater {
       try {
         this.latestConversionsRatesFromFeed = await this.currencyConversionFeed.$fetchLatestConversionRates();
         this.lastTimeConversionsRatesFetched = Math.round(new Date().getTime() / 1000);
-        logger.debug(`Fetched currencies conversion rates from external API: ${JSON.stringify(this.latestConversionsRatesFromFeed)}`);
+        logger.debug(`Fetched currencies conversion rates from conversions API: ${JSON.stringify(this.latestConversionsRatesFromFeed)}`);
       } catch (e) {
-        logger.err(`Cannot fetch conversion rates from the API. Reason: ${(e instanceof Error ? e.message : e)}`);
+        logger.err(`Cannot fetch conversion rates from conversions API. Reason: ${(e instanceof Error ? e.message : e)}`);
       }
     }
 
@@ -408,17 +408,17 @@ class PriceUpdater {
     try {
       const remainingQuota = await this.currencyConversionFeed?.$getQuota();
       if (remainingQuota['month']['remaining'] < 500) { // We need some calls left for the daily updates
-        logger.debug(`Not enough currency API credit to insert missing prices in ${priceTimesToFill.length} rows (${remainingQuota['month']['remaining']} calls left).`, logger.tags.mining);
+        logger.debug(`Not enough conversions API credit to insert missing prices in ${priceTimesToFill.length} rows (${remainingQuota['month']['remaining']} calls left).`, logger.tags.mining);
         this.additionalCurrenciesHistoryInserted = true; // Do not try again until next day
         return;
       }
     } catch (e) {
-      logger.err(`Cannot fetch currency API credit, insertion of missing prices aborted. Reason: ${(e instanceof Error ? e.message : e)}`);
+      logger.err(`Cannot fetch conversions API credit, insertion of missing prices aborted. Reason: ${(e instanceof Error ? e.message : e)}`);
       return;
     }
 
     this.additionalCurrenciesHistoryRunning = true;
-    logger.debug(`Fetching missing conversion rates from external API to fill ${priceTimesToFill.length} rows`, logger.tags.mining);
+    logger.debug(`Inserting missing historical conversion rates using conversions API to fill ${priceTimesToFill.length} rows`, logger.tags.mining);
 
     let conversionRates: { [timestamp: number]: ConversionRates } = {};
     let totalInserted = 0;
@@ -430,10 +430,23 @@ class PriceUpdater {
       const month = new Date(priceTime.time * 1000).getMonth();
       const yearMonthTimestamp = new Date(year, month, 1).getTime() / 1000;
       if (conversionRates[yearMonthTimestamp] === undefined) {
-        conversionRates[yearMonthTimestamp] = await this.currencyConversionFeed?.$fetchConversionRates(`${year}-${month + 1 < 10 ? `0${month + 1}` : `${month + 1}`}-01`) || { USD: -1 };
-        if (conversionRates[yearMonthTimestamp]['USD'] < 0) {
-          logger.err(`Cannot fetch conversion rates from the API for ${year}-${month + 1 < 10 ? `0${month + 1}` : `${month + 1}`}-01. Aborting insertion of missing prices.`, logger.tags.mining);
-          this.lastFailedHistoricalRun = Math.round(new Date().getTime() / 1000);
+        try {
+          if (year === new Date().getFullYear() && month === new Date().getMonth()) { // For rows in the current month, we use the latest conversion rates
+            conversionRates[yearMonthTimestamp] = this.latestConversionsRatesFromFeed;
+          } else {
+            conversionRates[yearMonthTimestamp] = await this.currencyConversionFeed?.$fetchConversionRates(`${year}-${month + 1 < 10 ? `0${month + 1}` : `${month + 1}`}-15`) || { USD: -1 };
+          }
+
+          if (conversionRates[yearMonthTimestamp]['USD'] < 0) {
+            throw new Error('Incorrect USD conversion rate');
+          }
+        } catch (e) {
+          if ((e instanceof Error ? e.message : '').includes('429')) { // Continue 60 seconds later if and only if error is 429
+            this.lastFailedHistoricalRun = Math.round(new Date().getTime() / 1000);
+            logger.info(`Got a 429 error from conversions API. This is expected to happen a few times during the initial historical price insertion, process will resume in 60 seconds.`, logger.tags.mining);
+          } else {
+            logger.err(`Cannot fetch conversion rates from conversions API for ${year}-${month + 1 < 10 ? `0${month + 1}` : `${month + 1}`}-01, trying again next day. Error: ${(e instanceof Error ? e.message : e)}`, logger.tags.mining);
+          }
           break;
         }
       }

backend/src/utils/axios-query.ts

@@ -5,7 +5,7 @@ import config from '../config';
 import logger from '../logger';
 import * as https from 'https';
 
-export async function query(path): Promise<object | undefined> {
+export async function query(path, throwOnFail: boolean = false): Promise<object | undefined> {
  type axiosOptions = {
    headers: {
      'User-Agent': string
@@ -21,6 +21,7 @@ export async function query(path): Promise<object | undefined> {
    timeout: config.SOCKS5PROXY.ENABLED ? 30000 : 10000
  };
  let retry = 0;
+ let lastError: any = null;
 
  while (retry < config.MEMPOOL.EXTERNAL_MAX_RETRY) {
    try {
@@ -50,6 +51,7 @@ export async function query(path): Promise<object | undefined> {
      }
      return data.data;
    } catch (e) {
+     lastError = e;
      logger.warn(`Could not connect to ${path} (Attempt ${retry + 1}/${config.MEMPOOL.EXTERNAL_MAX_RETRY}). Reason: ` + (e instanceof Error ? e.message : e));
      retry++;
    }
@@ -59,5 +61,10 @@ export async function query(path): Promise<object | undefined> {
  }
 
  logger.err(`Could not connect to ${path}. All ${config.MEMPOOL.EXTERNAL_MAX_RETRY} attempts failed`);
+
+ if (throwOnFail && lastError) {
+    throw lastError;
+  }
+
  return undefined;
 }