Add daily historical price - show USD in block fee reward charts

backend/src/api/database-migration.ts

@@ -279,7 +279,8 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `nodes` ADD longitude double NULL DEFAULT NULL');
       await this.$executeQuery('ALTER TABLE `nodes` ADD latitude double NULL DEFAULT NULL');
     }
-    if (databaseSchemaVersion < 25 && isBitcoin == true) { // Link blocks to prices
+
+    if (databaseSchemaVersion < 30 && isBitcoin == true) { // Link blocks to prices
       await this.$executeQuery('ALTER TABLE `prices` ADD `id` int NULL AUTO_INCREMENT UNIQUE');
       await this.$executeQuery('DROP TABLE IF EXISTS `blocks_prices`');
       await this.$executeQuery(this.getCreateBlocksPricesTableQuery(), await this.$checkIfTableExists('blocks_prices'));

backend/src/api/mining/mining.ts

@@ -1,4 +1,4 @@
-import { IndexedDifficultyAdjustment, PoolInfo, PoolStats, RewardStats } from '../../mempool.interfaces';
+import { BlockPrice, PoolInfo, PoolStats, RewardStats } from '../../mempool.interfaces';
 import BlocksRepository from '../../repositories/BlocksRepository';
 import PoolsRepository from '../../repositories/PoolsRepository';
 import HashratesRepository from '../../repositories/HashratesRepository';
@@ -7,11 +7,10 @@ import logger from '../../logger';
 import { Common } from '../common';
 import loadingIndicators from '../loading-indicators';
 import { escape } from 'mysql2';
-import indexer from '../../indexer';
 import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository';
 import config from '../../config';
 import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
-import PricesRepository from '../repositories/PricesRepository';
+import PricesRepository from '../../repositories/PricesRepository';
 
 class Mining {
   blocksPriceIndexingRunning = false;
@@ -34,7 +33,7 @@ class Mining {
    */
   public async $getHistoricalBlockFees(interval: string | null = null): Promise<any> {
     return await BlocksRepository.$getHistoricalBlockFees(
-      this.getTimeRangeForAmounts(interval),
+      this.getTimeRange(interval, 5),
       Common.getSqlInterval(interval)
     );
   }
@@ -44,7 +43,7 @@ class Mining {
    */
   public async $getHistoricalBlockRewards(interval: string | null = null): Promise<any> {
     return await BlocksRepository.$getHistoricalBlockRewards(
-      this.getTimeRangeForAmounts(interval),
+      this.getTimeRange(interval),
       Common.getSqlInterval(interval)
     );
   }
@@ -529,33 +528,18 @@ class Mining {
     return date;
   }
 
-  private getTimeRangeForAmounts(interval: string | null): number {
+  private getTimeRange(interval: string | null, scale = 1): number {
     switch (interval) {
-      case '3y': return 1296000;
-      case '2y': return 864000;
-      case '1y': return 432000;
-      case '6m': return 216000;
-      case '3m': return 108000;
-      case '1m': return 36000;
-      case '1w': return 8400;
-      case '3d': return 3600;
-      case '24h': return 1200; 
-      default: return 3888000;
-    }
-  }
-
-  private getTimeRange(interval: string | null): number {
-    switch (interval) {
-      case '3y': return 43200; // 12h
-      case '2y': return 28800; // 8h
-      case '1y': return 28800; // 8h
-      case '6m': return 10800; // 3h
-      case '3m': return 7200; // 2h
-      case '1m': return 1800; // 30min
-      case '1w': return 300; // 5min
-      case '3d': return 1;
-      case '24h': return 1;
-      default: return 86400;
+      case '3y': return 43200 * scale; // 12h
+      case '2y': return 28800 * scale; // 8h
+      case '1y': return 28800 * scale; // 8h
+      case '6m': return 10800 * scale; // 3h
+      case '3m': return 7200 * scale; // 2h
+      case '1m': return 1800 * scale; // 30min
+      case '1w': return 300 * scale; // 5min
+      case '3d': return 1 * scale;
+      case '24h': return 1 * scale;
+      default: return 86400 * scale;
     }
   }
 }

backend/src/index.ts

@@ -24,7 +24,6 @@ import icons from './api/liquid/icons';
 import { Common } from './api/common';
 import poolsUpdater from './tasks/pools-updater';
 import indexer from './indexer';
-import priceUpdater from './tasks/price-updater';
 import nodesRoutes from './api/explorer/nodes.routes';
 import channelsRoutes from './api/explorer/channels.routes';
 import generalLightningRoutes from './api/explorer/general.routes';
@@ -35,8 +34,6 @@ import miningRoutes from "./api/mining/mining-routes";
 import bisqRoutes from "./api/bisq/bisq.routes";
 import liquidRoutes from "./api/liquid/liquid.routes";
 import bitcoinRoutes from "./api/bitcoin/bitcoin.routes";
-import BlocksAuditsRepository from './repositories/BlocksAuditsRepository';
-import mining from "./api/mining";
 
 class Server {
   private wss: WebSocket.Server | undefined;
@@ -168,7 +165,6 @@ class Server {
       await blocks.$updateBlocks();
       await memPool.$updateMempool();
       indexer.$run();
-      priceUpdater.$run().then(mining.$indexBlockPrices.bind(this));
 
       setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
       this.currentBackendRetryInterval = 5;

backend/src/indexer.ts

@@ -5,6 +5,7 @@ import mining from './api/mining/mining';
 import logger from './logger';
 import HashratesRepository from './repositories/HashratesRepository';
 import bitcoinClient from './api/bitcoin/bitcoin-client';
+import priceUpdater from './tasks/price-updater';
 
 class Indexer {
   runIndexer = true;
@@ -38,6 +39,8 @@ class Indexer {
     logger.debug(`Running mining indexer`);
 
     try {
+      await priceUpdater.$run();
+
       const chainValid = await blocks.$generateBlockDatabase();
       if (chainValid === false) {
         // Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration
@@ -47,6 +50,7 @@ class Indexer {
         return;
       }
 
+      await mining.$indexBlockPrices();
       await mining.$indexDifficultyAdjustments();
       await this.$resetHashratesIndexingState(); // TODO - Remove this as it's not efficient
       await mining.$generateNetworkHashrateHistory();

backend/src/repositories/BlocksRepository.ts

@@ -274,7 +274,7 @@ class BlocksRepository {
       merkle_root,
       previous_block_hash as previousblockhash,
       avg_fee,
-      avg_fee_rate,
+      avg_fee_rate
       FROM blocks
       WHERE pool_id = ?`;
     params.push(pool.id);
@@ -288,6 +288,7 @@ class BlocksRepository {
       LIMIT 10`;
 
     try {
+      console.log(query, params);
       const [rows] = await DB.query(query, params);
 
       const blocks: BlockExtended[] = [];
@@ -360,12 +361,12 @@ class BlocksRepository {
         SELECT *, blocks.height, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, hash as id,
         pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug,
         pools.addresses as pool_addresses, pools.regexes as pool_regexes,
-        previous_block_hash as previousblockhash,
+        previous_block_hash as previousblockhash
         FROM blocks
         JOIN pools ON blocks.pool_id = pools.id
-        WHERE hash = '${hash}';
+        WHERE hash = '?';
       `;
-      const [rows]: any[] = await DB.query(query);
+      const [rows]: any[] = await DB.query(query, [hash]);
 
       if (rows.length <= 0) {
         return null;
@@ -488,8 +489,11 @@ class BlocksRepository {
       let query = `SELECT
         CAST(AVG(blocks.height) as INT) as avgHeight,
         CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
-        CAST(AVG(fees) as INT) as avgFees
+        CAST(AVG(fees) as INT) as avgFees,
+        prices.USD
         FROM blocks
+        JOIN blocks_prices on blocks_prices.height = blocks.height
+        JOIN prices on prices.id = blocks_prices.price_id
       `;
 
       if (interval !== null) {
@@ -514,8 +518,11 @@ class BlocksRepository {
       let query = `SELECT
         CAST(AVG(blocks.height) as INT) as avgHeight,
         CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
-        CAST(AVG(reward) as INT) as avgRewards
+        CAST(AVG(reward) as INT) as avgRewards,
+        prices.USD
         FROM blocks
+        JOIN blocks_prices on blocks_prices.height = blocks.height
+        JOIN prices on prices.id = blocks_prices.price_id
       `;
 
       if (interval !== null) {
@@ -663,22 +670,6 @@ class BlocksRepository {
     }
   }
 
-  /**
-   * Save block price
-   */
-   public async $saveBlockPrice(blockPrice: BlockPrice): Promise<void> {
-    try {
-      await DB.query(`INSERT INTO blocks_prices(height, price_id) VALUE (?, ?)`, [blockPrice.height, blockPrice.priceId]);
-    } catch (e: any) {
-      if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
-        logger.debug(`Cannot save block price for block ${blockPrice.height} because it has already been indexed, ignoring`);
-      } else {
-        logger.err(`Cannot save block price into db. Reason: ` + (e instanceof Error ? e.message : e));
-        throw e;
-      }
-    }
-  }
-
   /**
    * Save block price by batch
    */

backend/src/tasks/price-feeds/bitfinex-api.ts

@@ -16,7 +16,7 @@ class BitfinexApi implements PriceFeed {
     return response ? parseInt(response['last_price'], 10) : -1;
   }
 
-  public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
+  public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {
     const priceHistory: PriceHistory = {};
 
     for (const currency of currencies) {
@@ -24,7 +24,7 @@ class BitfinexApi implements PriceFeed {
         continue;
       }
 
-      const response = await query(this.urlHist.replace('{GRANULARITY}', '1h').replace('{CURRENCY}', currency));
+      const response = await query(this.urlHist.replace('{GRANULARITY}', type === 'hour' ? '1h' : '1D').replace('{CURRENCY}', currency));
       const pricesRaw = response ? response : [];
 
       for (const price of pricesRaw as any[]) {

backend/src/tasks/price-feeds/bitflyer-api.ts

@@ -16,7 +16,7 @@ class BitflyerApi implements PriceFeed {
     return response ? parseInt(response['ltp'], 10) : -1;
   }
 
-  public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
+  public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {
     return [];
   }
 }

backend/src/tasks/price-feeds/coinbase-api.ts

@@ -16,7 +16,7 @@ class CoinbaseApi implements PriceFeed {
     return response ? parseInt(response['data']['amount'], 10) : -1;
   }
 
-  public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
+  public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {
     const priceHistory: PriceHistory = {};
 
     for (const currency of currencies) {
@@ -24,7 +24,7 @@ class CoinbaseApi implements PriceFeed {
         continue;
       }
 
-      const response = await query(this.urlHist.replace('{GRANULARITY}', '3600').replace('{CURRENCY}', currency));
+      const response = await query(this.urlHist.replace('{GRANULARITY}', type === 'hour' ? '3600' : '86400').replace('{CURRENCY}', currency));
       const pricesRaw = response ? response : [];
 
       for (const price of pricesRaw as any[]) {

backend/src/tasks/price-feeds/ftx-api.ts

@@ -16,7 +16,7 @@ class FtxApi implements PriceFeed {
     return response ? parseInt(response['result']['last'], 10) : -1;
   }
 
-  public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
+  public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {
     const priceHistory: PriceHistory = {};
 
     for (const currency of currencies) {
@@ -24,7 +24,7 @@ class FtxApi implements PriceFeed {
         continue;
       }
 
-      const response = await query(this.urlHist.replace('{GRANULARITY}', '3600').replace('{CURRENCY}', currency));
+      const response = await query(this.urlHist.replace('{GRANULARITY}', type === 'hour' ? '3600' : '86400').replace('{CURRENCY}', currency));
       const pricesRaw = response ? response['result'] : [];
 
       for (const price of pricesRaw as any[]) {

backend/src/tasks/price-feeds/gemini-api.ts

@@ -16,7 +16,7 @@ class GeminiApi implements PriceFeed {
     return response ? parseInt(response['last'], 10) : -1;
   }
 
-  public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
+  public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {
     const priceHistory: PriceHistory = {};
 
     for (const currency of currencies) {
@@ -24,7 +24,7 @@ class GeminiApi implements PriceFeed {
         continue;
       }
 
-      const response = await query(this.urlHist.replace('{GRANULARITY}', '1hr').replace('{CURRENCY}', currency));
+      const response = await query(this.urlHist.replace('{GRANULARITY}', type === 'hour' ? '1hr' : '1day').replace('{CURRENCY}', currency));
       const pricesRaw = response ? response : [];
 
       for (const price of pricesRaw as any[]) {

backend/src/tasks/price-feeds/kraken-api.ts

@@ -26,7 +26,7 @@ class KrakenApi implements PriceFeed {
     return response ? parseInt(response['result'][this.getTicker(currency)]['c'][0], 10) : -1;
   }
 
-  public async $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory> {
+  public async $fetchRecentPrice(currencies: string[], type: 'hour' | 'day'): Promise<PriceHistory> {
     const priceHistory: PriceHistory = {};
 
     for (const currency of currencies) {

backend/src/tasks/price-updater.ts

@@ -16,7 +16,7 @@ export interface PriceFeed {
   currencies: string[];
 
   $fetchPrice(currency): Promise<number>;
-  $fetchRecentHourlyPrice(currencies: string[]): Promise<PriceHistory>;
+  $fetchRecentPrice(currencies: string[], type: string): Promise<PriceHistory>;
 }
 
 export interface PriceHistory {
@@ -185,7 +185,8 @@ class PriceUpdater {
     await new KrakenApi().$insertHistoricalPrice();
 
     // Insert missing recent hourly prices
-    await this.$insertMissingRecentPrices();
+    await this.$insertMissingRecentPrices('day');
+    await this.$insertMissingRecentPrices('hour');
 
     this.historyInserted = true;
     this.lastHistoricalRun = new Date().getTime();
@@ -195,17 +196,17 @@ class PriceUpdater {
    * Find missing hourly prices and insert them in the database
    * It has a limited backward range and it depends on which API are available
    */
-  private async $insertMissingRecentPrices(): Promise<void> {
+  private async $insertMissingRecentPrices(type: 'hour' | 'day'): Promise<void> {
     const existingPriceTimes = await PricesRepository.$getPricesTimes();
 
-    logger.info(`Fetching hourly price history from exchanges and saving missing ones into the database, this may take a while`);
+    logger.info(`Fetching ${type === 'day' ? 'dai' : 'hour'}ly price history from exchanges and saving missing ones into the database, this may take a while`);
 
     const historicalPrices: PriceHistory[] = [];
 
     // Fetch all historical hourly prices
     for (const feed of this.feeds) {
       try {
-        historicalPrices.push(await feed.$fetchRecentHourlyPrice(this.currencies));
+        historicalPrices.push(await feed.$fetchRecentPrice(this.currencies, type));
       } catch (e) {
         logger.err(`Cannot fetch hourly historical price from ${feed.name}. Ignoring this feed. Reason: ${e instanceof Error ? e.message : e}`);
       }
@@ -252,9 +253,9 @@ class PriceUpdater {
     }
 
     if (totalInserted > 0) {
-      logger.notice(`Inserted ${totalInserted} hourly historical prices into the db`);
+      logger.notice(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`);
     } else {
-      logger.debug(`Inserted ${totalInserted} hourly historical prices into the db`);
+      logger.debug(`Inserted ${totalInserted} ${type === 'day' ? 'dai' : 'hour'}ly historical prices into the db`);
     }
   }
 }