Merge pull request #3111 from mempool/nymkappa/bugfix/optimize-price-frontend

Redo/Fix completely failed PR #3092 + add PR #3105

backend/src/api/database-migration.ts

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 53;
+  private static currentVersion = 54;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -473,6 +473,14 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE statistics MODIFY mempool_byte_weight bigint(20) UNSIGNED NOT NULL');
       await this.updateToSchemaVersion(53);
     }
+
+    if (databaseSchemaVersion < 54) {
+      this.uniqueLog(logger.notice, `'prices' table has been truncated`);
+      this.uniqueLog(logger.notice, `'blocks_prices' table has been truncated`);
+      await this.$executeQuery(`TRUNCATE prices`);
+      await this.$executeQuery(`TRUNCATE blocks_prices`);
+      await this.updateToSchemaVersion(54);
+    }
   }
 
   /**

backend/src/api/mining/mining-routes.ts

@@ -38,7 +38,16 @@ class MiningRoutes {
 
   private async $getHistoricalPrice(req: Request, res: Response): Promise<void> {
     try {
-      res.status(200).send(await PricesRepository.$getHistoricalPrice());
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      if (req.query.timestamp) {
+        res.status(200).send(await PricesRepository.$getNearestHistoricalPrice(
+          parseInt(<string>req.query.timestamp ?? 0, 10)
+        ));
+      } else {
+        res.status(200).send(await PricesRepository.$getHistoricalPrices());
+      }
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
     }

backend/src/repositories/PricesRepository.ts

@@ -28,6 +28,16 @@ export interface Conversion {
   exchangeRates: ExchangeRates;
 }
 
+export const MAX_PRICES = {
+  USD: 100000000,
+  EUR: 100000000,
+  GBP: 100000000,
+  CAD: 100000000,
+  CHF: 100000000,
+  AUD: 100000000,
+  JPY: 10000000000,
+};
+
 class PricesRepository {
   public async $savePrices(time: number, prices: IConversionRates): Promise<void> {
     if (prices.USD === 0) {
@@ -36,6 +46,14 @@ class PricesRepository {
       return;
     }
 
+    // Sanity check
+    for (const currency of Object.keys(prices)) {
+      if (prices[currency] < -1 || prices[currency] > MAX_PRICES[currency]) { // We use -1 to mark a "missing data, so it's a valid entry"
+        logger.info(`Ignore BTC${currency} price of ${prices[currency]}`);
+        prices[currency] = 0;
+      }
+    }
+    
     try {
       await DB.query(`
         INSERT INTO prices(time,             USD, EUR, GBP, CAD, CHF, AUD, JPY)
@@ -86,9 +104,48 @@ class PricesRepository {
     return rates[0];
   }
 
-  public async $getHistoricalPrice(): Promise<Conversion | null> {
+  public async $getNearestHistoricalPrice(timestamp: number | undefined): Promise<Conversion | null> {
     try {
-      const [rates]: any[] = await DB.query(`SELECT *, UNIX_TIMESTAMP(time) as time FROM prices ORDER BY time DESC`);
+      const [rates]: any[] = await DB.query(`
+        SELECT *, UNIX_TIMESTAMP(time) AS time
+        FROM prices
+        WHERE UNIX_TIMESTAMP(time) < ?
+        ORDER BY time DESC
+        LIMIT 1`,
+        [timestamp]
+      );
+      if (!rates) {
+        throw Error(`Cannot get single historical price from the database`);
+      }
+
+      // Compute fiat exchange rates
+      const latestPrice = await this.$getLatestConversionRates();
+      const exchangeRates: ExchangeRates = {
+        USDEUR: Math.round(latestPrice.EUR / latestPrice.USD * 100) / 100,
+        USDGBP: Math.round(latestPrice.GBP / latestPrice.USD * 100) / 100,
+        USDCAD: Math.round(latestPrice.CAD / latestPrice.USD * 100) / 100,
+        USDCHF: Math.round(latestPrice.CHF / latestPrice.USD * 100) / 100,
+        USDAUD: Math.round(latestPrice.AUD / latestPrice.USD * 100) / 100,
+        USDJPY: Math.round(latestPrice.JPY / latestPrice.USD * 100) / 100,
+      };
+
+      return {
+        prices: rates,
+        exchangeRates: exchangeRates
+      };
+    } catch (e) {
+      logger.err(`Cannot fetch single historical prices from the db. Reason ${e instanceof Error ? e.message : e}`);
+      return null;
+    }
+  }
+
+  public async $getHistoricalPrices(): Promise<Conversion | null> {
+    try {
+      const [rates]: any[] = await DB.query(`
+        SELECT *, UNIX_TIMESTAMP(time) AS time
+        FROM prices
+        ORDER BY time DESC
+      `);
       if (!rates) {
         throw Error(`Cannot get average historical price from the database`);
       }
@@ -109,7 +166,7 @@ class PricesRepository {
         exchangeRates: exchangeRates
       };
     } catch (e) {
-      logger.err(`Cannot fetch averaged historical prices from the db. Reason ${e instanceof Error ? e.message : e}`);
+      logger.err(`Cannot fetch historical prices from the db. Reason ${e instanceof Error ? e.message : e}`);
       return null;
     }
   }

backend/src/tasks/price-updater.ts

@@ -3,7 +3,7 @@ import path from 'path';
 import config from '../config';
 import logger from '../logger';
 import { IConversionRates } from '../mempool.interfaces';
-import PricesRepository from '../repositories/PricesRepository';
+import PricesRepository, { MAX_PRICES } from '../repositories/PricesRepository';
 import BitfinexApi from './price-feeds/bitfinex-api';
 import BitflyerApi from './price-feeds/bitflyer-api';
 import CoinbaseApi from './price-feeds/coinbase-api';
@@ -46,13 +46,13 @@ class PriceUpdater {
 
   public getEmptyPricesObj(): IConversionRates {
     return {
-      USD: 0,
+      USD: -1,
-      EUR: 0,
+      EUR: -1,
-      GBP: 0,
+      GBP: -1,
-      CAD: 0,
+      CAD: -1,
-      CHF: 0,
+      CHF: -1,
-      AUD: 0,
+      AUD: -1,
-      JPY: 0,
+      JPY: -1,
     };
   }
 
@@ -115,7 +115,7 @@ class PriceUpdater {
         if (feed.currencies.includes(currency)) {
           try {
             const price = await feed.$fetchPrice(currency);
-            if (price > 0) {
+            if (price > -1 && price < MAX_PRICES[currency]) {
               prices.push(price);
             }
             logger.debug(`${feed.name} BTC/${currency} price: ${price}`, logger.tags.mining);
@@ -239,7 +239,7 @@ class PriceUpdater {
 
         for (const currency of this.currencies) {
           const price = historicalEntry[time][currency];
-          if (price > 0) {
+          if (price > -1 && price < MAX_PRICES[currency]) {
             grouped[time][currency].push(typeof price === 'string' ? parseInt(price, 10) : price);
           }
         }

frontend/src/app/components/amount/amount.component.html

@@ -3,7 +3,7 @@
     {{ addPlus && satoshis >= 0 ? '+' : '' }}
     {{
       (
-        (blockConversion.price[currency] > 0 ? blockConversion.price[currency] : null) ??
+        (blockConversion.price[currency] >= 0 ? blockConversion.price[currency] : null) ??
         (blockConversion.price['USD'] * blockConversion.exchangeRates['USD' + currency]) ?? 0
       ) * satoshis / 100000000 | fiatCurrency : digitsInfo : currency
     }}

frontend/src/app/components/block-overview-graph/block-overview-graph.component.html

@@ -10,5 +10,6 @@
     [cursorPosition]="tooltipPosition"
     [clickable]="!!selectedTx"
     [auditEnabled]="auditHighlighting"
+    [blockConversion]="blockConversion"
   ></app-block-overview-tooltip>
 </div>

frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts

@@ -5,6 +5,7 @@ import BlockScene from './block-scene';
 import TxSprite from './tx-sprite';
 import TxView from './tx-view';
 import { Position } from './sprite-types';
+import { Price } from 'src/app/services/price.service';
 
 @Component({
   selector: 'app-block-overview-graph',
@@ -21,6 +22,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
   @Input() mirrorTxid: string | void;
   @Input() unavailable: boolean = false;
   @Input() auditHighlighting: boolean = false;
+  @Input() blockConversion: Price;
   @Output() txClickEvent = new EventEmitter<TransactionStripped>();
   @Output() txHoverEvent = new EventEmitter<string>();
   @Output() readyEvent = new EventEmitter();

frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html

@@ -16,11 +16,11 @@
       </tr>
       <tr>
         <td class="td-width" i18n="dashboard.latest-transactions.amount">Amount</td>
-        <td><app-amount [satoshis]="value"></app-amount></td>
+        <td><app-amount [blockConversion]="blockConversion" [satoshis]="value"></app-amount></td>
       </tr>
       <tr>
         <td class="td-width" i18n="transaction.fee|Transaction fee">Fee</td>
-        <td>{{ fee | number }} <span class="symbol" i18n="shared.sat|sat">sat</span> &nbsp; <span class="fiat"><app-fiat [value]="fee"></app-fiat></span></td>
+        <td>{{ fee | number }} <span class="symbol" i18n="shared.sat|sat">sat</span> &nbsp; <span class="fiat"><app-fiat [blockConversion]="blockConversion" [value]="fee"></app-fiat></span></td>
       </tr>
       <tr>
         <td class="td-width" i18n="transaction.fee-rate|Transaction fee rate">Fee rate</td>

frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts

@@ -1,6 +1,7 @@
 import { Component, ElementRef, ViewChild, Input, OnChanges, ChangeDetectionStrategy } from '@angular/core';
 import { TransactionStripped } from '../../interfaces/websocket.interface';
 import { Position } from '../../components/block-overview-graph/sprite-types.js';
+import { Price } from 'src/app/services/price.service';
 
 @Component({
   selector: 'app-block-overview-tooltip',
@@ -12,6 +13,7 @@ export class BlockOverviewTooltipComponent implements OnChanges {
   @Input() cursorPosition: Position;
   @Input() clickable: boolean;
   @Input() auditEnabled: boolean = false;
+  @Input() blockConversion: Price;
 
   txid = '';
   fee = 0;

frontend/src/app/components/block/block.component.html

@@ -108,6 +108,7 @@
             [blockLimit]="stateService.blockVSize"
             [orientation]="'top'"
             [flip]="false"
+            [blockConversion]="blockConversion"
             (txClickEvent)="onTxClick($event)"
           ></app-block-overview-graph>
           <ng-container *ngTemplateOutlet="emptyBlockInfo"></ng-container>

frontend/src/app/components/block/block.component.ts

@@ -443,9 +443,9 @@ export class BlockComponent implements OnInit, OnDestroy {
     }
     this.priceSubscription = block$.pipe(
       switchMap((block) => {
-        return this.priceService.getPrices().pipe(
+        return this.priceService.getBlockPrice$(block.timestamp).pipe(
-          tap(() => {
+          tap((price) => {
-            this.blockConversion = this.priceService.getPriceForTimestamp(block.timestamp);
+            this.blockConversion = price;
           })
         );
       })
@@ -471,6 +471,7 @@ export class BlockComponent implements OnInit, OnDestroy {
     this.auditSubscription?.unsubscribe();
     this.unsubscribeNextBlockSubscriptions();
     this.childChangeSubscription?.unsubscribe();
+    this.priceSubscription?.unsubscribe();
   }
 
   unsubscribeNextBlockSubscriptions() {

frontend/src/app/components/transaction/transaction.component.ts

@@ -327,9 +327,9 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
             this.fetchRbfHistory$.next(this.tx.txid);
           }
 
-          this.priceService.getPrices().pipe(
+          this.priceService.getBlockPrice$(tx.status.block_time, true).pipe(
-            tap(() => {
+            tap((price) => {
-              this.blockConversion = this.priceService.getPriceForTimestamp(tx.status.block_time);
+              this.blockConversion = price;
             })
           ).subscribe();
       

frontend/src/app/components/transactions-list/transactions-list.component.ts

@@ -6,7 +6,7 @@ import { Outspend, Transaction, Vin, Vout } from '../../interfaces/electrs.inter
 import { ElectrsApiService } from '../../services/electrs-api.service';
 import { environment } from '../../../environments/environment';
 import { AssetsService } from '../../services/assets.service';
-import { filter, map, tap, switchMap } from 'rxjs/operators';
+import { filter, map, tap, switchMap, shareReplay } from 'rxjs/operators';
 import { BlockExtended } from '../../interfaces/node-api.interface';
 import { ApiService } from '../../services/api.service';
 import { PriceService } from 'src/app/services/price.service';
@@ -150,10 +150,8 @@ export class TransactionsListComponent implements OnInit, OnChanges {
           tx['addressValue'] = addressIn - addressOut;
         }
 
-        this.priceService.getPrices().pipe(
+        this.priceService.getBlockPrice$(tx.status.block_time).pipe(
-          tap(() => {
+          tap((price) => tx['price'] = price)
-            tx['price'] = this.priceService.getPriceForTimestamp(tx.status.block_time);
-          })
         ).subscribe();
       });
       const txIds = this.transactions.filter((tx) => !tx._outspends).map((tx) => tx.txid);

frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.html

@@ -56,7 +56,7 @@
           </ng-container>
       </ng-container>
       <p *ngIf="line.value == null && line.confidential" i18n="shared.confidential">Confidential</p>
-      <p *ngIf="line.value != null"><app-amount [satoshis]="line.value"></app-amount></p>
+      <p *ngIf="line.value != null"><app-amount [blockConversion]="blockConversion" [satoshis]="line.value"></app-amount></p>
       <p *ngIf="line.type !== 'fee' && line.address" class="address">
         <app-truncate [text]="line.address"></app-truncate>
       </p>

frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts

@@ -1,5 +1,6 @@
-import { Component, ElementRef, ViewChild, Input, OnChanges, ChangeDetectionStrategy } from '@angular/core';
+import { Component, ElementRef, ViewChild, Input, OnChanges, OnInit } from '@angular/core';
-import { TransactionStripped } from '../../interfaces/websocket.interface';
+import { tap } from 'rxjs';
+import { Price, PriceService } from 'src/app/services/price.service';
 
 interface Xput {
   type: 'input' | 'output' | 'fee';
@@ -14,6 +15,7 @@ interface Xput {
   pegin?: boolean;
   pegout?: string;
   confidential?: boolean;
+  timestamp?: number;
 }
 
 @Component({
@@ -27,12 +29,21 @@ export class TxBowtieGraphTooltipComponent implements OnChanges {
   @Input() isConnector: boolean = false;
 
   tooltipPosition = { x: 0, y: 0 };
+  blockConversion: Price;
 
   @ViewChild('tooltip') tooltipElement: ElementRef<HTMLCanvasElement>;
 
-  constructor() {}
+  constructor(private priceService: PriceService) {}
 
   ngOnChanges(changes): void {
+    if (changes.line?.currentValue) {
+      this.priceService.getBlockPrice$(changes.line?.currentValue.timestamp, true).pipe(
+        tap((price) => {
+          this.blockConversion = price;
+        })
+      ).subscribe();
+    }
+
     if (changes.cursorPosition && changes.cursorPosition.currentValue) {
       let x = Math.max(10, changes.cursorPosition.currentValue.x - 50);
       let y = changes.cursorPosition.currentValue.y + 20;

frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts

@@ -29,6 +29,7 @@ interface Xput {
   pegin?: boolean;
   pegout?: string;
   confidential?: boolean;
+  timestamp?: number;
 }
 
 @Component({
@@ -152,6 +153,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
         index: i,
         pegout: v?.pegout?.scriptpubkey_address,
         confidential: (this.isLiquid && v?.value === undefined),
+        timestamp: this.tx.status.block_time
       } as Xput;
     });
 
@@ -171,6 +173,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges {
         coinbase: v?.is_coinbase,
         pegin: v?.is_pegin,
         confidential: (this.isLiquid && v?.prevout?.value === undefined),
+        timestamp: this.tx.status.block_time
       } as Xput;
     });
 

frontend/src/app/fiat/fiat.component.html

@@ -1,7 +1,7 @@
 <span class="green-color" *ngIf="blockConversion; else noblockconversion">
   {{
     (
-      (blockConversion.price[currency] > 0 ? blockConversion.price[currency] : null) ??
+      (blockConversion.price[currency] >= 0 ? blockConversion.price[currency] : null) ??
       (blockConversion.price['USD'] * blockConversion.exchangeRates['USD' + currency]) ?? 0
     ) * value / 100000000 | fiatCurrency : digitsInfo : currency
   }}

frontend/src/app/services/api.service.ts

@@ -305,7 +305,10 @@ export class ApiService {
     );
   }
 
-  getHistoricalPrice$(): Observable<Conversion> {
+  getHistoricalPrice$(timestamp: number | undefined): Observable<Conversion> {
-    return this.httpClient.get<Conversion>( this.apiBaseUrl + this.apiBasePath + '/api/v1/historical-price');
+    return this.httpClient.get<Conversion>(
+      this.apiBaseUrl + this.apiBasePath + '/api/v1/historical-price' +
+        (timestamp ? `?timestamp=${timestamp}` : '')
+    );
   }
 }

frontend/src/app/services/price.service.ts

@@ -1,5 +1,5 @@
 import { Injectable } from '@angular/core';
-import { map, Observable, of, shareReplay } from 'rxjs';
+import { map, Observable, of, share, shareReplay, tap } from 'rxjs';
 import { ApiService } from './api.service';
 
 // nodejs backend interfaces
@@ -40,6 +40,12 @@ export interface ConversionDict {
   providedIn: 'root'
 })
 export class PriceService {
+  priceObservable$: Observable<Conversion>;
+  singlePriceObservable$: Observable<Conversion>;
+
+  lastQueriedTimestamp: number;
+  lastPriceHistoryUpdate: number;
+
   historicalPrice: ConversionDict = {
     prices: null,
     exchangeRates: null,
@@ -61,65 +67,86 @@ export class PriceService {
     };
   }
 
-  /**
+  getBlockPrice$(blockTimestamp: number, singlePrice = false): Observable<Price | undefined> {
-   * Fetch prices from the nodejs backend only once
+    const now = new Date().getTime() / 1000;
-   */
-  getPrices(): Observable<void> {
-    if (this.historicalPrice.prices) {
-      return of(null);
-    }
 
-    return this.apiService.getHistoricalPrice$().pipe(
+    /**
-      map((conversion: Conversion) => {
+     * Query nearest price for a specific blockTimestamp. The observable is invalidated if we
-        if (!this.historicalPrice.prices) {
+     * query a different timestamp than the last one
-          this.historicalPrice.prices = Object();
+     */
-        }
+    if (singlePrice) {
-        for (const price of conversion.prices) {
+      if (!this.singlePriceObservable$ || (this.singlePriceObservable$ && blockTimestamp !== this.lastQueriedTimestamp)) {
-          this.historicalPrice.prices[price.time] = {
+        this.singlePriceObservable$ = this.apiService.getHistoricalPrice$(blockTimestamp).pipe(shareReplay());
-            USD: price.USD, EUR: price.EUR, GBP: price.GBP, CAD: price.CAD,
+        this.lastQueriedTimestamp = blockTimestamp;
-            CHF: price.CHF, AUD: price.AUD, JPY: price.JPY
-          };
-        }
-        this.historicalPrice.exchangeRates = conversion.exchangeRates;
-        return;
-      }),
-      shareReplay(),
-    );
-  }
-
-  /**
-   * Note: The first block with a price we have is block 68952 (using MtGox price history)
-   * 
-   * @param blockTimestamp 
-   */
-  getPriceForTimestamp(blockTimestamp: number): Price | null {
-    if (!blockTimestamp) {
-      return undefined;
-    }
-
-    const priceTimestamps = Object.keys(this.historicalPrice.prices);
-    priceTimestamps.push(Number.MAX_SAFE_INTEGER.toString());
-    priceTimestamps.sort().reverse();
-    
-    // Small trick here. Because latest blocks have higher timestamps than our
-    // latest price timestamp (we only insert once every hour), we have no price for them.
-    // Therefore we want to fallback to the websocket price by returning an undefined `price` field.
-    // Since this.historicalPrice.prices[Number.MAX_SAFE_INTEGER] does not exists
-    // it will return `undefined` and automatically use the websocket price.
-    // This way we can differenciate blocks without prices like the genesis block
-    // vs ones without a price (yet) like the latest blocks
-
-    for (const t of priceTimestamps) {
-      const priceTimestamp = parseInt(t, 10);
-      if (blockTimestamp > priceTimestamp) {
-        return {
-          price: this.historicalPrice.prices[priceTimestamp],
-          exchangeRates: this.historicalPrice.exchangeRates,
-        };
       }
+
+      return this.singlePriceObservable$.pipe(
+        map((conversion) => {
+          if (conversion.prices.length <= 0) {
+            return this.getEmptyPrice();
+          }
+          return {
+            price: {
+              USD: conversion.prices[0].USD, EUR: conversion.prices[0].EUR, GBP: conversion.prices[0].GBP, CAD: conversion.prices[0].CAD,
+              CHF: conversion.prices[0].CHF, AUD: conversion.prices[0].AUD, JPY: conversion.prices[0].JPY
+            },
+            exchangeRates: conversion.exchangeRates,
+          };
+        })
+      );
     }
 
-    return this.getEmptyPrice();
+    /**
+     * Query all price history only once. The observable is invalidated after 1 hour
+     */
+    else {
+      if (!this.priceObservable$ || (this.priceObservable$ && (now - this.lastPriceHistoryUpdate > 3600))) {
+        this.priceObservable$ = this.apiService.getHistoricalPrice$(undefined).pipe(shareReplay());
+        this.lastPriceHistoryUpdate = new Date().getTime() / 1000;
+      }
+
+      return this.priceObservable$.pipe(
+        map((conversion) => {
+          if (!blockTimestamp) {
+            return undefined;
+          }
+
+          const historicalPrice = {
+            prices: {},
+            exchangeRates: conversion.exchangeRates,
+          };
+          for (const price of conversion.prices) {
+            historicalPrice.prices[price.time] = {
+              USD: price.USD, EUR: price.EUR, GBP: price.GBP, CAD: price.CAD,
+              CHF: price.CHF, AUD: price.AUD, JPY: price.JPY
+            };
+          }
+
+          const priceTimestamps = Object.keys(historicalPrice.prices);
+          priceTimestamps.push(Number.MAX_SAFE_INTEGER.toString());
+          priceTimestamps.sort().reverse();
+
+          // Small trick here. Because latest blocks have higher timestamps than our
+          // latest price timestamp (we only insert once every hour), we have no price for them.
+          // Therefore we want to fallback to the websocket price by returning an undefined `price` field.
+          // Since historicalPrice.prices[Number.MAX_SAFE_INTEGER] does not exists
+          // it will return `undefined` and automatically use the websocket price.
+          // This way we can differenciate blocks without prices like the genesis block
+          // vs ones without a price (yet) like the latest blocks
+
+          for (const t of priceTimestamps) {
+            const priceTimestamp = parseInt(t, 10);
+            if (blockTimestamp > priceTimestamp) {
+              return {
+                price: historicalPrice.prices[priceTimestamp],
+                exchangeRates: historicalPrice.exchangeRates,
+              };
+            }
+          }
+
+          return this.getEmptyPrice();
+        })
+      );
+    }
   }
 }
-