Merge pull request #4058 from mempool/mononaut/prices-api

Current fiat prices API

backend/mempool-config.sample.json

@@ -32,7 +32,8 @@
     "CPFP_INDEXING": false,
     "DISK_CACHE_BLOCK_INTERVAL": 6,
     "MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
-    "ALLOW_UNREACHABLE": true
+    "ALLOW_UNREACHABLE": true,
+    "PRICE_UPDATES_PER_HOUR": 1
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",
@@ -115,10 +116,6 @@
     "USERNAME": "",
     "PASSWORD": ""
   },
-  "PRICE_DATA_SERVER": {
-    "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices",
-    "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices"
-  },
   "EXTERNAL_DATA_SERVER": {
     "MEMPOOL_API": "https://mempool.space/api/v1",
     "MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1",

backend/src/__fixtures__/mempool-config.template.json

@@ -23,8 +23,8 @@
     "USER_AGENT": "__MEMPOOL_USER_AGENT__",
     "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
     "INDEXING_BLOCKS_AMOUNT": 14,
-    "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__",
+    "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__",
-    "POOLS_JSON_URL": "__POOLS_JSON_URL__",
+    "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__",
     "AUDIT": true,
     "ADVANCED_GBT_AUDIT": true,
     "ADVANCED_GBT_MEMPOOL": true,
@@ -33,7 +33,8 @@
     "MAX_BLOCKS_BULK_QUERY": 999,
     "DISK_CACHE_BLOCK_INTERVAL": 999,
     "MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
-    "ALLOW_UNREACHABLE": true
+    "ALLOW_UNREACHABLE": true,
+    "PRICE_UPDATES_PER_HOUR": 1
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",
@@ -92,10 +93,6 @@
     "USERNAME": "__SOCKS5PROXY_USERNAME__",
     "PASSWORD": "__SOCKS5PROXY_PASSWORD__"
   },
-  "PRICE_DATA_SERVER": {
-    "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__",
-    "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__"
-  },
   "EXTERNAL_DATA_SERVER": {
     "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__",
     "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__",

backend/src/__tests__/config.test.ts

@@ -47,6 +47,7 @@ describe('Mempool Backend Config', () => {
         DISK_CACHE_BLOCK_INTERVAL: 6,
         MAX_PUSH_TX_SIZE_WEIGHT: 400000,
         ALLOW_UNREACHABLE: true,
+        PRICE_UPDATES_PER_HOUR: 1,
       });
 
       expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
@@ -101,11 +102,6 @@ describe('Mempool Backend Config', () => {
         PASSWORD: ''
       });
 
-      expect(config.PRICE_DATA_SERVER).toStrictEqual({
-        TOR_URL: 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
-        CLEARNET_URL: 'https://price.bisq.wiz.biz/getAllMarketPrices'
-      });
-
       expect(config.EXTERNAL_DATA_SERVER).toStrictEqual({
         MEMPOOL_API: 'https://mempool.space/api/v1',
         MEMPOOL_ONION: 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
@@ -168,8 +164,6 @@ describe('Mempool Backend Config', () => {
 
       expect(config.SOCKS5PROXY).toStrictEqual(fixture.SOCKS5PROXY);
 
-      expect(config.PRICE_DATA_SERVER).toStrictEqual(fixture.PRICE_DATA_SERVER);
-
       expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER);
 
       expect(config.MEMPOOL_SERVICES).toStrictEqual(fixture.MEMPOOL_SERVICES);

backend/src/api/prices/prices.routes.ts

@@ -0,0 +1,19 @@
+import { Application, Request, Response } from 'express';
+import config from '../../config';
+import pricesUpdater from '../../tasks/price-updater';
+
+class PricesRoutes {
+  public initRoutes(app: Application): void {
+    app.get(config.MEMPOOL.API_URL_PREFIX + 'prices', this.$getCurrentPrices.bind(this));
+  }
+
+  private $getCurrentPrices(req: Request, res: Response): void {
+    res.header('Pragma', 'public');
+    res.header('Cache-control', 'public');
+    res.setHeader('Expires', new Date(Date.now() + 360_0000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR).toUTCString());
+
+    res.json(pricesUpdater.getLatestPrices());
+  }
+}
+
+export default new PricesRoutes();

backend/src/config.ts

@@ -38,6 +38,7 @@ interface IConfig {
     DISK_CACHE_BLOCK_INTERVAL: number;
     MAX_PUSH_TX_SIZE_WEIGHT: number;
     ALLOW_UNREACHABLE: boolean;
+    PRICE_UPDATES_PER_HOUR: number;
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -115,10 +116,6 @@ interface IConfig {
     USERNAME: string;
     PASSWORD: string;
   };
-  PRICE_DATA_SERVER: {
-    TOR_URL: string;
-    CLEARNET_URL: string;
-  };
   EXTERNAL_DATA_SERVER: {
     MEMPOOL_API: string;
     MEMPOOL_ONION: string;
@@ -185,6 +182,7 @@ const defaults: IConfig = {
     'DISK_CACHE_BLOCK_INTERVAL': 6,
     'MAX_PUSH_TX_SIZE_WEIGHT': 400000,
     'ALLOW_UNREACHABLE': true,
+    'PRICE_UPDATES_PER_HOUR': 1,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',
@@ -262,10 +260,6 @@ const defaults: IConfig = {
     'USERNAME': '',
     'PASSWORD': ''
   },
-  'PRICE_DATA_SERVER': {
-    'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
-    'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices'
-  },
   'EXTERNAL_DATA_SERVER': {
     'MEMPOOL_API': 'https://mempool.space/api/v1',
     'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
@@ -310,7 +304,6 @@ class Config implements IConfig {
   LND: IConfig['LND'];
   CLIGHTNING: IConfig['CLIGHTNING'];
   SOCKS5PROXY: IConfig['SOCKS5PROXY'];
-  PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
   EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
   MAXMIND: IConfig['MAXMIND'];
   REPLICATION: IConfig['REPLICATION'];
@@ -332,7 +325,6 @@ class Config implements IConfig {
     this.LND = configs.LND;
     this.CLIGHTNING = configs.CLIGHTNING;
     this.SOCKS5PROXY = configs.SOCKS5PROXY;
-    this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
     this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;
     this.MAXMIND = configs.MAXMIND;
     this.REPLICATION = configs.REPLICATION;

backend/src/index.ts

@@ -30,6 +30,7 @@ import generalLightningRoutes from './api/explorer/general.routes';
 import lightningStatsUpdater from './tasks/lightning/stats-updater.service';
 import networkSyncService from './tasks/lightning/network-sync.service';
 import statisticsRoutes from './api/statistics/statistics.routes';
+import pricesRoutes from './api/prices/prices.routes';
 import miningRoutes from './api/mining/mining-routes';
 import bisqRoutes from './api/bisq/bisq.routes';
 import liquidRoutes from './api/liquid/liquid.routes';
@@ -193,6 +194,7 @@ class Server {
         await memPool.$updateMempool(newMempool, pollRate);
       }
       indexer.$run();
+      priceUpdater.$run();
 
       // rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS
       const elapsed = Date.now() - start;
@@ -261,6 +263,7 @@ class Server {
   
   setUpHttpApiRoutes(): void {
     bitcoinRoutes.initRoutes(this.app);
+    pricesRoutes.initRoutes(this.app);
     if (config.STATISTICS.ENABLED && config.DATABASE.ENABLED && config.MEMPOOL.ENABLED) {
       statisticsRoutes.initRoutes(this.app);
     }

backend/src/indexer.ts

@@ -105,6 +105,12 @@ class Indexer {
       return;
     }
 
+    try {
+      await priceUpdater.$run();
+    } catch (e) {
+      logger.err(`Running priceUpdater failed. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+
     // Do not attempt to index anything unless Bitcoin Core is fully synced
     const blockchainInfo = await bitcoinClient.getBlockchainInfo();
     if (blockchainInfo.blocks !== blockchainInfo.headers) {
@@ -119,8 +125,6 @@ class Indexer {
     await this.checkAvailableCoreIndexes();
 
     try {
-      await priceUpdater.$run();
-
       const chainValid = await blocks.$generateBlockDatabase();
       if (chainValid === false) {
         // Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration

backend/src/tasks/price-updater.ts

@@ -25,7 +25,10 @@ export interface PriceHistory {
 
 class PriceUpdater {
   public historyInserted = false;
-  private lastRun = 0;
+  private timeBetweenUpdatesMs = 360_0000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR;
+  private cyclePosition = -1;
+  private firstRun = true;
+  private lastTime = -1;
   private lastHistoricalRun = 0;
   private running = false;
   private feeds: PriceFeed[] = [];
@@ -41,6 +44,8 @@ class PriceUpdater {
     this.feeds.push(new CoinbaseApi());
     this.feeds.push(new BitfinexApi());
     this.feeds.push(new GeminiApi());
+
+    this.setCyclePosition();
   }
 
   public getLatestPrices(): ApiPrice {
@@ -100,22 +105,48 @@ class PriceUpdater {
     this.running = false;
   }
 
+  private getMillisecondsSinceBeginningOfHour(): number {
+    const now = new Date();
+    const beginningOfHour = new Date(now);
+    beginningOfHour.setMinutes(0, 0, 0);
+    return now.getTime() - beginningOfHour.getTime();
+  }
+
+  private setCyclePosition(): void {
+    const millisecondsSinceBeginningOfHour = this.getMillisecondsSinceBeginningOfHour();
+    for (let i = 0; i < config.MEMPOOL.PRICE_UPDATES_PER_HOUR; i++) {
+      if (this.timeBetweenUpdatesMs * i > millisecondsSinceBeginningOfHour) {
+        this.cyclePosition = i;
+        return;
+      }
+    }
+    this.cyclePosition = config.MEMPOOL.PRICE_UPDATES_PER_HOUR;
+  }
+
   /**
    * Fetch last BTC price from exchanges, average them, and save it in the database once every hour
    */
   private async $updatePrice(): Promise<void> {
-    if (this.lastRun === 0 && config.DATABASE.ENABLED === true) {
+    let forceUpdate = false;
-      this.lastRun = await PricesRepository.$getLatestPriceTime();
+    if (this.firstRun === true && config.DATABASE.ENABLED === true) {
+      const lastUpdate = await PricesRepository.$getLatestPriceTime();
+      if (new Date().getTime() / 1000 - lastUpdate > this.timeBetweenUpdatesMs / 1000) {
+        forceUpdate = true;
+      }
+      this.firstRun = false;
     }
 
-    if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < 3600) {
+    const millisecondsSinceBeginningOfHour = this.getMillisecondsSinceBeginningOfHour();
-      // Refresh only once every hour
+
+    // Reset the cycle on new hour
+    if (this.lastTime > millisecondsSinceBeginningOfHour) {
+      this.cyclePosition = 0;
+    }
+    this.lastTime = millisecondsSinceBeginningOfHour;
+    if (millisecondsSinceBeginningOfHour < this.timeBetweenUpdatesMs * this.cyclePosition && !forceUpdate && this.cyclePosition !== 0) {
       return;
     }
 
-    const previousRun = this.lastRun;
-    this.lastRun = new Date().getTime() / 1000;
-
     for (const currency of this.currencies) {
       let prices: number[] = [];
 
@@ -146,26 +177,27 @@ class PriceUpdater {
       }
     }
 
-    logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`);
+    if (config.DATABASE.ENABLED === true && this.cyclePosition === 0) {
-
-    if (config.DATABASE.ENABLED === true) {
       // Save everything in db
       try {
         const p = 60 * 60 * 1000; // milliseconds in an hour
         const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042
-        this.latestPrices.time = nowRounded.getTime() / 1000;
         await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices);
       } catch (e) {
-        this.lastRun = previousRun + 5 * 60;
         logger.err(`Cannot save latest prices into db. Trying again in 5 minutes. Reason: ${(e instanceof Error ? e.message : e)}`);
       }
     }
 
+    this.latestPrices.time = Math.round(new Date().getTime() / 1000);
+    logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`);
+
     if (this.ratesChangedCallback) {
       this.ratesChangedCallback(this.latestPrices);
     }
 
-    this.lastRun = new Date().getTime() / 1000;
+    if (!forceUpdate) {
+      this.cyclePosition++;
+    }
 
     if (this.latestPrices.USD === -1) {
       this.latestPrices = await PricesRepository.$getLatestConversionRates();

docker/README.md

@@ -113,7 +113,8 @@ Below we list all settings from `mempool-config.json` and the corresponding over
     "ADVANCED_GBT_MEMPOOL": false,
     "CPFP_INDEXING": false,
     "MAX_BLOCKS_BULK_QUERY": 0,
-    "DISK_CACHE_BLOCK_INTERVAL": 6
+    "DISK_CACHE_BLOCK_INTERVAL": 6,
+    "PRICE_UPDATES_PER_HOUR": 1
   },
 ```
 
@@ -146,6 +147,7 @@ Corresponding `docker-compose.yml` overrides:
       MEMPOOL_CPFP_INDEXING: ""
       MEMPOOL_MAX_BLOCKS_BULK_QUERY: ""
       MEMPOOL_DISK_CACHE_BLOCK_INTERVAL: ""
+      MEMPOOL_PRICE_UPDATES_PER_HOUR: ""
       ...
 ```
 
@@ -363,25 +365,6 @@ Corresponding `docker-compose.yml` overrides:
 
 <br/>
 
-`mempool-config.json`:
-```json
-  "PRICE_DATA_SERVER": {
-    "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices",
-    "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices"
-  }
-```
-
-Corresponding `docker-compose.yml` overrides:
-```yaml
-  api:
-    environment:
-      PRICE_DATA_SERVER_TOR_URL: ""
-      PRICE_DATA_SERVER_CLEARNET_URL: ""
-      ...
-```
-
-<br/>
-
 `mempool-config.json`:
 ```json
   "LIGHTNING": {

docker/backend/mempool-config.json

@@ -33,7 +33,8 @@
     "MAX_PUSH_TX_SIZE_WEIGHT": __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__,
     "ALLOW_UNREACHABLE": __MEMPOOL_ALLOW_UNREACHABLE__,
     "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__",
-    "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__"
+    "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__",
+    "PRICE_UPDATES_PER_HOUR": __MEMPOOL_PRICE_UPDATES_PER_HOUR__
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",
@@ -111,10 +112,6 @@
     "USERNAME": "__SOCKS5PROXY_USERNAME__",
     "PASSWORD": "__SOCKS5PROXY_PASSWORD__"
   },
-  "PRICE_DATA_SERVER": {
-    "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__",
-    "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__"
-  },
   "EXTERNAL_DATA_SERVER": {
     "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__",
     "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__",

docker/backend/start.sh

@@ -35,7 +35,7 @@ __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
 __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6}
 __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__=${MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT:=4000000}
 __MEMPOOL_ALLOW_UNREACHABLE__=${MEMPOOL_ALLOW_UNREACHABLE:=true}
-
+__MEMPOOL_PRICE_UPDATES_PER_HOUR__=${MEMPOOL_PRICE_UPDATES_PER_HOUR:=1}
 
 # CORE_RPC
 __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
@@ -94,10 +94,6 @@ __SOCKS5PROXY_PORT__=${SOCKS5PROXY_PORT:=9050}
 __SOCKS5PROXY_USERNAME__=${SOCKS5PROXY_USERNAME:=""}
 __SOCKS5PROXY_PASSWORD__=${SOCKS5PROXY_PASSWORD:=""}
 
-# PRICE_DATA_SERVER
-__PRICE_DATA_SERVER_TOR_URL__=${PRICE_DATA_SERVER_TOR_URL:=http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices}
-__PRICE_DATA_SERVER_CLEARNET_URL__=${PRICE_DATA_SERVER_CLEARNET_URL:=https://price.bisq.wiz.biz/getAllMarketPrices}
-
 # EXTERNAL_DATA_SERVER
 __EXTERNAL_DATA_SERVER_MEMPOOL_API__=${EXTERNAL_DATA_SERVER_MEMPOOL_API:=https://mempool.space/api/v1}
 __EXTERNAL_DATA_SERVER_MEMPOOL_ONION__=${EXTERNAL_DATA_SERVER_MEMPOOL_ONION:=http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1}
@@ -181,6 +177,7 @@ sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}
 sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__!${__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_ALLOW_UNREACHABLE__!${__MEMPOOL_ALLOW_UNREACHABLE__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_PRICE_UPDATES_PER_HOUR__!${__MEMPOOL_PRICE_UPDATES_PER_HOUR__}!g" mempool-config.json
 
 sed -i "s!__CORE_RPC_HOST__!${__CORE_RPC_HOST__}!g" mempool-config.json
 sed -i "s!__CORE_RPC_PORT__!${__CORE_RPC_PORT__}!g" mempool-config.json
@@ -230,9 +227,6 @@ sed -i "s!__SOCKS5PROXY_PORT__!${__SOCKS5PROXY_PORT__}!g" mempool-config.json
 sed -i "s!__SOCKS5PROXY_USERNAME__!${__SOCKS5PROXY_USERNAME__}!g" mempool-config.json
 sed -i "s!__SOCKS5PROXY_PASSWORD__!${__SOCKS5PROXY_PASSWORD__}!g" mempool-config.json
 
-sed -i "s!__PRICE_DATA_SERVER_TOR_URL__!${__PRICE_DATA_SERVER_TOR_URL__}!g" mempool-config.json
-sed -i "s!__PRICE_DATA_SERVER_CLEARNET_URL__!${__PRICE_DATA_SERVER_CLEARNET_URL__}!g" mempool-config.json
-
 sed -i "s!__EXTERNAL_DATA_SERVER_MEMPOOL_API__!${__EXTERNAL_DATA_SERVER_MEMPOOL_API__}!g" mempool-config.json
 sed -i "s!__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__!${__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__}!g" mempool-config.json
 sed -i "s!__EXTERNAL_DATA_SERVER_LIQUID_API__!${__EXTERNAL_DATA_SERVER_LIQUID_API__}!g" mempool-config.json

production/mempool-config.mainnet.json

@@ -18,7 +18,8 @@
     "USE_SECOND_NODE_FOR_MINFEE": true,
     "DISK_CACHE_BLOCK_INTERVAL": 1,
     "MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
-    "ALLOW_UNREACHABLE": true
+    "ALLOW_UNREACHABLE": true,
+    "PRICE_UPDATES_PER_HOUR": 12
   },
   "SYSLOG" : {
     "MIN_PRIORITY": "debug"