Refactoring price update config. Fixing last price time.

backend/mempool-config.sample.json

@@ -32,7 +32,8 @@
     "CPFP_INDEXING": false,
     "DISK_CACHE_BLOCK_INTERVAL": 6,
     "MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
-    "ALLOW_UNREACHABLE": true
+    "ALLOW_UNREACHABLE": true,
+    "PRICE_UPDATE_FREQUENCY": 3600
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",
@@ -115,11 +116,6 @@
     "USERNAME": "",
     "PASSWORD": ""
   },
-  "PRICE_DATA_SERVER": {
-    "UPDATE_FREQUENCY": 300,
-    "TOR_URL": "http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices",
-    "CLEARNET_URL": "https://price.bisq.wiz.biz/getAllMarketPrices"
-  },
   "EXTERNAL_DATA_SERVER": {
     "MEMPOOL_API": "https://mempool.space/api/v1",
     "MEMPOOL_ONION": "http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1",

backend/src/__fixtures__/mempool-config.template.json

@@ -23,8 +23,8 @@
     "USER_AGENT": "__MEMPOOL_USER_AGENT__",
     "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
     "INDEXING_BLOCKS_AMOUNT": 14,
-    "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__",
+    "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__",
-    "POOLS_JSON_URL": "__POOLS_JSON_URL__",
+    "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__",
     "AUDIT": true,
     "ADVANCED_GBT_AUDIT": true,
     "ADVANCED_GBT_MEMPOOL": true,
@@ -33,7 +33,8 @@
     "MAX_BLOCKS_BULK_QUERY": 999,
     "DISK_CACHE_BLOCK_INTERVAL": 999,
     "MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
-    "ALLOW_UNREACHABLE": true
+    "ALLOW_UNREACHABLE": true,
+    "PRICE_UPDATE_FREQUENCY": "__MEMPOOL_PRICE_UPDATE_FREQUENCY__"
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",
@@ -92,11 +93,6 @@
     "USERNAME": "__SOCKS5PROXY_USERNAME__",
     "PASSWORD": "__SOCKS5PROXY_PASSWORD__"
   },
-  "PRICE_DATA_SERVER": {
-    "UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY__",
-    "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__",
-    "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__"
-  },
   "EXTERNAL_DATA_SERVER": {
     "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__",
     "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__",

backend/src/__tests__/config.test.ts

@@ -47,6 +47,7 @@ describe('Mempool Backend Config', () => {
         DISK_CACHE_BLOCK_INTERVAL: 6,
         MAX_PUSH_TX_SIZE_WEIGHT: 400000,
         ALLOW_UNREACHABLE: true,
+        PRICE_UPDATE_FREQUENCY: 3600,
       });
 
       expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
@@ -101,12 +102,6 @@ describe('Mempool Backend Config', () => {
         PASSWORD: ''
       });
 
-      expect(config.PRICE_DATA_SERVER).toStrictEqual({
-        UPDATE_FREQUENCY: 3600,
-        TOR_URL: 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
-        CLEARNET_URL: 'https://price.bisq.wiz.biz/getAllMarketPrices'
-      });
-
       expect(config.EXTERNAL_DATA_SERVER).toStrictEqual({
         MEMPOOL_API: 'https://mempool.space/api/v1',
         MEMPOOL_ONION: 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
@@ -164,8 +159,6 @@ describe('Mempool Backend Config', () => {
 
       expect(config.SOCKS5PROXY).toStrictEqual(fixture.SOCKS5PROXY);
 
-      expect(config.PRICE_DATA_SERVER).toStrictEqual(fixture.PRICE_DATA_SERVER);
-
       expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER);
 
       expect(config.REDIS).toStrictEqual(fixture.REDIS);

backend/src/config.ts

@@ -38,6 +38,7 @@ interface IConfig {
     DISK_CACHE_BLOCK_INTERVAL: number;
     MAX_PUSH_TX_SIZE_WEIGHT: number;
     ALLOW_UNREACHABLE: boolean;
+    PRICE_UPDATE_FREQUENCY: number;
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -115,11 +116,6 @@ interface IConfig {
     USERNAME: string;
     PASSWORD: string;
   };
-  PRICE_DATA_SERVER: {
-    UPDATE_FREQUENCY: number;
-    TOR_URL: string;
-    CLEARNET_URL: string;
-  };
   EXTERNAL_DATA_SERVER: {
     MEMPOOL_API: string;
     MEMPOOL_ONION: string;
@@ -182,6 +178,7 @@ const defaults: IConfig = {
     'DISK_CACHE_BLOCK_INTERVAL': 6,
     'MAX_PUSH_TX_SIZE_WEIGHT': 400000,
     'ALLOW_UNREACHABLE': true,
+    'PRICE_UPDATE_FREQUENCY': 3600,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',
@@ -259,11 +256,6 @@ const defaults: IConfig = {
     'USERNAME': '',
     'PASSWORD': ''
   },
-  'PRICE_DATA_SERVER': {
-    'UPDATE_FREQUENCY': 3600,
-    'TOR_URL': 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
-    'CLEARNET_URL': 'https://price.bisq.wiz.biz/getAllMarketPrices'
-  },
   'EXTERNAL_DATA_SERVER': {
     'MEMPOOL_API': 'https://mempool.space/api/v1',
     'MEMPOOL_ONION': 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
@@ -304,7 +296,6 @@ class Config implements IConfig {
   LND: IConfig['LND'];
   CLIGHTNING: IConfig['CLIGHTNING'];
   SOCKS5PROXY: IConfig['SOCKS5PROXY'];
-  PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
   EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
   MAXMIND: IConfig['MAXMIND'];
   REPLICATION: IConfig['REPLICATION'];
@@ -325,7 +316,6 @@ class Config implements IConfig {
     this.LND = configs.LND;
     this.CLIGHTNING = configs.CLIGHTNING;
     this.SOCKS5PROXY = configs.SOCKS5PROXY;
-    this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
     this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;
     this.MAXMIND = configs.MAXMIND;
     this.REPLICATION = configs.REPLICATION;

backend/src/indexer.ts

@@ -105,6 +105,12 @@ class Indexer {
       return;
     }
 
+    try {
+      await priceUpdater.$run();
+    } catch (e) {
+      logger.err(`Running priceUpdater failed. Reason: ` + (e instanceof Error ? e.message : e));
+    }
+
     // Do not attempt to index anything unless Bitcoin Core is fully synced
     const blockchainInfo = await bitcoinClient.getBlockchainInfo();
     if (blockchainInfo.blocks !== blockchainInfo.headers) {
@@ -119,8 +125,6 @@ class Indexer {
     await this.checkAvailableCoreIndexes();
 
     try {
-      await priceUpdater.$run();
-
       const chainValid = await blocks.$generateBlockDatabase();
       if (chainValid === false) {
         // Chain of block hash was invalid, so we need to reindex. Stop here and continue at the next iteration

backend/src/repositories/PricesRepository.ts

@@ -65,7 +65,7 @@ class PricesRepository {
     
     try {
       await DB.query(`
-        INSERT INTO prices(time,             USD, EUR, GBP, CAD, CHF, AUD, JPY)
+        INSERT IGNORE INTO prices(time,             USD, EUR, GBP, CAD, CHF, AUD, JPY)
         VALUE             (FROM_UNIXTIME(?), ?,   ?,   ?,   ?,   ?,   ?,   ?  )`,
         [time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY]
       );

backend/src/tasks/price-updater.ts

@@ -108,8 +108,8 @@ class PriceUpdater {
       this.lastRun = await PricesRepository.$getLatestPriceTime();
     }
 
-    if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < config.PRICE_DATA_SERVER.UPDATE_FREQUENCY) {
+    if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < config.MEMPOOL.PRICE_UPDATE_FREQUENCY) {
-      // Refresh every UPDATE_FREQUENCY seconds at most
+      // Refresh every PRICE_UPDATE_FREQUENCY seconds at most
       return;
     }
 
@@ -146,14 +146,11 @@ class PriceUpdater {
       }
     }
 
-    logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`);
-
     if (config.DATABASE.ENABLED === true) {
       // Save everything in db
       try {
         const p = 60 * 60 * 1000; // milliseconds in an hour
         const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042
-        this.latestPrices.time = nowRounded.getTime() / 1000;
         await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices);
       } catch (e) {
         this.lastRun = previousRun + 5 * 60;
@@ -161,6 +158,9 @@ class PriceUpdater {
       }
     }
 
+    this.latestPrices.time = Math.round(new Date().getTime() / 1000);
+    logger.info(`Latest BTC fiat averaged price: ${JSON.stringify(this.latestPrices)}`);
+
     if (this.ratesChangedCallback) {
       this.ratesChangedCallback(this.latestPrices);
     }

docker/backend/mempool-config.json

@@ -33,7 +33,8 @@
     "MAX_PUSH_TX_SIZE_WEIGHT": __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__,
     "ALLOW_UNREACHABLE": __MEMPOOL_ALLOW_UNREACHABLE__,
     "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__",
-    "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__"
+    "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__",
+    "PRICE_UPDATE_FREQUENCY": __MEMPOOL_PRICE_UPDATE_FREQUENCY__
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",
@@ -111,11 +112,6 @@
     "USERNAME": "__SOCKS5PROXY_USERNAME__",
     "PASSWORD": "__SOCKS5PROXY_PASSWORD__"
   },
-  "PRICE_DATA_SERVER": {
-    "UPDATE_FREQUENCY": "__PRICE_DATA_SERVER_UPDATE_FREQUENCY__",
-    "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__",
-    "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__"
-  },
   "EXTERNAL_DATA_SERVER": {
     "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__",
     "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__",

production/mempool-config.mainnet.json

@@ -18,7 +18,8 @@
     "USE_SECOND_NODE_FOR_MINFEE": true,
     "DISK_CACHE_BLOCK_INTERVAL": 1,
     "MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
-    "ALLOW_UNREACHABLE": true
+    "ALLOW_UNREACHABLE": true,
+    "PRICE_UPDATE_FREQUENCY": 300
   },
   "SYSLOG" : {
     "MIN_PRIORITY": "debug"
@@ -49,9 +50,6 @@
     "ENABLED": true,
     "TX_PER_SECOND_SAMPLE_PERIOD": 150
   },
-  "PRICE_DATA_SERVER": {
-    "UPDATE_FREQUENCY": 300
-  },
   "REPLICATION": {
     "ENABLED": true,
     "AUDIT": true,