Changing setting to per hour

backend/mempool-config.sample.json

@@ -33,7 +33,7 @@
     "DISK_CACHE_BLOCK_INTERVAL": 6,
     "MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
     "ALLOW_UNREACHABLE": true,
-    "PRICE_UPDATE_FREQUENCY": 3600
+    "PRICE_UPDATES_PER_HOUR": 3600
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",

backend/src/__fixtures__/mempool-config.template.json

@@ -34,7 +34,7 @@
     "DISK_CACHE_BLOCK_INTERVAL": 999,
     "MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
     "ALLOW_UNREACHABLE": true,
-    "PRICE_UPDATE_FREQUENCY": 3600
+    "PRICE_UPDATES_PER_HOUR": 1
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",

backend/src/__tests__/config.test.ts

@@ -47,7 +47,7 @@ describe('Mempool Backend Config', () => {
         DISK_CACHE_BLOCK_INTERVAL: 6,
         MAX_PUSH_TX_SIZE_WEIGHT: 400000,
         ALLOW_UNREACHABLE: true,
-        PRICE_UPDATE_FREQUENCY: 3600,
+        PRICE_UPDATES_PER_HOUR: 1,
       });
 
       expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });

backend/src/config.ts

@@ -38,7 +38,7 @@ interface IConfig {
     DISK_CACHE_BLOCK_INTERVAL: number;
     MAX_PUSH_TX_SIZE_WEIGHT: number;
     ALLOW_UNREACHABLE: boolean;
-    PRICE_UPDATE_FREQUENCY: number;
+    PRICE_UPDATES_PER_HOUR: number;
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -178,7 +178,7 @@ const defaults: IConfig = {
     'DISK_CACHE_BLOCK_INTERVAL': 6,
     'MAX_PUSH_TX_SIZE_WEIGHT': 400000,
     'ALLOW_UNREACHABLE': true,
-    'PRICE_UPDATE_FREQUENCY': 3600,
+    'PRICE_UPDATES_PER_HOUR': 1,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',

backend/src/indexer.ts

@@ -106,7 +106,7 @@ class Indexer {
     }
 
     try {
-      await priceUpdater.$run(true);
+      await priceUpdater.$run();
     } catch (e) {
       logger.err(`Running priceUpdater failed. Reason: ` + (e instanceof Error ? e.message : e));
     }

backend/src/repositories/PricesRepository.ts

@@ -65,7 +65,7 @@ class PricesRepository {
     
     try {
       await DB.query(`
-        INSERT IGNORE INTO prices(time,             USD, EUR, GBP, CAD, CHF, AUD, JPY)
+        INSERT INTO prices(time,             USD, EUR, GBP, CAD, CHF, AUD, JPY)
         VALUE             (FROM_UNIXTIME(?), ?,   ?,   ?,   ?,   ?,   ?,   ?  )`,
         [time, prices.USD, prices.EUR, prices.GBP, prices.CAD, prices.CHF, prices.AUD, prices.JPY]
       );

backend/src/tasks/price-updater.ts

@@ -25,7 +25,9 @@ export interface PriceHistory {
 
 class PriceUpdater {
   public historyInserted = false;
-  private lastRun = 0;
+  private timeBetweenUpdatesMs = 3600000 / config.MEMPOOL.PRICE_UPDATES_PER_HOUR;
+  private cyclePosition = -1;
+  private firstRun = true;
   private lastHistoricalRun = 0;
   private running = false;
   private feeds: PriceFeed[] = [];
@@ -41,6 +43,8 @@ class PriceUpdater {
     this.feeds.push(new CoinbaseApi());
     this.feeds.push(new BitfinexApi());
     this.feeds.push(new GeminiApi());
+
+    this.setCyclePosition();
   }
 
   public getLatestPrices(): ApiPrice {
@@ -72,7 +76,7 @@ class PriceUpdater {
     this.latestPrices = await PricesRepository.$getLatestConversionRates();
   }
 
-  public async $run(storeInDb: boolean = false): Promise<void> {
+  public async $run(): Promise<void> {
     if (config.MEMPOOL.NETWORK === 'signet' || config.MEMPOOL.NETWORK === 'testnet') {
       // Coins have no value on testnet/signet, so we want to always show 0
       return;
@@ -89,7 +93,7 @@ class PriceUpdater {
     }
 
     try {
-      await this.$updatePrice(storeInDb);
+      await this.$updatePrice();
       if (this.historyInserted === false && config.DATABASE.ENABLED === true) {
         await this.$insertHistoricalPrices();
       }
@@ -100,22 +104,42 @@ class PriceUpdater {
     this.running = false;
   }
 
+  private getMillisecondsSinceBeginningOfHour(): number {
+    const now = new Date();
+    const beginningOfHour = new Date(now);
+    beginningOfHour.setMinutes(0, 0, 0);
+    return now.getTime() - beginningOfHour.getTime();
+  }
+
+  private setCyclePosition(): void {
+    const millisecondsSinceBeginningOfHour = this.getMillisecondsSinceBeginningOfHour();
+    for (let i = 0; i < config.MEMPOOL.PRICE_UPDATES_PER_HOUR; i++) {
+      if (this.timeBetweenUpdatesMs * i > millisecondsSinceBeginningOfHour) {
+        this.cyclePosition = i;
+        return;
+      }
+    }
+    this.cyclePosition = config.MEMPOOL.PRICE_UPDATES_PER_HOUR;
+  }
+
   /**
    * Fetch last BTC price from exchanges, average them, and save it in the database once every hour
    */
-  private async $updatePrice(storeInDb: boolean): Promise<void> {
+  private async $updatePrice(): Promise<void> {
-    if (this.lastRun === 0 && config.DATABASE.ENABLED === true) {
+    let forceUpdate = false;
-      this.lastRun = await PricesRepository.$getLatestPriceTime();
+    if (this.firstRun === true && config.DATABASE.ENABLED === true) {
+      const lastUpdate = await PricesRepository.$getLatestPriceTime();
+      if (new Date().getTime() / 1000 - lastUpdate > this.timeBetweenUpdatesMs / 1000) {
+        forceUpdate = true;
+      }
+      this.firstRun = false;
     }
 
-    if ((Math.round(new Date().getTime() / 1000) - this.lastRun) < Math.min(config.MEMPOOL.PRICE_UPDATE_FREQUENCY, 3600)) {
+    const millisecondsSinceBeginningOfHour = this.getMillisecondsSinceBeginningOfHour();
-      // Refresh at least every hour or PRICE_UPDATE_FREQUENCY seconds
+    if (millisecondsSinceBeginningOfHour < this.timeBetweenUpdatesMs * this.cyclePosition && !forceUpdate) {
       return;
     }
 
-    const previousRun = this.lastRun;
-    this.lastRun = new Date().getTime() / 1000;
-
     for (const currency of this.currencies) {
       let prices: number[] = [];
 
@@ -146,14 +170,13 @@ class PriceUpdater {
       }
     }
 
-    if (config.DATABASE.ENABLED === true && storeInDb) {
+    if (config.DATABASE.ENABLED === true && this.cyclePosition === 0) {
       // Save everything in db
       try {
         const p = 60 * 60 * 1000; // milliseconds in an hour
         const nowRounded = new Date(Math.round(new Date().getTime() / p) * p); // https://stackoverflow.com/a/28037042
         await PricesRepository.$savePrices(nowRounded.getTime() / 1000, this.latestPrices);
       } catch (e) {
-        this.lastRun = previousRun + 5 * 60;
         logger.err(`Cannot save latest prices into db. Trying again in 5 minutes. Reason: ${(e instanceof Error ? e.message : e)}`);
       }
     }
@@ -165,7 +188,12 @@ class PriceUpdater {
       this.ratesChangedCallback(this.latestPrices);
     }
 
-    this.lastRun = new Date().getTime() / 1000;
+    if (!forceUpdate) {
+      this.cyclePosition++;
+      if (this.cyclePosition > config.MEMPOOL.PRICE_UPDATES_PER_HOUR) {
+        this.cyclePosition = 0;
+      }
+    }
 
     if (this.latestPrices.USD === -1) {
       this.latestPrices = await PricesRepository.$getLatestConversionRates();

docker/README.md

@@ -114,7 +114,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over
     "CPFP_INDEXING": false,
     "MAX_BLOCKS_BULK_QUERY": 0,
     "DISK_CACHE_BLOCK_INTERVAL": 6,
-    "PRICE_UPDATE_FREQUENCY": 3600
+    "PRICE_UPDATES_PER_HOUR": 1
   },
 ```
 
@@ -147,7 +147,7 @@ Corresponding `docker-compose.yml` overrides:
       MEMPOOL_CPFP_INDEXING: ""
       MEMPOOL_MAX_BLOCKS_BULK_QUERY: ""
       MEMPOOL_DISK_CACHE_BLOCK_INTERVAL: ""
-      MEMPOOL_PRICE_UPDATE_FREQUENCY: ""
+      MEMPOOL_PRICE_UPDATES_PER_HOUR: ""
       ...
 ```
 

docker/backend/mempool-config.json

@@ -34,7 +34,7 @@
     "ALLOW_UNREACHABLE": __MEMPOOL_ALLOW_UNREACHABLE__,
     "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__",
     "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__",
-    "PRICE_UPDATE_FREQUENCY": __MEMPOOL_PRICE_UPDATE_FREQUENCY__
+    "PRICE_UPDATES_PER_HOUR": __MEMPOOL_PRICE_UPDATES_PER_HOUR__
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",

docker/backend/start.sh

@@ -35,7 +35,7 @@ __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
 __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6}
 __MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__=${MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT:=4000000}
 __MEMPOOL_ALLOW_UNREACHABLE__=${MEMPOOL_ALLOW_UNREACHABLE:=true}
-__MEMPOOL_PRICE_UPDATE_FREQUENCY__=${MEMPOOL_PRICE_UPDATE_FREQUENCY:=3600}
+__MEMPOOL_PRICE_UPDATES_PER_HOUR__=${MEMPOOL_PRICE_UPDATES_PER_HOUR:=3600}
 
 # CORE_RPC
 __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
@@ -173,7 +173,7 @@ sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}
 sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__!${__MEMPOOL_MAX_PUSH_TX_SIZE_WEIGHT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_ALLOW_UNREACHABLE__!${__MEMPOOL_ALLOW_UNREACHABLE__}!g" mempool-config.json
-sed -i "s!__MEMPOOL_PRICE_UPDATE_FREQUENCY__!${__MEMPOOL_PRICE_UPDATE_FREQUENCY__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_PRICE_UPDATES_PER_HOUR__!${__MEMPOOL_PRICE_UPDATES_PER_HOUR__}!g" mempool-config.json
 
 sed -i "s!__CORE_RPC_HOST__!${__CORE_RPC_HOST__}!g" mempool-config.json
 sed -i "s!__CORE_RPC_PORT__!${__CORE_RPC_PORT__}!g" mempool-config.json

production/mempool-config.mainnet.json

@@ -19,7 +19,7 @@
     "DISK_CACHE_BLOCK_INTERVAL": 1,
     "MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
     "ALLOW_UNREACHABLE": true,
-    "PRICE_UPDATE_FREQUENCY": 300
+    "PRICE_UPDATES_PER_HOUR": 12
   },
   "SYSLOG" : {
     "MIN_PRIORITY": "debug"