Merge branch 'master' into nymkappa/bugfix/diff-adj-table-raw-db-data

backend/mempool-config.sample.json

@@ -20,7 +20,8 @@
     "EXTERNAL_MAX_RETRY": 1,
     "EXTERNAL_RETRY_INTERVAL": 0,
     "USER_AGENT": "mempool",
-    "STDOUT_LOG_MIN_PRIORITY": "debug"
+    "STDOUT_LOG_MIN_PRIORITY": "debug",
+    "AUTOMATIC_BLOCK_REINDEXING": false
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",

backend/src/api/pools-parser.ts

@@ -222,6 +222,10 @@ class PoolsParser {
    * Delete blocks which needs to be reindexed
    */
    private async $deleteBlocskToReindex(finalPoolDataUpdate: any[]) {
+    if (config.MEMPOOL.AUTOMATIC_BLOCK_REINDEXING === false) {
+      return;
+    }
+
     const blockCount = await BlocksRepository.$blockCount(null, null);
     if (blockCount === 0) {
       return;

backend/src/config.ts

@@ -23,6 +23,7 @@ interface IConfig {
     EXTERNAL_RETRY_INTERVAL: number;
     USER_AGENT: string;
     STDOUT_LOG_MIN_PRIORITY: 'emerg' | 'alert' | 'crit' | 'err' | 'warn' | 'notice' | 'info' | 'debug';
+    AUTOMATIC_BLOCK_REINDEXING: boolean;
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -113,6 +114,7 @@ const defaults: IConfig = {
     'EXTERNAL_RETRY_INTERVAL': 0,
     'USER_AGENT': 'mempool',
     'STDOUT_LOG_MIN_PRIORITY': 'debug',
+    'AUTOMATIC_BLOCK_REINDEXING': false,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',

backend/src/repositories/PricesRepository.ts

@@ -4,6 +4,12 @@ import { Prices } from '../tasks/price-updater';
 
 class PricesRepository {
   public async $savePrices(time: number, prices: Prices): Promise<void> {
+    if (prices.USD === -1) {
+      // Some historical price entries have not USD prices, so we just ignore them to avoid future UX issues
+      // As of today there are only 4 (on 2013-09-05, 2013-09-19, 2013-09-12 and 2013-09-26) so that's fine
+      return;
+    }
+
     try {
       await DB.query(`
         INSERT INTO prices(time,             USD, EUR, GBP, CAD, CHF, AUD, JPY)
@@ -17,17 +23,17 @@ class PricesRepository {
   }
 
   public async $getOldestPriceTime(): Promise<number> {
-    const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices ORDER BY time LIMIT 1`);
+    const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time LIMIT 1`);
     return oldestRow[0] ? oldestRow[0].time : 0;
   }
 
   public async $getLatestPriceTime(): Promise<number> {
-    const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices ORDER BY time DESC LIMIT 1`);
+    const [oldestRow] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time DESC LIMIT 1`);
     return oldestRow[0] ? oldestRow[0].time : 0;
   }
 
   public async $getPricesTimes(): Promise<number[]> {
-    const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices`);
+    const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1`);
     return times.map(time => time.time);
   }
 }

docker/backend/mempool-config.json

@@ -20,7 +20,8 @@
     "USER_AGENT": "__MEMPOOL_USER_AGENT__",
     "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
     "INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__,
-    "BLOCKS_SUMMARIES_INDEXING": __MEMPOOL_BLOCKS_SUMMARIES_INDEXING__
+    "BLOCKS_SUMMARIES_INDEXING": __MEMPOOL_BLOCKS_SUMMARIES_INDEXING__,
+    "AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",

docker/backend/start.sh

@@ -22,6 +22,8 @@ __MEMPOOL_EXTERNAL_MAX_RETRY__=${MEMPOOL_EXTERNAL_MAX_RETRY:=1}
 __MEMPOOL_EXTERNAL_RETRY_INTERVAL__=${MEMPOOL_EXTERNAL_RETRY_INTERVAL:=0}
 __MEMPOOL_USER_AGENT__=${MEMPOOL_USER_AGENT:=mempool}
 __MEMPOOL_STDOUT_LOG_MIN_PRIORITY__=${MEMPOOL_STDOUT_LOG_MIN_PRIORITY:=info}
+__MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=false}
+__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false}
 
 # CORE_RPC
 __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
@@ -110,6 +112,8 @@ sed -i "s!__MEMPOOL_EXTERNAL_MAX_RETRY__!${__MEMPOOL_EXTERNAL_MAX_RETRY__}!g" me
 sed -i "s!__MEMPOOL_EXTERNAL_RETRY_INTERVAL__!${__MEMPOOL_EXTERNAL_RETRY_INTERVAL__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_USER_AGENT__!${__MEMPOOL_USER_AGENT__}!g" mempool-config.json
 sed -i "s/__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__/${__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__}/g" mempool-config.json
+sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json
+sed -i "s/__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__/${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}/g" mempool-config.json
 
 sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json