Merge pull request #3448 from mempool/simon/fix-missing-temp-cache-disk-cache

Fix missing temp cache in disk cache

backend/mempool-config.sample.json

@@ -27,7 +27,8 @@
     "AUDIT": false,
     "ADVANCED_GBT_AUDIT": false,
     "ADVANCED_GBT_MEMPOOL": false,
-    "CPFP_INDEXING": false
+    "CPFP_INDEXING": false,
+    "DISK_CACHE_BLOCK_INTERVAL": 6
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",

backend/src/__fixtures__/mempool-config.template.json

@@ -28,7 +28,8 @@
     "ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__",
     "ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__",
     "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__",
-    "MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__"
+    "MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__",
+    "DISK_CACHE_BLOCK_INTERVAL": "__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__"
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",

backend/src/__tests__/config.test.ts

@@ -42,6 +42,7 @@ describe('Mempool Backend Config', () => {
         ADVANCED_GBT_MEMPOOL: false,
         CPFP_INDEXING: false,
         MAX_BLOCKS_BULK_QUERY: 0,
+        DISK_CACHE_BLOCK_INTERVAL: 6,
       });
 
       expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });

backend/src/api/blocks.ts

@@ -651,7 +651,7 @@ class Blocks {
       if (this.newBlockCallbacks.length) {
         this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
       }
-      if (!memPool.hasPriority() && (block.height % 6 === 0)) {
+      if (!memPool.hasPriority() && (block.height % config.MEMPOOL.DISK_CACHE_BLOCK_INTERVAL === 0)) {
         diskCache.$saveCacheToDisk();
       }
 

backend/src/api/disk-cache.ts

@@ -19,20 +19,16 @@ class DiskCache {
   private isWritingCache = false;
 
   constructor() {
-    if (!cluster.isMaster) {
+    if (!cluster.isPrimary) {
       return;
     }
     process.on('SIGINT', (e) => {
-      this.saveCacheToDiskSync();
+      this.$saveCacheToDisk(true);
-      process.exit(2);
+      process.exit(0);
-    });
-    process.on('SIGTERM', (e) => {
-      this.saveCacheToDiskSync();
-      process.exit(2);
     });
   }
 
-  async $saveCacheToDisk(): Promise<void> {
+  async $saveCacheToDisk(sync: boolean = false): Promise<void> {
     if (!cluster.isPrimary) {
       return;
     }
@@ -41,51 +37,7 @@ class DiskCache {
       return;
     }
     try {
-      logger.debug('Writing mempool and blocks data to disk cache (async)...');
+      logger.debug(`Writing mempool and blocks data to disk cache (${ sync ? 'sync' : 'async' })...`);
-      this.isWritingCache = true;
-
-      const mempool = memPool.getMempool();
-      const mempoolArray: TransactionExtended[] = [];
-      for (const tx in mempool) {
-        mempoolArray.push(mempool[tx]);
-      }
-
-      Common.shuffleArray(mempoolArray);
-
-      const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
-
-      await fsPromises.writeFile(DiskCache.FILE_NAME, JSON.stringify({
-        network: config.MEMPOOL.NETWORK,
-        cacheSchemaVersion: this.cacheSchemaVersion,
-        blocks: blocks.getBlocks(),
-        blockSummaries: blocks.getBlockSummaries(),
-        mempool: {},
-        mempoolArray: mempoolArray.splice(0, chunkSize),
-      }), { flag: 'w' });
-      for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-        await fsPromises.writeFile(DiskCache.FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
-          mempool: {},
-          mempoolArray: mempoolArray.splice(0, chunkSize),
-        }), { flag: 'w' });
-      }
-      logger.debug('Mempool and blocks data saved to disk cache');
-      this.isWritingCache = false;
-    } catch (e) {
-      logger.warn('Error writing to cache file: ' + (e instanceof Error ? e.message : e));
-      this.isWritingCache = false;
-    }
-  }
-
-  saveCacheToDiskSync(): void {
-    if (!cluster.isPrimary) {
-      return;
-    }
-    if (this.isWritingCache) {
-      logger.debug('Saving cache already in progress. Skipping.');
-      return;
-    }
-    try {
-      logger.debug('Writing mempool and blocks data to disk cache (sync)...');
       this.isWritingCache = true;
 
       const mempool = memPool.getMempool();
@@ -98,6 +50,7 @@ class DiskCache {
 
       const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
 
+      if (sync) {
         fs.writeFileSync(DiskCache.TMP_FILE_NAME, JSON.stringify({
           network: config.MEMPOOL.NETWORK,
           cacheSchemaVersion: this.cacheSchemaVersion,
@@ -117,6 +70,27 @@ class DiskCache {
         for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
           fs.renameSync(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
         }
+      } else {
+        await fsPromises.writeFile(DiskCache.TMP_FILE_NAME, JSON.stringify({
+          network: config.MEMPOOL.NETWORK,
+          cacheSchemaVersion: this.cacheSchemaVersion,
+          blocks: blocks.getBlocks(),
+          blockSummaries: blocks.getBlockSummaries(),
+          mempool: {},
+          mempoolArray: mempoolArray.splice(0, chunkSize),
+        }), { flag: 'w' });
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          await fsPromises.writeFile(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
+            mempool: {},
+            mempoolArray: mempoolArray.splice(0, chunkSize),
+          }), { flag: 'w' });
+        }
+
+        await fsPromises.rename(DiskCache.TMP_FILE_NAME, DiskCache.FILE_NAME);
+        for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
+          await fsPromises.rename(DiskCache.TMP_FILE_NAMES.replace('{number}', i.toString()), DiskCache.FILE_NAMES.replace('{number}', i.toString()));
+        }
+      }
 
       logger.debug('Mempool and blocks data saved to disk cache');
       this.isWritingCache = false;

backend/src/config.ts

@@ -33,6 +33,7 @@ interface IConfig {
     ADVANCED_GBT_MEMPOOL: boolean;
     CPFP_INDEXING: boolean;
     MAX_BLOCKS_BULK_QUERY: number;
+    DISK_CACHE_BLOCK_INTERVAL: number;
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -155,6 +156,7 @@ const defaults: IConfig = {
     'ADVANCED_GBT_MEMPOOL': false,
     'CPFP_INDEXING': false,
     'MAX_BLOCKS_BULK_QUERY': 0,
+    'DISK_CACHE_BLOCK_INTERVAL': 6,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',

docker/README.md

@@ -112,6 +112,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over
     "ADVANCED_GBT_MEMPOOL": false,
     "CPFP_INDEXING": false,
     "MAX_BLOCKS_BULK_QUERY": 0,
+    "DISK_CACHE_BLOCK_INTERVAL": 6
   },
 ```
 
@@ -143,6 +144,7 @@ Corresponding `docker-compose.yml` overrides:
       MEMPOOL_ADVANCED_GBT_MEMPOOL: ""
       MEMPOOL_CPFP_INDEXING: ""
       MAX_BLOCKS_BULK_QUERY: ""
+      DISK_CACHE_BLOCK_INTERVAL: ""
       ...
 ```
 

docker/backend/mempool-config.json

@@ -26,7 +26,8 @@
     "ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__,
     "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__,
     "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__,
-    "MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__
+    "MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__,
+    "DISK_CACHE_BLOCK_INTERVAL": __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",

docker/backend/start.sh

@@ -30,6 +30,7 @@ __MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false}
 __MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false}
 __MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
 __MEMPOOL_MAX_BLOCKS_BULK_QUERY__=${MEMPOOL_MAX_BLOCKS_BULK_QUERY:=0}
+__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__=${MEMPOOL_DISK_CACHE_BLOCK_INTERVAL:=6}
 
 # CORE_RPC
 __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1}
@@ -149,6 +150,7 @@ sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g
 sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json
 
 sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json
 sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json

production/mempool-config.liquid.json

@@ -6,7 +6,8 @@
     "MINED_BLOCKS_CACHE": 144,
     "SPAWN_CLUSTER_PROCS": 0,
     "API_URL_PREFIX": "/api/v1/",
-    "POLL_RATE_MS": 1000
+    "POLL_RATE_MS": 1000,
+    "DISK_CACHE_BLOCK_INTERVAL": 1
   },
   "SYSLOG" : {
     "MIN_PRIORITY": "debug"

production/mempool-config.liquidtestnet.json

@@ -6,7 +6,8 @@
     "MINED_BLOCKS_CACHE": 144,
     "SPAWN_CLUSTER_PROCS": 0,
     "API_URL_PREFIX": "/api/v1/",
-    "POLL_RATE_MS": 1000
+    "POLL_RATE_MS": 1000,
+    "DISK_CACHE_BLOCK_INTERVAL": 1
   },
   "SYSLOG" : {
     "MIN_PRIORITY": "debug"

production/mempool-config.mainnet.json

@@ -14,7 +14,8 @@
     "CPFP_INDEXING": true,
     "ADVANCED_GBT_AUDIT": true,
     "ADVANCED_GBT_MEMPOOL": true,
-    "USE_SECOND_NODE_FOR_MINFEE": true
+    "USE_SECOND_NODE_FOR_MINFEE": true,
+    "DISK_CACHE_BLOCK_INTERVAL": 1
   },
   "SYSLOG" : {
     "MIN_PRIORITY": "debug"

production/mempool-config.signet.json

@@ -10,7 +10,8 @@
     "AUDIT": true,
     "ADVANCED_GBT_AUDIT": true,
     "ADVANCED_GBT_MEMPOOL": true,
-    "POLL_RATE_MS": 1000
+    "POLL_RATE_MS": 1000,
+    "DISK_CACHE_BLOCK_INTERVAL": 1
   },
   "SYSLOG" : {
     "MIN_PRIORITY": "debug"

production/mempool-config.testnet.json

@@ -10,7 +10,8 @@
     "AUDIT": true,
     "ADVANCED_GBT_AUDIT": true,
     "ADVANCED_GBT_MEMPOOL": true,
-    "POLL_RATE_MS": 1000
+    "POLL_RATE_MS": 1000,
+    "DISK_CACHE_BLOCK_INTERVAL": 1
   },
   "SYSLOG" : {
     "MIN_PRIORITY": "debug"