Fix: Split mempool disk cache into 50K txs per file

backend/.gitignore

@@ -43,3 +43,4 @@ testem.log
 Thumbs.db
 
 cache.json
+cache2.json

backend/src/api/blocks.ts

@@ -99,7 +99,7 @@ class Blocks {
       if (this.newBlockCallbacks.length) {
         this.newBlockCallbacks.forEach((cb) => cb(block, txIds, transactions));
       }
-      diskCache.$saveCacheToDiskAsync();
+      diskCache.$saveCacheToDisk();
     }
   }
 

backend/src/api/disk-cache.ts

@@ -1,85 +1,74 @@
-import config from '../config';
 import * as fs from 'fs';
+const fsPromises = fs.promises;
 import * as process from 'process';
+import * as cluster from 'cluster';
 import memPool from './mempool';
 import blocks from './blocks';
 import logger from '../logger';
 
 class DiskCache {
   private static FILE_NAME = './cache.json';
-  private enabled = true;
+  private static FILE_NAME_2 = './cache2.json';
-
   constructor() {
-    if (process.env.workerId === '0' || !config.MEMPOOL.SPAWN_CLUSTER_PROCS) {
+    if (!cluster.isMaster) {
-      if (!fs.existsSync(DiskCache.FILE_NAME)) {
+      return;
-        fs.closeSync(fs.openSync(DiskCache.FILE_NAME, 'w'));
-        logger.info('Disk cache file created');
-      }
-
-      process.on('SIGINT', () => {
-        this.saveCacheToDisk();
-        process.exit(2);
-      });
-
-      process.on('SIGTERM', () => {
-        this.saveCacheToDisk();
-        process.exit(2);
-      });
-    } else {
-      this.enabled = false;
     }
+    /*
+    if (!fs.existsSync(DiskCache.FILE_NAME)) {
+      fs.closeSync(fs.openSync(DiskCache.FILE_NAME, 'w'));
+      logger.info('Disk cache file created');
+    }
+    */
+    process.on('SIGINT', async () => {
+      await this.$saveCacheToDisk();
+      process.exit(2);
+    });
+
+    process.on('SIGTERM', async () => {
+      await this.$saveCacheToDisk();
+      process.exit(2);
+    });
   }
 
-  async $saveCacheToDiskAsync(): Promise<void> {
+  async $saveCacheToDisk(): Promise<void> {
-    if (!this.enabled) {
+    if (!cluster.isMaster) {
       return;
     }
     try {
-      await this.$saveDataAsync(JSON.stringify({
+      logger.debug('Writing mempool and blocks data to disk cache...');
-        mempool: memPool.getMempool(),
+      const mempoolChunk_1 = Object.fromEntries(Object.entries(memPool.getMempool()).splice(0, 50000));
+      const mempoolChunk_2 = Object.fromEntries(Object.entries(memPool.getMempool()).splice(50000));
+      await fsPromises.writeFile(DiskCache.FILE_NAME, JSON.stringify({
         blocks: blocks.getBlocks(),
-      }));
+        mempool: mempoolChunk_1
+      }), {flag: 'w'});
+      await fsPromises.writeFile(DiskCache.FILE_NAME_2, JSON.stringify({
+        mempool: mempoolChunk_2
+      }), {flag: 'w'});
       logger.debug('Mempool and blocks data saved to disk cache');
     } catch (e) {
-      logger.warn('Error writing to cache file asynchronously: ' + e.message || e);
+      logger.warn('Error writing to cache file: ' + e.message || e);
     }
   }
 
   loadMempoolCache() {
-    const cacheData = this.loadDataSync();
+    if (!fs.existsSync(DiskCache.FILE_NAME)) {
+      return;
+    }
+    let data: any = {};
+    const cacheData = fs.readFileSync(DiskCache.FILE_NAME, 'utf8');
     if (cacheData) {
       logger.info('Restoring mempool and blocks data from disk cache');
-      const data = JSON.parse(cacheData);
+      data = JSON.parse(cacheData);
-      memPool.setMempool(data.mempool);
-      blocks.setBlocks(data.blocks);
     }
-  }
 
-  private saveCacheToDisk() {
+    if (fs.existsSync(DiskCache.FILE_NAME_2)) {
-    this.saveDataSync(JSON.stringify({
+      const cacheData2 = JSON.parse(fs.readFileSync(DiskCache.FILE_NAME_2, 'utf8'));
-      mempool: memPool.getMempool(),
+      Object.assign(data.mempool, cacheData2.mempool);
-      blocks: blocks.getBlocks(),
+    }
-    }));
-    logger.info('Mempool and blocks data saved to disk cache');
-  }
 
-  private $saveDataAsync(dataBlob: string): Promise<void> {
+    memPool.setMempool(data.mempool);
-    return new Promise((resolve, reject) => {
+    blocks.setBlocks(data.blocks);
-      fs.writeFile(DiskCache.FILE_NAME, dataBlob, (err) => {
-        if (err) {
-          return reject(err);
-        }
-        resolve();
-      });
-    });
-  }
-
-  private saveDataSync(dataBlob: string) {
-    fs.writeFileSync(DiskCache.FILE_NAME, dataBlob, 'utf8');
-  }
-
-  private loadDataSync(): string {
-    return fs.readFileSync(DiskCache.FILE_NAME, 'utf8');
   }
 }
 

backend/tsconfig.json

@@ -1,7 +1,8 @@
 {
   "compilerOptions": {
     "module": "commonjs",
-    "target": "es2015",
+    "target": "esnext",
+    "lib": ["es2019"],
     "strict": true,
     "noImplicitAny": false,
     "sourceMap": false,