Merge pull request #342 from mempool/simon/optimize-disk-cache

Optimized cache chunks. Default cache files to cache/ directory.

backend/.gitignore

@@ -41,14 +41,3 @@ testem.log
 #System Files
 .DS_Store
 Thumbs.db
-
-cache.json
-cache1.json
-cache2.json
-cache3.json
-cache4.json
-cache5.json
-cache6.json
-cache7.json
-cache8.json
-cache9.json

backend/cache/.gitignore

@@ -0,0 +1 @@
+*.json

backend/mempool-config.sample.json

@@ -6,7 +6,7 @@
     "SPAWN_CLUSTER_PROCS": 0,
     "API_URL_PREFIX": "/api/v1/",
     "POLL_RATE_MS": 2000,
-    "CACHE_DIR": "./"
+    "CACHE_DIR": "./cache"
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",

backend/src/api/disk-cache.ts

@@ -5,11 +5,13 @@ import memPool from './mempool';
 import blocks from './blocks';
 import logger from '../logger';
 import config from '../config';
+import { TransactionExtended } from '../mempool.interfaces';
 
 class DiskCache {
-  private static FILE_NAME = config.MEMPOOL.CACHE_DIR + 'cache.json';
+  private static FILE_NAME = config.MEMPOOL.CACHE_DIR + '/cache.json';
-  private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + 'cache{number}.json';
+  private static FILE_NAMES = config.MEMPOOL.CACHE_DIR + '/cache{number}.json';
-  private static CHUNK_SIZE = 10000;
+  private static CHUNK_FILES = 25;
+
   constructor() { }
 
   async $saveCacheToDisk(): Promise<void> {
@@ -18,19 +20,24 @@ class DiskCache {
     }
     try {
       logger.debug('Writing mempool and blocks data to disk cache (async)...');
-      const mempoolChunk_1 = Object.fromEntries(Object.entries(memPool.getMempool()).slice(0, DiskCache.CHUNK_SIZE));
+
+      const mempool = memPool.getMempool();
+      const mempoolArray: TransactionExtended[] = [];
+      for (const tx in mempool) {
+        mempoolArray.push(mempool[tx]);
+      }
+
+      const chunkSize = Math.floor(mempoolArray.length / DiskCache.CHUNK_FILES);
+
       await fsPromises.writeFile(DiskCache.FILE_NAME, JSON.stringify({
         blocks: blocks.getBlocks(),
-        mempool: mempoolChunk_1
+        mempool: {},
+        mempoolArray: mempoolArray.splice(0, chunkSize),
       }), {flag: 'w'});
-      for (let i = 1; i < 10; i++) {
+      for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
-        const mempoolChunk = Object.fromEntries(
-          Object.entries(memPool.getMempool()).slice(
-            DiskCache.CHUNK_SIZE * i, i === 9 ? undefined : DiskCache.CHUNK_SIZE * i + DiskCache.CHUNK_SIZE
-          )
-        );
         await fsPromises.writeFile(DiskCache.FILE_NAMES.replace('{number}', i.toString()), JSON.stringify({
-          mempool: mempoolChunk
+          mempool: {},
+          mempoolArray: mempoolArray.splice(0, chunkSize),
         }), {flag: 'w'});
       }
       logger.debug('Mempool and blocks data saved to disk cache');
@@ -49,15 +56,26 @@ class DiskCache {
       if (cacheData) {
         logger.info('Restoring mempool and blocks data from disk cache');
         data = JSON.parse(cacheData);
+        if (data.mempoolArray) {
+          for (const tx of data.mempoolArray) {
+            data.mempool[tx.txid] = tx;
+          }
+        }
       }
 
-      for (let i = 1; i < 10; i++) {
+      for (let i = 1; i < DiskCache.CHUNK_FILES; i++) {
         const fileName = DiskCache.FILE_NAMES.replace('{number}', i.toString());
         if (fs.existsSync(fileName)) {
           const cacheData2 = JSON.parse(fs.readFileSync(fileName, 'utf8'));
+          if (cacheData2.mempoolArray) {
+            for (const tx of cacheData2.mempoolArray) {
+              data.mempool[tx.txid] = tx;
+            }
+          } else {
             Object.assign(data.mempool, cacheData2.mempool);
           }
         }
+      }
 
       memPool.setMempool(data.mempool);
       blocks.setBlocks(data.blocks);

backend/src/config.ts

@@ -61,7 +61,7 @@ const defaults: IConfig = {
     'SPAWN_CLUSTER_PROCS': 0,
     'API_URL_PREFIX': '/api/v1/',
     'POLL_RATE_MS': 2000,
-    'CACHE_DIR': './'
+    'CACHE_DIR': './cache'
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',