attempt to recover from stalling the main loop

backend/src/api/blocks.ts

@@ -530,7 +530,7 @@ class Blocks {
   }
 
   public async $updateBlocks() {
-    // warn if this run stalls the main loop for more than 2 minutes
+    // throw an error if this stalls the main loop for more than 2 minutes
     const timer = this.startTimer();
 
     let fastForwarded = false;
@@ -714,6 +714,11 @@ class Blocks {
 
   private updateTimerProgress(state, msg) {
     state.progress = msg;
+    if (Date.now() - state.start > this.mainLoopTimeout) {
+      // abort the function if it already timed out
+      logger.err(`$updateBlocks attempted to resume after "${state.progress}"`);
+      throw new Error(`$updateBlocks attempted to resume after "${state.progress}"`);
+    }
   }
 
   private clearTimer(state) {

backend/src/api/mempool.ts

@@ -122,7 +122,7 @@ class Mempool {
   public async $updateMempool(): Promise<void> {
     logger.debug(`Updating mempool...`);
 
-    // warn if this run stalls the main loop for more than 2 minutes
+    // throw an error if this stalls the main loop for more than 2 minutes
     const timer = this.startTimer();
 
     const start = new Date().getTime();
@@ -257,6 +257,11 @@ class Mempool {
 
   private updateTimerProgress(state, msg) {
     state.progress = msg;
+    if (Date.now() - state.start > this.mainLoopTimeout) {
+      // abort the function if it already timed out
+      logger.err(`$updateMempool attempted to resume after "${state.progress}"`);
+      throw new Error(`$updateMempool attempted to resume after "${state.progress}"`);
+    }
   }
 
   private clearTimer(state) {

backend/src/index.ts

@@ -179,8 +179,14 @@ class Server {
         }
       }
       memPool.deleteExpiredTransactions();
-      await blocks.$updateBlocks();
-      await memPool.$updateMempool();
+      await Promise.race([
+        blocks.$updateBlocks(),
+        new Promise((_, reject) => setTimeout(() => reject(new Error('$updateBlocks timed out')), 180000))
+      ]);
+      await Promise.race([
+        memPool.$updateMempool(),
+        new Promise((_, reject) => setTimeout(() => reject(new Error('$updateMempool timed out')), 180000))
+      ]);
       indexer.$run();
 
       setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);