[main loop] retry every seconds upon exception - warn after 5 attempts

This commit is contained in:
nymkappa 2023-04-05 22:44:01 +09:00
parent ab7cb5f681
commit c23e529f0a
No known key found for this signature in database
GPG Key ID: E155910B16E8BD04

View File

@ -45,7 +45,8 @@ class Server {
private wss: WebSocket.Server | undefined; private wss: WebSocket.Server | undefined;
private server: http.Server | undefined; private server: http.Server | undefined;
private app: Application; private app: Application;
private currentBackendRetryInterval = 5; private currentBackendRetryInterval = 1;
private backendRetryCount = 0;
private maxHeapSize: number = 0; private maxHeapSize: number = 0;
private heapLogInterval: number = 60; private heapLogInterval: number = 60;
@ -184,17 +185,17 @@ class Server {
indexer.$run(); indexer.$run();
setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS); setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
this.currentBackendRetryInterval = 5; this.backendRetryCount = 0;
} catch (e: any) { } catch (e: any) {
let loggerMsg = `Exception in runMainUpdateLoop(). Retrying in ${this.currentBackendRetryInterval} sec.`; this.backendRetryCount++;
let loggerMsg = `Exception in runMainUpdateLoop() (count: ${this.backendRetryCount}). Retrying in ${this.currentBackendRetryInterval} sec.`;
loggerMsg += ` Reason: ${(e instanceof Error ? e.message : e)}.`; loggerMsg += ` Reason: ${(e instanceof Error ? e.message : e)}.`;
if (e?.stack) { if (e?.stack) {
loggerMsg += ` Stack trace: ${e.stack}`; loggerMsg += ` Stack trace: ${e.stack}`;
} }
// When we get a first Exception, only `logger.debug` it and retry after 5 seconds // When we get a first Exception, only `logger.debug` it and retry after 5 seconds
// From the second Exception, `logger.warn` the Exception and increase the retry delay // From the second Exception, `logger.warn` the Exception and increase the retry delay
// Maximum retry delay is 60 seconds if (this.backendRetryCount >= 5) {
if (this.currentBackendRetryInterval > 5) {
logger.warn(loggerMsg); logger.warn(loggerMsg);
mempool.setOutOfSync(); mempool.setOutOfSync();
} else { } else {
@ -204,8 +205,6 @@ class Server {
logger.debug(`AxiosError: ${e?.message}`); logger.debug(`AxiosError: ${e?.message}`);
} }
setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.currentBackendRetryInterval); setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.currentBackendRetryInterval);
this.currentBackendRetryInterval *= 2;
this.currentBackendRetryInterval = Math.min(this.currentBackendRetryInterval, 60);
} }
} }