[main loop] retry every seconds upon exception - warn after 5 attempts
This commit is contained in:
parent
ab7cb5f681
commit
c23e529f0a
@ -45,7 +45,8 @@ class Server {
|
|||||||
private wss: WebSocket.Server | undefined;
|
private wss: WebSocket.Server | undefined;
|
||||||
private server: http.Server | undefined;
|
private server: http.Server | undefined;
|
||||||
private app: Application;
|
private app: Application;
|
||||||
private currentBackendRetryInterval = 5;
|
private currentBackendRetryInterval = 1;
|
||||||
|
private backendRetryCount = 0;
|
||||||
|
|
||||||
private maxHeapSize: number = 0;
|
private maxHeapSize: number = 0;
|
||||||
private heapLogInterval: number = 60;
|
private heapLogInterval: number = 60;
|
||||||
@ -184,17 +185,17 @@ class Server {
|
|||||||
indexer.$run();
|
indexer.$run();
|
||||||
|
|
||||||
setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
|
setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
|
||||||
this.currentBackendRetryInterval = 5;
|
this.backendRetryCount = 0;
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
let loggerMsg = `Exception in runMainUpdateLoop(). Retrying in ${this.currentBackendRetryInterval} sec.`;
|
this.backendRetryCount++;
|
||||||
|
let loggerMsg = `Exception in runMainUpdateLoop() (count: ${this.backendRetryCount}). Retrying in ${this.currentBackendRetryInterval} sec.`;
|
||||||
loggerMsg += ` Reason: ${(e instanceof Error ? e.message : e)}.`;
|
loggerMsg += ` Reason: ${(e instanceof Error ? e.message : e)}.`;
|
||||||
if (e?.stack) {
|
if (e?.stack) {
|
||||||
loggerMsg += ` Stack trace: ${e.stack}`;
|
loggerMsg += ` Stack trace: ${e.stack}`;
|
||||||
}
|
}
|
||||||
// When we get a first Exception, only `logger.debug` it and retry after 5 seconds
|
// When we get a first Exception, only `logger.debug` it and retry after 5 seconds
|
||||||
// From the second Exception, `logger.warn` the Exception and increase the retry delay
|
// From the second Exception, `logger.warn` the Exception and increase the retry delay
|
||||||
// Maximum retry delay is 60 seconds
|
if (this.backendRetryCount >= 5) {
|
||||||
if (this.currentBackendRetryInterval > 5) {
|
|
||||||
logger.warn(loggerMsg);
|
logger.warn(loggerMsg);
|
||||||
mempool.setOutOfSync();
|
mempool.setOutOfSync();
|
||||||
} else {
|
} else {
|
||||||
@ -204,8 +205,6 @@ class Server {
|
|||||||
logger.debug(`AxiosError: ${e?.message}`);
|
logger.debug(`AxiosError: ${e?.message}`);
|
||||||
}
|
}
|
||||||
setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.currentBackendRetryInterval);
|
setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.currentBackendRetryInterval);
|
||||||
this.currentBackendRetryInterval *= 2;
|
|
||||||
this.currentBackendRetryInterval = Math.min(this.currentBackendRetryInterval, 60);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user