Fix failover debug prints

This commit is contained in:
Mononaut 2023-08-05 19:55:33 +09:00
parent e512feef74
commit ae5e1e6d29
No known key found for this signature in database
GPG Key ID: A3F058E41374C04E

View File

@ -7,8 +7,8 @@ import logger from '../../logger';
interface FailoverHost { interface FailoverHost {
host: string, host: string,
latencies: number[], rtts: number[],
latency: number rtt: number
failures: number, failures: number,
socket?: boolean, socket?: boolean,
outOfSync?: boolean, outOfSync?: boolean,
@ -33,15 +33,15 @@ class FailoverRouter {
this.hosts = (config.ESPLORA.FALLBACK || []).map(domain => { this.hosts = (config.ESPLORA.FALLBACK || []).map(domain => {
return { return {
host: 'https://' + domain + '/api', host: 'https://' + domain + '/api',
latencies: [], rtts: [],
latency: Infinity, rtt: Infinity,
failures: 0, failures: 0,
}; };
}); });
this.activeHost = { this.activeHost = {
host: config.ESPLORA.UNIX_SOCKET_PATH || config.ESPLORA.REST_API_URL, host: config.ESPLORA.UNIX_SOCKET_PATH || config.ESPLORA.REST_API_URL,
latencies: [], rtts: [],
latency: 0, rtt: 0,
failures: 0, failures: 0,
socket: !!config.ESPLORA.UNIX_SOCKET_PATH, socket: !!config.ESPLORA.UNIX_SOCKET_PATH,
preferred: true, preferred: true,
@ -52,13 +52,13 @@ class FailoverRouter {
} }
public startHealthChecks(): void { public startHealthChecks(): void {
// use axios interceptors to measure request latency // use axios interceptors to measure request rtt
this.pollConnection.interceptors.request.use((config) => { this.pollConnection.interceptors.request.use((config) => {
config['meta'] = { startTime: Date.now() }; config['meta'] = { startTime: Date.now() };
return config; return config;
}); });
this.pollConnection.interceptors.response.use((response) => { this.pollConnection.interceptors.response.use((response) => {
response.config['meta'].latency = Date.now() - response.config['meta'].startTime; response.config['meta'].rtt = Date.now() - response.config['meta'].startTime;
return response; return response;
}); });
@ -67,7 +67,7 @@ class FailoverRouter {
} }
} }
// start polling hosts to measure availability & latency // start polling hosts to measure availability & rtt
private async pollHosts(): Promise<void> { private async pollHosts(): Promise<void> {
if (this.pollTimer) { if (this.pollTimer) {
clearTimeout(this.pollTimer); clearTimeout(this.pollTimer);
@ -82,16 +82,16 @@ class FailoverRouter {
})); }));
const maxHeight = results.reduce((max, result) => Math.max(max, result.status === 'fulfilled' ? result.value?.data || 0 : 0), 0); const maxHeight = results.reduce((max, result) => Math.max(max, result.status === 'fulfilled' ? result.value?.data || 0 : 0), 0);
// update latencies & sync status // update rtts & sync status
for (let i = 0; i < results.length; i++) { for (let i = 0; i < results.length; i++) {
const host = this.hosts[i]; const host = this.hosts[i];
const result = results[i].status === 'fulfilled' ? (results[i] as PromiseFulfilledResult<AxiosResponse<number, any>>).value : null; const result = results[i].status === 'fulfilled' ? (results[i] as PromiseFulfilledResult<AxiosResponse<number, any>>).value : null;
if (result) { if (result) {
const height = result.data; const height = result.data;
const latency = result.config['meta'].latency; const rtt = result.config['meta'].rtt;
host.latencies.unshift(latency); host.rtts.unshift(rtt);
host.latencies.slice(0, 5); host.rtts.slice(0, 5);
host.latency = host.latencies.reduce((acc, l) => acc + l, 0) / host.latencies.length; host.rtt = host.rtts.reduce((acc, l) => acc + l, 0) / host.rtts.length;
if (height == null || isNaN(height) || (maxHeight - height > 2)) { if (height == null || isNaN(height) || (maxHeight - height > 2)) {
host.outOfSync = true; host.outOfSync = true;
} else { } else {
@ -105,10 +105,10 @@ class FailoverRouter {
this.sortHosts(); this.sortHosts();
logger.debug(`Tomahawk ranking: ${this.hosts.map(host => '\navg latency ' + Math.round(host.latency).toString().padStart(5, ' ') + ' | reachable? ' + !(host.unreachable || false).toString().padStart(5, ' ') + ' | in sync? ' + !(host.outOfSync || false).toString().padStart(5, ' ') + ` | ${host.host}`).join('')}`); logger.debug(`Tomahawk ranking: ${this.hosts.map(host => '\navg rtt ' + Math.round(host.rtt).toString().padStart(5, ' ') + ' | reachable? ' + (!host.unreachable || false).toString().padStart(5, ' ') + ' | in sync? ' + (!host.outOfSync || false).toString().padStart(5, ' ') + ` | ${host.host}`).join('')}`);
// switch if the current host is out of sync or significantly slower than the next best alternative // switch if the current host is out of sync or significantly slower than the next best alternative
if (this.activeHost.outOfSync || this.activeHost.unreachable || (!this.activeHost.preferred && this.activeHost.latency > (this.hosts[0].latency * 2) + 50)) { if (this.activeHost.outOfSync || this.activeHost.unreachable || (!this.activeHost.preferred && this.activeHost.rtt > (this.hosts[0].rtt * 2) + 50)) {
if (this.activeHost.unreachable) { if (this.activeHost.unreachable) {
logger.warn(`Unable to reach ${this.activeHost.host}, failing over to next best alternative`); logger.warn(`Unable to reach ${this.activeHost.host}, failing over to next best alternative`);
} else if (this.activeHost.outOfSync) { } else if (this.activeHost.outOfSync) {
@ -128,8 +128,8 @@ class FailoverRouter {
this.hosts.sort((a, b) => { this.hosts.sort((a, b) => {
if ((a.unreachable || a.outOfSync) === (b.unreachable || b.outOfSync)) { if ((a.unreachable || a.outOfSync) === (b.unreachable || b.outOfSync)) {
if (a.preferred === b.preferred) { if (a.preferred === b.preferred) {
// lower latency is best // lower rtt is best
return a.latency - b.latency; return a.rtt - b.rtt;
} else { // unless we have a preferred host } else { // unless we have a preferred host
return a.preferred ? -1 : 1; return a.preferred ? -1 : 1;
} }