From 44a0913b818713acc0da935570628ab4e1822d6b Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Wed, 5 Apr 2023 16:27:13 +0900 Subject: [PATCH 1/6] [esplora] fallback to tcp socket if unix socket fails --- backend/src/api/bitcoin/esplora-api.ts | 76 +++++++++++++++++--------- backend/src/config.ts | 2 + 2 files changed, 53 insertions(+), 25 deletions(-) diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index ff6219587..73e810a5c 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -3,68 +3,96 @@ import axios, { AxiosRequestConfig } from 'axios'; import http from 'http'; import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; import { IEsploraApi } from './esplora-api.interface'; +import logger from '../../logger'; const axiosConnection = axios.create({ httpAgent: new http.Agent({ keepAlive: true, }) }); class ElectrsApi implements AbstractBitcoinApi { - axiosConfig: AxiosRequestConfig = config.ESPLORA.UNIX_SOCKET_PATH ? { + private axiosConfigWithUnixSocket: AxiosRequestConfig = config.ESPLORA.UNIX_SOCKET_PATH ? { socketPath: config.ESPLORA.UNIX_SOCKET_PATH, timeout: 10000, } : { timeout: 10000, }; + private axiosConfigTcpSocketOnly: AxiosRequestConfig = { + timeout: 10000, + }; - constructor() { } + unixSocketRetryTimeout; + activeAxiosConfig; + + fallbackToTcpSocket() { + if (!this.unixSocketRetryTimeout) { + logger.err(`Unable to connect to esplora unix socket. Falling back to tcp socket. Retrying unix socket in ${config.ESPLORA.RETRY_UNIX_SOCKET_AFTER} seconds`); + // Retry the unix socket after a few seconds + this.unixSocketRetryTimeout = setTimeout(() => { + this.activeAxiosConfig = this.axiosConfigWithUnixSocket; + }, config.ESPLORA.RETRY_UNIX_SOCKET_AFTER); + } + + // Use the TCP socket (reach a different esplora instance through nginx) + this.activeAxiosConfig = this.axiosConfigTcpSocketOnly; + } + + $queryWrapper(url, responseType = 'json'): Promise { + return axiosConnection.get(url, { ...this.activeAxiosConfig, responseType: responseType }) + .then((response) => response.data) + .catch((e) => { + if (e?.code === 'ECONNREFUSED' || e?.code === 'ETIMEDOUT') { + this.fallbackToTcpSocket(); + // Retry immediately + return axiosConnection.get(url, this.activeAxiosConfig) + .then((response) => response.data) + .catch((e) => { + logger.warn(`Cannot query esplora through the unix socket nor the tcp socket. Exception ${e}`); + throw e; + }); + } else { + throw e; + } + }); + } $getRawMempool(): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig) - .then((response) => response.data); + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/mempool/txids'); } $getRawTransaction(txId: string): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig) - .then((response) => response.data); + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/tx/' + txId); } $getTransactionHex(txId: string): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex', this.axiosConfig) - .then((response) => response.data); + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex'); } $getBlockHeightTip(): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig) - .then((response) => response.data); + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/blocks/tip/height'); } $getBlockHashTip(): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig) - .then((response) => response.data); + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/blocks/tip/hash'); } $getTxIdsForBlock(hash: string): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig) - .then((response) => response.data); + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids'); } $getBlockHash(height: number): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig) - .then((response) => response.data); + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/block-height/' + height); } $getBlockHeader(hash: string): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig) - .then((response) => response.data); + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header'); } $getBlock(hash: string): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig) - .then((response) => response.data); + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/block/' + hash); } $getRawBlock(hash: string): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' }) + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", 'arraybuffer') .then((response) => { return Buffer.from(response.data); }); } @@ -85,13 +113,11 @@ class ElectrsApi implements AbstractBitcoinApi { } $getOutspend(txId: string, vout: number): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig) - .then((response) => response.data); + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout); } $getOutspends(txId: string): Promise { - return axiosConnection.get(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig) - .then((response) => response.data); + return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends'); } async $getBatchedOutspends(txId: string[]): Promise { diff --git a/backend/src/config.ts b/backend/src/config.ts index 81db16081..7c0e4e950 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -38,6 +38,7 @@ interface IConfig { ESPLORA: { REST_API_URL: string; UNIX_SOCKET_PATH: string | void | null; + RETRY_UNIX_SOCKET_AFTER: number; }; LIGHTNING: { ENABLED: boolean; @@ -165,6 +166,7 @@ const defaults: IConfig = { 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', 'UNIX_SOCKET_PATH': null, + 'RETRY_UNIX_SOCKET_AFTER': 30000, }, 'ELECTRUM': { 'HOST': '127.0.0.1', From b6f1fd5a4a8ec59c7e12248c76b9eb731ef9a477 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Wed, 5 Apr 2023 16:38:37 +0900 Subject: [PATCH 2/6] [esplora] initialize default socket config to axiosConfigWithUnixSocket --- backend/src/api/bitcoin/esplora-api.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index 73e810a5c..15a6a6844 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -23,6 +23,10 @@ class ElectrsApi implements AbstractBitcoinApi { unixSocketRetryTimeout; activeAxiosConfig; + constructor() { + this.activeAxiosConfig = this.axiosConfigWithUnixSocket; + } + fallbackToTcpSocket() { if (!this.unixSocketRetryTimeout) { logger.err(`Unable to connect to esplora unix socket. Falling back to tcp socket. Retrying unix socket in ${config.ESPLORA.RETRY_UNIX_SOCKET_AFTER} seconds`); From 66109afb0d5dadcd7ea736423a8cb471e0974f13 Mon Sep 17 00:00:00 2001 From: wiz Date: Wed, 5 Apr 2023 16:48:26 +0900 Subject: [PATCH 3/6] ops: Enable unix socket for esplora on mainnet-lightning --- production/mempool-config.mainnet-lightning.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/mempool-config.mainnet-lightning.json b/production/mempool-config.mainnet-lightning.json index 546c58163..37e407aeb 100644 --- a/production/mempool-config.mainnet-lightning.json +++ b/production/mempool-config.mainnet-lightning.json @@ -15,7 +15,7 @@ "PASSWORD": "__BITCOIN_RPC_PASS__" }, "ESPLORA": { - "REST_API_URL": "http://127.0.0.1:4000" + "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-mainnet" }, "LIGHTNING": { "ENABLED": true, From db27e5a92c8feb98d1ddc0ee38b3bcaa4cc98444 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Wed, 5 Apr 2023 17:00:41 +0900 Subject: [PATCH 4/6] [esplora] print log when retrying unix socket - don't fallback to tcp socket on ETIMEDOUT --- backend/src/api/bitcoin/esplora-api.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index 15a6a6844..e471e4cde 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -29,9 +29,10 @@ class ElectrsApi implements AbstractBitcoinApi { fallbackToTcpSocket() { if (!this.unixSocketRetryTimeout) { - logger.err(`Unable to connect to esplora unix socket. Falling back to tcp socket. Retrying unix socket in ${config.ESPLORA.RETRY_UNIX_SOCKET_AFTER} seconds`); + logger.err(`Unable to connect to esplora unix socket. Falling back to tcp socket. Retrying unix socket in ${config.ESPLORA.RETRY_UNIX_SOCKET_AFTER / 1000} seconds`); // Retry the unix socket after a few seconds this.unixSocketRetryTimeout = setTimeout(() => { + logger.info(`Retrying to use unix socket for esplora now (applied for the next query)`); this.activeAxiosConfig = this.axiosConfigWithUnixSocket; }, config.ESPLORA.RETRY_UNIX_SOCKET_AFTER); } @@ -44,7 +45,7 @@ class ElectrsApi implements AbstractBitcoinApi { return axiosConnection.get(url, { ...this.activeAxiosConfig, responseType: responseType }) .then((response) => response.data) .catch((e) => { - if (e?.code === 'ECONNREFUSED' || e?.code === 'ETIMEDOUT') { + if (e?.code === 'ECONNREFUSED') { this.fallbackToTcpSocket(); // Retry immediately return axiosConnection.get(url, this.activeAxiosConfig) From ab7cb5f68148335d6baed475ef9595f9dcfe225d Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Wed, 5 Apr 2023 17:05:23 +0900 Subject: [PATCH 5/6] [esplora] reset timeout variable when retrying unix socket --- backend/src/api/bitcoin/esplora-api.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index e471e4cde..ee7fa4765 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -34,6 +34,7 @@ class ElectrsApi implements AbstractBitcoinApi { this.unixSocketRetryTimeout = setTimeout(() => { logger.info(`Retrying to use unix socket for esplora now (applied for the next query)`); this.activeAxiosConfig = this.axiosConfigWithUnixSocket; + this.unixSocketRetryTimeout = undefined; }, config.ESPLORA.RETRY_UNIX_SOCKET_AFTER); } From c23e529f0ac404646871d1c45949836ecc57f52c Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Wed, 5 Apr 2023 22:44:01 +0900 Subject: [PATCH 6/6] [main loop] retry every seconds upon exception - warn after 5 attempts --- backend/src/index.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/backend/src/index.ts b/backend/src/index.ts index a7f805313..abaec9cef 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -45,7 +45,8 @@ class Server { private wss: WebSocket.Server | undefined; private server: http.Server | undefined; private app: Application; - private currentBackendRetryInterval = 5; + private currentBackendRetryInterval = 1; + private backendRetryCount = 0; private maxHeapSize: number = 0; private heapLogInterval: number = 60; @@ -184,17 +185,17 @@ class Server { indexer.$run(); setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS); - this.currentBackendRetryInterval = 5; + this.backendRetryCount = 0; } catch (e: any) { - let loggerMsg = `Exception in runMainUpdateLoop(). Retrying in ${this.currentBackendRetryInterval} sec.`; + this.backendRetryCount++; + let loggerMsg = `Exception in runMainUpdateLoop() (count: ${this.backendRetryCount}). Retrying in ${this.currentBackendRetryInterval} sec.`; loggerMsg += ` Reason: ${(e instanceof Error ? e.message : e)}.`; if (e?.stack) { loggerMsg += ` Stack trace: ${e.stack}`; } // When we get a first Exception, only `logger.debug` it and retry after 5 seconds // From the second Exception, `logger.warn` the Exception and increase the retry delay - // Maximum retry delay is 60 seconds - if (this.currentBackendRetryInterval > 5) { + if (this.backendRetryCount >= 5) { logger.warn(loggerMsg); mempool.setOutOfSync(); } else { @@ -204,8 +205,6 @@ class Server { logger.debug(`AxiosError: ${e?.message}`); } setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.currentBackendRetryInterval); - this.currentBackendRetryInterval *= 2; - this.currentBackendRetryInterval = Math.min(this.currentBackendRetryInterval, 60); } }