Merge pull request #3643 from mempool/nymkappa/esplora-socket-fallback

[esplora] fallback to tcp socket if unix socket fails

backend/src/api/bitcoin/esplora-api.ts

@@ -3,68 +3,102 @@ import axios, { AxiosRequestConfig } from 'axios';
 import http from 'http';
 import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory';
 import { IEsploraApi } from './esplora-api.interface';
+import logger from '../../logger';
 
 const axiosConnection = axios.create({
   httpAgent: new http.Agent({ keepAlive: true, })
 });
 
 class ElectrsApi implements AbstractBitcoinApi {
-  axiosConfig: AxiosRequestConfig = config.ESPLORA.UNIX_SOCKET_PATH ? {
+  private axiosConfigWithUnixSocket: AxiosRequestConfig = config.ESPLORA.UNIX_SOCKET_PATH ? {
     socketPath: config.ESPLORA.UNIX_SOCKET_PATH,
     timeout: 10000,
   } : {
     timeout: 10000,
   };
+  private axiosConfigTcpSocketOnly: AxiosRequestConfig = {
+    timeout: 10000,
+  };
 
-  constructor() { }
+  unixSocketRetryTimeout;
+  activeAxiosConfig;
+
+  constructor() {
+    this.activeAxiosConfig = this.axiosConfigWithUnixSocket;
+  }
+
+  fallbackToTcpSocket() {
+    if (!this.unixSocketRetryTimeout) {
+      logger.err(`Unable to connect to esplora unix socket. Falling back to tcp socket. Retrying unix socket in ${config.ESPLORA.RETRY_UNIX_SOCKET_AFTER / 1000} seconds`);
+      // Retry the unix socket after a few seconds
+      this.unixSocketRetryTimeout = setTimeout(() => {
+        logger.info(`Retrying to use unix socket for esplora now (applied for the next query)`);
+        this.activeAxiosConfig = this.axiosConfigWithUnixSocket;
+        this.unixSocketRetryTimeout = undefined;
+      }, config.ESPLORA.RETRY_UNIX_SOCKET_AFTER);
+    }
+
+    // Use the TCP socket (reach a different esplora instance through nginx)
+    this.activeAxiosConfig = this.axiosConfigTcpSocketOnly;
+  }
+
+  $queryWrapper<T>(url, responseType = 'json'): Promise<T> {
+    return axiosConnection.get<T>(url, { ...this.activeAxiosConfig, responseType: responseType })
+      .then((response) => response.data)
+      .catch((e) => {
+        if (e?.code === 'ECONNREFUSED') {
+          this.fallbackToTcpSocket();
+          // Retry immediately
+          return axiosConnection.get<T>(url, this.activeAxiosConfig)
+            .then((response) => response.data)
+            .catch((e) => {
+              logger.warn(`Cannot query esplora through the unix socket nor the tcp socket. Exception ${e}`);
+              throw e;
+            });
+        } else {
+          throw e;
+        }
+      });
+  }
 
   $getRawMempool(): Promise<IEsploraApi.Transaction['txid'][]> {
-    return axiosConnection.get<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids', this.axiosConfig)
+    return this.$queryWrapper<IEsploraApi.Transaction['txid'][]>(config.ESPLORA.REST_API_URL + '/mempool/txids');
-      .then((response) => response.data);
   }
 
   $getRawTransaction(txId: string): Promise<IEsploraApi.Transaction> {
-    return axiosConnection.get<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId, this.axiosConfig)
+    return this.$queryWrapper<IEsploraApi.Transaction>(config.ESPLORA.REST_API_URL + '/tx/' + txId);
-      .then((response) => response.data);
   }
 
   $getTransactionHex(txId: string): Promise<string> {
-    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex', this.axiosConfig)
+    return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex');
-      .then((response) => response.data);
   }
 
   $getBlockHeightTip(): Promise<number> {
-    return axiosConnection.get<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height', this.axiosConfig)
+    return this.$queryWrapper<number>(config.ESPLORA.REST_API_URL + '/blocks/tip/height');
-      .then((response) => response.data);
   }
 
   $getBlockHashTip(): Promise<string> {
-    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash', this.axiosConfig)
+    return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/blocks/tip/hash');
-      .then((response) => response.data);
   }
 
   $getTxIdsForBlock(hash: string): Promise<string[]> {
-    return axiosConnection.get<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids', this.axiosConfig)
+    return this.$queryWrapper<string[]>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids');
-      .then((response) => response.data);
   }
 
   $getBlockHash(height: number): Promise<string> {
-    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height, this.axiosConfig)
+    return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/block-height/' + height);
-      .then((response) => response.data);
   }
 
   $getBlockHeader(hash: string): Promise<string> {
-    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header', this.axiosConfig)
+    return this.$queryWrapper<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header');
-      .then((response) => response.data);
   }
 
   $getBlock(hash: string): Promise<IEsploraApi.Block> {
-    return axiosConnection.get<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash, this.axiosConfig)
+    return this.$queryWrapper<IEsploraApi.Block>(config.ESPLORA.REST_API_URL + '/block/' + hash);
-      .then((response) => response.data);
   }
 
   $getRawBlock(hash: string): Promise<Buffer> {
-    return axiosConnection.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", { ...this.axiosConfig, responseType: 'arraybuffer' })
+    return this.$queryWrapper<any>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", 'arraybuffer')
       .then((response) => { return Buffer.from(response.data); });
   }
 
@@ -85,13 +119,11 @@ class ElectrsApi implements AbstractBitcoinApi {
   }
 
   $getOutspend(txId: string, vout: number): Promise<IEsploraApi.Outspend> {
-    return axiosConnection.get<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout, this.axiosConfig)
+    return this.$queryWrapper<IEsploraApi.Outspend>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout);
-      .then((response) => response.data);
   }
 
   $getOutspends(txId: string): Promise<IEsploraApi.Outspend[]> {
-    return axiosConnection.get<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends', this.axiosConfig)
+    return this.$queryWrapper<IEsploraApi.Outspend[]>(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends');
-      .then((response) => response.data);
   }
 
   async $getBatchedOutspends(txId: string[]): Promise<IEsploraApi.Outspend[][]> {

backend/src/config.ts

@@ -38,6 +38,7 @@ interface IConfig {
   ESPLORA: {
     REST_API_URL: string;
     UNIX_SOCKET_PATH: string | void | null;
+    RETRY_UNIX_SOCKET_AFTER: number;
   };
   LIGHTNING: {
     ENABLED: boolean;
@@ -165,6 +166,7 @@ const defaults: IConfig = {
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',
     'UNIX_SOCKET_PATH': null,
+    'RETRY_UNIX_SOCKET_AFTER': 30000,
   },
   'ELECTRUM': {
     'HOST': '127.0.0.1',

backend/src/index.ts

@@ -45,7 +45,8 @@ class Server {
   private wss: WebSocket.Server | undefined;
   private server: http.Server | undefined;
   private app: Application;
-  private currentBackendRetryInterval = 5;
+  private currentBackendRetryInterval = 1;
+  private backendRetryCount = 0;
 
   private maxHeapSize: number = 0;
   private heapLogInterval: number = 60;
@@ -184,17 +185,17 @@ class Server {
       indexer.$run();
 
       setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
-      this.currentBackendRetryInterval = 5;
+      this.backendRetryCount = 0;
     } catch (e: any) {
-      let loggerMsg = `Exception in runMainUpdateLoop(). Retrying in ${this.currentBackendRetryInterval} sec.`;
+      this.backendRetryCount++;
+      let loggerMsg = `Exception in runMainUpdateLoop() (count: ${this.backendRetryCount}). Retrying in ${this.currentBackendRetryInterval} sec.`;
       loggerMsg += ` Reason: ${(e instanceof Error ? e.message : e)}.`;
       if (e?.stack) {
         loggerMsg += ` Stack trace: ${e.stack}`;
       }
       // When we get a first Exception, only `logger.debug` it and retry after 5 seconds
       // From the second Exception, `logger.warn` the Exception and increase the retry delay
-      // Maximum retry delay is 60 seconds
+      if (this.backendRetryCount >= 5) {
-      if (this.currentBackendRetryInterval > 5) {
         logger.warn(loggerMsg);
         mempool.setOutOfSync();
       } else {
@@ -204,8 +205,6 @@ class Server {
         logger.debug(`AxiosError: ${e?.message}`);
       }
       setTimeout(this.runMainUpdateLoop.bind(this), 1000 * this.currentBackendRetryInterval);
-      this.currentBackendRetryInterval *= 2;
-      this.currentBackendRetryInterval = Math.min(this.currentBackendRetryInterval, 60);
     }
   }
 

production/mempool-config.mainnet-lightning.json

@@ -15,7 +15,7 @@
     "PASSWORD": "__BITCOIN_RPC_PASS__"
   },
   "ESPLORA": {
-    "REST_API_URL": "http://127.0.0.1:4000"
+    "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-mainnet"
   },
   "LIGHTNING": {
     "ENABLED": true,