diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 80c2b4e28..76b27d630 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -52,6 +52,7 @@ "ESPLORA": { "REST_API_URL": "http://127.0.0.1:3000", "UNIX_SOCKET_PATH": "/tmp/esplora-bitcoin-mainnet", + "BATCH_QUERY_BASE_SIZE": 1000, "RETRY_UNIX_SOCKET_AFTER": 30000, "REQUEST_TIMEOUT": 10000, "FALLBACK_TIMEOUT": 5000, @@ -132,6 +133,11 @@ "BISQ_URL": "https://bisq.markets/api", "BISQ_ONION": "http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api" }, + "REDIS": { + "ENABLED": false, + "UNIX_SOCKET_PATH": "/tmp/redis.sock", + "BATCH_QUERY_BASE_SIZE": 5000 + }, "REPLICATION": { "ENABLED": false, "AUDIT": false, diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 7d5a14de6..0c30651ce 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -53,6 +53,7 @@ "ESPLORA": { "REST_API_URL": "__ESPLORA_REST_API_URL__", "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__", + "BATCH_QUERY_BASE_SIZE": 1000, "RETRY_UNIX_SOCKET_AFTER": 888, "REQUEST_TIMEOUT": 10000, "FALLBACK_TIMEOUT": 5000, @@ -140,6 +141,7 @@ }, "REDIS": { "ENABLED": false, - "UNIX_SOCKET_PATH": "/tmp/redis.sock" + "UNIX_SOCKET_PATH": "/tmp/redis.sock", + "BATCH_QUERY_BASE_SIZE": 5000 } } diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index a565c64af..2991162e9 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -55,6 +55,7 @@ describe('Mempool Backend Config', () => { expect(config.ESPLORA).toStrictEqual({ REST_API_URL: 'http://127.0.0.1:3000', UNIX_SOCKET_PATH: null, + BATCH_QUERY_BASE_SIZE: 1000, RETRY_UNIX_SOCKET_AFTER: 30000, REQUEST_TIMEOUT: 10000, FALLBACK_TIMEOUT: 5000, @@ -144,7 +145,8 @@ describe('Mempool Backend Config', () => { expect(config.REDIS).toStrictEqual({ ENABLED: false, - UNIX_SOCKET_PATH: '' + UNIX_SOCKET_PATH: '', + BATCH_QUERY_BASE_SIZE: 5000, }); }); }); diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index f5e788f98..a11e0d504 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -235,7 +235,7 @@ class Mempool { if (!loaded) { const remainingTxids = transactions.filter(txid => !this.mempoolCache[txid]); - const sliceLength = 10000; + const sliceLength = config.ESPLORA.BATCH_QUERY_BASE_SIZE; for (let i = 0; i < Math.ceil(remainingTxids.length / sliceLength); i++) { const slice = remainingTxids.slice(i * sliceLength, (i + 1) * sliceLength); const txs = await transactionUtils.$getMempoolTransactionsExtended(slice, false, false, false); diff --git a/backend/src/api/rbf-cache.ts b/backend/src/api/rbf-cache.ts index c573d3291..d2b7ddf6c 100644 --- a/backend/src/api/rbf-cache.ts +++ b/backend/src/api/rbf-cache.ts @@ -480,7 +480,7 @@ class RbfCache { }; if (config.MEMPOOL.BACKEND === 'esplora') { - const sliceLength = 250; + const sliceLength = Math.floor(config.ESPLORA.BATCH_QUERY_BASE_SIZE / 40); for (let i = 0; i < Math.ceil(txids.length / sliceLength); i++) { const slice = txids.slice(i * sliceLength, (i + 1) * sliceLength); try { diff --git a/backend/src/api/redis-cache.ts b/backend/src/api/redis-cache.ts index 00b280274..82ce34ad1 100644 --- a/backend/src/api/redis-cache.ts +++ b/backend/src/api/redis-cache.ts @@ -122,8 +122,9 @@ class RedisCache { async $removeTransactions(transactions: string[]) { try { await this.$ensureConnected(); - for (let i = 0; i < Math.ceil(transactions.length / 10000); i++) { - const slice = transactions.slice(i * 10000, (i + 1) * 10000); + const sliceLength = config.REDIS.BATCH_QUERY_BASE_SIZE; + for (let i = 0; i < Math.ceil(transactions.length / sliceLength); i++) { + const slice = transactions.slice(i * sliceLength, (i + 1) * sliceLength); await this.client.unlink(slice.map(txid => `mempool:tx:${txid}`)); logger.debug(`Deleted ${slice.length} transactions from the Redis cache`); } diff --git a/backend/src/config.ts b/backend/src/config.ts index 37d5a2de9..4115149e6 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -43,6 +43,7 @@ interface IConfig { ESPLORA: { REST_API_URL: string; UNIX_SOCKET_PATH: string | void | null; + BATCH_QUERY_BASE_SIZE: number; RETRY_UNIX_SOCKET_AFTER: number; REQUEST_TIMEOUT: number; FALLBACK_TIMEOUT: number; @@ -151,6 +152,7 @@ interface IConfig { REDIS: { ENABLED: boolean; UNIX_SOCKET_PATH: string; + BATCH_QUERY_BASE_SIZE: number; }, } @@ -195,6 +197,7 @@ const defaults: IConfig = { 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', 'UNIX_SOCKET_PATH': null, + 'BATCH_QUERY_BASE_SIZE': 1000, 'RETRY_UNIX_SOCKET_AFTER': 30000, 'REQUEST_TIMEOUT': 10000, 'FALLBACK_TIMEOUT': 5000, @@ -303,6 +306,7 @@ const defaults: IConfig = { 'REDIS': { 'ENABLED': false, 'UNIX_SOCKET_PATH': '', + 'BATCH_QUERY_BASE_SIZE': 5000, }, }; diff --git a/backend/src/tasks/lightning/forensics.service.ts b/backend/src/tasks/lightning/forensics.service.ts index 584dd3c79..a21c4c280 100644 --- a/backend/src/tasks/lightning/forensics.service.ts +++ b/backend/src/tasks/lightning/forensics.service.ts @@ -79,7 +79,7 @@ class ForensicsService { } let progress = 0; - const sliceLength = 1000; + const sliceLength = Math.floor(config.ESPLORA.BATCH_QUERY_BASE_SIZE / 10); // process batches of 1000 channels for (let i = 0; i < Math.ceil(allChannels.length / sliceLength); i++) { const channels = allChannels.slice(i * sliceLength, (i + 1) * sliceLength); diff --git a/backend/src/tasks/lightning/network-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts index dc0d609fa..09c8b912e 100644 --- a/backend/src/tasks/lightning/network-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -290,7 +290,7 @@ class NetworkSyncService { const allChannels = await channelsApi.$getChannelsByStatus([0, 1]); - const sliceLength = 5000; + const sliceLength = Math.floor(config.ESPLORA.BATCH_QUERY_BASE_SIZE / 2); // process batches of 5000 channels for (let i = 0; i < Math.ceil(allChannels.length / sliceLength); i++) { const channels = allChannels.slice(i * sliceLength, (i + 1) * sliceLength); diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index a1359db97..e73fa1929 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -53,6 +53,7 @@ "ESPLORA": { "REST_API_URL": "__ESPLORA_REST_API_URL__", "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__", + "BATCH_QUERY_BASE_SIZE": __ESPLORA_BATCH_QUERY_BASE_SIZE__, "RETRY_UNIX_SOCKET_AFTER": __ESPLORA_RETRY_UNIX_SOCKET_AFTER__, "REQUEST_TIMEOUT": __ESPLORA_REQUEST_TIMEOUT__, "FALLBACK_TIMEOUT": __ESPLORA_FALLBACK_TIMEOUT__, @@ -146,6 +147,7 @@ }, "REDIS": { "ENABLED": __REDIS_ENABLED__, - "UNIX_SOCKET_PATH": "__REDIS_UNIX_SOCKET_PATH__" + "UNIX_SOCKET_PATH": "__REDIS_UNIX_SOCKET_PATH__", + "BATCH_QUERY_BASE_SIZE": __REDIS_BATCH_QUERY_BASE_SIZE__ } } diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 23c578efe..232cf7284 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -54,6 +54,7 @@ __ELECTRUM_TLS_ENABLED__=${ELECTRUM_TLS_ENABLED:=false} # ESPLORA __ESPLORA_REST_API_URL__=${ESPLORA_REST_API_URL:=http://127.0.0.1:3000} __ESPLORA_UNIX_SOCKET_PATH__=${ESPLORA_UNIX_SOCKET_PATH:="null"} +__ESPLORA_BATCH_QUERY_BASE_SIZE__=${ESPLORA_BATCH_QUERY_BASE_SIZE:=1000} __ESPLORA_RETRY_UNIX_SOCKET_AFTER__=${ESPLORA_RETRY_UNIX_SOCKET_AFTER:=30000} __ESPLORA_REQUEST_TIMEOUT__=${ESPLORA_REQUEST_TIMEOUT:=5000} __ESPLORA_FALLBACK_TIMEOUT__=${ESPLORA_FALLBACK_TIMEOUT:=5000} @@ -148,6 +149,7 @@ __MEMPOOL_SERVICES_ACCELERATIONS__=${MEMPOOL_SERVICES_ACCELERATIONS:=false} # REDIS __REDIS_ENABLED__=${REDIS_ENABLED:=false} __REDIS_UNIX_SOCKET_PATH__=${REDIS_UNIX_SOCKET_PATH:=true} +__REDIS_BATCH_QUERY_BASE_SIZE__=${REDIS_BATCH_QUERY_BASE_SIZE:=5000} mkdir -p "${__MEMPOOL_CACHE_DIR__}" @@ -201,6 +203,7 @@ sed -i "s!__ELECTRUM_TLS_ENABLED__!${__ELECTRUM_TLS_ENABLED__}!g" mempool-config sed -i "s!__ESPLORA_REST_API_URL__!${__ESPLORA_REST_API_URL__}!g" mempool-config.json sed -i "s!__ESPLORA_UNIX_SOCKET_PATH__!${__ESPLORA_UNIX_SOCKET_PATH__}!g" mempool-config.json +sed -i "s!__ESPLORA_BATCH_QUERY_BASE_SIZE__!${__ESPLORA_BATCH_QUERY_BASE_SIZE__}!g" mempool-config.json sed -i "s!__ESPLORA_RETRY_UNIX_SOCKET_AFTER__!${__ESPLORA_RETRY_UNIX_SOCKET_AFTER__}!g" mempool-config.json sed -i "s!__ESPLORA_REQUEST_TIMEOUT__!${__ESPLORA_REQUEST_TIMEOUT__}!g" mempool-config.json sed -i "s!__ESPLORA_FALLBACK_TIMEOUT__!${__ESPLORA_FALLBACK_TIMEOUT__}!g" mempool-config.json @@ -288,5 +291,6 @@ sed -i "s!__MEMPOOL_SERVICES_ACCELERATIONS__!${__MEMPOOL_SERVICES_ACCELERATIONS_ # REDIS sed -i "s!__REDIS_ENABLED__!${__REDIS_ENABLED__}!g" mempool-config.json sed -i "s!__REDIS_UNIX_SOCKET_PATH__!${__REDIS_UNIX_SOCKET_PATH__}!g" mempool-config.json +sed -i "s!__REDIS_BATCH_QUERY_BASE_SIZE__!${__REDIS_BATCH_QUERY_BASE_SIZE__}!g" mempool-config.json node /backend/package/index.js