From a23cd5ad2982fc8e25eb6567207f42569f511086 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Mon, 31 Jul 2023 14:44:59 -0700 Subject: [PATCH 01/23] Update Cypress deps --- frontend/package-lock.json | 34 +++++++++++++++++++--------------- frontend/package.json | 4 ++-- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 561b61096..d82eecd5b 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -58,9 +58,9 @@ }, "optionalDependencies": { "@cypress/schematic": "^2.5.0", - "cypress": "^12.17.1", + "cypress": "^12.17.2", "cypress-fail-on-console-error": "~4.0.3", - "cypress-wait-until": "^1.7.2", + "cypress-wait-until": "^2.0.0", "mock-socket": "~9.2.1", "start-server-and-test": "~2.0.0" } @@ -6641,9 +6641,9 @@ "peer": true }, "node_modules/cypress": { - "version": "12.17.1", - "resolved": "https://registry.npmjs.org/cypress/-/cypress-12.17.1.tgz", - "integrity": "sha512-eKfBgO6t8waEyhegL4gxD7tcI6uTCGttu+ZU7y9Hq8BlpMztd7iLeIF4AJFAnbZH1xjX+wwgg4cRKFNSvv3VWQ==", + "version": "12.17.2", + "resolved": "https://registry.npmjs.org/cypress/-/cypress-12.17.2.tgz", + "integrity": "sha512-hxWAaWbqQBzzMuadSGSuQg5PDvIGOovm6xm0hIfpCVcORsCAj/gF2p0EvfnJ4f+jK2PCiDgP6D2eeE9/FK4Mjg==", "hasInstallScript": true, "optional": true, "dependencies": { @@ -6710,10 +6710,14 @@ } }, "node_modules/cypress-wait-until": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/cypress-wait-until/-/cypress-wait-until-1.7.2.tgz", - "integrity": "sha512-uZ+M8/MqRcpf+FII/UZrU7g1qYZ4aVlHcgyVopnladyoBrpoaMJ4PKZDrdOJ05H5RHbr7s9Tid635X3E+ZLU/Q==", - "optional": true + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/cypress-wait-until/-/cypress-wait-until-2.0.0.tgz", + "integrity": "sha512-ulUZyrWBn+OuC8oiQuGKAScDYfpaWnE3dEE/raUo64w4RHQxZrQ/iMIWT4ZjGMMPr3P+BFEALCRnjQeRqzZj6g==", + "optional": true, + "engines": { + "node": ">=18.16.0", + "npm": ">=9.5.1" + } }, "node_modules/cypress/node_modules/@types/node": { "version": "14.18.53", @@ -20968,9 +20972,9 @@ "peer": true }, "cypress": { - "version": "12.17.1", - "resolved": "https://registry.npmjs.org/cypress/-/cypress-12.17.1.tgz", - "integrity": "sha512-eKfBgO6t8waEyhegL4gxD7tcI6uTCGttu+ZU7y9Hq8BlpMztd7iLeIF4AJFAnbZH1xjX+wwgg4cRKFNSvv3VWQ==", + "version": "12.17.2", + "resolved": "https://registry.npmjs.org/cypress/-/cypress-12.17.2.tgz", + "integrity": "sha512-hxWAaWbqQBzzMuadSGSuQg5PDvIGOovm6xm0hIfpCVcORsCAj/gF2p0EvfnJ4f+jK2PCiDgP6D2eeE9/FK4Mjg==", "optional": true, "requires": { "@cypress/request": "^2.88.11", @@ -21151,9 +21155,9 @@ } }, "cypress-wait-until": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/cypress-wait-until/-/cypress-wait-until-1.7.2.tgz", - "integrity": "sha512-uZ+M8/MqRcpf+FII/UZrU7g1qYZ4aVlHcgyVopnladyoBrpoaMJ4PKZDrdOJ05H5RHbr7s9Tid635X3E+ZLU/Q==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/cypress-wait-until/-/cypress-wait-until-2.0.0.tgz", + "integrity": "sha512-ulUZyrWBn+OuC8oiQuGKAScDYfpaWnE3dEE/raUo64w4RHQxZrQ/iMIWT4ZjGMMPr3P+BFEALCRnjQeRqzZj6g==", "optional": true }, "d": { diff --git a/frontend/package.json b/frontend/package.json index 29f57538e..db6ee0f3a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -110,9 +110,9 @@ }, "optionalDependencies": { "@cypress/schematic": "^2.5.0", - "cypress": "^12.17.1", + "cypress": "^12.17.2", "cypress-fail-on-console-error": "~4.0.3", - "cypress-wait-until": "^1.7.2", + "cypress-wait-until": "^2.0.0", "mock-socket": "~9.2.1", "start-server-and-test": "~2.0.0" }, From 3074d814e7c7bdd5f189d7557e010961d60b3478 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 4 Aug 2023 10:52:15 +0900 Subject: [PATCH 02/23] precompute address transactions for websocket msg loop --- backend/src/api/websocket-handler.ts | 156 ++++++++++++--------------- 1 file changed, 66 insertions(+), 90 deletions(-) diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 0d0332523..1a3d3c21f 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -192,7 +192,7 @@ class WebsocketHandler { if (/^04[a-fA-F0-9]{128}$/.test(parsedMessage['track-address'])) { client['track-address'] = null; client['track-scriptpubkey'] = '41' + matchedAddress + 'ac'; - } else if (/^|(02|03)[a-fA-F0-9]{64}$/.test(parsedMessage['track-address'])) { + } else if (/^(02|03)[a-fA-F0-9]{64}$/.test(parsedMessage['track-address'])) { client['track-address'] = null; client['track-scriptpubkey'] = '21' + matchedAddress + 'ac'; } else { @@ -480,6 +480,9 @@ class WebsocketHandler { } } + // pre-compute address transactions + const addressCache = this.makeAddressCache(newTransactions); + this.wss.clients.forEach(async (client) => { if (client.readyState !== WebSocket.OPEN) { return; @@ -519,78 +522,23 @@ class WebsocketHandler { } if (client['track-address']) { - const foundTransactions: TransactionExtended[] = []; + const foundTransactions = Array.from(addressCache[client['track-address']]?.values() || []); + // txs may be missing prevouts in non-esplora backends + // so fetch the full transactions now + const fullTransactions = (config.MEMPOOL.BACKEND !== 'esplora') ? await this.getFullTransactions(foundTransactions) : foundTransactions; - for (const tx of newTransactions) { - const someVin = tx.vin.some((vin) => !!vin.prevout && vin.prevout.scriptpubkey_address === client['track-address']); - if (someVin) { - if (config.MEMPOOL.BACKEND !== 'esplora') { - try { - const fullTx = await transactionUtils.$getMempoolTransactionExtended(tx.txid, true); - foundTransactions.push(fullTx); - } catch (e) { - logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e)); - } - } else { - foundTransactions.push(tx); - } - return; - } - const someVout = tx.vout.some((vout) => vout.scriptpubkey_address === client['track-address']); - if (someVout) { - if (config.MEMPOOL.BACKEND !== 'esplora') { - try { - const fullTx = await transactionUtils.$getMempoolTransactionExtended(tx.txid, true); - foundTransactions.push(fullTx); - } catch (e) { - logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e)); - } - } else { - foundTransactions.push(tx); - } - } - } - - if (foundTransactions.length) { - response['address-transactions'] = JSON.stringify(foundTransactions); + if (fullTransactions.length) { + response['address-transactions'] = JSON.stringify(fullTransactions); } } if (client['track-scriptpubkey']) { - const foundTransactions: TransactionExtended[] = []; - - for (const tx of newTransactions) { - const someVin = tx.vin.some((vin) => !!vin.prevout && vin.prevout.scriptpubkey_type === 'p2pk' && vin.prevout.scriptpubkey === client['track-scriptpubkey']); - if (someVin) { - if (config.MEMPOOL.BACKEND !== 'esplora') { - try { - const fullTx = await transactionUtils.$getMempoolTransactionExtended(tx.txid, true); - foundTransactions.push(fullTx); - } catch (e) { - logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e)); - } - } else { - foundTransactions.push(tx); - } - return; - } - const someVout = tx.vout.some((vout) => vout.scriptpubkey_type === 'p2pk' && vout.scriptpubkey === client['track-scriptpubkey']); - if (someVout) { - if (config.MEMPOOL.BACKEND !== 'esplora') { - try { - const fullTx = await transactionUtils.$getMempoolTransactionExtended(tx.txid, true); - foundTransactions.push(fullTx); - } catch (e) { - logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e)); - } - } else { - foundTransactions.push(tx); - } - } - } - + const foundTransactions = Array.from(addressCache[client['track-address']]?.values() || []); + // txs may be missing prevouts in non-esplora backends + // so fetch the full transactions now + const fullTransactions = (config.MEMPOOL.BACKEND !== 'esplora') ? await this.getFullTransactions(foundTransactions) : foundTransactions; if (foundTransactions.length) { - response['address-transactions'] = JSON.stringify(foundTransactions); + response['address-transactions'] = JSON.stringify(fullTransactions); } } @@ -598,7 +546,6 @@ class WebsocketHandler { const foundTransactions: TransactionExtended[] = []; newTransactions.forEach((tx) => { - if (client['track-asset'] === Common.nativeAssetId) { if (tx.vin.some((vin) => !!vin.is_pegin)) { foundTransactions.push(tx); @@ -784,6 +731,9 @@ class WebsocketHandler { const fees = feeApi.getRecommendedFee(); const mempoolInfo = memPool.getMempoolInfo(); + // pre-compute address transactions + const addressCache = this.makeAddressCache(transactions); + // update init data this.updateSocketDataFields({ 'mempoolInfo': mempoolInfo, @@ -843,17 +793,7 @@ class WebsocketHandler { } if (client['track-address']) { - const foundTransactions: TransactionExtended[] = []; - - transactions.forEach((tx) => { - if (tx.vin && tx.vin.some((vin) => !!vin.prevout && vin.prevout.scriptpubkey_address === client['track-address'])) { - foundTransactions.push(tx); - return; - } - if (tx.vout && tx.vout.some((vout) => vout.scriptpubkey_address === client['track-address'])) { - foundTransactions.push(tx); - } - }); + const foundTransactions: TransactionExtended[] = Array.from(addressCache[client['track-address']]?.values() || []); if (foundTransactions.length) { foundTransactions.forEach((tx) => { @@ -870,17 +810,7 @@ class WebsocketHandler { } if (client['track-scriptpubkey']) { - const foundTransactions: TransactionExtended[] = []; - - transactions.forEach((tx) => { - if (tx.vin && tx.vin.some((vin) => !!vin.prevout && vin.prevout.scriptpubkey_type === 'p2pk' && vin.prevout.scriptpubkey === client['track-scriptpubkey'])) { - foundTransactions.push(tx); - return; - } - if (tx.vout && tx.vout.some((vout) => vout.scriptpubkey_type === 'p2pk' && vout.scriptpubkey === client['track-scriptpubkey'])) { - foundTransactions.push(tx); - } - }); + const foundTransactions: TransactionExtended[] = Array.from(addressCache[client['track-scriptpubkey']]?.values() || []); if (foundTransactions.length) { foundTransactions.forEach((tx) => { @@ -958,6 +888,52 @@ class WebsocketHandler { + '}'; } + private makeAddressCache(transactions: MempoolTransactionExtended[]): { [address: string]: Set } { + const addressCache: { [address: string]: Set } = {}; + for (const tx of transactions) { + for (const vin of tx.vin) { + if (vin?.prevout?.scriptpubkey_address) { + if (!addressCache[vin.prevout.scriptpubkey_address]) { + addressCache[vin.prevout.scriptpubkey_address] = new Set(); + } + addressCache[vin.prevout.scriptpubkey_address].add(tx); + } + if (vin?.prevout?.scriptpubkey) { + if (!addressCache[vin.prevout.scriptpubkey]) { + addressCache[vin.prevout.scriptpubkey] = new Set(); + } + addressCache[vin.prevout.scriptpubkey].add(tx); + } + } + for (const vout of tx.vout) { + if (vout?.scriptpubkey_address) { + if (!addressCache[vout?.scriptpubkey_address]) { + addressCache[vout?.scriptpubkey_address] = new Set(); + } + addressCache[vout?.scriptpubkey_address].add(tx); + } + if (vout?.scriptpubkey) { + if (!addressCache[vout.scriptpubkey]) { + addressCache[vout.scriptpubkey] = new Set(); + } + addressCache[vout.scriptpubkey].add(tx); + } + } + } + return addressCache; + } + + private async getFullTransactions(transactions: MempoolTransactionExtended[]): Promise { + for (let i = 0; i < transactions.length; i++) { + try { + transactions[i] = await transactionUtils.$getMempoolTransactionExtended(transactions[i].txid, true); + } catch (e) { + logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e)); + } + } + return transactions; + } + private printLogs(): void { if (this.wss) { const count = this.wss?.clients?.size || 0; From e4a43fcca5191d99f6b8e1651943d259a386020b Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Fri, 4 Aug 2023 08:19:19 -0700 Subject: [PATCH 03/23] Update tsconfig for the tests --- frontend/tsconfig.spec.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frontend/tsconfig.spec.json b/frontend/tsconfig.spec.json index 1db2e6ee9..3467e0d0a 100644 --- a/frontend/tsconfig.spec.json +++ b/frontend/tsconfig.spec.json @@ -4,8 +4,10 @@ "outDir": "./out-tsc/spec", "types": [ "jasmine", - "node" - ] + "node", + "cypress", + "cypress-wait-until" +] }, "files": [ "src/test.ts", From da4f891e2fe6e92d77b851f8ae54ccd930adb03d Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Fri, 4 Aug 2023 08:23:29 -0700 Subject: [PATCH 04/23] Use full path for import --- frontend/cypress/support/commands.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/cypress/support/commands.ts b/frontend/cypress/support/commands.ts index 0d698fca5..25a3b6a7e 100644 --- a/frontend/cypress/support/commands.ts +++ b/frontend/cypress/support/commands.ts @@ -44,7 +44,7 @@ 'use strict' -import 'cypress-wait-until'; +import '../../node_modules/cypress-wait-until'; import { PageIdleDetector } from './PageIdleDetector'; import { mockWebSocket } from './websocket'; From e3114144e1dfa6befc25c657389d9369c0f5c6fb Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Fri, 4 Aug 2023 08:36:05 -0700 Subject: [PATCH 05/23] Tweak imports again --- frontend/cypress/support/commands.ts | 3 --- frontend/cypress/support/e2e.ts | 1 + frontend/cypress/tsconfig.json | 2 +- frontend/package-lock.json | 20 ++++++++++++++++++++ frontend/package.json | 1 + 5 files changed, 23 insertions(+), 4 deletions(-) diff --git a/frontend/cypress/support/commands.ts b/frontend/cypress/support/commands.ts index 25a3b6a7e..3c32df517 100644 --- a/frontend/cypress/support/commands.ts +++ b/frontend/cypress/support/commands.ts @@ -42,9 +42,6 @@ // -- This will overwrite an existing command -- // Cypress.Commands.overwrite("visit", (originalFn, url, options) => { ... }) -'use strict' - -import '../../node_modules/cypress-wait-until'; import { PageIdleDetector } from './PageIdleDetector'; import { mockWebSocket } from './websocket'; diff --git a/frontend/cypress/support/e2e.ts b/frontend/cypress/support/e2e.ts index 71e9ea589..8b9e29027 100644 --- a/frontend/cypress/support/e2e.ts +++ b/frontend/cypress/support/e2e.ts @@ -14,6 +14,7 @@ // *********************************************************** // When a command from ./commands is ready to use, import with `import './commands'` syntax +import 'cypress-wait-until'; import './commands'; import failOnConsoleError from 'cypress-fail-on-console-error'; diff --git a/frontend/cypress/tsconfig.json b/frontend/cypress/tsconfig.json index 8f044958a..20e0cc894 100644 --- a/frontend/cypress/tsconfig.json +++ b/frontend/cypress/tsconfig.json @@ -2,7 +2,7 @@ "extends": "../tsconfig.json", "include": ["**/*.ts"], "compilerOptions": { - "types": ["cypress"], + "types": ["cypress", "node", "cypress-wait-until"], "lib": ["es2015", "dom"], "allowJs": true, "noEmit": true, diff --git a/frontend/package-lock.json b/frontend/package-lock.json index d82eecd5b..97c023be0 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -58,6 +58,7 @@ }, "optionalDependencies": { "@cypress/schematic": "^2.5.0", + "@types/cypress": "^1.1.3", "cypress": "^12.17.2", "cypress-fail-on-console-error": "~4.0.3", "cypress-wait-until": "^2.0.0", @@ -3925,6 +3926,16 @@ "@types/node": "*" } }, + "node_modules/@types/cypress": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@types/cypress/-/cypress-1.1.3.tgz", + "integrity": "sha512-OXe0Gw8LeCflkG1oPgFpyrYWJmEKqYncBsD/J0r17r0ETx/TnIGDNLwXt/pFYSYuYTpzcq1q3g62M9DrfsBL4g==", + "deprecated": "This is a stub types definition for cypress (https://cypress.io). cypress provides its own type definitions, so you don't need @types/cypress installed!", + "optional": true, + "dependencies": { + "cypress": "*" + } + }, "node_modules/@types/eslint": { "version": "8.4.1", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.4.1.tgz", @@ -18866,6 +18877,15 @@ "@types/node": "*" } }, + "@types/cypress": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@types/cypress/-/cypress-1.1.3.tgz", + "integrity": "sha512-OXe0Gw8LeCflkG1oPgFpyrYWJmEKqYncBsD/J0r17r0ETx/TnIGDNLwXt/pFYSYuYTpzcq1q3g62M9DrfsBL4g==", + "optional": true, + "requires": { + "cypress": "*" + } + }, "@types/eslint": { "version": "8.4.1", "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-8.4.1.tgz", diff --git a/frontend/package.json b/frontend/package.json index db6ee0f3a..7f6f34060 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -110,6 +110,7 @@ }, "optionalDependencies": { "@cypress/schematic": "^2.5.0", + "@types/cypress": "^1.1.3", "cypress": "^12.17.2", "cypress-fail-on-console-error": "~4.0.3", "cypress-wait-until": "^2.0.0", From 367bcbda83fefcb6addc3e480e5833ba25760688 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Fri, 4 Aug 2023 08:40:15 -0700 Subject: [PATCH 06/23] Update cypress action to node 18 --- .github/workflows/cypress.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cypress.yml b/.github/workflows/cypress.yml index bc66678d4..d067136bf 100644 --- a/.github/workflows/cypress.yml +++ b/.github/workflows/cypress.yml @@ -38,7 +38,7 @@ jobs: - name: Setup node uses: actions/setup-node@v3 with: - node-version: 16.15.0 + node-version: 18 cache: "npm" cache-dependency-path: ${{ matrix.module }}/frontend/package-lock.json From 2095f90262859c1a2c7667d66cf0afe7c2f15e75 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 5 Aug 2023 13:08:47 +0900 Subject: [PATCH 07/23] new health-check based esplora failover mechanism --- backend/mempool-config.sample.json | 3 +- .../__fixtures__/mempool-config.template.json | 3 +- backend/src/__tests__/config.test.ts | 7 +- .../bitcoin/bitcoin-api-abstract-factory.ts | 2 + backend/src/api/bitcoin/bitcoin-api.ts | 1 + backend/src/api/bitcoin/esplora-api.ts | 271 +++++++++++++----- backend/src/config.ts | 2 + backend/src/index.ts | 4 + docker/backend/mempool-config.json | 3 +- 9 files changed, 220 insertions(+), 76 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 47ec6898a..00fe95cc5 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -50,7 +50,8 @@ "ESPLORA": { "REST_API_URL": "http://127.0.0.1:3000", "UNIX_SOCKET_PATH": "/tmp/esplora-bitcoin-mainnet", - "RETRY_UNIX_SOCKET_AFTER": 30000 + "RETRY_UNIX_SOCKET_AFTER": 30000, + "FALLBACK": [] }, "SECOND_CORE_RPC": { "HOST": "127.0.0.1", diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 658b1a6c2..1b6c8d411 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -51,7 +51,8 @@ "ESPLORA": { "REST_API_URL": "__ESPLORA_REST_API_URL__", "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__", - "RETRY_UNIX_SOCKET_AFTER": 888 + "RETRY_UNIX_SOCKET_AFTER": 888, + "FALLBACK": [] }, "SECOND_CORE_RPC": { "HOST": "__SECOND_CORE_RPC_HOST__", diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 23ad0e4a6..655898917 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -52,7 +52,12 @@ describe('Mempool Backend Config', () => { expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true }); - expect(config.ESPLORA).toStrictEqual({ REST_API_URL: 'http://127.0.0.1:3000', UNIX_SOCKET_PATH: null, RETRY_UNIX_SOCKET_AFTER: 30000 }); + expect(config.ESPLORA).toStrictEqual({ + REST_API_URL: 'http://127.0.0.1:3000', + UNIX_SOCKET_PATH: null, + RETRY_UNIX_SOCKET_AFTER: 30000, + FALLBACK: [], + }); expect(config.CORE_RPC).toStrictEqual({ HOST: '127.0.0.1', diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts index f14c5525d..c44653a3d 100644 --- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -23,6 +23,8 @@ export interface AbstractBitcoinApi { $getOutspend(txId: string, vout: number): Promise; $getOutspends(txId: string): Promise; $getBatchedOutspends(txId: string[]): Promise; + + startHealthChecks(): void; } export interface BitcoinRpcCredentials { host: string; diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index b315ed0f7..807baae2e 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -355,6 +355,7 @@ class BitcoinApi implements AbstractBitcoinApi { return transaction; } + public startHealthChecks(): void {}; } export default BitcoinApi; diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index 77c6d80fc..75d38ea8c 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -1,135 +1,258 @@ import config from '../../config'; -import axios, { AxiosRequestConfig } from 'axios'; +import axios, { AxiosResponse } from 'axios'; import http from 'http'; import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; import { IEsploraApi } from './esplora-api.interface'; import logger from '../../logger'; -const axiosConnection = axios.create({ - httpAgent: new http.Agent({ keepAlive: true, }) -}); +interface FailoverHost { + host: string, + latencies: number[], + latency: number + failures: number, + socket?: boolean, + outOfSync?: boolean, + unreachable?: boolean, + preferred?: boolean, +} -class ElectrsApi implements AbstractBitcoinApi { - private axiosConfigWithUnixSocket: AxiosRequestConfig = config.ESPLORA.UNIX_SOCKET_PATH ? { - socketPath: config.ESPLORA.UNIX_SOCKET_PATH, - timeout: 10000, - } : { - timeout: 10000, - }; - private axiosConfigTcpSocketOnly: AxiosRequestConfig = { - timeout: 10000, - }; - - unixSocketRetryTimeout; - activeAxiosConfig; +class FailoverRouter { + activeHost: FailoverHost; + fallbackHost: FailoverHost; + hosts: FailoverHost[]; + multihost: boolean; + pollInterval: number = 60000; + pollTimer: NodeJS.Timeout | null = null; + pollConnection = axios.create(); + requestConnection = axios.create({ + httpAgent: new http.Agent({ keepAlive: true }) + }); constructor() { - this.activeAxiosConfig = this.axiosConfigWithUnixSocket; + // setup list of hosts + this.hosts = (config.ESPLORA.FALLBACK || []).map(domain => { + return { + host: 'https://' + domain + '/api', + latencies: [], + latency: Infinity, + failures: 0, + }; + }); + this.activeHost = { + host: config.ESPLORA.UNIX_SOCKET_PATH || config.ESPLORA.REST_API_URL, + latencies: [], + latency: 0, + failures: 0, + socket: !!config.ESPLORA.UNIX_SOCKET_PATH, + preferred: true, + }; + this.fallbackHost = this.activeHost; + this.hosts.unshift(this.activeHost); + this.multihost = this.hosts.length > 1; } - fallbackToTcpSocket() { - if (!this.unixSocketRetryTimeout) { - logger.err(`Unable to connect to esplora unix socket. Falling back to tcp socket. Retrying unix socket in ${config.ESPLORA.RETRY_UNIX_SOCKET_AFTER / 1000} seconds`); - // Retry the unix socket after a few seconds - this.unixSocketRetryTimeout = setTimeout(() => { - logger.info(`Retrying to use unix socket for esplora now (applied for the next query)`); - this.activeAxiosConfig = this.axiosConfigWithUnixSocket; - this.unixSocketRetryTimeout = undefined; - }, config.ESPLORA.RETRY_UNIX_SOCKET_AFTER); + public startHealthChecks(): void { + // use axios interceptors to measure request latency + this.pollConnection.interceptors.request.use((config) => { + config['meta'] = { startTime: Date.now() }; + return config; + }); + this.pollConnection.interceptors.response.use((response) => { + response.config['meta'].latency = Date.now() - response.config['meta'].startTime; + return response; + }); + + if (this.multihost) { + this.pollHosts(); + } + } + + // start polling hosts to measure availability & latency + private async pollHosts(): Promise { + if (this.pollTimer) { + clearTimeout(this.pollTimer); } - // Use the TCP socket (reach a different esplora instance through nginx) - this.activeAxiosConfig = this.axiosConfigTcpSocketOnly; + const results = await Promise.allSettled(this.hosts.map(async (host) => { + if (host.socket) { + return this.pollConnection.get('/blocks/tip/height', { socketPath: host.host, timeout: 2000 }); + } else { + return this.pollConnection.get(host.host + '/blocks/tip/height', { timeout: 2000 }); + } + })); + const maxHeight = results.reduce((max, result) => Math.max(max, result.status === 'fulfilled' ? result.value?.data || 0 : 0), 0); + + // update latencies & sync status + for (let i = 0; i < results.length; i++) { + const host = this.hosts[i]; + const result = results[i].status === 'fulfilled' ? (results[i] as PromiseFulfilledResult>).value : null; + if (result) { + const height = result.data; + const latency = result.config['meta'].latency; + host.latencies.unshift(latency); + host.latencies.slice(0, 5); + host.latency = host.latencies.reduce((acc, l) => acc + l, 0) / host.latencies.length; + if (height == null || isNaN(height) || (maxHeight - height > 2)) { + host.outOfSync = true; + } else { + host.outOfSync = false; + } + host.unreachable = false; + } else { + host.unreachable = true; + } + } + + this.sortHosts(); + + // switch if the current host is out of sync or significantly slower than the next best alternative + if (this.activeHost.outOfSync || this.activeHost.unreachable || (!this.activeHost.preferred && this.activeHost.latency > (this.hosts[0].latency * 2) + 50)) { + if (this.activeHost.unreachable) { + logger.warn(`Unable to reach ${this.activeHost.host}, failing over to next best alternative`); + } else if (this.activeHost.outOfSync) { + logger.warn(`${this.activeHost.host} has fallen behind, failing over to next best alternative`); + } else { + logger.debug(`${this.activeHost.host} is no longer the best esplora host`); + } + this.electHost(); + } + + this.pollTimer = setTimeout(() => { this.pollHosts(); }, this.pollInterval); } - $queryWrapper(url, responseType = 'json'): Promise { - return axiosConnection.get(url, { ...this.activeAxiosConfig, responseType: responseType }) - .then((response) => response.data) + // sort hosts by connection quality, and update default fallback + private sortHosts(): void { + // sort by connection quality + this.hosts.sort((a, b) => { + if ((a.unreachable || a.outOfSync) === (b.unreachable || b.outOfSync)) { + if (a.preferred === b.preferred) { + // lower latency is best + return a.latency - b.latency; + } else { // unless we have a preferred host + return a.preferred ? -1 : 1; + } + } else { // or the host is out of sync + return (a.unreachable || a.outOfSync) ? 1 : -1; + } + }); + if (this.hosts.length > 1 && this.hosts[0] === this.activeHost) { + this.fallbackHost = this.hosts[1]; + } else { + this.fallbackHost = this.hosts[0]; + } + } + + // depose the active host and choose the next best replacement + private electHost(): void { + this.activeHost.outOfSync = true; + this.activeHost.failures = 0; + this.sortHosts(); + this.activeHost = this.hosts[0]; + logger.warn(`Switching esplora host to ${this.activeHost.host}`); + } + + private addFailure(host: FailoverHost): FailoverHost { + host.failures++; + if (host.failures > 5 && this.multihost) { + logger.warn(`Too many esplora failures on ${this.activeHost.host}, falling back to next best alternative`); + this.electHost(); + return this.activeHost; + } else { + return this.fallbackHost; + } + } + + private async $query(method: 'get'| 'post', path, data: any, responseType = 'json', host = this.activeHost, retry: boolean = true): Promise { + let axiosConfig; + let url; + if (host.socket) { + axiosConfig = { socketPath: host.host, timeout: 10000, responseType }; + url = path; + } else { + axiosConfig = { timeout: 10000, responseType }; + url = host.host + path; + } + return (method === 'post' + ? this.requestConnection.post(url, data, axiosConfig) + : this.requestConnection.get(url, axiosConfig) + ).then((response) => { host.failures = Math.max(0, host.failures - 1); return response.data; }) .catch((e) => { - if (e?.code === 'ECONNREFUSED') { - this.fallbackToTcpSocket(); + let fallbackHost = this.fallbackHost; + if (e?.response?.status !== 404) { + logger.warn(`esplora request failed ${e?.response?.status || 500} ${host.host}${path}`); + fallbackHost = this.addFailure(host); + } + if (retry && e?.code === 'ECONNREFUSED' && this.multihost) { // Retry immediately - return axiosConnection.get(url, this.activeAxiosConfig) - .then((response) => response.data) - .catch((e) => { - logger.warn(`Cannot query esplora through the unix socket nor the tcp socket. Exception ${e}`); - throw e; - }); + return this.$query(method, path, data, responseType, fallbackHost, false); } else { throw e; } }); } - $postWrapper(url, body, responseType = 'json', params: any = undefined): Promise { - return axiosConnection.post(url, body, { ...this.activeAxiosConfig, responseType: responseType, params }) - .then((response) => response.data) - .catch((e) => { - if (e?.code === 'ECONNREFUSED') { - this.fallbackToTcpSocket(); - // Retry immediately - return axiosConnection.post(url, body, this.activeAxiosConfig) - .then((response) => response.data) - .catch((e) => { - logger.warn(`Cannot query esplora through the unix socket nor the tcp socket. Exception ${e}`); - throw e; - }); - } else { - throw e; - } - }); + public async $get(path, responseType = 'json'): Promise { + return this.$query('get', path, null, responseType); } + public async $post(path, data: any, responseType = 'json'): Promise { + return this.$query('post', path, null, responseType); + } +} + +class ElectrsApi implements AbstractBitcoinApi { + private failoverRouter = new FailoverRouter(); + $getRawMempool(): Promise { - return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/mempool/txids'); + return this.failoverRouter.$get('/mempool/txids'); } $getRawTransaction(txId: string): Promise { - return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/tx/' + txId); + return this.failoverRouter.$get('/tx/' + txId); } async $getMempoolTransactions(txids: string[]): Promise { - return this.$postWrapper(config.ESPLORA.REST_API_URL + '/mempool/txs', txids, 'json'); + return this.failoverRouter.$post('/mempool/txs', txids, 'json'); } async $getAllMempoolTransactions(lastSeenTxid?: string): Promise { - return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/mempool/txs' + (lastSeenTxid ? '/' + lastSeenTxid : '')); + return this.failoverRouter.$get('/mempool/txs' + (lastSeenTxid ? '/' + lastSeenTxid : '')); } $getTransactionHex(txId: string): Promise { - return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/hex'); + return this.failoverRouter.$get('/tx/' + txId + '/hex'); } $getBlockHeightTip(): Promise { - return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/blocks/tip/height'); + return this.failoverRouter.$get('/blocks/tip/height'); } $getBlockHashTip(): Promise { - return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/blocks/tip/hash'); + return this.failoverRouter.$get('/blocks/tip/hash'); } $getTxIdsForBlock(hash: string): Promise { - return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txids'); + return this.failoverRouter.$get('/block/' + hash + '/txids'); } $getTxsForBlock(hash: string): Promise { - return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/block/' + hash + '/txs'); + return this.failoverRouter.$get('/block/' + hash + '/txs'); } $getBlockHash(height: number): Promise { - return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/block-height/' + height); + return this.failoverRouter.$get('/block-height/' + height); } $getBlockHeader(hash: string): Promise { - return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/block/' + hash + '/header'); + return this.failoverRouter.$get('/block/' + hash + '/header'); } $getBlock(hash: string): Promise { - return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/block/' + hash); + return this.failoverRouter.$get('/block/' + hash); } $getRawBlock(hash: string): Promise { - return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", 'arraybuffer') + return this.failoverRouter.$get('/block/' + hash + '/raw', 'arraybuffer') .then((response) => { return Buffer.from(response.data); }); } @@ -158,11 +281,11 @@ class ElectrsApi implements AbstractBitcoinApi { } $getOutspend(txId: string, vout: number): Promise { - return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspend/' + vout); + return this.failoverRouter.$get('/tx/' + txId + '/outspend/' + vout); } $getOutspends(txId: string): Promise { - return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/tx/' + txId + '/outspends'); + return this.failoverRouter.$get('/tx/' + txId + '/outspends'); } async $getBatchedOutspends(txId: string[]): Promise { @@ -173,6 +296,10 @@ class ElectrsApi implements AbstractBitcoinApi { } return outspends; } + + public startHealthChecks(): void { + this.failoverRouter.startHealthChecks(); + } } export default ElectrsApi; diff --git a/backend/src/config.ts b/backend/src/config.ts index 982e17b34..ed320d957 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -44,6 +44,7 @@ interface IConfig { REST_API_URL: string; UNIX_SOCKET_PATH: string | void | null; RETRY_UNIX_SOCKET_AFTER: number; + FALLBACK: string[]; }; LIGHTNING: { ENABLED: boolean; @@ -188,6 +189,7 @@ const defaults: IConfig = { 'REST_API_URL': 'http://127.0.0.1:3000', 'UNIX_SOCKET_PATH': null, 'RETRY_UNIX_SOCKET_AFTER': 30000, + 'FALLBACK': [], }, 'ELECTRUM': { 'HOST': '127.0.0.1', diff --git a/backend/src/index.ts b/backend/src/index.ts index adb3f2e02..9d0fa07f5 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -91,6 +91,10 @@ class Server { async startServer(worker = false): Promise { logger.notice(`Starting Mempool Server${worker ? ' (worker)' : ''}... (${backendInfo.getShortCommitHash()})`); + if (config.MEMPOOL.BACKEND === 'esplora') { + bitcoinApi.startHealthChecks(); + } + if (config.DATABASE.ENABLED) { await DB.checkDbConnection(); try { diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 70ff0d283..e283d1171 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -51,7 +51,8 @@ "ESPLORA": { "REST_API_URL": "__ESPLORA_REST_API_URL__", "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__", - "RETRY_UNIX_SOCKET_AFTER": __ESPLORA_RETRY_UNIX_SOCKET_AFTER__ + "RETRY_UNIX_SOCKET_AFTER": __ESPLORA_RETRY_UNIX_SOCKET_AFTER__, + "FALLBACK": __ESPLORA_FALLBACK__, }, "SECOND_CORE_RPC": { "HOST": "__SECOND_CORE_RPC_HOST__", From b1ed05e95e9333789a5bdcda669836786eb40b57 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Sat, 5 Aug 2023 19:06:25 +0900 Subject: [PATCH 08/23] Add space below logo in footer --- .../components/global-footer/global-footer.component.scss | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.scss b/frontend/src/app/shared/components/global-footer/global-footer.component.scss index 6b67711b6..81a8c2a57 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.scss +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.scss @@ -22,7 +22,7 @@ footer .row.main .branding { } footer .row.main .branding > p { - margin-bottom: 25px; + margin-bottom: 45px; } footer .row.main .branding .btn { From e512feef7431963dd04e2bc4ce57b0fc8d553434 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 5 Aug 2023 19:17:20 +0900 Subject: [PATCH 09/23] Add debug prints, fix POST request --- backend/src/api/bitcoin/esplora-api.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index 75d38ea8c..3377ddd52 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -105,6 +105,8 @@ class FailoverRouter { this.sortHosts(); + logger.debug(`Tomahawk ranking: ${this.hosts.map(host => '\navg latency ' + Math.round(host.latency).toString().padStart(5, ' ') + ' | reachable? ' + !(host.unreachable || false).toString().padStart(5, ' ') + ' | in sync? ' + !(host.outOfSync || false).toString().padStart(5, ' ') + ` | ${host.host}`).join('')}`); + // switch if the current host is out of sync or significantly slower than the next best alternative if (this.activeHost.outOfSync || this.activeHost.unreachable || (!this.activeHost.preferred && this.activeHost.latency > (this.hosts[0].latency * 2) + 50)) { if (this.activeHost.unreachable) { @@ -196,7 +198,7 @@ class FailoverRouter { } public async $post(path, data: any, responseType = 'json'): Promise { - return this.$query('post', path, null, responseType); + return this.$query('post', path, data, responseType); } } From ae5e1e6d29c9f72445cd07164b24a8c7c40ac299 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 5 Aug 2023 19:55:33 +0900 Subject: [PATCH 10/23] Fix failover debug prints --- backend/src/api/bitcoin/esplora-api.ts | 36 +++++++++++++------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index 3377ddd52..13687ad53 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -7,8 +7,8 @@ import logger from '../../logger'; interface FailoverHost { host: string, - latencies: number[], - latency: number + rtts: number[], + rtt: number failures: number, socket?: boolean, outOfSync?: boolean, @@ -33,15 +33,15 @@ class FailoverRouter { this.hosts = (config.ESPLORA.FALLBACK || []).map(domain => { return { host: 'https://' + domain + '/api', - latencies: [], - latency: Infinity, + rtts: [], + rtt: Infinity, failures: 0, }; }); this.activeHost = { host: config.ESPLORA.UNIX_SOCKET_PATH || config.ESPLORA.REST_API_URL, - latencies: [], - latency: 0, + rtts: [], + rtt: 0, failures: 0, socket: !!config.ESPLORA.UNIX_SOCKET_PATH, preferred: true, @@ -52,13 +52,13 @@ class FailoverRouter { } public startHealthChecks(): void { - // use axios interceptors to measure request latency + // use axios interceptors to measure request rtt this.pollConnection.interceptors.request.use((config) => { config['meta'] = { startTime: Date.now() }; return config; }); this.pollConnection.interceptors.response.use((response) => { - response.config['meta'].latency = Date.now() - response.config['meta'].startTime; + response.config['meta'].rtt = Date.now() - response.config['meta'].startTime; return response; }); @@ -67,7 +67,7 @@ class FailoverRouter { } } - // start polling hosts to measure availability & latency + // start polling hosts to measure availability & rtt private async pollHosts(): Promise { if (this.pollTimer) { clearTimeout(this.pollTimer); @@ -82,16 +82,16 @@ class FailoverRouter { })); const maxHeight = results.reduce((max, result) => Math.max(max, result.status === 'fulfilled' ? result.value?.data || 0 : 0), 0); - // update latencies & sync status + // update rtts & sync status for (let i = 0; i < results.length; i++) { const host = this.hosts[i]; const result = results[i].status === 'fulfilled' ? (results[i] as PromiseFulfilledResult>).value : null; if (result) { const height = result.data; - const latency = result.config['meta'].latency; - host.latencies.unshift(latency); - host.latencies.slice(0, 5); - host.latency = host.latencies.reduce((acc, l) => acc + l, 0) / host.latencies.length; + const rtt = result.config['meta'].rtt; + host.rtts.unshift(rtt); + host.rtts.slice(0, 5); + host.rtt = host.rtts.reduce((acc, l) => acc + l, 0) / host.rtts.length; if (height == null || isNaN(height) || (maxHeight - height > 2)) { host.outOfSync = true; } else { @@ -105,10 +105,10 @@ class FailoverRouter { this.sortHosts(); - logger.debug(`Tomahawk ranking: ${this.hosts.map(host => '\navg latency ' + Math.round(host.latency).toString().padStart(5, ' ') + ' | reachable? ' + !(host.unreachable || false).toString().padStart(5, ' ') + ' | in sync? ' + !(host.outOfSync || false).toString().padStart(5, ' ') + ` | ${host.host}`).join('')}`); + logger.debug(`Tomahawk ranking: ${this.hosts.map(host => '\navg rtt ' + Math.round(host.rtt).toString().padStart(5, ' ') + ' | reachable? ' + (!host.unreachable || false).toString().padStart(5, ' ') + ' | in sync? ' + (!host.outOfSync || false).toString().padStart(5, ' ') + ` | ${host.host}`).join('')}`); // switch if the current host is out of sync or significantly slower than the next best alternative - if (this.activeHost.outOfSync || this.activeHost.unreachable || (!this.activeHost.preferred && this.activeHost.latency > (this.hosts[0].latency * 2) + 50)) { + if (this.activeHost.outOfSync || this.activeHost.unreachable || (!this.activeHost.preferred && this.activeHost.rtt > (this.hosts[0].rtt * 2) + 50)) { if (this.activeHost.unreachable) { logger.warn(`Unable to reach ${this.activeHost.host}, failing over to next best alternative`); } else if (this.activeHost.outOfSync) { @@ -128,8 +128,8 @@ class FailoverRouter { this.hosts.sort((a, b) => { if ((a.unreachable || a.outOfSync) === (b.unreachable || b.outOfSync)) { if (a.preferred === b.preferred) { - // lower latency is best - return a.latency - b.latency; + // lower rtt is best + return a.rtt - b.rtt; } else { // unless we have a preferred host return a.preferred ? -1 : 1; } From 85935d8f904afeae77d1b22c4047ed275c9cf331 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 5 Aug 2023 20:06:19 +0900 Subject: [PATCH 11/23] allow protocol, port & path in fallback urls --- backend/src/api/bitcoin/esplora-api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index 13687ad53..f3af6bf90 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -32,7 +32,7 @@ class FailoverRouter { // setup list of hosts this.hosts = (config.ESPLORA.FALLBACK || []).map(domain => { return { - host: 'https://' + domain + '/api', + host: domain, rtts: [], rtt: Infinity, failures: 0, From 2ceafcacc605f2c2ce64f912911cea4062c2af31 Mon Sep 17 00:00:00 2001 From: softsimon Date: Sat, 5 Aug 2023 20:29:00 +0900 Subject: [PATCH 12/23] Disable historical prices on testnets --- backend/src/api/mining/mining-routes.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/src/api/mining/mining-routes.ts b/backend/src/api/mining/mining-routes.ts index 1c9a0de30..77cb63e96 100644 --- a/backend/src/api/mining/mining-routes.ts +++ b/backend/src/api/mining/mining-routes.ts @@ -41,6 +41,10 @@ class MiningRoutes { res.header('Pragma', 'public'); res.header('Cache-control', 'public'); res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); + if (['testnet', 'signet', 'liquidtestnet'].includes(config.MEMPOOL.NETWORK)) { + res.status(400).send('Prices are not available on testnets.'); + return; + } if (req.query.timestamp) { res.status(200).send(await PricesRepository.$getNearestHistoricalPrice( parseInt(req.query.timestamp ?? 0, 10) From 9138c3b676aa053566882f0595d6ee9dabdd8b6f Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 5 Aug 2023 20:39:02 +0900 Subject: [PATCH 13/23] always switch back to local if available --- backend/src/api/bitcoin/esplora-api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index f3af6bf90..a44720d83 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -108,7 +108,7 @@ class FailoverRouter { logger.debug(`Tomahawk ranking: ${this.hosts.map(host => '\navg rtt ' + Math.round(host.rtt).toString().padStart(5, ' ') + ' | reachable? ' + (!host.unreachable || false).toString().padStart(5, ' ') + ' | in sync? ' + (!host.outOfSync || false).toString().padStart(5, ' ') + ` | ${host.host}`).join('')}`); // switch if the current host is out of sync or significantly slower than the next best alternative - if (this.activeHost.outOfSync || this.activeHost.unreachable || (!this.activeHost.preferred && this.activeHost.rtt > (this.hosts[0].rtt * 2) + 50)) { + if (this.activeHost.outOfSync || this.activeHost.unreachable || (this.activeHost !== this.hosts[0] && this.hosts[0].preferred) || (!this.activeHost.preferred && this.activeHost.rtt > (this.hosts[0].rtt * 2) + 50)) { if (this.activeHost.unreachable) { logger.warn(`Unable to reach ${this.activeHost.host}, failing over to next best alternative`); } else if (this.activeHost.outOfSync) { From 53ba48de9f95ebd481010bee3fbddfee8334c808 Mon Sep 17 00:00:00 2001 From: wiz Date: Sat, 5 Aug 2023 21:57:43 +0900 Subject: [PATCH 14/23] ops: Enable replication for production --- production/mempool-config.liquid.json | 23 ++++++++++++++++++-- production/mempool-config.liquidtestnet.json | 23 ++++++++++++++++++-- production/mempool-config.mainnet.json | 23 ++++++++++++++++++-- production/mempool-config.signet.json | 23 ++++++++++++++++++-- production/mempool-config.testnet.json | 23 ++++++++++++++++++-- 5 files changed, 105 insertions(+), 10 deletions(-) diff --git a/production/mempool-config.liquid.json b/production/mempool-config.liquid.json index 29223fa07..d67d7b794 100644 --- a/production/mempool-config.liquid.json +++ b/production/mempool-config.liquid.json @@ -23,8 +23,27 @@ "PASSWORD": "__BITCOIN_RPC_PASS__" }, "ESPLORA": { - "REST_API_URL": "http://127.0.0.1:5001", - "UNIX_SOCKET_PATH": "/elements/socket/esplora-liquid-mainnet" + "UNIX_SOCKET_PATH": "/elements/socket/esplora-liquid-mainnet", + "FALLBACK": [ + "http://node201.fmt.mempool.space:3001", + "http://node202.fmt.mempool.space:3001", + "http://node203.fmt.mempool.space:3001", + "http://node204.fmt.mempool.space:3001", + "http://node205.fmt.mempool.space:3001", + "http://node206.fmt.mempool.space:3001", + "http://node201.fra.mempool.space:3001", + "http://node202.fra.mempool.space:3001", + "http://node203.fra.mempool.space:3001", + "http://node204.fra.mempool.space:3001", + "http://node205.fra.mempool.space:3001", + "http://node206.fra.mempool.space:3001", + "http://node201.tk7.mempool.space:3001", + "http://node202.tk7.mempool.space:3001", + "http://node203.tk7.mempool.space:3001", + "http://node204.tk7.mempool.space:3001", + "http://node205.tk7.mempool.space:3001", + "http://node206.tk7.mempool.space:3001" + ] }, "DATABASE": { "ENABLED": true, diff --git a/production/mempool-config.liquidtestnet.json b/production/mempool-config.liquidtestnet.json index 82b41e07f..3a76b4c86 100644 --- a/production/mempool-config.liquidtestnet.json +++ b/production/mempool-config.liquidtestnet.json @@ -23,8 +23,27 @@ "PASSWORD": "__BITCOIN_RPC_PASS__" }, "ESPLORA": { - "REST_API_URL": "http://127.0.0.1:5004", - "UNIX_SOCKET_PATH": "/elements/socket/esplora-liquid-testnet" + "UNIX_SOCKET_PATH": "/elements/socket/esplora-liquid-testnet", + "FALLBACK": [ + "http://node201.fmt.mempool.space:3004", + "http://node202.fmt.mempool.space:3004", + "http://node203.fmt.mempool.space:3004", + "http://node204.fmt.mempool.space:3004", + "http://node205.fmt.mempool.space:3004", + "http://node206.fmt.mempool.space:3004", + "http://node201.fra.mempool.space:3004", + "http://node202.fra.mempool.space:3004", + "http://node203.fra.mempool.space:3004", + "http://node204.fra.mempool.space:3004", + "http://node205.fra.mempool.space:3004", + "http://node206.fra.mempool.space:3004", + "http://node201.tk7.mempool.space:3004", + "http://node202.tk7.mempool.space:3004", + "http://node203.tk7.mempool.space:3004", + "http://node204.tk7.mempool.space:3004", + "http://node205.tk7.mempool.space:3004", + "http://node206.tk7.mempool.space:3004" + ] }, "DATABASE": { "ENABLED": true, diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index f54635415..d4222bd05 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -35,8 +35,27 @@ "PASSWORD": "__BITCOIN_RPC_PASS__" }, "ESPLORA": { - "REST_API_URL": "http://127.0.0.1:5000", - "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-mainnet" + "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-mainnet", + "FALLBACK": [ + "http://node201.fmt.mempool.space:3000", + "http://node202.fmt.mempool.space:3000", + "http://node203.fmt.mempool.space:3000", + "http://node204.fmt.mempool.space:3000", + "http://node205.fmt.mempool.space:3000", + "http://node206.fmt.mempool.space:3000", + "http://node201.fra.mempool.space:3000", + "http://node202.fra.mempool.space:3000", + "http://node203.fra.mempool.space:3000", + "http://node204.fra.mempool.space:3000", + "http://node205.fra.mempool.space:3000", + "http://node206.fra.mempool.space:3000", + "http://node201.tk7.mempool.space:3000", + "http://node202.tk7.mempool.space:3000", + "http://node203.tk7.mempool.space:3000", + "http://node204.tk7.mempool.space:3000", + "http://node205.tk7.mempool.space:3000", + "http://node206.tk7.mempool.space:3000" + ] }, "DATABASE": { "ENABLED": true, diff --git a/production/mempool-config.signet.json b/production/mempool-config.signet.json index 957b36101..38d59c0e9 100644 --- a/production/mempool-config.signet.json +++ b/production/mempool-config.signet.json @@ -25,8 +25,27 @@ "PASSWORD": "__BITCOIN_RPC_PASS__" }, "ESPLORA": { - "REST_API_URL": "http://127.0.0.1:5003", - "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-signet" + "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-signet", + "FALLBACK": [ + "http://node201.fmt.mempool.space:3003", + "http://node202.fmt.mempool.space:3003", + "http://node203.fmt.mempool.space:3003", + "http://node204.fmt.mempool.space:3003", + "http://node205.fmt.mempool.space:3003", + "http://node206.fmt.mempool.space:3003", + "http://node201.fra.mempool.space:3003", + "http://node202.fra.mempool.space:3003", + "http://node203.fra.mempool.space:3003", + "http://node204.fra.mempool.space:3003", + "http://node205.fra.mempool.space:3003", + "http://node206.fra.mempool.space:3003", + "http://node201.tk7.mempool.space:3003", + "http://node202.tk7.mempool.space:3003", + "http://node203.tk7.mempool.space:3003", + "http://node204.tk7.mempool.space:3003", + "http://node205.tk7.mempool.space:3003", + "http://node206.tk7.mempool.space:3003" + ] }, "DATABASE": { "ENABLED": true, diff --git a/production/mempool-config.testnet.json b/production/mempool-config.testnet.json index 8943e987f..c5bdfc8d7 100644 --- a/production/mempool-config.testnet.json +++ b/production/mempool-config.testnet.json @@ -25,8 +25,27 @@ "PASSWORD": "__BITCOIN_RPC_PASS__" }, "ESPLORA": { - "REST_API_URL": "http://127.0.0.1:5002", - "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-testnet" + "UNIX_SOCKET_PATH": "/bitcoin/socket/esplora-bitcoin-testnet", + "FALLBACK": [ + "http://node201.fmt.mempool.space:3002", + "http://node202.fmt.mempool.space:3002", + "http://node203.fmt.mempool.space:3002", + "http://node204.fmt.mempool.space:3002", + "http://node205.fmt.mempool.space:3002", + "http://node206.fmt.mempool.space:3002", + "http://node201.fra.mempool.space:3002", + "http://node202.fra.mempool.space:3002", + "http://node203.fra.mempool.space:3002", + "http://node204.fra.mempool.space:3002", + "http://node205.fra.mempool.space:3002", + "http://node206.fra.mempool.space:3002", + "http://node201.tk7.mempool.space:3002", + "http://node202.tk7.mempool.space:3002", + "http://node203.tk7.mempool.space:3002", + "http://node204.tk7.mempool.space:3002", + "http://node205.tk7.mempool.space:3002", + "http://node206.tk7.mempool.space:3002" + ] }, "DATABASE": { "ENABLED": true, From cbebbd40f16e48c7efa220854441c695ad69ee0c Mon Sep 17 00:00:00 2001 From: softsimon Date: Sun, 6 Aug 2023 15:41:57 +0900 Subject: [PATCH 15/23] Handle price API in the frontend when testnet --- frontend/src/app/services/api.service.ts | 15 ++++++++++++++- frontend/src/app/services/state.service.ts | 4 ++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index 1ed9d2f5c..798df72c1 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -2,7 +2,7 @@ import { Injectable } from '@angular/core'; import { HttpClient, HttpParams, HttpResponse } from '@angular/common/http'; import { CpfpInfo, OptimizedMempoolStats, AddressInformation, LiquidPegs, ITranslators, PoolStat, BlockExtended, TransactionStripped, RewardStats, AuditScore, BlockSizesAndWeights, RbfTree, BlockAudit } from '../interfaces/node-api.interface'; -import { Observable } from 'rxjs'; +import { Observable, of } from 'rxjs'; import { StateService } from './state.service'; import { WebsocketResponse } from '../interfaces/websocket.interface'; import { Outspend, Transaction } from '../interfaces/electrs.interface'; @@ -312,6 +312,19 @@ export class ApiService { } getHistoricalPrice$(timestamp: number | undefined): Observable { + if (this.stateService.isAnyTestnet()) { + return of({ + prices: [], + exchangeRates: { + USDEUR: 0, + USDGBP: 0, + USDCAD: 0, + USDCHF: 0, + USDAUD: 0, + USDJPY: 0, + } + }); + } return this.httpClient.get( this.apiBaseUrl + this.apiBasePath + '/api/v1/historical-price' + (timestamp ? `?timestamp=${timestamp}` : '') diff --git a/frontend/src/app/services/state.service.ts b/frontend/src/app/services/state.service.ts index 675cf88d1..91e4d7475 100644 --- a/frontend/src/app/services/state.service.ts +++ b/frontend/src/app/services/state.service.ts @@ -339,6 +339,10 @@ export class StateService { return this.network === 'liquid' || this.network === 'liquidtestnet'; } + isAnyTestnet(): boolean { + return ['testnet', 'signet', 'liquidtestnet'].includes(this.network); + } + resetChainTip() { this.latestBlockHeight = -1; this.chainTip$.next(-1); From e4fca3c2b7362a48bd30d678d211d4c93999a3ed Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Sun, 6 Aug 2023 08:00:30 -0700 Subject: [PATCH 16/23] Log verbose Docker checks only when running on CI --- backend/src/__tests__/config.test.ts | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 655898917..2370fe7a1 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -186,7 +186,9 @@ describe('Mempool Backend Config', () => { for (const [key, value] of Object.entries(jsonObj)) { // We have a few cases where we can't follow the pattern if (root === 'MEMPOOL' && key === 'HTTP_PORT') { - console.log('skipping check for MEMPOOL_HTTP_PORT'); + if (process.env.CI) { + console.log('skipping check for MEMPOOL_HTTP_PORT'); + } continue; } switch (typeof value) { @@ -208,13 +210,17 @@ describe('Mempool Backend Config', () => { //The string used as the default value, to be checked as a regex, i.e, __MEMPOOL_ENABLED__=${MEMPOOL_ENABLED:=(.*)} const defaultEntry = replaceStr + '=' + '\\${' + envVarStr + ':=(.*)' + '}'; - console.log(`looking for ${defaultEntry} in the start.sh script`); + if (process.env.CI) { + console.log(`looking for ${defaultEntry} in the start.sh script`); + } const re = new RegExp(defaultEntry); expect(startSh).toMatch(re); //The string that actually replaces the values in the config file const sedStr = 'sed -i "s!' + replaceStr + '!${' + replaceStr + '}!g" mempool-config.json'; - console.log(`looking for ${sedStr} in the start.sh script`); + if (process.env.CI) { + console.log(`looking for ${sedStr} in the start.sh script`); + } expect(startSh).toContain(sedStr); break; } From 8912bac0ac498f672a029351ca36f3366ea94246 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Sun, 6 Aug 2023 08:00:49 -0700 Subject: [PATCH 17/23] Add test:ci task --- backend/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/package.json b/backend/package.json index d1cdce286..500cbf93c 100644 --- a/backend/package.json +++ b/backend/package.json @@ -31,6 +31,7 @@ "reindex-updated-pools": "npm run start-production --update-pools", "reindex-all-blocks": "npm run start-production --update-pools --reindex-blocks", "test": "./node_modules/.bin/jest --coverage", + "test:ci": "CI=true ./node_modules/.bin/jest --coverage", "lint": "./node_modules/.bin/eslint . --ext .ts", "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix", "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\"", From adc46b6ae5a8e0b50352eb1be2a0384c966dcfbf Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Sun, 6 Aug 2023 08:01:28 -0700 Subject: [PATCH 18/23] Update Github workflow to run with the CI flag --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6947a0f00..6b9b1594b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,7 @@ jobs: - name: Unit Tests if: ${{ matrix.flavor == 'dev'}} - run: npm run test + run: npm run test:ci working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend - name: Build From 38e9021e8c284a883f3bead2e033daf5708a2d30 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 7 Aug 2023 10:43:42 +0900 Subject: [PATCH 19/23] simplify scriptpubkey tracking --- backend/src/api/websocket-handler.ts | 35 ++-------------------------- 1 file changed, 2 insertions(+), 33 deletions(-) diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 1a3d3c21f..977db9393 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -190,18 +190,14 @@ class WebsocketHandler { matchedAddress = matchedAddress.toLowerCase(); } if (/^04[a-fA-F0-9]{128}$/.test(parsedMessage['track-address'])) { - client['track-address'] = null; - client['track-scriptpubkey'] = '41' + matchedAddress + 'ac'; + client['track-address'] = '41' + matchedAddress + 'ac'; } else if (/^(02|03)[a-fA-F0-9]{64}$/.test(parsedMessage['track-address'])) { - client['track-address'] = null; - client['track-scriptpubkey'] = '21' + matchedAddress + 'ac'; + client['track-address'] = '21' + matchedAddress + 'ac'; } else { client['track-address'] = matchedAddress; - client['track-scriptpubkey'] = null; } } else { client['track-address'] = null; - client['track-scriptpubkey'] = null; } } @@ -532,16 +528,6 @@ class WebsocketHandler { } } - if (client['track-scriptpubkey']) { - const foundTransactions = Array.from(addressCache[client['track-address']]?.values() || []); - // txs may be missing prevouts in non-esplora backends - // so fetch the full transactions now - const fullTransactions = (config.MEMPOOL.BACKEND !== 'esplora') ? await this.getFullTransactions(foundTransactions) : foundTransactions; - if (foundTransactions.length) { - response['address-transactions'] = JSON.stringify(fullTransactions); - } - } - if (client['track-asset']) { const foundTransactions: TransactionExtended[] = []; @@ -809,23 +795,6 @@ class WebsocketHandler { } } - if (client['track-scriptpubkey']) { - const foundTransactions: TransactionExtended[] = Array.from(addressCache[client['track-scriptpubkey']]?.values() || []); - - if (foundTransactions.length) { - foundTransactions.forEach((tx) => { - tx.status = { - confirmed: true, - block_height: block.height, - block_hash: block.id, - block_time: block.timestamp, - }; - }); - - response['block-transactions'] = JSON.stringify(foundTransactions); - } - } - if (client['track-asset']) { const foundTransactions: TransactionExtended[] = []; From e4c17e501148b14ffa47c3dd5ad98623d68be5ed Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Mon, 7 Aug 2023 12:16:01 +0900 Subject: [PATCH 20/23] [mining] add /api/v1/mining/pools API to list mining pools --- backend/src/api/mining/mining-routes.ts | 24 ++++++++++++++++++++++++ backend/src/api/mining/mining.ts | 22 ++++++++++++++++++---- 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/backend/src/api/mining/mining-routes.ts b/backend/src/api/mining/mining-routes.ts index 1c9a0de30..fffd57377 100644 --- a/backend/src/api/mining/mining-routes.ts +++ b/backend/src/api/mining/mining-routes.ts @@ -12,6 +12,7 @@ import PricesRepository from '../../repositories/PricesRepository'; class MiningRoutes { public initRoutes(app: Application) { app + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools', this.$listPools) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/:interval', this.$getPools) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/hashrate', this.$getPoolHistoricalHashrate) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:slug/blocks', this.$getPoolBlocks) @@ -88,6 +89,29 @@ class MiningRoutes { } } + private async $listPools(req: Request, res: Response): Promise { + try { + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + + const pools = await mining.$listPools(); + if (!pools) { + res.status(500).end(); + return; + } + + res.header('X-total-count', pools.length.toString()); + if (pools.length === 0) { + res.status(204).send(); + } else { + res.json(pools); + } + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + private async $getPools(req: Request, res: Response) { try { const stats = await mining.$getPoolsStats(req.params.interval); diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index 6e87d70b8..d9d5995da 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -26,7 +26,7 @@ class Mining { /** * Get historical blocks health */ - public async $getBlocksHealthHistory(interval: string | null = null): Promise { + public async $getBlocksHealthHistory(interval: string | null = null): Promise { return await BlocksAuditsRepository.$getBlocksHealthHistory( this.getTimeRange(interval), Common.getSqlInterval(interval) @@ -56,7 +56,7 @@ class Mining { /** * Get historical block fee rates percentiles */ - public async $getHistoricalBlockFeeRates(interval: string | null = null): Promise { + public async $getHistoricalBlockFeeRates(interval: string | null = null): Promise { return await BlocksRepository.$getHistoricalBlockFeeRates( this.getTimeRange(interval), Common.getSqlInterval(interval) @@ -66,7 +66,7 @@ class Mining { /** * Get historical block sizes */ - public async $getHistoricalBlockSizes(interval: string | null = null): Promise { + public async $getHistoricalBlockSizes(interval: string | null = null): Promise { return await BlocksRepository.$getHistoricalBlockSizes( this.getTimeRange(interval), Common.getSqlInterval(interval) @@ -76,7 +76,7 @@ class Mining { /** * Get historical block weights */ - public async $getHistoricalBlockWeights(interval: string | null = null): Promise { + public async $getHistoricalBlockWeights(interval: string | null = null): Promise { return await BlocksRepository.$getHistoricalBlockWeights( this.getTimeRange(interval), Common.getSqlInterval(interval) @@ -595,6 +595,20 @@ class Mining { } } + /** + * List existing mining pools + */ + public async $listPools(): Promise<{name: string, slug: string, unique_id: number}[] | null> { + const [rows] = await database.query(` + SELECT + name, + slug, + unique_id + FROM pools` + ); + return rows as {name: string, slug: string, unique_id: number}[]; + } + private getDateMidnight(date: Date): Date { date.setUTCHours(0); date.setUTCMinutes(0); From c5bdcec164a6f8df2a3af6f9b7ac50356fc6ff7d Mon Sep 17 00:00:00 2001 From: wiz Date: Wed, 9 Aug 2023 23:43:49 +0900 Subject: [PATCH 21/23] ops: Add delay before starting bitcoind/elementsd/electrs --- production/bitcoin.crontab | 10 +++++----- production/elements.crontab | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/production/bitcoin.crontab b/production/bitcoin.crontab index cca0443c7..ba1fa9701 100644 --- a/production/bitcoin.crontab +++ b/production/bitcoin.crontab @@ -1,5 +1,5 @@ -@reboot screen -dmS mainnet /bitcoin/electrs/electrs-start-mainnet -@reboot /usr/local/bin/bitcoind -testnet >/dev/null 2>&1 -@reboot screen -dmS testnet /bitcoin/electrs/electrs-start-testnet -@reboot /usr/local/bin/bitcoind -signet >/dev/null 2>&1 -@reboot screen -dmS signet /bitcoin/electrs/electrs-start-signet +@reboot sleep 5 ; /usr/local/bin/bitcoind -testnet >/dev/null 2>&1 +@reboot sleep 5 ; /usr/local/bin/bitcoind -signet >/dev/null 2>&1 +@reboot sleep 10 ; screen -dmS mainnet /bitcoin/electrs/electrs-start-mainnet +@reboot sleep 10 ; screen -dmS testnet /bitcoin/electrs/electrs-start-testnet +@reboot sleep 10 ; screen -dmS signet /bitcoin/electrs/electrs-start-signet diff --git a/production/elements.crontab b/production/elements.crontab index c0194ac4f..f4a42ed39 100644 --- a/production/elements.crontab +++ b/production/elements.crontab @@ -1,10 +1,10 @@ # start elements on reboot -@reboot /usr/local/bin/elementsd -chain=liquidv1 >/dev/null 2>&1 -@reboot /usr/local/bin/elementsd -chain=liquidtestnet >/dev/null 2>&1 +@reboot sleep 5 ; /usr/local/bin/elementsd -chain=liquidv1 >/dev/null 2>&1 +@reboot sleep 5 ; /usr/local/bin/elementsd -chain=liquidtestnet >/dev/null 2>&1 # start electrs on reboot -@reboot screen -dmS liquidv1 /elements/electrs/electrs-start-liquid -@reboot screen -dmS liquidtestnet /elements/electrs/electrs-start-liquidtestnet +@reboot sleep 20 ; screen -dmS liquidv1 /elements/electrs/electrs-start-liquid +@reboot sleep 20 ; screen -dmS liquidtestnet /elements/electrs/electrs-start-liquidtestnet # hourly asset update and electrs restart 6 * * * * cd $HOME/asset_registry_db && git pull --quiet origin master && cd $HOME/asset_registry_testnet_db && git pull --quiet origin master && killall electrs From 9f007d78f8b096365543745e7f19c721913fd62d Mon Sep 17 00:00:00 2001 From: wiz Date: Wed, 9 Aug 2023 23:44:03 +0900 Subject: [PATCH 22/23] ops: Fix fonts config from installer --- production/install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/install b/production/install index addb4dc16..5ade5b8c4 100755 --- a/production/install +++ b/production/install @@ -1449,7 +1449,7 @@ if [ "${UNFURL_INSTALL}" = ON ];then echo "[*] Installing color emoji" osSudo "${ROOT_USER}" curl "https://github.com/samuelngs/apple-emoji-linux/releases/download/ios-15.4/AppleColorEmoji.ttf" -o /usr/local/share/fonts/TTF/AppleColorEmoji.ttf - cat >> /usr/local/etc/fonts/conf.d/01-emoji.conf < /usr/local/etc/fonts/conf.d/01-emoji.conf < From fb3cb127b00aee616bd26d65b7651869c25bb2dc Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Wed, 9 Aug 2023 21:51:11 -0700 Subject: [PATCH 23/23] Set dependabot to fix the package.json file in addition to the lock file --- .github/dependabot.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e0aee68c5..a824900f4 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,6 +1,7 @@ version: 2 updates: - package-ecosystem: npm + versioning-strategy: increase directory: "/backend" schedule: interval: daily @@ -14,6 +15,7 @@ updates: - package-ecosystem: npm directory: "/frontend" + versioning-strategy: increase schedule: interval: daily open-pull-requests-limit: 10