diff --git a/.github/workflows/get_backend_block_height.yml b/.github/workflows/get_backend_block_height.yml new file mode 100644 index 000000000..52f3b038c --- /dev/null +++ b/.github/workflows/get_backend_block_height.yml @@ -0,0 +1,19 @@ +name: 'Check if servers are in sync' + +on: [workflow_dispatch] + +jobs: + print-backend-sha: + runs-on: 'ubuntu-latest' + name: Get block height + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + path: repo + + - name: Run script + working-directory: repo + run: | + chmod +x ./scripts/get_block_tip_height.sh + sh ./scripts/get_block_tip_height.sh diff --git a/Cargo.lock b/Cargo.lock index 30a0d97ab..0b51ea544 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -57,9 +57,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1586fa608b1dab41f667475b4a41faec5ba680aee428bfa5de4ea520fdc6e901" dependencies = [ "quote", - "syn 2.0.20", + "syn", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "gbt" version = "1.0.0" @@ -71,15 +77,15 @@ dependencies = [ "napi-derive", "priority-queue", "tracing", - "tracing-log", + "tracing-log 0.2.0", "tracing-subscriber", ] [[package]] name = "hashbrown" -version = "0.12.3" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" [[package]] name = "hermit-abi" @@ -92,11 +98,11 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.9.3" +version = "2.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" dependencies = [ - "autocfg", + "equivalent", "hashbrown", ] @@ -114,12 +120,12 @@ checksum = "f92be4933c13fd498862a9e02a3055f8a8d9c039ce33db97306fd5a6caa7f29b" [[package]] name = "libloading" -version = "0.7.4" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +checksum = "2caa5afb8bf9f3a2652760ce7d4f62d21c4d5a423e68466fca30df82f2330164" dependencies = [ "cfg-if", - "winapi", + "windows-targets", ] [[package]] @@ -145,9 +151,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "napi" -version = "2.13.2" +version = "2.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ede2d12cd6fce44da537a4be1f5510c73be2506c2e32dfaaafd1f36968f3a0e" +checksum = "54a63d0570e4c3e0daf7a8d380563610e159f538e20448d6c911337246f40e84" dependencies = [ "bitflags", "ctor", @@ -159,29 +165,29 @@ dependencies = [ [[package]] name = "napi-build" -version = "2.0.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "882a73d9ef23e8dc2ebbffb6a6ae2ef467c0f18ac10711e4cc59c5485d41df0e" +checksum = "2f9130fccc5f763cf2069b34a089a18f0d0883c66aceb81f2fad541a3d823c43" [[package]] name = "napi-derive" -version = "2.13.0" +version = "2.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da1c6a8fa84d549aa8708fcd062372bf8ec6e849de39016ab921067d21bde367" +checksum = "05bb7c37e3c1dda9312fdbe4a9fc7507fca72288ba154ec093e2d49114e727ce" dependencies = [ "cfg-if", "convert_case", "napi-derive-backend", "proc-macro2", "quote", - "syn 1.0.109", + "syn", ] [[package]] name = "napi-derive-backend" -version = "1.0.52" +version = "1.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20bbc7c69168d06a848f925ec5f0e0997f98e8c8d4f2cc30157f0da51c009e17" +checksum = "f785a8b8d7b83e925f5aa6d2ae3c159d17fe137ac368dc185bef410e7acdaeb4" dependencies = [ "convert_case", "once_cell", @@ -189,14 +195,14 @@ dependencies = [ "quote", "regex", "semver", - "syn 1.0.109", + "syn", ] [[package]] name = "napi-sys" -version = "2.2.3" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "166b5ef52a3ab5575047a9fe8d4a030cdd0f63c96f071cd6907674453b07bae3" +checksum = "2503fa6af34dc83fb74888df8b22afe933b58d37daf7d80424b1c60c68196b8b" dependencies = [ "libloading", ] @@ -223,9 +229,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "overload" @@ -241,11 +247,12 @@ checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "priority-queue" -version = "1.3.2" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fff39edfcaec0d64e8d0da38564fad195d2d51b680940295fcc307366e101e61" +checksum = "509354d8a769e8d0b567d6821b84495c60213162761a732d68ce87c964bd347f" dependencies = [ "autocfg", + "equivalent", "indexmap", ] @@ -320,17 +327,6 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.20" @@ -384,7 +380,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.20", + "syn", ] [[package]] @@ -408,6 +404,17 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.17" @@ -423,7 +430,7 @@ dependencies = [ "thread_local", "tracing", "tracing-core", - "tracing-log", + "tracing-log 0.1.3", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 59562297c..2f70699f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,5 @@ [workspace] +resolver = "2" members = [ "./backend/rust-gbt", ] diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index 3c2fccfb7..5d2cf1fba 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -1,5 +1,6 @@ { "MEMPOOL": { + "OFFICIAL": false, "NETWORK": "mainnet", "BACKEND": "electrum", "ENABLED": true, diff --git a/backend/package-lock.json b/backend/package-lock.json index 8c18dc8c2..95a949ef5 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -9,6 +9,7 @@ "version": "3.0.0-dev", "license": "GNU Affero General Public License v3.0", "dependencies": { + "@babel/core": "^7.24.0", "@mempool/electrum-client": "1.1.9", "@types/node": "^18.15.3", "axios": "~1.6.1", @@ -1499,9 +1500,9 @@ } }, "node_modules/@napi-rs/cli": { - "version": "2.16.1", - "resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.16.1.tgz", - "integrity": "sha512-L0Gr5iEQIDEbvWdDr1HUaBOxBSHL1VZhWSk1oryawoT8qJIY+KGfLFelU+Qma64ivCPbxYpkfPoKYVG3rcoGIA==", + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.18.0.tgz", + "integrity": "sha512-lfSRT7cs3iC4L+kv9suGYQEezn5Nii7Kpu+THsYVI0tA1Vh59LH45p4QADaD7hvIkmOz79eEGtoKQ9nAkAPkzA==", "bin": { "napi": "scripts/index.js" }, @@ -7669,7 +7670,7 @@ "version": "3.0.1", "hasInstallScript": true, "dependencies": { - "@napi-rs/cli": "2.16.1" + "@napi-rs/cli": "2.18.0" }, "engines": { "node": ">= 12" @@ -8774,9 +8775,9 @@ "integrity": "sha512-mlvPiCzUlaETpYW3i6V87A24jjMYgsebaXtUo3WQyyLnYUuxs0KiXQ2mnKh3h15j8Xg/hfxeGIi+5OC9u0nftQ==" }, "@napi-rs/cli": { - "version": "2.16.1", - "resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.16.1.tgz", - "integrity": "sha512-L0Gr5iEQIDEbvWdDr1HUaBOxBSHL1VZhWSk1oryawoT8qJIY+KGfLFelU+Qma64ivCPbxYpkfPoKYVG3rcoGIA==" + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.18.0.tgz", + "integrity": "sha512-lfSRT7cs3iC4L+kv9suGYQEezn5Nii7Kpu+THsYVI0tA1Vh59LH45p4QADaD7hvIkmOz79eEGtoKQ9nAkAPkzA==" }, "@noble/hashes": { "version": "1.3.0", @@ -12702,7 +12703,7 @@ "rust-gbt": { "version": "file:rust-gbt", "requires": { - "@napi-rs/cli": "2.16.1" + "@napi-rs/cli": "2.18.0" } }, "safe-buffer": { diff --git a/backend/rust-gbt/Cargo.toml b/backend/rust-gbt/Cargo.toml index 4d0a5b45d..10c572bf9 100644 --- a/backend/rust-gbt/Cargo.toml +++ b/backend/rust-gbt/Cargo.toml @@ -12,14 +12,14 @@ crate-type = ["cdylib"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -priority-queue = "1.3.2" +priority-queue = "2.0.2" bytes = "1.4.0" -napi = { version = "2.13.2", features = ["napi8", "tokio_rt"] } -napi-derive = "2.13.0" +napi = { version = "2.16.0", features = ["napi8", "tokio_rt"] } +napi-derive = "2.16.0" bytemuck = "1.13.1" tracing = "0.1.36" -tracing-log = "0.1.3" +tracing-log = "0.2.0" tracing-subscriber = { version = "0.3.15", features = ["env-filter"]} [build-dependencies] -napi-build = "2.0.1" +napi-build = "2.1.2" diff --git a/backend/rust-gbt/index.js b/backend/rust-gbt/index.js index 8680501d1..dd58a8b76 100644 --- a/backend/rust-gbt/index.js +++ b/backend/rust-gbt/index.js @@ -237,6 +237,49 @@ switch (platform) { loadError = e } break + case 'riscv64': + if (isMusl()) { + localFileExisted = existsSync( + join(__dirname, 'gbt.linux-riscv64-musl.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./gbt.linux-riscv64-musl.node') + } else { + nativeBinding = require('gbt-linux-riscv64-musl') + } + } catch (e) { + loadError = e + } + } else { + localFileExisted = existsSync( + join(__dirname, 'gbt.linux-riscv64-gnu.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./gbt.linux-riscv64-gnu.node') + } else { + nativeBinding = require('gbt-linux-riscv64-gnu') + } + } catch (e) { + loadError = e + } + } + break + case 's390x': + localFileExisted = existsSync( + join(__dirname, 'gbt.linux-s390x-gnu.node') + ) + try { + if (localFileExisted) { + nativeBinding = require('./gbt.linux-s390x-gnu.node') + } else { + nativeBinding = require('gbt-linux-s390x-gnu') + } + } catch (e) { + loadError = e + } + break default: throw new Error(`Unsupported architecture on Linux: ${arch}`) } diff --git a/backend/rust-gbt/package-lock.json b/backend/rust-gbt/package-lock.json index ab3d72e52..e351c82f8 100644 --- a/backend/rust-gbt/package-lock.json +++ b/backend/rust-gbt/package-lock.json @@ -9,16 +9,16 @@ "version": "3.0.1", "hasInstallScript": true, "dependencies": { - "@napi-rs/cli": "2.16.1" + "@napi-rs/cli": "2.18.0" }, "engines": { "node": ">= 12" } }, "node_modules/@napi-rs/cli": { - "version": "2.16.1", - "resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.16.1.tgz", - "integrity": "sha512-L0Gr5iEQIDEbvWdDr1HUaBOxBSHL1VZhWSk1oryawoT8qJIY+KGfLFelU+Qma64ivCPbxYpkfPoKYVG3rcoGIA==", + "version": "2.18.0", + "resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.18.0.tgz", + "integrity": "sha512-lfSRT7cs3iC4L+kv9suGYQEezn5Nii7Kpu+THsYVI0tA1Vh59LH45p4QADaD7hvIkmOz79eEGtoKQ9nAkAPkzA==", "bin": { "napi": "scripts/index.js" }, diff --git a/backend/rust-gbt/package.json b/backend/rust-gbt/package.json index aa98313ed..b0dd96698 100644 --- a/backend/rust-gbt/package.json +++ b/backend/rust-gbt/package.json @@ -25,7 +25,7 @@ } }, "dependencies": { - "@napi-rs/cli": "2.16.1" + "@napi-rs/cli": "2.18.0" }, "engines": { "node": ">= 12" diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 9ee2bd0bc..26ae6fb28 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -1,6 +1,7 @@ { "MEMPOOL": { "ENABLED": true, + "OFFICIAL": false, "NETWORK": "__MEMPOOL_NETWORK__", "BACKEND": "__MEMPOOL_BACKEND__", "BLOCKS_SUMMARIES_INDEXING": true, @@ -79,7 +80,8 @@ "USERNAME": "__DATABASE_USERNAME__", "PASSWORD": "__DATABASE_PASSWORD__", "PID_DIR": "__DATABASE_PID_FILE__", - "TIMEOUT": 3000 + "TIMEOUT": 3000, + "POOL_SIZE": 100 }, "SYSLOG": { "ENABLED": false, diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 6af0ce32f..5066e0ef7 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -14,6 +14,7 @@ describe('Mempool Backend Config', () => { expect(config.MEMPOOL).toStrictEqual({ ENABLED: true, + OFFICIAL: false, NETWORK: 'mainnet', BACKEND: 'none', BLOCKS_SUMMARIES_INDEXING: false, @@ -93,7 +94,8 @@ describe('Mempool Backend Config', () => { USERNAME: 'mempool', PASSWORD: 'mempool', TIMEOUT: 180000, - PID_DIR: '' + PID_DIR: '', + POOL_SIZE: 100, }); expect(config.SYSLOG).toStrictEqual({ diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts index 02640efc0..cc0c801b5 100644 --- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -29,6 +29,7 @@ export interface AbstractBitcoinApi { $getOutSpendsByOutpoint(outpoints: { txid: string, vout: number }[]): Promise; startHealthChecks(): void; + getHealthStatus(): HealthCheckHost[]; } export interface BitcoinRpcCredentials { host: string; @@ -38,3 +39,15 @@ export interface BitcoinRpcCredentials { timeout: number; cookie?: string; } + +export interface HealthCheckHost { + host: string; + active: boolean; + rtt: number; + latestHeight: number; + socket: boolean; + outOfSync: boolean; + unreachable: boolean; + checked: boolean; + lastChecked: number; +} diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index f54c836f8..d19eb06ac 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -1,5 +1,5 @@ import * as bitcoinjs from 'bitcoinjs-lib'; -import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; +import { AbstractBitcoinApi, HealthCheckHost } from './bitcoin-api-abstract-factory'; import { IBitcoinApi } from './bitcoin-api.interface'; import { IEsploraApi } from './esplora-api.interface'; import blocks from '../blocks'; @@ -382,6 +382,10 @@ class BitcoinApi implements AbstractBitcoinApi { } public startHealthChecks(): void {}; + + public getHealthStatus() { + return []; + } } export default BitcoinApi; diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index 2f4bcee85..a9dadf4a0 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -1,7 +1,7 @@ import config from '../../config'; -import axios, { AxiosResponse } from 'axios'; +import axios, { AxiosResponse, isAxiosError } from 'axios'; import http from 'http'; -import { AbstractBitcoinApi } from './bitcoin-api-abstract-factory'; +import { AbstractBitcoinApi, HealthCheckHost } from './bitcoin-api-abstract-factory'; import { IEsploraApi } from './esplora-api.interface'; import logger from '../../logger'; import { Common } from '../common'; @@ -10,6 +10,7 @@ interface FailoverHost { host: string, rtts: number[], rtt: number, + timedOut?: boolean, failures: number, latestHeight?: number, socket?: boolean, @@ -17,6 +18,7 @@ interface FailoverHost { unreachable?: boolean, preferred?: boolean, checked: boolean, + lastChecked?: number, } class FailoverRouter { @@ -108,14 +110,20 @@ class FailoverRouter { host.rtts = []; host.rtt = Infinity; } + host.timedOut = false; } catch (e) { host.outOfSync = true; host.unreachable = true; host.rtts = []; host.rtt = Infinity; + if (isAxiosError(e) && (e.code === 'ECONNABORTED' || e.code === 'ETIMEDOUT')) { + host.timedOut = true; + } else { + host.timedOut = false; + } } host.checked = true; - + host.lastChecked = Date.now(); // switch if the current host is out of sync or significantly slower than the next best alternative const rankOrder = this.sortHosts(); @@ -143,7 +151,7 @@ class FailoverRouter { private formatRanking(index: number, host: FailoverHost, active: FailoverHost, maxHeight: number): string { const heightStatus = !host.checked ? '⏳' : (host.outOfSync ? '🚫' : (host.latestHeight && host.latestHeight < maxHeight ? '🟧' : '✅')); - return `${host === active ? '⭐️' : ' '} ${host.rtt < Infinity ? Math.round(host.rtt).toString().padStart(5, ' ') + 'ms' : ' - '} ${!host.checked ? '⏳' : (host.unreachable ? '🔥' : '✅')} | block: ${host.latestHeight || '??????'} ${heightStatus} | ${host.host} ${host === active ? '⭐️' : ' '}`; + return `${host === active ? '⭐️' : ' '} ${host.rtt < Infinity ? Math.round(host.rtt).toString().padStart(5, ' ') + 'ms' : (host.timedOut ? ' ⌛️💥 ' : ' - ')} ${!host.checked ? '⏳' : (host.unreachable ? '🔥' : '✅')} | block: ${host.latestHeight || '??????'} ${heightStatus} | ${host.host} ${host === active ? '⭐️' : ' '}`; } private updateFallback(): FailoverHost[] { @@ -157,7 +165,7 @@ class FailoverRouter { } // sort hosts by connection quality, and update default fallback - private sortHosts(): FailoverHost[] { + public sortHosts(): FailoverHost[] { // sort by connection quality return this.hosts.slice().sort((a, b) => { if ((a.unreachable || a.outOfSync) === (b.unreachable || b.outOfSync)) { @@ -342,6 +350,24 @@ class ElectrsApi implements AbstractBitcoinApi { public startHealthChecks(): void { this.failoverRouter.startHealthChecks(); } + + public getHealthStatus(): HealthCheckHost[] { + if (config.MEMPOOL.OFFICIAL) { + return this.failoverRouter.sortHosts().map(host => ({ + host: host.host, + active: host === this.failoverRouter.activeHost, + rtt: host.rtt, + latestHeight: host.latestHeight || 0, + socket: !!host.socket, + outOfSync: !!host.outOfSync, + unreachable: !!host.unreachable, + checked: !!host.checked, + lastChecked: host.lastChecked || 0, + })); + } else { + return []; + } + } } export default ElectrsApi; diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 5c007fadd..6711c88fb 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -26,6 +26,7 @@ import mempool from './mempool'; import statistics from './statistics/statistics'; import accelerationCosts from './acceleration'; import accelerationRepository from '../repositories/AccelerationRepository'; +import bitcoinApi from './bitcoin/bitcoin-api-factory'; interface AddressTransactions { mempool: MempoolTransactionExtended[], @@ -39,6 +40,7 @@ const wantable = [ 'mempool-blocks', 'live-2h-chart', 'stats', + 'tomahawk', ]; class WebsocketHandler { @@ -123,7 +125,7 @@ class WebsocketHandler { for (const sub of wantable) { const key = `want-${sub}`; const wants = parsedMessage.data.includes(sub); - if (wants && client['wants'] && !client[key]) { + if (wants && !client[key]) { wantNow[key] = true; } client[key] = wants; @@ -147,6 +149,10 @@ class WebsocketHandler { response['da'] = this.socketData['da']; } + if (wantNow['want-tomahawk']) { + response['tomahawk'] = JSON.stringify(bitcoinApi.getHealthStatus()); + } + if (parsedMessage && parsedMessage['track-tx']) { if (/^[a-fA-F0-9]{64}$/.test(parsedMessage['track-tx'])) { client['track-tx'] = parsedMessage['track-tx']; @@ -546,6 +552,10 @@ class WebsocketHandler { response['mempool-blocks'] = getCachedResponse('mempool-blocks', mBlocks); } + if (client['want-tomahawk']) { + response['tomahawk'] = getCachedResponse('tomahawk', bitcoinApi.getHealthStatus()); + } + if (client['track-mempool-tx']) { const tx = newTransactions.find((t) => t.txid === client['track-mempool-tx']); if (tx) { @@ -909,6 +919,10 @@ class WebsocketHandler { response['mempool-blocks'] = getCachedResponse('mempool-blocks', mBlocks); } + if (client['want-tomahawk']) { + response['tomahawk'] = getCachedResponse('tomahawk', bitcoinApi.getHealthStatus()); + } + if (client['track-tx']) { const trackTxid = client['track-tx']; if (trackTxid && confirmedTxids[trackTxid]) { diff --git a/backend/src/config.ts b/backend/src/config.ts index 32a7af3df..3330adca0 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -5,6 +5,7 @@ const configFromFile = require( interface IConfig { MEMPOOL: { ENABLED: boolean; + OFFICIAL: boolean; NETWORK: 'mainnet' | 'testnet' | 'signet' | 'liquid' | 'liquidtestnet'; BACKEND: 'esplora' | 'electrum' | 'none'; HTTP_PORT: number; @@ -103,6 +104,7 @@ interface IConfig { PASSWORD: string; TIMEOUT: number; PID_DIR: string; + POOL_SIZE: number; }; SYSLOG: { ENABLED: boolean; @@ -161,6 +163,7 @@ interface IConfig { const defaults: IConfig = { 'MEMPOOL': { 'ENABLED': true, + 'OFFICIAL': false, 'NETWORK': 'mainnet', 'BACKEND': 'none', 'HTTP_PORT': 8999, @@ -240,6 +243,7 @@ const defaults: IConfig = { 'PASSWORD': 'mempool', 'TIMEOUT': 180000, 'PID_DIR': '', + 'POOL_SIZE': 100, }, 'SYSLOG': { 'ENABLED': true, diff --git a/backend/src/database.ts b/backend/src/database.ts index dc543bbbc..05f624ff4 100644 --- a/backend/src/database.ts +++ b/backend/src/database.ts @@ -21,7 +21,7 @@ import { execSync } from 'child_process'; database: config.DATABASE.DATABASE, user: config.DATABASE.USERNAME, password: config.DATABASE.PASSWORD, - connectionLimit: 10, + connectionLimit: config.DATABASE.POOL_SIZE, supportBigNumbers: true, timezone: '+00:00', }; diff --git a/backend/src/index.ts b/backend/src/index.ts index 3a8449131..213319946 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -155,11 +155,17 @@ class Server { } if (Common.isLiquid()) { - try { - icons.loadIcons(); - } catch (e) { - logger.err('Cannot load liquid icons. Ignoring. Reason: ' + (e instanceof Error ? e.message : e)); - } + const refreshIcons = () => { + try { + icons.loadIcons(); + } catch (e) { + logger.err('Cannot load liquid icons. Ignoring. Reason: ' + (e instanceof Error ? e.message : e)); + } + }; + // Run once on startup. + refreshIcons(); + // Matches crontab refresh interval for asset db. + setInterval(refreshIcons, 3600_000); } priceUpdater.$run(); diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 8f69fd0c1..eca4cf14c 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -3,6 +3,7 @@ "NETWORK": "__MEMPOOL_NETWORK__", "BACKEND": "__MEMPOOL_BACKEND__", "ENABLED": __MEMPOOL_ENABLED__, + "OFFICIAL": __MEMPOOL_OFFICIAL__, "HTTP_PORT": __MEMPOOL_HTTP_PORT__, "SPAWN_CLUSTER_PROCS": __MEMPOOL_SPAWN_CLUSTER_PROCS__, "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__", @@ -79,7 +80,8 @@ "USERNAME": "__DATABASE_USERNAME__", "PASSWORD": "__DATABASE_PASSWORD__", "TIMEOUT": __DATABASE_TIMEOUT__, - "PID_DIR": "__DATABASE_PID_DIR__" + "PID_DIR": "__DATABASE_PID_DIR__", + "POOL_SIZE": __DATABASE_POOL_SIZE__ }, "SYSLOG": { "ENABLED": __SYSLOG_ENABLED__, diff --git a/docker/backend/start.sh b/docker/backend/start.sh index ba9b99233..b700bba32 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -4,6 +4,7 @@ __MEMPOOL_NETWORK__=${MEMPOOL_NETWORK:=mainnet} __MEMPOOL_BACKEND__=${MEMPOOL_BACKEND:=electrum} __MEMPOOL_ENABLED__=${MEMPOOL_ENABLED:=true} +__MEMPOOL_OFFICIAL__=${MEMPOOL_OFFICIAL:=false} __MEMPOOL_HTTP_PORT__=${BACKEND_HTTP_PORT:=8999} __MEMPOOL_SPAWN_CLUSTER_PROCS__=${MEMPOOL_SPAWN_CLUSTER_PROCS:=0} __MEMPOOL_API_URL_PREFIX__=${MEMPOOL_API_URL_PREFIX:=/api/v1/} @@ -81,6 +82,7 @@ __DATABASE_USERNAME__=${DATABASE_USERNAME:=mempool} __DATABASE_PASSWORD__=${DATABASE_PASSWORD:=mempool} __DATABASE_TIMEOUT__=${DATABASE_TIMEOUT:=180000} __DATABASE_PID_DIR__=${DATABASE_PID_DIR:=""} +__DATABASE_POOL_SIZE__=${DATABASE_POOL_SIZE:=100} # SYSLOG __SYSLOG_ENABLED__=${SYSLOG_ENABLED:=false} @@ -158,6 +160,7 @@ mkdir -p "${__MEMPOOL_CACHE_DIR__}" sed -i "s!__MEMPOOL_NETWORK__!${__MEMPOOL_NETWORK__}!g" mempool-config.json sed -i "s!__MEMPOOL_BACKEND__!${__MEMPOOL_BACKEND__}!g" mempool-config.json sed -i "s!__MEMPOOL_ENABLED__!${__MEMPOOL_ENABLED__}!g" mempool-config.json +sed -i "s!__MEMPOOL_OFFICIAL__!${__MEMPOOL_OFFICIAL__}!g" mempool-config.json sed -i "s!__MEMPOOL_HTTP_PORT__!${__MEMPOOL_HTTP_PORT__}!g" mempool-config.json sed -i "s!__MEMPOOL_SPAWN_CLUSTER_PROCS__!${__MEMPOOL_SPAWN_CLUSTER_PROCS__}!g" mempool-config.json sed -i "s!__MEMPOOL_API_URL_PREFIX__!${__MEMPOOL_API_URL_PREFIX__}!g" mempool-config.json @@ -230,6 +233,7 @@ sed -i "s!__DATABASE_USERNAME__!${__DATABASE_USERNAME__}!g" mempool-config.json sed -i "s!__DATABASE_PASSWORD__!${__DATABASE_PASSWORD__}!g" mempool-config.json sed -i "s!__DATABASE_TIMEOUT__!${__DATABASE_TIMEOUT__}!g" mempool-config.json sed -i "s!__DATABASE_PID_DIR__!${__DATABASE_PID_DIR__}!g" mempool-config.json +sed -i "s!__DATABASE_POOL_SIZE__!${__DATABASE_POOL_SIZE__}!g" mempool-config.json sed -i "s!__SYSLOG_ENABLED__!${__SYSLOG_ENABLED__}!g" mempool-config.json sed -i "s!__SYSLOG_HOST__!${__SYSLOG_HOST__}!g" mempool-config.json diff --git a/docker/init.sh b/docker/init.sh index ee9ac9542..3c5ec6aa3 100755 --- a/docker/init.sh +++ b/docker/init.sh @@ -1,7 +1,7 @@ #!/bin/sh #backend -cp ./docker/backend/* ./backend/ +cp -r ./docker/backend/* ./backend/ #geoip-data mkdir -p ./backend/GeoIP/ @@ -13,8 +13,8 @@ localhostIP="127.0.0.1" cp ./docker/frontend/* ./frontend cp ./nginx.conf ./frontend/ cp ./nginx-mempool.conf ./frontend/ -sed -i "s/${localhostIP}:80/0.0.0.0:__MEMPOOL_FRONTEND_HTTP_PORT__/g" ./frontend/nginx.conf -sed -i "s/${localhostIP}/0.0.0.0/g" ./frontend/nginx.conf -sed -i "s/user nobody;//g" ./frontend/nginx.conf -sed -i "s!/etc/nginx/nginx-mempool.conf!/etc/nginx/conf.d/nginx-mempool.conf!g" ./frontend/nginx.conf -sed -i "s/${localhostIP}:8999/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/g" ./frontend/nginx-mempool.conf +sed -i"" -e "s/${localhostIP}:80/0.0.0.0:__MEMPOOL_FRONTEND_HTTP_PORT__/g" ./frontend/nginx.conf +sed -i"" -e "s/${localhostIP}/0.0.0.0/g" ./frontend/nginx.conf +sed -i"" -e "s/user nobody;//g" ./frontend/nginx.conf +sed -i"" -e "s!/etc/nginx/nginx-mempool.conf!/etc/nginx/conf.d/nginx-mempool.conf!g" ./frontend/nginx.conf +sed -i"" -e "s/${localhostIP}:8999/__MEMPOOL_BACKEND_MAINNET_HTTP_HOST__:__MEMPOOL_BACKEND_MAINNET_HTTP_PORT__/g" ./frontend/nginx-mempool.conf diff --git a/frontend/src/app/app-routing.module.ts b/frontend/src/app/app-routing.module.ts index e123a1525..1fe196090 100644 --- a/frontend/src/app/app-routing.module.ts +++ b/frontend/src/app/app-routing.module.ts @@ -6,6 +6,7 @@ import { EightBlocksComponent } from './components/eight-blocks/eight-blocks.com import { MempoolBlockViewComponent } from './components/mempool-block-view/mempool-block-view.component'; import { ClockComponent } from './components/clock/clock.component'; import { StatusViewComponent } from './components/status-view/status-view.component'; +import { AddressGroupComponent } from './components/address-group/address-group.component'; const browserWindow = window || {}; // @ts-ignore @@ -26,6 +27,14 @@ let routes: Routes = [ loadChildren: () => import('./master-page.module').then(m => m.MasterPageModule), data: { preload: true }, }, + { + path: 'wallet', + children: [], + component: AddressGroupComponent, + data: { + networkSpecific: true, + } + }, { path: 'status', data: { networks: ['bitcoin', 'liquid'] }, @@ -61,6 +70,14 @@ let routes: Routes = [ loadChildren: () => import('./master-page.module').then(m => m.MasterPageModule), data: { preload: true }, }, + { + path: 'wallet', + children: [], + component: AddressGroupComponent, + data: { + networkSpecific: true, + } + }, { path: 'status', data: { networks: ['bitcoin', 'liquid'] }, @@ -88,6 +105,14 @@ let routes: Routes = [ loadChildren: () => import('./master-page.module').then(m => m.MasterPageModule), data: { preload: true }, }, + { + path: 'wallet', + children: [], + component: AddressGroupComponent, + data: { + networkSpecific: true, + } + }, { path: 'preview', children: [ @@ -168,6 +193,14 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') { loadChildren: () => import ('./liquid/liquid-master-page.module').then(m => m.LiquidMasterPageModule), data: { preload: true }, }, + { + path: 'wallet', + children: [], + component: AddressGroupComponent, + data: { + networkSpecific: true, + } + }, { path: 'status', data: { networks: ['bitcoin', 'liquid'] }, @@ -195,6 +228,14 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') { loadChildren: () => import ('./liquid/liquid-master-page.module').then(m => m.LiquidMasterPageModule), data: { preload: true }, }, + { + path: 'wallet', + children: [], + component: AddressGroupComponent, + data: { + networkSpecific: true, + } + }, { path: 'preview', children: [ diff --git a/frontend/src/app/components/address-group/address-group.component.html b/frontend/src/app/components/address-group/address-group.component.html new file mode 100644 index 000000000..174853600 --- /dev/null +++ b/frontend/src/app/components/address-group/address-group.component.html @@ -0,0 +1,24 @@ +
+
+ +

Balances

+
+
+ + + + + + + + + + + +
Total
+ +
+ +
diff --git a/frontend/src/app/components/address-group/address-group.component.scss b/frontend/src/app/components/address-group/address-group.component.scss new file mode 100644 index 000000000..fedb57d2d --- /dev/null +++ b/frontend/src/app/components/address-group/address-group.component.scss @@ -0,0 +1,101 @@ +.frame { + position: relative; + background: #24273e; + padding: 0.5rem; + height: calc(100% + 60px); +} + +.heading { + display: flex; + flex-direction: row; + justify-content: space-between; + align-items: start; + + & > * { + flex-basis: 0; + flex-grow: 1; + } + + h3 { + text-align: center; + margin: 0 1em; + } +} + +.pagination { + position: absolute; + bottom: 0.5rem; + right: 0.5rem; +} + +.table { + margin-top: 0.5em; + + td, th { + padding: 0.15rem 0.5rem; + + &.address { + width: auto; + } + &.btc { + width: 140px; + text-align: right; + } + &.fiat { + width: 142px; + text-align: right; + } + } + + tr { + border-collapse: collapse; + + &:first-child { + border-bottom: solid 1px white; + td, th { + padding-bottom: 0.3rem; + } + } + &:nth-child(2) { + td, th { + padding-top: 0.3rem; + } + } + &:nth-child(even) { + background: #181b2d; + } + } + + @media (min-width: 528px) { + td, th { + &.btc { + width: 160px; + } + &.fiat { + width: 140px; + } + } + } + + @media (min-width: 576px) { + td, th { + &.btc { + width: 170px; + } + &.fiat { + width: 140px; + } + } + } + + @media (min-width: 992px) { + td, th { + &.btc { + width: 210px; + } + &.fiat { + width: 140px; + } + } + } +} \ No newline at end of file diff --git a/frontend/src/app/components/address-group/address-group.component.ts b/frontend/src/app/components/address-group/address-group.component.ts new file mode 100644 index 000000000..30bee7543 --- /dev/null +++ b/frontend/src/app/components/address-group/address-group.component.ts @@ -0,0 +1,212 @@ +import { Component, OnInit, OnDestroy, ChangeDetectorRef, HostListener } from '@angular/core'; +import { ActivatedRoute, ParamMap } from '@angular/router'; +import { ElectrsApiService } from '../../services/electrs-api.service'; +import { switchMap, catchError } from 'rxjs/operators'; +import { Address, Transaction } from '../../interfaces/electrs.interface'; +import { WebsocketService } from '../../services/websocket.service'; +import { StateService } from '../../services/state.service'; +import { AudioService } from '../../services/audio.service'; +import { ApiService } from '../../services/api.service'; +import { of, Subscription, forkJoin } from 'rxjs'; +import { SeoService } from '../../services/seo.service'; +import { AddressInformation } from '../../interfaces/node-api.interface'; + +@Component({ + selector: 'app-address-group', + templateUrl: './address-group.component.html', + styleUrls: ['./address-group.component.scss'] +}) +export class AddressGroupComponent implements OnInit, OnDestroy { + network = ''; + + balance = 0; + confirmed = 0; + mempool = 0; + addresses: { [address: string]: number | null }; + addressStrings: string[] = []; + addressInfo: { [address: string]: AddressInformation | null }; + seenTxs: { [txid: string ]: boolean } = {}; + isLoadingAddress = true; + error: any; + mainSubscription: Subscription; + wsSubscription: Subscription; + + page: string[] = []; + pageIndex: number = 1; + itemsPerPage: number = 10; + + screenSize: 'lg' | 'md' | 'sm' = 'lg'; + digitsInfo: string = '1.8-8'; + + constructor( + private route: ActivatedRoute, + private electrsApiService: ElectrsApiService, + private websocketService: WebsocketService, + private stateService: StateService, + private audioService: AudioService, + private apiService: ApiService, + private seoService: SeoService, + private cd: ChangeDetectorRef, + ) { } + + ngOnInit(): void { + this.onResize(); + this.stateService.networkChanged$.subscribe((network) => this.network = network); + this.websocketService.want(['blocks']); + + this.mainSubscription = this.route.queryParamMap + .pipe( + switchMap((params: ParamMap) => { + this.error = undefined; + this.isLoadingAddress = true; + this.addresses = {}; + this.addressInfo = {}; + this.balance = 0; + + this.addressStrings = params.get('addresses').split(',').map(address => { + if (/^[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}|04[a-fA-F0-9]{128}|(02|03)[a-fA-F0-9]{64}$/.test(address)) { + return address.toLowerCase(); + } else { + return address; + } + }); + + return forkJoin(this.addressStrings.map(address => { + const getLiquidInfo = ((this.stateService.network === 'liquid' || this.stateService.network === 'liquidtestnet') && /^([a-zA-HJ-NP-Z1-9]{26,35}|[a-z]{2,5}1[ac-hj-np-z02-9]{8,100}|[a-km-zA-HJ-NP-Z1-9]{80})$/.test(address)); + return forkJoin([ + of(address), + this.electrsApiService.getAddress$(address), + (getLiquidInfo ? this.apiService.validateAddress$(address) : of(null)), + ]); + })); + }), + catchError(e => { + this.error = e; + return of([]); + }) + ).subscribe((addresses) => { + for (const addressData of addresses) { + const address = addressData[0]; + const addressBalance = addressData[1] as Address; + if (addressBalance) { + this.addresses[address] = addressBalance.chain_stats.funded_txo_sum + + addressBalance.mempool_stats.funded_txo_sum + - addressBalance.chain_stats.spent_txo_sum + - addressBalance.mempool_stats.spent_txo_sum; + this.balance += this.addresses[address]; + this.confirmed += (addressBalance.chain_stats.funded_txo_sum - addressBalance.chain_stats.spent_txo_sum); + } + this.addressInfo[address] = addressData[2] ? addressData[2] as AddressInformation : null; + } + this.websocketService.startTrackAddresses(this.addressStrings); + this.isLoadingAddress = false; + this.pageChange(this.pageIndex); + }); + + this.wsSubscription = this.stateService.multiAddressTransactions$.subscribe(update => { + for (const address of Object.keys(update)) { + for (const tx of update[address].mempool) { + this.addTransaction(tx, false, false); + } + for (const tx of update[address].confirmed) { + this.addTransaction(tx, true, false); + } + for (const tx of update[address].removed) { + this.removeTransaction(tx, tx.status.confirmed); + } + } + }); + } + + pageChange(index): void { + this.page = this.addressStrings.slice((index - 1) * this.itemsPerPage, index * this.itemsPerPage); + } + + addTransaction(transaction: Transaction, confirmed = false, playSound: boolean = true): boolean { + if (this.seenTxs[transaction.txid]) { + this.removeTransaction(transaction, false); + } + this.seenTxs[transaction.txid] = true; + + let balance = 0; + transaction.vin.forEach((vin) => { + if (this.addressStrings.includes(vin?.prevout?.scriptpubkey_address)) { + this.addresses[vin?.prevout?.scriptpubkey_address] -= vin.prevout.value; + balance -= vin.prevout.value; + this.balance -= vin.prevout.value; + if (confirmed) { + this.confirmed -= vin.prevout.value; + } + } + }); + transaction.vout.forEach((vout) => { + if (this.addressStrings.includes(vout?.scriptpubkey_address)) { + this.addresses[vout?.scriptpubkey_address] += vout.value; + balance += vout.value; + this.balance += vout.value; + if (confirmed) { + this.confirmed += vout.value; + } + } + }); + + if (playSound) { + if (balance > 0) { + this.audioService.playSound('cha-ching'); + } else { + this.audioService.playSound('chime'); + } + } + + return true; + } + + removeTransaction(transaction: Transaction, confirmed = false): boolean { + transaction.vin.forEach((vin) => { + if (this.addressStrings.includes(vin?.prevout?.scriptpubkey_address)) { + this.addresses[vin?.prevout?.scriptpubkey_address] += vin.prevout.value; + this.balance += vin.prevout.value; + if (confirmed) { + this.confirmed += vin.prevout.value; + } + } + }); + transaction.vout.forEach((vout) => { + if (this.addressStrings.includes(vout?.scriptpubkey_address)) { + this.addresses[vout?.scriptpubkey_address] -= vout.value; + this.balance -= vout.value; + if (confirmed) { + this.confirmed -= vout.value; + } + } + }); + + return true; + } + + @HostListener('window:resize', ['$event']) + onResize(): void { + if (window.innerWidth >= 992) { + this.screenSize = 'lg'; + this.digitsInfo = '1.8-8'; + } else if (window.innerWidth >= 528) { + this.screenSize = 'md'; + this.digitsInfo = '1.4-4'; + } else { + this.screenSize = 'sm'; + this.digitsInfo = '1.2-2'; + } + const newItemsPerPage = Math.floor((window.innerHeight - 150) / 30); + if (newItemsPerPage !== this.itemsPerPage) { + this.itemsPerPage = newItemsPerPage; + this.pageIndex = 1; + this.pageChange(this.pageIndex); + } + } + + ngOnDestroy(): void { + this.mainSubscription.unsubscribe(); + this.wsSubscription.unsubscribe(); + this.websocketService.stopTrackingAddresses(); + } +} diff --git a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html index 443fc1946..e9f64b9b8 100644 --- a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html +++ b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.html @@ -56,7 +56,7 @@ -
+
{{ block.extras.pool.name}} diff --git a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.scss b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.scss index 795e1f4df..c1cc6809d 100644 --- a/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.scss +++ b/frontend/src/app/components/blockchain-blocks/blockchain-blocks.component.scss @@ -166,7 +166,7 @@ opacity: 1; } .hide { - opacity: 0; + opacity: 0.4; pointer-events : none; } diff --git a/frontend/src/app/components/footer/footer.component.html b/frontend/src/app/components/footer/footer.component.html index 9c0369827..f89e780ff 100644 --- a/frontend/src/app/components/footer/footer.component.html +++ b/frontend/src/app/components/footer/footer.component.html @@ -1,4 +1,4 @@ -