From 9c09c00fab0a1d3f2f409dc8d54975ec82753d96 Mon Sep 17 00:00:00 2001 From: softsimon Date: Fri, 19 Aug 2022 17:54:52 +0400 Subject: [PATCH 01/62] Updated mempool debug log --- backend/src/api/mempool.ts | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 43aea6059..76c8b169f 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -103,12 +103,11 @@ class Mempool { return txTimes; } - public async $updateMempool() { - logger.debug('Updating mempool'); + public async $updateMempool(): Promise { + logger.debug(`Updating mempool...`); const start = new Date().getTime(); let hasChange: boolean = false; const currentMempoolSize = Object.keys(this.mempoolCache).length; - let txCount = 0; const transactions = await bitcoinApi.$getRawMempool(); const diff = transactions.length - currentMempoolSize; const newTransactions: TransactionExtended[] = []; @@ -124,7 +123,6 @@ class Mempool { try { const transaction = await transactionUtils.$getTransactionExtended(txid); this.mempoolCache[txid] = transaction; - txCount++; if (this.inSync) { this.txPerSecondArray.push(new Date().getTime()); this.vBytesPerSecondArray.push({ @@ -133,14 +131,9 @@ class Mempool { }); } hasChange = true; - if (diff > 0) { - logger.debug('Fetched transaction ' + txCount + ' / ' + diff); - } else { - logger.debug('Fetched transaction ' + txCount); - } newTransactions.push(transaction); } catch (e) { - logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e)); + logger.debug(`Error finding transaction '${txid}' in the mempool: ` + (e instanceof Error ? e.message : e)); } } @@ -197,8 +190,7 @@ class Mempool { const end = new Date().getTime(); const time = end - start; - logger.debug(`New mempool size: ${Object.keys(this.mempoolCache).length} Change: ${diff}`); - logger.debug('Mempool updated in ' + time / 1000 + ' seconds'); + logger.debug(`Mempool updated in ${time / 1000} seconds. New size: ${Object.keys(this.mempoolCache).length} (${diff > 0 ? '+' + diff : diff})`); } public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended; }) { From 19467de809301ddecbc414676e9c5bbfe29b2977 Mon Sep 17 00:00:00 2001 From: junderw Date: Sun, 18 Sep 2022 22:30:09 +0900 Subject: [PATCH 02/62] Backend: Add block height from timestamp endpoint --- backend/src/api/mining/mining-routes.ts | 20 +++++++++++++ backend/src/repositories/BlocksRepository.ts | 30 ++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/backend/src/api/mining/mining-routes.ts b/backend/src/api/mining/mining-routes.ts index f52d42d1f..c9ace12e5 100644 --- a/backend/src/api/mining/mining-routes.ts +++ b/backend/src/api/mining/mining-routes.ts @@ -27,6 +27,7 @@ class MiningRoutes { .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments/:interval', this.$getDifficultyAdjustments) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/predictions/:interval', this.$getHistoricalBlockPrediction) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/:hash', this.$getBlockAudit) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/timestamp/:timestamp', this.$getHeightFromTimestamp) ; } @@ -246,6 +247,25 @@ class MiningRoutes { res.status(500).send(e instanceof Error ? e.message : e); } } + + private async $getHeightFromTimestamp(req: Request, res: Response) { + try { + const timestamp = parseInt(req.params.timestamp, 10); + // Prevent non-integers that are not seconds + if (!/^[1-9][0-9]*$/.test(req.params.timestamp) || timestamp >= 2 ** 32) { + throw new Error(`Invalid timestamp, value must be Unix seconds`); + } + const result = await BlocksRepository.$getBlockHeightFromTimestamp( + timestamp, + ); + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString()); + res.json(result); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } } export default new MiningRoutes(); diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 40f670833..c5a1a2ae4 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -392,6 +392,36 @@ class BlocksRepository { } } + /** + * Get the first block at or directly after a given timestamp + * @param timestamp number unix time in seconds + * @returns The height and timestamp of a block (timestamp might vary from given timestamp) + */ + public async $getBlockHeightFromTimestamp( + timestamp: number, + ): Promise<{ height: number; timestamp: number }> { + try { + // Get first block at or after the given timestamp + const query = `SELECT height, blockTimestamp as timestamp FROM blocks + WHERE blockTimestamp >= FROM_UNIXTIME(?) + ORDER BY blockTimestamp ASC + LIMIT 1`; + const params = [timestamp]; + const [rows]: any[][] = await DB.query(query, params); + if (rows.length === 0) { + throw new Error(`No block was found after timestamp ${timestamp}`); + } + + return rows[0]; + } catch (e) { + logger.err( + 'Cannot get block height from timestamp from the db. Reason: ' + + (e instanceof Error ? e.message : e), + ); + throw e; + } + } + /** * Return blocks height */ From 5d1c5b51dd23c7ef6d4445d4f71aee74f2858370 Mon Sep 17 00:00:00 2001 From: junderw Date: Mon, 19 Sep 2022 16:44:53 +0900 Subject: [PATCH 03/62] Fix: Add hash and reverse search order --- backend/src/api/mining/mining-routes.ts | 6 +++++- backend/src/repositories/BlocksRepository.ts | 10 +++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/backend/src/api/mining/mining-routes.ts b/backend/src/api/mining/mining-routes.ts index c9ace12e5..ac4b82363 100644 --- a/backend/src/api/mining/mining-routes.ts +++ b/backend/src/api/mining/mining-routes.ts @@ -251,8 +251,12 @@ class MiningRoutes { private async $getHeightFromTimestamp(req: Request, res: Response) { try { const timestamp = parseInt(req.params.timestamp, 10); + // This will prevent people from entering milliseconds etc. + // Block timestamps are allowed to be up to 2 hours off, so 24 hours + // will never put the maximum value before the most recent block + const nowPlus1day = Math.floor(Date.now() / 1000) + 60 * 60 * 24; // Prevent non-integers that are not seconds - if (!/^[1-9][0-9]*$/.test(req.params.timestamp) || timestamp >= 2 ** 32) { + if (!/^[1-9][0-9]*$/.test(req.params.timestamp) || timestamp > nowPlus1day) { throw new Error(`Invalid timestamp, value must be Unix seconds`); } const result = await BlocksRepository.$getBlockHeightFromTimestamp( diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index c5a1a2ae4..590e9de37 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -399,17 +399,17 @@ class BlocksRepository { */ public async $getBlockHeightFromTimestamp( timestamp: number, - ): Promise<{ height: number; timestamp: number }> { + ): Promise<{ height: number; hash: string; timestamp: number }> { try { // Get first block at or after the given timestamp - const query = `SELECT height, blockTimestamp as timestamp FROM blocks - WHERE blockTimestamp >= FROM_UNIXTIME(?) - ORDER BY blockTimestamp ASC + const query = `SELECT height, hash, blockTimestamp as timestamp FROM blocks + WHERE blockTimestamp <= FROM_UNIXTIME(?) + ORDER BY blockTimestamp DESC LIMIT 1`; const params = [timestamp]; const [rows]: any[][] = await DB.query(query, params); if (rows.length === 0) { - throw new Error(`No block was found after timestamp ${timestamp}`); + throw new Error(`No block was found before timestamp ${timestamp}`); } return rows[0]; From 8ef88e9f3974563eade08d3557f24cd672175125 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sat, 8 Oct 2022 13:47:33 -0700 Subject: [PATCH 04/62] Ignore the new config files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 687e9e8cb..b41b0db08 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ data docker-compose.yml backend/mempool-config.json *.swp +frontend/src/resources/config.template.js +frontend/src/resources/config.js From 5d21a61840aeac10524e6bd8e2db481d3576527b Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sat, 8 Oct 2022 13:48:29 -0700 Subject: [PATCH 05/62] Serve the frontend config from resources, stop bundling the generated file --- frontend/angular.json | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/frontend/angular.json b/frontend/angular.json index 1ed29cad9..672b84417 100644 --- a/frontend/angular.json +++ b/frontend/angular.json @@ -152,15 +152,14 @@ "assets": [ "src/favicon.ico", "src/resources", - "src/robots.txt" + "src/robots.txt", + "src/config.js", + "src/config.template.js" ], "styles": [ "src/styles.scss", "node_modules/@fortawesome/fontawesome-svg-core/styles.css" ], - "scripts": [ - "generated-config.js" - ], "vendorChunk": true, "extractLicenses": false, "buildOptimizer": false, From 71e00f66c950c63e4324d33b5025c2ac05c9151e Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sat, 8 Oct 2022 14:51:32 -0700 Subject: [PATCH 06/62] Update config generator to output the template and new config file --- frontend/generate-config.js | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/frontend/generate-config.js b/frontend/generate-config.js index 1f37953b7..4b7f80cd8 100644 --- a/frontend/generate-config.js +++ b/frontend/generate-config.js @@ -2,7 +2,8 @@ var fs = require('fs'); const { spawnSync } = require('child_process'); const CONFIG_FILE_NAME = 'mempool-frontend-config.json'; -const GENERATED_CONFIG_FILE_NAME = 'generated-config.js'; +const GENERATED_CONFIG_FILE_NAME = 'src/resources/config.js'; +const GENERATED_TEMPLATE_CONFIG_FILE_NAME = 'src/resources/config.template.js'; let settings = []; let configContent = {}; @@ -67,10 +68,17 @@ if (process.env.DOCKER_COMMIT_HASH) { const newConfig = `(function (window) { window.__env = window.__env || {};${settings.reduce((str, obj) => `${str} - window.__env.${obj.key} = ${ typeof obj.value === 'string' ? `'${obj.value}'` : obj.value };`, '')} + window.__env.${obj.key} = ${typeof obj.value === 'string' ? `'${obj.value}'` : obj.value};`, '')} window.__env.GIT_COMMIT_HASH = '${gitCommitHash}'; window.__env.PACKAGE_JSON_VERSION = '${packetJsonVersion}'; - }(global || this));`; + }(this));`; + +const newConfigTemplate = `(function (window) { + window.__env = window.__env || {};${settings.reduce((str, obj) => `${str} + window.__env.${obj.key} = ${typeof obj.value === 'string' ? `'\${${obj.key}}'` : `\${${obj.key}}`};`, '')} + window.__env.GIT_COMMIT_HASH = '${gitCommitHash}'; + window.__env.PACKAGE_JSON_VERSION = '${packetJsonVersion}'; + }(this));`; function readConfig(path) { try { @@ -89,6 +97,16 @@ function writeConfig(path, config) { } } +function writeConfigTemplate(path, config) { + try { + fs.writeFileSync(path, config, 'utf8'); + } catch (e) { + throw new Error(e); + } +} + +writeConfigTemplate(GENERATED_TEMPLATE_CONFIG_FILE_NAME, newConfigTemplate); + const currentConfig = readConfig(GENERATED_CONFIG_FILE_NAME); if (currentConfig && currentConfig === newConfig) { @@ -106,4 +124,4 @@ if (currentConfig && currentConfig === newConfig) { console.log('NEW CONFIG: ', newConfig); writeConfig(GENERATED_CONFIG_FILE_NAME, newConfig); console.log(`${GENERATED_CONFIG_FILE_NAME} file updated`); -}; +} From ad7e7795f9809df1c37c12d37f3112e00a6b2f63 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sat, 8 Oct 2022 14:58:09 -0700 Subject: [PATCH 07/62] Update index files to read the new config file --- frontend/src/index.bisq.html | 9 ++++++++- frontend/src/index.liquid.html | 9 ++++++++- frontend/src/index.mempool.html | 9 ++++++++- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/frontend/src/index.bisq.html b/frontend/src/index.bisq.html index 8da1e77e0..82f491d50 100644 --- a/frontend/src/index.bisq.html +++ b/frontend/src/index.bisq.html @@ -1,10 +1,15 @@ + mempool - Bisq Markets + + @@ -31,11 +36,13 @@ - + + + diff --git a/frontend/src/index.liquid.html b/frontend/src/index.liquid.html index 89a6984ba..78a78aaf2 100644 --- a/frontend/src/index.liquid.html +++ b/frontend/src/index.liquid.html @@ -1,10 +1,15 @@ + mempool - Liquid Network + + @@ -17,7 +22,7 @@ - + @@ -33,7 +38,9 @@ + + diff --git a/frontend/src/index.mempool.html b/frontend/src/index.mempool.html index 1176a3da2..b6514ed9a 100644 --- a/frontend/src/index.mempool.html +++ b/frontend/src/index.mempool.html @@ -1,10 +1,15 @@ + mempool - Bitcoin Explorer + + @@ -17,7 +22,7 @@ - + @@ -32,7 +37,9 @@ + + From 81d35d9401f102f9de9455953014272151a99f80 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sat, 15 Oct 2022 19:44:34 -0700 Subject: [PATCH 08/62] Update nginx cache settings for the frontend config files --- nginx-mempool.conf | 7 +++++++ production/nginx/server-common.conf | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/nginx-mempool.conf b/nginx-mempool.conf index a6f701478..d4f111d49 100644 --- a/nginx-mempool.conf +++ b/nginx-mempool.conf @@ -21,6 +21,13 @@ try_files $uri @index-redirect; expires 1h; } + + # only cache /resources/config.* for 5 minutes since it changes often + location /resources/config. { + try_files $uri =404; + expires 5m; + } + location @index-redirect { rewrite (.*) /$lang/index.html; } diff --git a/production/nginx/server-common.conf b/production/nginx/server-common.conf index dedd36411..1d1bcbcd1 100644 --- a/production/nginx/server-common.conf +++ b/production/nginx/server-common.conf @@ -81,6 +81,13 @@ location /resources { try_files $uri /en-US/index.html; expires 1w; } + +# only cache /resources/config.* for 5 minutes since it changes often +location /resources/config. { + try_files $uri =404; + expires 5m; +} + # cache /main.f40e91d908a068a2.js forever since they never change location ~* ^/.+\..+\.(js|css) { try_files /$lang/$uri /en-US/$uri =404; From b77fe0dca202e5c113a8cb2848702adddf8f859a Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sat, 15 Oct 2022 19:45:15 -0700 Subject: [PATCH 09/62] Change template keys in generate-config script --- frontend/generate-config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/generate-config.js b/frontend/generate-config.js index 4b7f80cd8..3cc173e00 100644 --- a/frontend/generate-config.js +++ b/frontend/generate-config.js @@ -75,7 +75,7 @@ const newConfig = `(function (window) { const newConfigTemplate = `(function (window) { window.__env = window.__env || {};${settings.reduce((str, obj) => `${str} - window.__env.${obj.key} = ${typeof obj.value === 'string' ? `'\${${obj.key}}'` : `\${${obj.key}}`};`, '')} + window.__env.${obj.key} = ${typeof obj.value === 'string' ? `'\${__${obj.key}__}'` : `\${__${obj.key}__}`};`, '')} window.__env.GIT_COMMIT_HASH = '${gitCommitHash}'; window.__env.PACKAGE_JSON_VERSION = '${packetJsonVersion}'; }(this));`; From cfa8a9a7d6cd70f1e690b065a1c4353b89a18a19 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sat, 15 Oct 2022 19:46:30 -0700 Subject: [PATCH 10/62] Update the Docker frontend startup script to read and replace runtime config values --- docker/frontend/entrypoint.sh | 56 +++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/docker/frontend/entrypoint.sh b/docker/frontend/entrypoint.sh index 7ebe5632c..29064a396 100644 --- a/docker/frontend/entrypoint.sh +++ b/docker/frontend/entrypoint.sh @@ -10,4 +10,60 @@ cp /etc/nginx/nginx.conf /patch/nginx.conf sed -i "s/__MEMPOOL_FRONTEND_HTTP_PORT__/${__MEMPOOL_FRONTEND_HTTP_PORT__}/g" /patch/nginx.conf cat /patch/nginx.conf > /etc/nginx/nginx.conf +# Runtime overrides - read env vars defined in docker compose + +__TESTNET_ENABLED__=${TESTNET_ENABLED:=false} +__SIGNET_ENABLED__=${SIGNET_ENABLED:=false} +__LIQUID_ENABLED__=${LIQUID_EANBLED:=false} +__LIQUID_TESTNET_ENABLED__=${LIQUID_TESTNET_ENABLED:=false} +__BISQ_ENABLED__=${BISQ_ENABLED:=false} +__BISQ_SEPARATE_BACKEND__=${BISQ_SEPARATE_BACKEND:=false} +__ITEMS_PER_PAGE__=${ITEMS_PER_PAGE:=10} +__KEEP_BLOCKS_AMOUNT__=${KEEP_BLOCKS_AMOUNT:=8} +__NGINX_PROTOCOL__=${NGINX_PROTOCOL:=http} +__NGINX_HOSTNAME__=${NGINX_HOSTNAME:=localhost} +__NGINX_PORT__=${NGINX_PORT:=8999} +__BLOCK_WEIGHT_UNITS__=${BLOCK_WEIGHT_UNITS:=4000000} +__MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_BLOCKS_AMOUNT:=8} +__BASE_MODULE__=${BASE_MODULE:=mempool} +__MEMPOOL_WEBSITE_URL__=${MEMPOOL_WEBSITE_URL:=https://mempool.space} +__LIQUID_WEBSITE_URL__=${LIQUID_WEBSITE_URL:=https://liquid.network} +__BISQ_WEBSITE_URL__=${BISQ_WEBSITE_URL:=https://bisq.markets} +__MINING_DASHBOARD__=${MINING_DASHBOARD:=true} +__LIGHTNING__=${LIGHTNING:=false} + +# Export as environment variables to be used by envsubst +export __TESTNET_ENABLED__ +export __SIGNET_ENABLED__ +export __LIQUID_ENABLED__ +export __LIQUID_TESTNET_ENABLED__ +export __BISQ_ENABLED__ +export __BISQ_SEPARATE_BACKEND__ +export __ITEMS_PER_PAGE__ +export __KEEP_BLOCKS_AMOUNT__ +export __NGINX_PROTOCOL__ +export __NGINX_HOSTNAME__ +export __NGINX_PORT__ +export __BLOCK_WEIGHT_UNITS__ +export __MEMPOOL_BLOCKS_AMOUNT__ +export __BASE_MODULE__ +export __MEMPOOL_WEBSITE_URL__ +export __LIQUID_WEBSITE_URL__ +export __BISQ_WEBSITE_URL__ +export __MINING_DASHBOARD__ +export __LIGHTNING__ + +# This is not an array right now but that might change in the future +files=() +while IFS= read -r -d $'\0'; do + files+=("$REPLY") +done < <(find /var/www/mempool -name "config.js" -print0) + +for file in "${files[@]}" +do + folder=$(dirname ${file}) + echo ${folder} + envsubst < ${folder}/config.template.js > ${folder}/config.js +done + exec "$@" From 82a4212b729d0affe0824a0bfe8b67a146459df8 Mon Sep 17 00:00:00 2001 From: softsimon Date: Mon, 12 Sep 2022 17:31:36 +0200 Subject: [PATCH 11/62] Click to close search dropdown --- .../search-form/search-form.component.html | 4 +--- .../components/search-form/search-form.component.ts | 13 ++++++++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/frontend/src/app/components/search-form/search-form.component.html b/frontend/src/app/components/search-form/search-form.component.html index 1303f4a62..4e38ea6e0 100644 --- a/frontend/src/app/components/search-form/search-form.component.html +++ b/frontend/src/app/components/search-form/search-form.component.html @@ -2,9 +2,7 @@
- - - +
-
-
From 2022d3f6d5b45b655affcaf8f29bd128a813acac Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 31 Oct 2022 09:09:32 -0600 Subject: [PATCH 38/62] Block audit UX adjustments --- .../block-audit/block-audit.component.html | 26 ++++++------------ .../block-audit/block-audit.component.ts | 15 +++++++++-- .../block-overview-graph.component.ts | 27 +++++++++++++++++++ .../block-overview-graph/tx-view.ts | 4 +-- .../block-overview-tooltip.component.html | 4 +-- 5 files changed, 52 insertions(+), 24 deletions(-) diff --git a/frontend/src/app/components/block-audit/block-audit.component.html b/frontend/src/app/components/block-audit/block-audit.component.html index ca28c6707..a3f2e2ada 100644 --- a/frontend/src/app/components/block-audit/block-audit.component.html +++ b/frontend/src/app/components/block-audit/block-audit.component.html @@ -41,10 +41,6 @@
- - Transactions - {{ blockAudit.tx_count }} - Size @@ -61,6 +57,10 @@
+ + + + @@ -69,18 +69,10 @@ - - - - - - - -
Transactions{{ blockAudit.tx_count }}
Block health {{ blockAudit.matchRate }}%Removed txs {{ blockAudit.missingTxs.length }}
Omitted txs{{ numMissing }}
Added txs {{ blockAudit.addedTxs.length }}
Included txs{{ numUnexpected }}
@@ -108,7 +100,6 @@ -
@@ -121,7 +112,6 @@ -
@@ -165,16 +155,16 @@

Projected Block

+ [blockLimit]="stateService.blockVSize" [orientation]="'top'" [flip]="false" [mirrorTxid]="hoverTx" + (txClickEvent)="onTxClick($event)" (txHoverEvent)="onTxHover($event)">

Actual Block

+ [blockLimit]="stateService.blockVSize" [orientation]="'top'" [flip]="false" [mirrorTxid]="hoverTx" + (txClickEvent)="onTxClick($event)" (txHoverEvent)="onTxHover($event)">
diff --git a/frontend/src/app/components/block-audit/block-audit.component.ts b/frontend/src/app/components/block-audit/block-audit.component.ts index f8ce8d9bb..ab85b84ff 100644 --- a/frontend/src/app/components/block-audit/block-audit.component.ts +++ b/frontend/src/app/components/block-audit/block-audit.component.ts @@ -37,6 +37,7 @@ export class BlockAuditComponent implements OnInit, AfterViewInit, OnDestroy { isLoading = true; webGlEnabled = true; isMobile = window.innerWidth <= 767.98; + hoverTx: string; childChangeSubscription: Subscription; @@ -117,9 +118,11 @@ export class BlockAuditComponent implements OnInit, AfterViewInit, OnDestroy { } } for (const [index, tx] of blockAudit.transactions.entries()) { - if (isAdded[tx.txid]) { + if (index === 0) { + tx.status = null; + } else if (isAdded[tx.txid]) { tx.status = 'added'; - } else if (index === 0 || inTemplate[tx.txid]) { + } else if (inTemplate[tx.txid]) { tx.status = 'found'; } else { tx.status = 'selected'; @@ -189,4 +192,12 @@ export class BlockAuditComponent implements OnInit, AfterViewInit, OnDestroy { const url = new RelativeUrlPipe(this.stateService).transform(`/tx/${event.txid}`); this.router.navigate([url]); } + + onTxHover(txid: string): void { + if (txid && txid.length) { + this.hoverTx = txid; + } else { + this.hoverTx = null; + } + } } diff --git a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts index 14607f398..751781d19 100644 --- a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts +++ b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts @@ -18,7 +18,9 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On @Input() orientation = 'left'; @Input() flip = true; @Input() disableSpinner = false; + @Input() mirrorTxid: string | void; @Output() txClickEvent = new EventEmitter(); + @Output() txHoverEvent = new EventEmitter(); @Output() readyEvent = new EventEmitter(); @ViewChild('blockCanvas') @@ -37,6 +39,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On scene: BlockScene; hoverTx: TxView | void; selectedTx: TxView | void; + mirrorTx: TxView | void; tooltipPosition: Position; readyNextFrame = false; @@ -63,6 +66,9 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On this.scene.setOrientation(this.orientation, this.flip); } } + if (changes.mirrorTxid) { + this.setMirror(this.mirrorTxid); + } } ngOnDestroy(): void { @@ -76,6 +82,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On this.exit(direction); this.hoverTx = null; this.selectedTx = null; + this.onTxHover(null); this.start(); } @@ -301,6 +308,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On } this.hoverTx = null; this.selectedTx = null; + this.onTxHover(null); } } @@ -352,17 +360,20 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On this.selectedTx = selected; } else { this.hoverTx = selected; + this.onTxHover(this.hoverTx ? this.hoverTx.txid : null); } } else { if (clicked) { this.selectedTx = null; } this.hoverTx = null; + this.onTxHover(null); } } else if (clicked) { if (selected === this.selectedTx) { this.hoverTx = this.selectedTx; this.selectedTx = null; + this.onTxHover(this.hoverTx ? this.hoverTx.txid : null); } else { this.selectedTx = selected; } @@ -370,6 +381,18 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On } } + setMirror(txid: string | void) { + if (this.mirrorTx) { + this.scene.setHover(this.mirrorTx, false); + this.start(); + } + if (txid && this.scene.txs[txid]) { + this.mirrorTx = this.scene.txs[txid]; + this.scene.setHover(this.mirrorTx, true); + this.start(); + } + } + onTxClick(cssX: number, cssY: number) { const x = cssX * window.devicePixelRatio; const y = cssY * window.devicePixelRatio; @@ -378,6 +401,10 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On this.txClickEvent.emit(selected); } } + + onTxHover(hoverId: string) { + this.txHoverEvent.emit(hoverId); + } } // WebGL shader attributes diff --git a/frontend/src/app/components/block-overview-graph/tx-view.ts b/frontend/src/app/components/block-overview-graph/tx-view.ts index ac2a4655a..f07d96eb0 100644 --- a/frontend/src/app/components/block-overview-graph/tx-view.ts +++ b/frontend/src/app/components/block-overview-graph/tx-view.ts @@ -12,8 +12,8 @@ const auditFeeColors = feeColors.map((color) => desaturate(color, 0.3)); const auditColors = { censored: hexToColor('f344df'), missing: darken(desaturate(hexToColor('f344df'), 0.3), 0.7), - added: hexToColor('03E1E5'), - selected: darken(desaturate(hexToColor('039BE5'), 0.3), 0.7), + added: hexToColor('0099ff'), + selected: darken(desaturate(hexToColor('0099ff'), 0.3), 0.7), } // convert from this class's update format to TxSprite's update format diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html index b19b67b06..8c1002025 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html @@ -37,9 +37,9 @@ match removed - missing + omitted added - included + extra From 1b3bc0ef4efaf7636389001b264e49b8a729a79b Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 31 Oct 2022 09:31:24 -0600 Subject: [PATCH 39/62] Handle block height or hash in audit page --- .../block-audit/block-audit.component.ts | 146 +++++++++++------- 1 file changed, 86 insertions(+), 60 deletions(-) diff --git a/frontend/src/app/components/block-audit/block-audit.component.ts b/frontend/src/app/components/block-audit/block-audit.component.ts index ab85b84ff..3787796fd 100644 --- a/frontend/src/app/components/block-audit/block-audit.component.ts +++ b/frontend/src/app/components/block-audit/block-audit.component.ts @@ -1,9 +1,10 @@ import { Component, OnDestroy, OnInit, AfterViewInit, ViewChildren, QueryList } from '@angular/core'; import { ActivatedRoute, ParamMap, Router } from '@angular/router'; -import { Subscription, combineLatest } from 'rxjs'; -import { map, switchMap, startWith, catchError } from 'rxjs/operators'; +import { Subscription, combineLatest, of } from 'rxjs'; +import { map, switchMap, startWith, catchError, filter } from 'rxjs/operators'; import { BlockAudit, TransactionStripped } from '../../interfaces/node-api.interface'; import { ApiService } from '../../services/api.service'; +import { ElectrsApiService } from '../../services/electrs-api.service'; import { StateService } from '../../services/state.service'; import { detectWebGL } from '../../shared/graphs.utils'; import { RelativeUrlPipe } from '../../shared/pipes/relative-url/relative-url.pipe'; @@ -52,7 +53,8 @@ export class BlockAuditComponent implements OnInit, AfterViewInit, OnDestroy { private route: ActivatedRoute, public stateService: StateService, private router: Router, - private apiService: ApiService + private apiService: ApiService, + private electrsApiService: ElectrsApiService, ) { this.webGlEnabled = detectWebGL(); } @@ -77,71 +79,95 @@ export class BlockAuditComponent implements OnInit, AfterViewInit, OnDestroy { this.auditSubscription = this.route.paramMap.pipe( switchMap((params: ParamMap) => { - this.blockHash = params.get('id') || null; - if (!this.blockHash) { + const blockHash = params.get('id') || null; + if (!blockHash) { return null; } + + let isBlockHeight = false; + if (/^[0-9]+$/.test(blockHash)) { + isBlockHeight = true; + } else { + this.blockHash = blockHash; + } + + if (isBlockHeight) { + return this.electrsApiService.getBlockHashFromHeight$(parseInt(blockHash, 10)) + .pipe( + switchMap((hash: string) => { + if (hash) { + this.blockHash = hash; + return this.apiService.getBlockAudit$(this.blockHash) + } else { + return null; + } + }), + catchError((err) => { + this.error = err; + return of(null); + }), + ); + } return this.apiService.getBlockAudit$(this.blockHash) - .pipe( - map((response) => { - const blockAudit = response.body; - const inTemplate = {}; - const inBlock = {}; - const isAdded = {}; - const isCensored = {}; - const isMissing = {}; - const isSelected = {}; - this.numMissing = 0; - this.numUnexpected = 0; - for (const tx of blockAudit.template) { - inTemplate[tx.txid] = true; - } - for (const tx of blockAudit.transactions) { - inBlock[tx.txid] = true; - } - for (const txid of blockAudit.addedTxs) { - isAdded[txid] = true; - } - for (const txid of blockAudit.missingTxs) { - isCensored[txid] = true; - } - // set transaction statuses - for (const tx of blockAudit.template) { - if (isCensored[tx.txid]) { - tx.status = 'censored'; - } else if (inBlock[tx.txid]) { - tx.status = 'found'; - } else { - tx.status = 'missing'; - isMissing[tx.txid] = true; - this.numMissing++; - } - } - for (const [index, tx] of blockAudit.transactions.entries()) { - if (index === 0) { - tx.status = null; - } else if (isAdded[tx.txid]) { - tx.status = 'added'; - } else if (inTemplate[tx.txid]) { - tx.status = 'found'; - } else { - tx.status = 'selected'; - isSelected[tx.txid] = true; - this.numUnexpected++; - } - } - for (const tx of blockAudit.transactions) { - inBlock[tx.txid] = true; - } - return blockAudit; - }) - ); + }), + filter((response) => response != null), + map((response) => { + const blockAudit = response.body; + const inTemplate = {}; + const inBlock = {}; + const isAdded = {}; + const isCensored = {}; + const isMissing = {}; + const isSelected = {}; + this.numMissing = 0; + this.numUnexpected = 0; + for (const tx of blockAudit.template) { + inTemplate[tx.txid] = true; + } + for (const tx of blockAudit.transactions) { + inBlock[tx.txid] = true; + } + for (const txid of blockAudit.addedTxs) { + isAdded[txid] = true; + } + for (const txid of blockAudit.missingTxs) { + isCensored[txid] = true; + } + // set transaction statuses + for (const tx of blockAudit.template) { + if (isCensored[tx.txid]) { + tx.status = 'censored'; + } else if (inBlock[tx.txid]) { + tx.status = 'found'; + } else { + tx.status = 'missing'; + isMissing[tx.txid] = true; + this.numMissing++; + } + } + for (const [index, tx] of blockAudit.transactions.entries()) { + if (index === 0) { + tx.status = null; + } else if (isAdded[tx.txid]) { + tx.status = 'added'; + } else if (inTemplate[tx.txid]) { + tx.status = 'found'; + } else { + tx.status = 'selected'; + isSelected[tx.txid] = true; + this.numUnexpected++; + } + } + for (const tx of blockAudit.transactions) { + inBlock[tx.txid] = true; + } + return blockAudit; }), catchError((err) => { console.log(err); this.error = err; this.isLoading = false; - return null; + return of(null); }), ).subscribe((blockAudit) => { this.blockAudit = blockAudit; From 5b6f713ef3ed7c99143ce512f4eab4caca155399 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 1 Nov 2022 14:01:50 -0600 Subject: [PATCH 40/62] Fetch missing block audit scores --- backend/src/api/audit.ts | 50 +++++++++++++++- backend/src/api/blocks.ts | 6 +- backend/src/api/mining/mining-routes.ts | 27 +++++++++ backend/src/mempool.interfaces.ts | 5 ++ .../repositories/BlocksAuditsRepository.ts | 6 +- .../app/components/block/block.component.html | 2 +- .../app/components/block/block.component.ts | 26 +++++++- .../blocks-list/blocks-list.component.html | 15 ++--- .../blocks-list/blocks-list.component.scss | 4 ++ .../blocks-list/blocks-list.component.ts | 59 +++++++++++++++++-- .../src/app/interfaces/node-api.interface.ts | 5 ++ frontend/src/app/services/api.service.ts | 15 ++++- 12 files changed, 194 insertions(+), 26 deletions(-) diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index 9ad37c798..1cbfe7a84 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -1,5 +1,10 @@ import config from '../config'; -import { BlockExtended, TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces'; +import bitcoinApi from './bitcoin/bitcoin-api-factory'; +import { Common } from './common'; +import { TransactionExtended, MempoolBlockWithTransactions, AuditScore } from '../mempool.interfaces'; +import blocksRepository from '../repositories/BlocksRepository'; +import blocksAuditsRepository from '../repositories/BlocksAuditsRepository'; +import blocks from '../api/blocks'; const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners @@ -81,7 +86,7 @@ class Audit { } overflowWeight += tx.weight; } - totalWeight += tx.weight + totalWeight += tx.weight; } // transactions missing from near the end of our template are probably not being censored @@ -97,7 +102,7 @@ class Audit { } if (mempool[txid].effectiveFeePerVsize > maxOverflowRate) { maxOverflowRate = mempool[txid].effectiveFeePerVsize; - rateThreshold = (Math.ceil(maxOverflowRate * 100) / 100) + 0.005 + rateThreshold = (Math.ceil(maxOverflowRate * 100) / 100) + 0.005; } } else if (mempool[txid].effectiveFeePerVsize <= rateThreshold) { // tolerance of 0.01 sat/vb + rounding if (isCensored[txid]) { @@ -117,6 +122,45 @@ class Audit { score }; } + + public async $getBlockAuditScores(fromHeight?: number, limit: number = 15): Promise { + let currentHeight = fromHeight !== undefined ? fromHeight : await blocksRepository.$mostRecentBlockHeight(); + const returnScores: AuditScore[] = []; + + if (currentHeight < 0) { + return returnScores; + } + + for (let i = 0; i < limit && currentHeight >= 0; i++) { + const block = blocks.getBlocks().find((b) => b.height === currentHeight); + if (block?.extras?.matchRate != null) { + returnScores.push({ + hash: block.id, + matchRate: block.extras.matchRate + }); + } else { + let currentHash; + if (!currentHash && Common.indexingEnabled()) { + const dbBlock = await blocksRepository.$getBlockByHeight(currentHeight); + if (dbBlock && dbBlock['id']) { + currentHash = dbBlock['id']; + } + } + if (!currentHash) { + currentHash = await bitcoinApi.$getBlockHash(currentHeight); + } + if (currentHash) { + const auditScore = await blocksAuditsRepository.$getBlockAuditScore(currentHash); + returnScores.push({ + hash: currentHash, + matchRate: auditScore?.matchRate + }); + } + } + currentHeight--; + } + return returnScores; + } } export default new Audit(); \ No newline at end of file diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index f536ce3d5..ea2aff78b 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -195,9 +195,9 @@ class Blocks { }; } - const auditSummary = await BlocksAuditsRepository.$getShortBlockAudit(block.id); - if (auditSummary) { - blockExtended.extras.matchRate = auditSummary.matchRate; + const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id); + if (auditScore != null) { + blockExtended.extras.matchRate = auditScore.matchRate; } } diff --git a/backend/src/api/mining/mining-routes.ts b/backend/src/api/mining/mining-routes.ts index 591af3f90..73d38d841 100644 --- a/backend/src/api/mining/mining-routes.ts +++ b/backend/src/api/mining/mining-routes.ts @@ -1,6 +1,7 @@ import { Application, Request, Response } from 'express'; import config from "../../config"; import logger from '../../logger'; +import audits from '../audit'; import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository'; import BlocksRepository from '../../repositories/BlocksRepository'; import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository'; @@ -26,6 +27,9 @@ class MiningRoutes { .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', this.$getHistoricalBlockSizeAndWeight) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments/:interval', this.$getDifficultyAdjustments) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/predictions/:interval', this.$getHistoricalBlockPrediction) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/scores', this.$getBlockAuditScores) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/scores/:height', this.$getBlockAuditScores) + .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/score/:hash', this.$getBlockAuditScore) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/:hash', this.$getBlockAudit) .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/timestamp/:timestamp', this.$getHeightFromTimestamp) ; @@ -276,6 +280,29 @@ class MiningRoutes { res.status(500).send(e instanceof Error ? e.message : e); } } + + private async $getBlockAuditScores(req: Request, res: Response) { + try { + const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10); + res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString()); + res.json(await audits.$getBlockAuditScores(height, 15)); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + public async $getBlockAuditScore(req: Request, res: Response) { + try { + const audit = await BlocksAuditsRepository.$getBlockAuditScore(req.params.hash); + + res.header('Pragma', 'public'); + res.header('Cache-control', 'public'); + res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString()); + res.json(audit || 'null'); + } catch (e) { + res.status(500).send(e instanceof Error ? e.message : e); + } + } } export default new MiningRoutes(); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 32d87f3dc..24bfa1565 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -32,6 +32,11 @@ export interface BlockAudit { matchRate: number, } +export interface AuditScore { + hash: string, + matchRate?: number, +} + export interface MempoolBlock { blockSize: number; blockVSize: number; diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index 188cf4c38..2aa1fb260 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -1,6 +1,6 @@ import DB from '../database'; import logger from '../logger'; -import { BlockAudit } from '../mempool.interfaces'; +import { BlockAudit, AuditScore } from '../mempool.interfaces'; class BlocksAuditRepositories { public async $saveAudit(audit: BlockAudit): Promise { @@ -72,10 +72,10 @@ class BlocksAuditRepositories { } } - public async $getShortBlockAudit(hash: string): Promise { + public async $getBlockAuditScore(hash: string): Promise { try { const [rows]: any[] = await DB.query( - `SELECT hash as id, match_rate as matchRate + `SELECT hash, match_rate as matchRate FROM blocks_audits WHERE blocks_audits.hash = "${hash}" `); diff --git a/frontend/src/app/components/block/block.component.html b/frontend/src/app/components/block/block.component.html index 819b05c81..ba8f3aef3 100644 --- a/frontend/src/app/components/block/block.component.html +++ b/frontend/src/app/components/block/block.component.html @@ -114,7 +114,7 @@ Block health {{ block.extras.matchRate }}% - Unknown + Unknown diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 8f977b81d..aff07a95e 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -4,7 +4,7 @@ import { ActivatedRoute, ParamMap, Router } from '@angular/router'; import { ElectrsApiService } from '../../services/electrs-api.service'; import { switchMap, tap, throttleTime, catchError, map, shareReplay, startWith, pairwise } from 'rxjs/operators'; import { Transaction, Vout } from '../../interfaces/electrs.interface'; -import { Observable, of, Subscription, asyncScheduler, EMPTY } from 'rxjs'; +import { Observable, of, Subscription, asyncScheduler, EMPTY, Subject } from 'rxjs'; import { StateService } from '../../services/state.service'; import { SeoService } from '../../services/seo.service'; import { WebsocketService } from '../../services/websocket.service'; @@ -60,6 +60,8 @@ export class BlockComponent implements OnInit, OnDestroy { nextBlockTxListSubscription: Subscription = undefined; timeLtrSubscription: Subscription; timeLtr: boolean; + fetchAuditScore$ = new Subject(); + fetchAuditScoreSubscription: Subscription; @ViewChild('blockGraph') blockGraph: BlockOverviewGraphComponent; @@ -105,12 +107,30 @@ export class BlockComponent implements OnInit, OnDestroy { if (block.id === this.blockHash) { this.block = block; + if (this.block.id && this.block?.extras?.matchRate == null) { + this.fetchAuditScore$.next(this.block.id); + } if (block?.extras?.reward != undefined) { this.fees = block.extras.reward / 100000000 - this.blockSubsidy; } } }); + if (this.indexingAvailable) { + this.fetchAuditScoreSubscription = this.fetchAuditScore$ + .pipe( + switchMap((hash) => this.apiService.getBlockAuditScore$(hash)), + catchError(() => EMPTY), + ) + .subscribe((score) => { + if (score && score.hash === this.block.id) { + this.block.extras.matchRate = score.matchRate || null; + } else { + this.block.extras.matchRate = null; + } + }); + } + const block$ = this.route.paramMap.pipe( switchMap((params: ParamMap) => { const blockHash: string = params.get('id') || ''; @@ -209,6 +229,9 @@ export class BlockComponent implements OnInit, OnDestroy { this.fees = block.extras.reward / 100000000 - this.blockSubsidy; } this.stateService.markBlock$.next({ blockHeight: this.blockHeight }); + if (this.block.id && this.block?.extras?.matchRate == null) { + this.fetchAuditScore$.next(this.block.id); + } this.isLoadingTransactions = true; this.transactions = null; this.transactionsError = null; @@ -311,6 +334,7 @@ export class BlockComponent implements OnInit, OnDestroy { this.networkChangedSubscription.unsubscribe(); this.queryParamsSubscription.unsubscribe(); this.timeLtrSubscription.unsubscribe(); + this.fetchAuditScoreSubscription?.unsubscribe(); this.unsubscribeNextBlockSubscriptions(); } diff --git a/frontend/src/app/components/blocks-list/blocks-list.component.html b/frontend/src/app/components/blocks-list/blocks-list.component.html index 68acf71ea..69bcf3141 100644 --- a/frontend/src/app/components/blocks-list/blocks-list.component.html +++ b/frontend/src/app/components/blocks-list/blocks-list.component.html @@ -46,22 +46,17 @@ - +
+ [ngStyle]="{'width': (100 - (auditScores[block.id] || 0)) + '%' }">
- {{ block.extras.matchRate }}% + {{ auditScores[block.id] }}% + + ~
-
-
-
- ~ -
-
diff --git a/frontend/src/app/components/blocks-list/blocks-list.component.scss b/frontend/src/app/components/blocks-list/blocks-list.component.scss index 6617cec58..713e59640 100644 --- a/frontend/src/app/components/blocks-list/blocks-list.component.scss +++ b/frontend/src/app/components/blocks-list/blocks-list.component.scss @@ -196,6 +196,10 @@ tr, td, th { @media (max-width: 950px) { display: none; } + + .progress-text .skeleton-loader { + top: -8.5px; + } } .health.widget { width: 25%; diff --git a/frontend/src/app/components/blocks-list/blocks-list.component.ts b/frontend/src/app/components/blocks-list/blocks-list.component.ts index 7e4c34eb4..700032225 100644 --- a/frontend/src/app/components/blocks-list/blocks-list.component.ts +++ b/frontend/src/app/components/blocks-list/blocks-list.component.ts @@ -1,6 +1,6 @@ -import { Component, OnInit, ChangeDetectionStrategy, Input } from '@angular/core'; -import { BehaviorSubject, combineLatest, concat, Observable, timer } from 'rxjs'; -import { delayWhen, map, retryWhen, scan, skip, switchMap, tap } from 'rxjs/operators'; +import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, Input } from '@angular/core'; +import { BehaviorSubject, combineLatest, concat, Observable, timer, EMPTY, Subscription, of } from 'rxjs'; +import { catchError, delayWhen, map, retryWhen, scan, skip, switchMap, tap } from 'rxjs/operators'; import { BlockExtended } from '../../interfaces/node-api.interface'; import { ApiService } from '../../services/api.service'; import { StateService } from '../../services/state.service'; @@ -12,10 +12,14 @@ import { WebsocketService } from '../../services/websocket.service'; styleUrls: ['./blocks-list.component.scss'], changeDetection: ChangeDetectionStrategy.OnPush, }) -export class BlocksList implements OnInit { +export class BlocksList implements OnInit, OnDestroy { @Input() widget: boolean = false; blocks$: Observable = undefined; + auditScores: { [hash: string]: number | void } = {}; + + auditScoreSubscription: Subscription; + latestScoreSubscription: Subscription; indexingAvailable = false; isLoading = true; @@ -105,6 +109,53 @@ export class BlocksList implements OnInit { return acc; }, []) ); + + if (this.indexingAvailable) { + this.auditScoreSubscription = this.fromHeightSubject.pipe( + switchMap((fromBlockHeight) => { + return this.apiService.getBlockAuditScores$(this.page === 1 ? undefined : fromBlockHeight) + .pipe( + catchError(() => { + return EMPTY; + }) + ); + }) + ).subscribe((scores) => { + Object.values(scores).forEach(score => { + this.auditScores[score.hash] = score?.matchRate != null ? score.matchRate : null; + }); + }); + + this.latestScoreSubscription = this.stateService.blocks$.pipe( + switchMap((block) => { + if (block[0]?.extras?.matchRate != null) { + return of({ + hash: block[0].id, + matchRate: block[0]?.extras?.matchRate, + }); + } + else if (block[0]?.id && this.auditScores[block[0].id] === undefined) { + return this.apiService.getBlockAuditScore$(block[0].id) + .pipe( + catchError(() => { + return EMPTY; + }) + ); + } else { + return EMPTY; + } + }), + ).subscribe((score) => { + if (score && score.hash) { + this.auditScores[score.hash] = score?.matchRate != null ? score.matchRate : null; + } + }); + } + } + + ngOnDestroy(): void { + this.auditScoreSubscription?.unsubscribe(); + this.latestScoreSubscription?.unsubscribe(); } pageChange(page: number) { diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 8e04c8635..39d0c3d5d 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -152,6 +152,11 @@ export interface RewardStats { totalTx: number; } +export interface AuditScore { + hash: string; + matchRate?: number; +} + export interface ITopNodesPerChannels { publicKey: string, alias: string, diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index 8c0f5ecd0..dfed35d72 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -1,7 +1,7 @@ import { Injectable } from '@angular/core'; import { HttpClient, HttpParams } from '@angular/common/http'; import { CpfpInfo, OptimizedMempoolStats, AddressInformation, LiquidPegs, ITranslators, - PoolStat, BlockExtended, TransactionStripped, RewardStats } from '../interfaces/node-api.interface'; + PoolStat, BlockExtended, TransactionStripped, RewardStats, AuditScore } from '../interfaces/node-api.interface'; import { Observable } from 'rxjs'; import { StateService } from './state.service'; import { WebsocketResponse } from '../interfaces/websocket.interface'; @@ -234,6 +234,19 @@ export class ApiService { ); } + getBlockAuditScores$(from: number): Observable { + return this.httpClient.get( + this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/blocks/audit/scores` + + (from !== undefined ? `/${from}` : ``) + ); + } + + getBlockAuditScore$(hash: string) : Observable { + return this.httpClient.get( + this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/blocks/audit/score/` + hash + ); + } + getRewardStats$(blockCount: number = 144): Observable { return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/reward-stats/${blockCount}`); } From 373e02a5b05fd811e28f556e03108d680af382fe Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 4 Nov 2022 18:21:08 -0600 Subject: [PATCH 41/62] Store & expose node extension TLV data in backend --- backend/src/api/database-migration.ts | 19 +++++- backend/src/api/explorer/nodes.api.ts | 12 ++++ .../clightning/clightning-convert.ts | 10 +++ .../api/lightning/lightning-api.interface.ts | 1 + .../src/repositories/NodeRecordsRepository.ts | 67 +++++++++++++++++++ .../tasks/lightning/network-sync.service.ts | 19 +++++- 6 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 backend/src/repositories/NodeRecordsRepository.ts diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 20e5ab339..3bbd501fc 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 42; + private static currentVersion = 43; private queryTimeout = 120000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -356,6 +356,10 @@ class DatabaseMigration { if (databaseSchemaVersion < 42 && isBitcoin === true) { await this.$executeQuery('ALTER TABLE `channels` ADD closing_resolved tinyint(1) DEFAULT 0'); } + + if (databaseSchemaVersion < 43 && isBitcoin === true) { + await this.$executeQuery(this.getCreateLNNodeRecordsTableQuery(), await this.$checkIfTableExists('nodes_records')); + } } /** @@ -791,6 +795,19 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } + private getCreateLNNodeRecordsTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS nodes_records ( + public_key varchar(66) NOT NULL, + type int(10) unsigned NOT NULL, + payload blob NOT NULL, + UNIQUE KEY public_key_type (public_key, type), + INDEX (public_key), + FOREIGN KEY (public_key) + REFERENCES nodes (public_key) + ON DELETE CASCADE + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } + public async $truncateIndexedData(tables: string[]) { const allowedTables = ['blocks', 'hashrates', 'prices']; diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index d8dceab19..b21544e4b 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -105,6 +105,18 @@ class NodesApi { node.closed_channel_count = rows[0].closed_channel_count; } + // Custom records + query = ` + SELECT type, payload + FROM nodes_records + WHERE public_key = ? + `; + [rows] = await DB.query(query, [public_key]); + node.custom_records = {}; + for (const record of rows) { + node.custom_records[record.type] = Buffer.from(record.payload, 'binary').toString('hex'); + } + return node; } catch (e) { logger.err(`Cannot get node information for ${public_key}. Reason: ${(e instanceof Error ? e.message : e)}`); diff --git a/backend/src/api/lightning/clightning/clightning-convert.ts b/backend/src/api/lightning/clightning/clightning-convert.ts index 9b3c62f04..92ae1f0a7 100644 --- a/backend/src/api/lightning/clightning/clightning-convert.ts +++ b/backend/src/api/lightning/clightning/clightning-convert.ts @@ -7,6 +7,15 @@ import { Common } from '../../common'; * Convert a clightning "listnode" entry to a lnd node entry */ export function convertNode(clNode: any): ILightningApi.Node { + let custom_records: { [type: number]: string } | undefined = undefined; + if (clNode.option_will_fund) { + try { + custom_records = { '1': Buffer.from(clNode.option_will_fund.compact_lease || '', 'hex').toString('base64') }; + } catch (e) { + logger.err(`Cannot decode option_will_fund compact_lease for ${clNode.nodeid}). Reason: ` + (e instanceof Error ? e.message : e)); + custom_records = undefined; + } + } return { alias: clNode.alias ?? '', color: `#${clNode.color ?? ''}`, @@ -23,6 +32,7 @@ export function convertNode(clNode: any): ILightningApi.Node { }; }) ?? [], last_update: clNode?.last_timestamp ?? 0, + custom_records }; } diff --git a/backend/src/api/lightning/lightning-api.interface.ts b/backend/src/api/lightning/lightning-api.interface.ts index 1a5e2793f..6e3ea0de3 100644 --- a/backend/src/api/lightning/lightning-api.interface.ts +++ b/backend/src/api/lightning/lightning-api.interface.ts @@ -49,6 +49,7 @@ export namespace ILightningApi { }[]; color: string; features: { [key: number]: Feature }; + custom_records?: { [type: number]: string }; } export interface Info { diff --git a/backend/src/repositories/NodeRecordsRepository.ts b/backend/src/repositories/NodeRecordsRepository.ts new file mode 100644 index 000000000..cf676e35e --- /dev/null +++ b/backend/src/repositories/NodeRecordsRepository.ts @@ -0,0 +1,67 @@ +import { ResultSetHeader, RowDataPacket } from 'mysql2'; +import DB from '../database'; +import logger from '../logger'; + +export interface NodeRecord { + publicKey: string; // node public key + type: number; // TLV extension record type + payload: string; // base64 record payload +} + +class NodesRecordsRepository { + public async $saveRecord(record: NodeRecord): Promise { + try { + const payloadBytes = Buffer.from(record.payload, 'base64'); + await DB.query(` + INSERT INTO nodes_records(public_key, type, payload) + VALUE (?, ?, ?) + ON DUPLICATE KEY UPDATE + payload = ? + `, [record.publicKey, record.type, payloadBytes, payloadBytes]); + } catch (e: any) { + if (e.errno !== 1062) { // ER_DUP_ENTRY - Not an issue, just ignore this + logger.err(`Cannot save node record (${[record.publicKey, record.type, record.payload]}) into db. Reason: ` + (e instanceof Error ? e.message : e)); + // We don't throw, not a critical issue if we miss some nodes records + } + } + } + + public async $getRecordTypes(publicKey: string): Promise { + try { + const query = ` + SELECT type FROM nodes_records + WHERE public_key = ? + `; + const [rows] = await DB.query(query, [publicKey]); + return rows.map(row => row['type']); + } catch (e) { + logger.err(`Cannot retrieve custom records for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e)); + return []; + } + } + + public async $deleteUnusedRecords(publicKey: string, recordTypes: number[]): Promise { + try { + let query; + if (recordTypes.length) { + query = ` + DELETE FROM nodes_records + WHERE public_key = ? + AND type NOT IN (${recordTypes.map(type => `${type}`).join(',')}) + `; + } else { + query = ` + DELETE FROM nodes_records + WHERE public_key = ? + `; + } + const [result] = await DB.query(query, [publicKey]); + return result.affectedRows; + } catch (e) { + logger.err(`Cannot delete unused custom records for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e)); + return 0; + } + } +} + +export default new NodesRecordsRepository(); diff --git a/backend/src/tasks/lightning/network-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts index 70173d6bc..2910f0f9c 100644 --- a/backend/src/tasks/lightning/network-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -13,6 +13,7 @@ import fundingTxFetcher from './sync-tasks/funding-tx-fetcher'; import NodesSocketsRepository from '../../repositories/NodesSocketsRepository'; import { Common } from '../../api/common'; import blocks from '../../api/blocks'; +import NodeRecordsRepository from '../../repositories/NodeRecordsRepository'; class NetworkSyncService { loggerTimer = 0; @@ -63,6 +64,7 @@ class NetworkSyncService { let progress = 0; let deletedSockets = 0; + let deletedRecords = 0; const graphNodesPubkeys: string[] = []; for (const node of nodes) { const latestUpdated = await channelsApi.$getLatestChannelUpdateForNode(node.pub_key); @@ -84,8 +86,23 @@ class NetworkSyncService { addresses.push(socket.addr); } deletedSockets += await NodesSocketsRepository.$deleteUnusedSockets(node.pub_key, addresses); + + const oldRecordTypes = await NodeRecordsRepository.$getRecordTypes(node.pub_key); + const customRecordTypes: number[] = []; + for (const [type, payload] of Object.entries(node.custom_records || {})) { + const numericalType = parseInt(type); + await NodeRecordsRepository.$saveRecord({ + publicKey: node.pub_key, + type: numericalType, + payload, + }); + customRecordTypes.push(numericalType); + } + if (oldRecordTypes.reduce((changed, type) => changed || customRecordTypes.indexOf(type) === -1, false)) { + deletedRecords += await NodeRecordsRepository.$deleteUnusedRecords(node.pub_key, customRecordTypes); + } } - logger.info(`${progress} nodes updated. ${deletedSockets} sockets deleted`); + logger.info(`${progress} nodes updated. ${deletedSockets} sockets deleted. ${deletedRecords} custom records deleted.`); // If a channel if not present in the graph, mark it as inactive await nodesApi.$setNodesInactive(graphNodesPubkeys); From 010e9f2bb148854de8ea4b8263b0eca7484ac912 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 4 Nov 2022 21:59:54 -0600 Subject: [PATCH 42/62] Display extension TLV data on node page --- .../app/lightning/node/node.component.html | 22 +++++++++++++++++++ .../app/lightning/node/node.component.scss | 15 +++++++++++++ .../src/app/lightning/node/node.component.ts | 11 +++++++++- 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/lightning/node/node.component.html b/frontend/src/app/lightning/node/node.component.html index 7d506e6b0..9c8fc6149 100644 --- a/frontend/src/app/lightning/node/node.component.html +++ b/frontend/src/app/lightning/node/node.component.html @@ -125,6 +125,28 @@ +
+
+
TLV extension records
+
+
+ + + + + + + +
{{ recordItem.key }}{{ recordItem.value }}
+
+
+
+
+ +
+ +
+
diff --git a/frontend/src/app/lightning/node/node.component.scss b/frontend/src/app/lightning/node/node.component.scss index 3803ce2fb..fe9737b85 100644 --- a/frontend/src/app/lightning/node/node.component.scss +++ b/frontend/src/app/lightning/node/node.component.scss @@ -72,3 +72,18 @@ app-fiat { height: 28px !important; }; } + +.details tbody { + font-size: 12px; + + .tlv-type { + color: #ffffff66; + } + + .tlv-payload { + width: 100%; + word-break: break-all; + white-space: normal; + font-family: "Courier New", Courier, monospace; + } +} diff --git a/frontend/src/app/lightning/node/node.component.ts b/frontend/src/app/lightning/node/node.component.ts index e2a8123ac..161a53e37 100644 --- a/frontend/src/app/lightning/node/node.component.ts +++ b/frontend/src/app/lightning/node/node.component.ts @@ -1,7 +1,7 @@ import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; import { ActivatedRoute, ParamMap } from '@angular/router'; import { Observable } from 'rxjs'; -import { catchError, map, switchMap } from 'rxjs/operators'; +import { catchError, map, switchMap, tap } from 'rxjs/operators'; import { SeoService } from '../../services/seo.service'; import { LightningApiService } from '../lightning-api.service'; import { GeolocationData } from '../../shared/components/geolocation/geolocation.component'; @@ -24,6 +24,8 @@ export class NodeComponent implements OnInit { channelListLoading = false; clearnetSocketCount = 0; torSocketCount = 0; + hasDetails = false; + showDetails = false; constructor( private lightningApiService: LightningApiService, @@ -79,6 +81,9 @@ export class NodeComponent implements OnInit { return node; }), + tap((node) => { + this.hasDetails = Object.keys(node.custom_records).length > 0; + }), catchError(err => { this.error = err; return [{ @@ -89,6 +94,10 @@ export class NodeComponent implements OnInit { ); } + toggleShowDetails(): void { + this.showDetails = !this.showDetails; + } + changeSocket(index: number) { this.selectedSocketIndex = index; } From c2ab0bc7155ad2aa0cbc10a578e270904359536c Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 4 Nov 2022 23:24:44 -0600 Subject: [PATCH 43/62] Parse & display liquidity ads on node page --- .../src/app/lightning/node/liquidity-ad.ts | 31 +++++++ .../app/lightning/node/node.component.html | 89 ++++++++++++++++--- .../app/lightning/node/node.component.scss | 18 +++- .../src/app/lightning/node/node.component.ts | 27 ++++++ 4 files changed, 151 insertions(+), 14 deletions(-) create mode 100644 frontend/src/app/lightning/node/liquidity-ad.ts diff --git a/frontend/src/app/lightning/node/liquidity-ad.ts b/frontend/src/app/lightning/node/liquidity-ad.ts new file mode 100644 index 000000000..4b0e04b0b --- /dev/null +++ b/frontend/src/app/lightning/node/liquidity-ad.ts @@ -0,0 +1,31 @@ +export interface ILiquidityAd { + funding_weight: number; + lease_fee_basis: number; // lease fee rate in parts-per-thousandth + lease_fee_base_sat: number; // fixed lease fee in sats + channel_fee_max_rate: number; // max routing fee rate in parts-per-thousandth + channel_fee_max_base: number; // max routing base fee in milli-sats + compact_lease?: string; +} + +export function parseLiquidityAdHex(compact_lease: string): ILiquidityAd | false { + if (!compact_lease || compact_lease.length < 20 || compact_lease.length > 28) { + return false; + } + try { + const liquidityAd: ILiquidityAd = { + funding_weight: parseInt(compact_lease.slice(0, 4), 16), + lease_fee_basis: parseInt(compact_lease.slice(4, 8), 16), + channel_fee_max_rate: parseInt(compact_lease.slice(8, 12), 16), + lease_fee_base_sat: parseInt(compact_lease.slice(12, 20), 16), + channel_fee_max_base: compact_lease.length > 20 ? parseInt(compact_lease.slice(20), 16) : 0, + } + if (Object.values(liquidityAd).reduce((valid: boolean, value: number): boolean => (valid && !isNaN(value) && value >= 0), true)) { + liquidityAd.compact_lease = compact_lease; + return liquidityAd; + } else { + return false; + } + } catch (err) { + return false; + } +} \ No newline at end of file diff --git a/frontend/src/app/lightning/node/node.component.html b/frontend/src/app/lightning/node/node.component.html index 9c8fc6149..858aa9b48 100644 --- a/frontend/src/app/lightning/node/node.component.html +++ b/frontend/src/app/lightning/node/node.component.html @@ -127,19 +127,84 @@
-
TLV extension records
-
-
- - - - - - - -
{{ recordItem.key }}{{ recordItem.value }}
+ +
+
Liquidity ad
+
+
+ + + + + + + + + + + + + + + +
Lease fee rate + + {{ liquidityAd.lease_fee_basis !== null ? ((liquidityAd.lease_fee_basis * 1000) | amountShortener : 2 : undefined : true) : '-' }} ppm {{ liquidityAd.lease_fee_basis !== null ? '(' + (liquidityAd.lease_fee_basis / 10 | amountShortener : 2 : undefined : true) + '%)' : '' }} + +
Lease base fee + +
Funding weight
+
+
+ + + + + + + + + + + + + + + +
Channel fee rate + + {{ liquidityAd.channel_fee_max_rate !== null ? ((liquidityAd.channel_fee_max_rate * 1000) | amountShortener : 2 : undefined : true) : '-' }} ppm {{ liquidityAd.channel_fee_max_rate !== null ? '(' + (liquidityAd.channel_fee_max_rate / 10 | amountShortener : 2 : undefined : true) + '%)' : '' }} + +
Channel base fee + + {{ liquidityAd.channel_fee_max_base | amountShortener : 0 }} + mSats + + + - + +
Compact lease{{ liquidityAd.compact_lease }}
+
+
-
+ + +
+
TLV extension records
+
+
+ + + + + + + +
{{ recordItem.type }}{{ recordItem.payload }}
+
+
+
+
diff --git a/frontend/src/app/lightning/node/node.component.scss b/frontend/src/app/lightning/node/node.component.scss index fe9737b85..d54b1851b 100644 --- a/frontend/src/app/lightning/node/node.component.scss +++ b/frontend/src/app/lightning/node/node.component.scss @@ -73,17 +73,31 @@ app-fiat { }; } -.details tbody { - font-size: 12px; +.details { + + .detail-section { + margin-bottom: 1.5rem; + &:last-child { + margin-bottom: 0; + } + } .tlv-type { + font-size: 12px; color: #ffffff66; } .tlv-payload { + font-size: 12px; width: 100%; word-break: break-all; white-space: normal; font-family: "Courier New", Courier, monospace; } + + .compact-lease { + word-break: break-all; + white-space: normal; + font-family: "Courier New", Courier, monospace; + } } diff --git a/frontend/src/app/lightning/node/node.component.ts b/frontend/src/app/lightning/node/node.component.ts index 161a53e37..ec2edd252 100644 --- a/frontend/src/app/lightning/node/node.component.ts +++ b/frontend/src/app/lightning/node/node.component.ts @@ -5,6 +5,12 @@ import { catchError, map, switchMap, tap } from 'rxjs/operators'; import { SeoService } from '../../services/seo.service'; import { LightningApiService } from '../lightning-api.service'; import { GeolocationData } from '../../shared/components/geolocation/geolocation.component'; +import { ILiquidityAd, parseLiquidityAdHex } from './liquidity-ad'; + +interface CustomRecord { + type: string; + payload: string; +} @Component({ selector: 'app-node', @@ -26,6 +32,8 @@ export class NodeComponent implements OnInit { torSocketCount = 0; hasDetails = false; showDetails = false; + liquidityAd: ILiquidityAd; + tlvRecords: CustomRecord[]; constructor( private lightningApiService: LightningApiService, @@ -38,6 +46,8 @@ export class NodeComponent implements OnInit { .pipe( switchMap((params: ParamMap) => { this.publicKey = params.get('public_key'); + this.tlvRecords = []; + this.liquidityAd = null; return this.lightningApiService.getNode$(params.get('public_key')); }), map((node) => { @@ -83,6 +93,23 @@ export class NodeComponent implements OnInit { }), tap((node) => { this.hasDetails = Object.keys(node.custom_records).length > 0; + for (const [type, payload] of Object.entries(node.custom_records)) { + if (typeof payload !== 'string') { + break; + } + + let parsed = false; + if (type === '1') { + const ad = parseLiquidityAdHex(payload); + if (ad) { + parsed = true; + this.liquidityAd = ad; + } + } + if (!parsed) { + this.tlvRecords.push({ type, payload }); + } + } }), catchError(err => { this.error = err; From eb2abefabc4673b2fbacfd23427bd04f526a5000 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 21 Nov 2022 12:28:26 +0900 Subject: [PATCH 44/62] Add shapes to flow diagram to indicate spent txos --- .../transaction-preview.component.html | 2 +- .../transaction-preview.component.scss | 2 +- .../transaction/transaction.component.html | 1 + .../transaction/transaction.component.scss | 2 +- .../transaction/transaction.component.ts | 2 +- .../tx-bowtie-graph.component.html | 24 +++++++++++++ .../tx-bowtie-graph.component.scss | 11 ++++++ .../tx-bowtie-graph.component.ts | 35 ++++++++++++++----- 8 files changed, 67 insertions(+), 12 deletions(-) diff --git a/frontend/src/app/components/transaction/transaction-preview.component.html b/frontend/src/app/components/transaction/transaction-preview.component.html index f023a77b1..cb273b16c 100644 --- a/frontend/src/app/components/transaction/transaction-preview.component.html +++ b/frontend/src/app/components/transaction/transaction-preview.component.html @@ -29,7 +29,7 @@
- +

diff --git a/frontend/src/app/components/transaction/transaction-preview.component.scss b/frontend/src/app/components/transaction/transaction-preview.component.scss index 65c0ca75e..75eceb99e 100644 --- a/frontend/src/app/components/transaction/transaction-preview.component.scss +++ b/frontend/src/app/components/transaction/transaction-preview.component.scss @@ -69,7 +69,7 @@ .graph-wrapper { position: relative; background: #181b2d; - padding: 10px; + padding: 10px 0; padding-bottom: 0; .above-bow { diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index ec0e824c8..2f4fb7ba2 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -209,6 +209,7 @@ [maxStrands]="graphExpanded ? maxInOut : 24" [network]="network" [tooltip]="true" + [connectors]="true" [inputIndex]="inputIndex" [outputIndex]="outputIndex" > diff --git a/frontend/src/app/components/transaction/transaction.component.scss b/frontend/src/app/components/transaction/transaction.component.scss index 7127a898a..619cac89a 100644 --- a/frontend/src/app/components/transaction/transaction.component.scss +++ b/frontend/src/app/components/transaction/transaction.component.scss @@ -86,7 +86,7 @@ position: relative; width: 100%; background: #181b2d; - padding: 10px; + padding: 10px 0; padding-bottom: 0; } diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index 0235dd887..787abe935 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -402,7 +402,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { @HostListener('window:resize', ['$event']) setGraphSize(): void { if (this.graphContainer) { - this.graphWidth = this.graphContainer.nativeElement.clientWidth - 24; + this.graphWidth = this.graphContainer.nativeElement.clientWidth; } } diff --git a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html index a85f62c65..c09064201 100644 --- a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html +++ b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html @@ -29,6 +29,14 @@ + + + + + + + + @@ -65,6 +73,14 @@ + + = new ReplaySubject(); gradientColors = { - '': ['#9339f4', '#105fb0'], - bisq: ['#9339f4', '#105fb0'], + '': ['#9339f4', '#105fb0', '#9339f433'], + bisq: ['#9339f4', '#105fb0', '#9339f433'], // liquid: ['#116761', '#183550'], - liquid: ['#09a197', '#0f62af'], + liquid: ['#09a197', '#0f62af', '#09a19733'], // 'liquidtestnet': ['#494a4a', '#272e46'], - 'liquidtestnet': ['#d2d2d2', '#979797'], + 'liquidtestnet': ['#d2d2d2', '#979797', '#d2d2d233'], // testnet: ['#1d486f', '#183550'], - testnet: ['#4edf77', '#10a0af'], + testnet: ['#4edf77', '#10a0af', '#4edf7733'], // signet: ['#6f1d5d', '#471850'], - signet: ['#d24fc8', '#a84fd2'], + signet: ['#d24fc8', '#a84fd2', '#d24fc833'], }; gradient: string[] = ['#105fb0', '#105fb0']; @@ -308,13 +310,14 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { return { path: this.makePath(side, line.outerY, line.innerY, line.thickness, line.offset, pad + maxOffset), style: this.makeStyle(line.thickness, xputs[i].type), - class: xputs[i].type + class: xputs[i].type, + connectorPath: this.connectors ? this.makeConnectorPath(side, line.outerY, line.innerY, line.thickness): null, }; }); } makePath(side: 'in' | 'out', outer: number, inner: number, weight: number, offset: number, pad: number): string { - const start = (weight * 0.5); + const start = (weight * 0.5) + 10; const curveStart = Math.max(start + 1, pad - offset); const end = this.width / 2 - (this.midWidth * 0.9) + 1; const curveEnd = end - offset - 10; @@ -332,6 +335,22 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { } } + makeConnectorPath(side: 'in' | 'out', y: number, inner, weight: number): string { + const halfWidth = weight * 0.5; + const offset = Math.max(2, halfWidth * 0.2); + + // align with for svg horizontal gradient bug correction + if (Math.round(y) === Math.round(inner)) { + y -= 1; + } + + if (side === 'in') { + return `M ${10 - offset} ${y - halfWidth} L ${halfWidth + 10 - offset} ${y} L ${10 - offset} ${y + halfWidth} L -10 ${ y + halfWidth} L -10 ${y - halfWidth}`; + } else { + return `M ${this.width - halfWidth - 10 + offset} ${y - halfWidth} L ${this.width - 10 + offset} ${y} L ${this.width - halfWidth - 10 + offset} ${y + halfWidth} L ${this.width + 10} ${ y + halfWidth} L ${this.width + 10} ${y - halfWidth}`; + } + } + makeStyle(minWeight, type): string { if (type === 'fee') { return `stroke-width: ${minWeight}`; From 2c1f38aa9d7e277bdbb5d2e26ede3e673fe116fd Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 21 Nov 2022 12:28:39 +0900 Subject: [PATCH 45/62] Fix clash w/ liquid unblinding and vin/vout syntax --- .../src/app/components/transaction/liquid-ublinding.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/components/transaction/liquid-ublinding.ts b/frontend/src/app/components/transaction/liquid-ublinding.ts index 338e9013c..a53edeb4c 100644 --- a/frontend/src/app/components/transaction/liquid-ublinding.ts +++ b/frontend/src/app/components/transaction/liquid-ublinding.ts @@ -126,9 +126,13 @@ export class LiquidUnblinding { } async checkUnblindedTx(tx: Transaction) { - const windowLocationHash = window.location.hash.substring('#blinded='.length); - if (windowLocationHash.length > 0) { - const blinders = this.parseBlinders(windowLocationHash); + if (!window.location.hash?.length) { + return tx; + } + const fragmentParams = new URLSearchParams(window.location.hash.slice(1) || ''); + const blinderStr = fragmentParams.get('blinded'); + if (blinderStr && blinderStr.length) { + const blinders = this.parseBlinders(blinderStr); if (blinders) { this.commitments = await this.makeCommitmentMap(blinders); return this.tryUnblindTx(tx); From 14ec427f5ef372365e04e78045c4a1d8d8708625 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 21 Nov 2022 12:28:46 +0900 Subject: [PATCH 46/62] Mouse events for flow diagram endcaps & connectors --- .../tx-bowtie-graph.component.html | 16 +++++++++ .../tx-bowtie-graph.component.scss | 33 +++++++++++-------- .../tx-bowtie-graph.component.ts | 18 ++++++++++ 3 files changed, 54 insertions(+), 13 deletions(-) diff --git a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html index c09064201..c736e5548 100644 --- a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html +++ b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html @@ -81,6 +81,14 @@ (pointerout)="onBlur($event, 'input', i);" (click)="onClick($event, 'input', inputData[i].index);" /> + + Date: Tue, 22 Nov 2022 10:05:14 +0900 Subject: [PATCH 47/62] longer input/output connectors on flow diagram & new nav logic --- .../tx-bowtie-graph-tooltip.component.html | 17 +++++- .../tx-bowtie-graph-tooltip.component.ts | 4 ++ .../tx-bowtie-graph.component.html | 21 +++++-- .../tx-bowtie-graph.component.scss | 11 +++- .../tx-bowtie-graph.component.ts | 57 ++++++++++++------- 5 files changed, 80 insertions(+), 30 deletions(-) diff --git a/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.html b/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.html index cbf2f7d5a..6262c56a7 100644 --- a/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.html +++ b/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.html @@ -22,13 +22,13 @@ -

Peg In

+

Peg In

-

Peg Out

+

Peg Out

{{ line.pegout.slice(0, -4) }} @@ -38,7 +38,7 @@ -

+

Input Output @@ -46,6 +46,17 @@ #{{ line.index + 1 }}

+ +

+ Transaction  + {{ line.txid.slice(0, 8) }}... + {{ line.txid.slice(-4) }} +

+ +

Output  #{{ line.vout + 1 }}

+

Input  #{{ line.vin + 1 }}

+ +

Confidential

diff --git a/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts b/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts index 4e450a7dc..54c58ffab 100644 --- a/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts +++ b/frontend/src/app/components/tx-bowtie-graph-tooltip/tx-bowtie-graph-tooltip.component.ts @@ -5,6 +5,9 @@ interface Xput { type: 'input' | 'output' | 'fee'; value?: number; index?: number; + txid?: string; + vin?: number; + vout?: number; address?: string; rest?: number; coinbase?: boolean; @@ -21,6 +24,7 @@ interface Xput { export class TxBowtieGraphTooltipComponent implements OnChanges { @Input() line: Xput | void; @Input() cursorPosition: { x: number, y: number }; + @Input() isConnector: boolean = false; tooltipPosition = { x: 0, y: 0 }; diff --git a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html index c736e5548..23346f405 100644 --- a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html +++ b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html @@ -49,6 +49,14 @@ + + + + + + + + @@ -77,9 +85,9 @@ [attr.d]="input.connectorPath" class="input connector {{input.class}}" [class.highlight]="inputData[i].index === inputIndex" - (pointerover)="onHover($event, 'input', i);" - (pointerout)="onBlur($event, 'input', i);" - (click)="onClick($event, 'input', inputData[i].index);" + (pointerover)="onHover($event, 'input-connector', i);" + (pointerout)="onBlur($event, 'input-connector', i);" + (click)="onClick($event, 'input-connector', inputData[i].index);" />

diff --git a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.scss b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.scss index 71bf5e97b..7c9ecf0ce 100644 --- a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.scss +++ b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.scss @@ -24,7 +24,7 @@ } } - .line:hover, .connector:hover + .marker-target + .line, .marker-target:hover + .line { + .line:hover, .marker-target:hover + .line { z-index: 10; cursor: pointer; &.input { @@ -50,6 +50,15 @@ } } + .connector:hover { + &.input { + fill: url(#input-hover-connector-gradient); + } + &.output { + fill: url(#output-hover-connector-gradient); + } + } + .marker-target { stroke: none; fill: transparent; diff --git a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts index f6deb496c..39164314a 100644 --- a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts +++ b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts @@ -19,6 +19,9 @@ interface Xput { type: 'input' | 'output' | 'fee'; value?: number; index?: number; + txid?: string; + vin?: number; + vout?: number; address?: string; rest?: number; coinbase?: boolean; @@ -52,9 +55,12 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { outputs: SvgLine[]; middle: SvgLine; midWidth: number; + txWidth: number; + connectorWidth: number; combinedWeight: number; isLiquid: boolean = false; hoverLine: Xput | void = null; + hoverConnector: boolean = false; tooltipPosition = { x: 0, y: 0 }; outspends: Outspend[] = []; @@ -62,16 +68,16 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { refreshOutspends$: ReplaySubject = new ReplaySubject(); gradientColors = { - '': ['#9339f4', '#105fb0', '#9339f433'], - bisq: ['#9339f4', '#105fb0', '#9339f433'], + '': ['#9339f4', '#105fb0', '#9339f400'], + bisq: ['#9339f4', '#105fb0', '#9339f400'], // liquid: ['#116761', '#183550'], - liquid: ['#09a197', '#0f62af', '#09a19733'], + liquid: ['#09a197', '#0f62af', '#09a19700'], // 'liquidtestnet': ['#494a4a', '#272e46'], - 'liquidtestnet': ['#d2d2d2', '#979797', '#d2d2d233'], + 'liquidtestnet': ['#d2d2d2', '#979797', '#d2d2d200'], // testnet: ['#1d486f', '#183550'], - testnet: ['#4edf77', '#10a0af', '#4edf7733'], + testnet: ['#4edf77', '#10a0af', '#4edf7700'], // signet: ['#6f1d5d', '#471850'], - signet: ['#d24fc8', '#a84fd2', '#d24fc833'], + signet: ['#d24fc8', '#a84fd2', '#d24fc800'], }; gradient: string[] = ['#105fb0', '#105fb0']; @@ -121,7 +127,9 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { this.isLiquid = (this.network === 'liquid' || this.network === 'liquidtestnet'); this.gradient = this.gradientColors[this.network]; this.midWidth = Math.min(10, Math.ceil(this.width / 100)); - this.combinedWeight = Math.min(this.maxCombinedWeight, Math.floor((this.width - (2 * this.midWidth)) / 6)); + this.txWidth = this.connectors ? Math.max(this.width - 200, this.width * 0.8) : this.width - 20; + this.combinedWeight = Math.min(this.maxCombinedWeight, Math.floor((this.txWidth - (2 * this.midWidth)) / 6)); + this.connectorWidth = (this.width - this.txWidth) / 2; const totalValue = this.calcTotalValue(this.tx); let voutWithFee = this.tx.vout.map((v, i) => { @@ -144,6 +152,8 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { return { type: 'input', value: v?.prevout?.value, + txid: v.txid, + vout: v.vout, address: v?.prevout?.scriptpubkey_address || v?.prevout?.scriptpubkey_type?.toUpperCase(), index: i, coinbase: v?.is_coinbase, @@ -271,7 +281,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { // required to prevent this line overlapping its neighbor if (this.tooltip || !xputs[i].rest) { - const w = (this.width - Math.max(lastWeight, line.weight)) / 2; // approximate horizontal width of the curved section of the line + const w = (this.width - Math.max(lastWeight, line.weight) - (2 * this.connectorWidth)) / 2; // approximate horizontal width of the curved section of the line const y1 = line.outerY; const y2 = line.innerY; const t = (lastWeight + line.weight) / 2; // distance between center of this line and center of previous line @@ -319,7 +329,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { } makePath(side: 'in' | 'out', outer: number, inner: number, weight: number, offset: number, pad: number): string { - const start = (weight * 0.5) + 10; + const start = (weight * 0.5) + this.connectorWidth; const curveStart = Math.max(start + 1, pad - offset); const end = this.width / 2 - (this.midWidth * 0.9) + 1; const curveEnd = end - offset - 10; @@ -339,7 +349,8 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { makeConnectorPath(side: 'in' | 'out', y: number, inner, weight: number): string { const halfWidth = weight * 0.5; - const offset = Math.max(2, halfWidth * 0.2); + const offset = 10; //Math.max(2, halfWidth * 0.2); + const lineEnd = this.connectorWidth; // align with for svg horizontal gradient bug correction if (Math.round(y) === Math.round(inner)) { @@ -347,15 +358,16 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { } if (side === 'in') { - return `M ${10 - offset} ${y - halfWidth} L ${halfWidth + 10 - offset} ${y} L ${10 - offset} ${y + halfWidth} L -10 ${ y + halfWidth} L -10 ${y - halfWidth}`; + return `M ${lineEnd - offset} ${y - halfWidth} L ${halfWidth + lineEnd - offset} ${y} L ${lineEnd - offset} ${y + halfWidth} L -${10} ${ y + halfWidth} L -${10} ${y - halfWidth}`; } else { - return `M ${this.width - halfWidth - 10 + offset} ${y - halfWidth} L ${this.width - 10 + offset} ${y} L ${this.width - halfWidth - 10 + offset} ${y + halfWidth} L ${this.width + 10} ${ y + halfWidth} L ${this.width + 10} ${y - halfWidth}`; + return `M ${this.width - halfWidth - lineEnd + offset} ${y - halfWidth} L ${this.width - lineEnd + offset} ${y} L ${this.width - halfWidth - lineEnd + offset} ${y + halfWidth} L ${this.width + 10} ${ y + halfWidth} L ${this.width + 10} ${y - halfWidth}`; } } makeMarkerPath(side: 'in' | 'out', y: number, inner, weight: number): string { const halfWidth = weight * 0.5; - const offset = Math.max(2, halfWidth * 0.2); + const offset = 10; //Math.max(2, halfWidth * 0.2); + const lineEnd = this.connectorWidth; // align with for svg horizontal gradient bug correction if (Math.round(y) === Math.round(inner)) { @@ -363,9 +375,9 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { } if (side === 'in') { - return `M ${10 - offset} ${y - halfWidth} L ${halfWidth + 10 - offset} ${y} L ${10 - offset} ${y + halfWidth} L ${offset + weight} ${ y + halfWidth} L ${offset + weight} ${y - halfWidth}`; + return `M ${lineEnd - offset} ${y - halfWidth} L ${halfWidth + lineEnd - offset} ${y} L ${lineEnd - offset} ${y + halfWidth} L ${weight + lineEnd} ${ y + halfWidth} L ${weight + lineEnd} ${y - halfWidth}`; } else { - return `M ${this.width - halfWidth - 10 + offset} ${y - halfWidth} L ${this.width - 10 + offset} ${y} L ${this.width - halfWidth - 10 + offset} ${y + halfWidth} L ${this.width - halfWidth - 10} ${ y + halfWidth} L ${this.width - halfWidth - 10} ${y - halfWidth}`; + return `M ${this.width - halfWidth - lineEnd + offset} ${y - halfWidth} L ${this.width - lineEnd + offset} ${y} L ${this.width - halfWidth - lineEnd + offset} ${y + halfWidth} L ${this.width - halfWidth - lineEnd} ${ y + halfWidth} L ${this.width - halfWidth - lineEnd} ${y - halfWidth}`; } } @@ -383,26 +395,31 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { } onHover(event, side, index): void { - if (side === 'input') { + if (side.startsWith('input')) { this.hoverLine = { ...this.inputData[index], index }; + this.hoverConnector = (side === 'input-connector'); + } else { this.hoverLine = { - ...this.outputData[index] + ...this.outputData[index], + ...this.outspends[this.outputData[index].index] }; + this.hoverConnector = (side === 'output-connector'); } } onBlur(event, side, index): void { this.hoverLine = null; + this.hoverConnector = false; } onClick(event, side, index): void { - if (side === 'input') { + if (side.startsWith('input')) { const input = this.tx.vin[index]; - if (input && !input.is_coinbase && !input.is_pegin && input.txid && input.vout != null) { + if (side === 'input-connector' && input && !input.is_coinbase && !input.is_pegin && input.txid && input.vout != null) { this.router.navigate([this.relativeUrlPipe.transform('/tx'), input.txid], { queryParamsHandling: 'merge', fragment: (new URLSearchParams({ @@ -422,7 +439,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { } else { const output = this.tx.vout[index]; const outspend = this.outspends[index]; - if (output && outspend && outspend.spent && outspend.txid) { + if (side === 'output-connector' && output && outspend && outspend.spent && outspend.txid) { this.router.navigate([this.relativeUrlPipe.transform('/tx'), outspend.txid], { queryParamsHandling: 'merge', fragment: (new URLSearchParams({ From 672001af72baf555881ebf22e5b7b51321c978c3 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 22 Nov 2022 11:03:28 +0900 Subject: [PATCH 48/62] Update mining pool color --- frontend/src/app/app.constants.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/app.constants.ts b/frontend/src/app/app.constants.ts index 232578e6b..9cd374cd0 100644 --- a/frontend/src/app/app.constants.ts +++ b/frontend/src/app/app.constants.ts @@ -79,7 +79,7 @@ export const poolsColor = { 'binancepool': '#1E88E5', 'viabtc': '#039BE5', 'btccom': '#00897B', - 'slushpool': '#00ACC1', + 'braiinspool': '#00ACC1', 'sbicrypto': '#43A047', 'marapool': '#7CB342', 'luxor': '#C0CA33', From 9d5717f30de14b2aefe6e3091a6321600b35043a Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 22 Nov 2022 11:58:16 +0900 Subject: [PATCH 49/62] Make sure we handle all isp id in the queried list --- backend/src/api/explorer/nodes.api.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/backend/src/api/explorer/nodes.api.ts b/backend/src/api/explorer/nodes.api.ts index 4d49b1d67..f7b283213 100644 --- a/backend/src/api/explorer/nodes.api.ts +++ b/backend/src/api/explorer/nodes.api.ts @@ -524,15 +524,20 @@ class NodesApi { ORDER BY short_id DESC `; - const [rows]: any = await DB.query(query, [ISPId.split(','), ISPId.split(',')]); + const IPSIds = ISPId.split(','); + const [rows]: any = await DB.query(query, [IPSIds, IPSIds]); const nodes = {}; - const intIspId = parseInt(ISPId); + const intISPIds: number[] = []; + for (const ispId of IPSIds) { + intISPIds.push(parseInt(ispId, 10)); + } + for (const channel of rows) { - if (channel.isp1ID === intIspId) { + if (intISPIds.includes(channel.isp1ID)) { nodes[channel.node1PublicKey] = true; } - if (channel.isp2ID === intIspId) { + if (intISPIds.includes(channel.isp2ID)) { nodes[channel.node2PublicKey] = true; } } From ed184824d43b15d931ed09df114d236696975f20 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 15 Nov 2022 11:04:33 -0600 Subject: [PATCH 50/62] calculate & show avg channel distance on node page --- .../app/lightning/node/node.component.html | 4 +++ .../app/lightning/node/node.component.scss | 4 +++ .../src/app/lightning/node/node.component.ts | 26 +++++++++++++++++++ frontend/src/app/shared/common.utils.ts | 18 +++++++++++++ 4 files changed, 52 insertions(+) diff --git a/frontend/src/app/lightning/node/node.component.html b/frontend/src/app/lightning/node/node.component.html index 858aa9b48..b5d472d2c 100644 --- a/frontend/src/app/lightning/node/node.component.html +++ b/frontend/src/app/lightning/node/node.component.html @@ -52,6 +52,10 @@ Unknown + + Avg channel distance + {{ avgDistance | number : '1.0-0' }} km / {{ kmToMiles(avgDistance) | number : '1.0-0' }} mi +
diff --git a/frontend/src/app/lightning/node/node.component.scss b/frontend/src/app/lightning/node/node.component.scss index d54b1851b..77b9cabb7 100644 --- a/frontend/src/app/lightning/node/node.component.scss +++ b/frontend/src/app/lightning/node/node.component.scss @@ -101,3 +101,7 @@ app-fiat { font-family: "Courier New", Courier, monospace; } } + +.separator { + margin: 0 1em; +} diff --git a/frontend/src/app/lightning/node/node.component.ts b/frontend/src/app/lightning/node/node.component.ts index ec2edd252..411246d2b 100644 --- a/frontend/src/app/lightning/node/node.component.ts +++ b/frontend/src/app/lightning/node/node.component.ts @@ -3,9 +3,11 @@ import { ActivatedRoute, ParamMap } from '@angular/router'; import { Observable } from 'rxjs'; import { catchError, map, switchMap, tap } from 'rxjs/operators'; import { SeoService } from '../../services/seo.service'; +import { ApiService } from '../../services/api.service'; import { LightningApiService } from '../lightning-api.service'; import { GeolocationData } from '../../shared/components/geolocation/geolocation.component'; import { ILiquidityAd, parseLiquidityAdHex } from './liquidity-ad'; +import { haversineDistance, kmToMiles } from 'src/app/shared/common.utils'; interface CustomRecord { type: string; @@ -34,8 +36,12 @@ export class NodeComponent implements OnInit { showDetails = false; liquidityAd: ILiquidityAd; tlvRecords: CustomRecord[]; + avgChannelDistance$: Observable; + + kmToMiles = kmToMiles; constructor( + private apiService: ApiService, private lightningApiService: LightningApiService, private activatedRoute: ActivatedRoute, private seoService: SeoService, @@ -119,6 +125,26 @@ export class NodeComponent implements OnInit { }]; }) ); + + this.avgChannelDistance$ = this.activatedRoute.paramMap + .pipe( + switchMap((params: ParamMap) => { + return this.apiService.getChannelsGeo$(params.get('public_key'), 'nodepage'); + }), + map((channelsGeo) => { + if (channelsGeo?.length) { + const totalDistance = channelsGeo.reduce((sum, chan) => { + return sum + haversineDistance(chan[3], chan[2], chan[7], chan[6]); + }, 0); + return totalDistance / channelsGeo.length; + } else { + return null; + } + }), + catchError(() => { + return null; + }) + ) as Observable; } toggleShowDetails(): void { diff --git a/frontend/src/app/shared/common.utils.ts b/frontend/src/app/shared/common.utils.ts index d38583217..7d206f4b5 100644 --- a/frontend/src/app/shared/common.utils.ts +++ b/frontend/src/app/shared/common.utils.ts @@ -118,3 +118,21 @@ export function convertRegion(input, to: 'name' | 'abbreviated'): string { } } } + +export function haversineDistance(lat1: number, lon1: number, lat2: number, lon2: number): number { + const rlat1 = lat1 * Math.PI / 180; + const rlon1 = lon1 * Math.PI / 180; + const rlat2 = lat2 * Math.PI / 180; + const rlon2 = lon2 * Math.PI / 180; + + const dlat = Math.sin((rlat2 - rlat1) / 2); + const dlon = Math.sin((rlon2 - rlon1) / 2); + const a = Math.min(1, Math.max(0, (dlat * dlat) + (Math.cos(rlat1) * Math.cos(rlat2) * dlon * dlon))); + const d = 2 * 6371 * Math.asin(Math.sqrt(a)); + + return d; +} + +export function kmToMiles(km: number): number { + return km * 0.62137119; +} \ No newline at end of file From a32f960c4af1b4a11d306d9b2865a19c38830c34 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 31 Oct 2022 11:07:28 -0600 Subject: [PATCH 51/62] db migration to clear obsolete audit data --- backend/src/api/database-migration.ts | 7 ++++++- backend/src/api/websocket-handler.ts | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 3bbd501fc..441ad2785 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 43; + private static currentVersion = 44; private queryTimeout = 120000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -360,6 +360,11 @@ class DatabaseMigration { if (databaseSchemaVersion < 43 && isBitcoin === true) { await this.$executeQuery(this.getCreateLNNodeRecordsTableQuery(), await this.$checkIfTableExists('nodes_records')); } + + if (databaseSchemaVersion < 44 && isBitcoin === true) { + await this.$executeQuery('TRUNCATE TABLE `blocks_audits`'); + await this.$executeQuery('UPDATE blocks_summaries SET template = NULL'); + } } /** diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 4bd7cfc8d..ed29646ef 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -416,7 +416,7 @@ class WebsocketHandler { let matchRate; const _memPool = memPool.getMempool(); - if (Common.indexingEnabled()) { + if (Common.indexingEnabled() && memPool.isInSync()) { const mempoolCopy = cloneMempool(_memPool); const projectedBlocks = mempoolBlocks.makeBlockTemplates(mempoolCopy, 2); From 24dba5a2ef1f5cab603dfacdd50257eda517be09 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 22 Nov 2022 14:25:57 +0900 Subject: [PATCH 52/62] Bump db migration query timeout to 900s --- backend/src/api/database-migration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 441ad2785..6dbfab723 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -5,7 +5,7 @@ import { Common } from './common'; class DatabaseMigration { private static currentVersion = 44; - private queryTimeout = 120000; + private queryTimeout = 900_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; From 08ad6a0da31b0bfa733f16df695a73025dea8d3f Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 16 Nov 2022 18:17:07 -0600 Subject: [PATCH 53/62] move new tx selection algorithm into thread worker --- backend/src/api/mempool-blocks.ts | 307 +++------------------- backend/src/api/tx-selection-worker.ts | 336 +++++++++++++++++++++++++ backend/src/api/websocket-handler.ts | 15 +- 3 files changed, 366 insertions(+), 292 deletions(-) create mode 100644 backend/src/api/tx-selection-worker.ts diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index d0c2a4f63..334362458 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,12 +1,17 @@ import logger from '../logger'; -import { MempoolBlock, TransactionExtended, AuditTransaction, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor } from '../mempool.interfaces'; +import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta } from '../mempool.interfaces'; import { Common } from './common'; import config from '../config'; -import { PairingHeap } from '../utils/pairing-heap'; +import { StaticPool } from 'node-worker-threads-pool'; +import path from 'path'; class MempoolBlocks { private mempoolBlocks: MempoolBlockWithTransactions[] = []; private mempoolBlockDeltas: MempoolBlockDelta[] = []; + private makeTemplatesPool = new StaticPool({ + size: 1, + task: path.resolve(__dirname, './tx-selection-worker.js'), + }); constructor() {} @@ -72,16 +77,15 @@ class MempoolBlocks { const time = end - start; logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds'); - const { blocks, deltas } = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks); + const blocks = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks); + const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks); this.mempoolBlocks = blocks; this.mempoolBlockDeltas = deltas; } - private calculateMempoolBlocks(transactionsSorted: TransactionExtended[], prevBlocks: MempoolBlockWithTransactions[]): - { blocks: MempoolBlockWithTransactions[], deltas: MempoolBlockDelta[] } { + private calculateMempoolBlocks(transactionsSorted: TransactionExtended[], prevBlocks: MempoolBlockWithTransactions[]): MempoolBlockWithTransactions[] { const mempoolBlocks: MempoolBlockWithTransactions[] = []; - const mempoolBlockDeltas: MempoolBlockDelta[] = []; let blockWeight = 0; let blockSize = 0; let transactions: TransactionExtended[] = []; @@ -102,7 +106,11 @@ class MempoolBlocks { mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length)); } - // Calculate change from previous block states + return mempoolBlocks; + } + + private calculateMempoolDeltas(prevBlocks: MempoolBlockWithTransactions[], mempoolBlocks: MempoolBlockWithTransactions[]): MempoolBlockDelta[] { + const mempoolBlockDeltas: MempoolBlockDelta[] = []; for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) { let added: TransactionStripped[] = []; let removed: string[] = []; @@ -135,284 +143,25 @@ class MempoolBlocks { removed }); } - - return { - blocks: mempoolBlocks, - deltas: mempoolBlockDeltas - }; + return mempoolBlockDeltas; } - /* - * Build projected mempool blocks using an approximation of the transaction selection algorithm from Bitcoin Core - * (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp) - * - * blockLimit: number of blocks to build in total. - * weightLimit: maximum weight of transactions to consider using the selection algorithm. - * if weightLimit is significantly lower than the mempool size, results may start to diverge from getBlockTemplate - * condenseRest: whether to ignore excess transactions or append them to the final block. - */ - public makeBlockTemplates(mempool: { [txid: string]: TransactionExtended }, blockLimit: number, weightLimit: number | null = null, condenseRest = false): MempoolBlockWithTransactions[] { - const start = Date.now(); - const auditPool: { [txid: string]: AuditTransaction } = {}; - const mempoolArray: AuditTransaction[] = []; - const restOfArray: TransactionExtended[] = []; - - let weight = 0; - const maxWeight = weightLimit ? Math.max(4_000_000 * blockLimit, weightLimit) : Infinity; - // grab the top feerate txs up to maxWeight - Object.values(mempool).sort((a, b) => b.feePerVsize - a.feePerVsize).forEach(tx => { - weight += tx.weight; - if (weight >= maxWeight) { - restOfArray.push(tx); - return; - } - // initializing everything up front helps V8 optimize property access later - auditPool[tx.txid] = { - txid: tx.txid, - fee: tx.fee, - size: tx.size, - weight: tx.weight, - feePerVsize: tx.feePerVsize, - vin: tx.vin, - relativesSet: false, - ancestorMap: new Map(), - children: new Set(), - ancestorFee: 0, - ancestorWeight: 0, - score: 0, - used: false, - modified: false, - modifiedNode: null, - } - mempoolArray.push(auditPool[tx.txid]); - }) + public async makeBlockTemplates(newMempool: { [txid: string]: TransactionExtended }, blockLimit: number, weightLimit: number | null = null, condenseRest = false): Promise { + const { mempool, blocks } = await this.makeTemplatesPool.exec({ mempool: newMempool, blockLimit, weightLimit, condenseRest }); + const deltas = this.calculateMempoolDeltas(this.mempoolBlocks, blocks); - // Build relatives graph & calculate ancestor scores - for (const tx of mempoolArray) { - if (!tx.relativesSet) { - this.setRelatives(tx, auditPool); - } - } - - // Sort by descending ancestor score - mempoolArray.sort((a, b) => (b.score || 0) - (a.score || 0)); - - // Build blocks by greedily choosing the highest feerate package - // (i.e. the package rooted in the transaction with the best ancestor score) - const blocks: MempoolBlockWithTransactions[] = []; - let blockWeight = 4000; - let blockSize = 0; - let transactions: AuditTransaction[] = []; - const modified: PairingHeap = new PairingHeap((a, b): boolean => (a.score || 0) > (b.score || 0)); - let overflow: AuditTransaction[] = []; - let failures = 0; - let top = 0; - while ((top < mempoolArray.length || !modified.isEmpty()) && (condenseRest || blocks.length < blockLimit)) { - // skip invalid transactions - while (top < mempoolArray.length && (mempoolArray[top].used || mempoolArray[top].modified)) { - top++; - } - - // Select best next package - let nextTx; - const nextPoolTx = mempoolArray[top]; - const nextModifiedTx = modified.peek(); - if (nextPoolTx && (!nextModifiedTx || (nextPoolTx.score || 0) > (nextModifiedTx.score || 0))) { - nextTx = nextPoolTx; - top++; - } else { - modified.pop(); - if (nextModifiedTx) { - nextTx = nextModifiedTx; - nextTx.modifiedNode = undefined; - } - } - - if (nextTx && !nextTx?.used) { - // Check if the package fits into this block - if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) { - blockWeight += nextTx.ancestorWeight; - const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values()); - // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count) - const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx]; - const effectiveFeeRate = nextTx.ancestorFee / (nextTx.ancestorWeight / 4); - sortedTxSet.forEach((ancestor, i, arr) => { - const mempoolTx = mempool[ancestor.txid]; - if (ancestor && !ancestor?.used) { - ancestor.used = true; - // update original copy of this tx with effective fee rate & relatives data - mempoolTx.effectiveFeePerVsize = effectiveFeeRate; - mempoolTx.ancestors = (Array.from(ancestor.ancestorMap?.values()) as AuditTransaction[]).map((a) => { - return { - txid: a.txid, - fee: a.fee, - weight: a.weight, - } - }) - if (i < arr.length - 1) { - mempoolTx.bestDescendant = { - txid: arr[arr.length - 1].txid, - fee: arr[arr.length - 1].fee, - weight: arr[arr.length - 1].weight, - }; - } - transactions.push(ancestor); - blockSize += ancestor.size; - } - }); - - // remove these as valid package ancestors for any descendants remaining in the mempool - if (sortedTxSet.length) { - sortedTxSet.forEach(tx => { - this.updateDescendants(tx, auditPool, modified); - }); - } - - failures = 0; - } else { - // hold this package in an overflow list while we check for smaller options - overflow.push(nextTx); - failures++; - } - } - - // this block is full - const exceededPackageTries = failures > 1000 && blockWeight > (config.MEMPOOL.BLOCK_WEIGHT_UNITS - 4000); - if (exceededPackageTries && (!condenseRest || blocks.length < blockLimit - 1)) { - // construct this block - if (transactions.length) { - blocks.push(this.dataToMempoolBlocks(transactions.map(t => mempool[t.txid]), blockSize, blockWeight, blocks.length)); - } - // reset for the next block - transactions = []; - blockSize = 0; - blockWeight = 4000; - - // 'overflow' packages didn't fit in this block, but are valid candidates for the next - for (const overflowTx of overflow.reverse()) { - if (overflowTx.modified) { - overflowTx.modifiedNode = modified.add(overflowTx); - } else { - top--; - mempoolArray[top] = overflowTx; - } - } - overflow = []; - } - } - if (condenseRest) { - // pack any leftover transactions into the last block - for (const tx of overflow) { - if (!tx || tx?.used) { - continue; - } - blockWeight += tx.weight; - blockSize += tx.size; - transactions.push(tx); - tx.used = true; - } - const blockTransactions = transactions.map(t => mempool[t.txid]) - restOfArray.forEach(tx => { - blockWeight += tx.weight; - blockSize += tx.size; - blockTransactions.push(tx); - }); - if (blockTransactions.length) { - blocks.push(this.dataToMempoolBlocks(blockTransactions, blockSize, blockWeight, blocks.length)); - } - transactions = []; - } else if (transactions.length) { - blocks.push(this.dataToMempoolBlocks(transactions.map(t => mempool[t.txid]), blockSize, blockWeight, blocks.length)); - } - - const end = Date.now(); - const time = end - start; - logger.debug('Mempool templates calculated in ' + time / 1000 + ' seconds'); - - return blocks; - } - - // traverse in-mempool ancestors - // recursion unavoidable, but should be limited to depth < 25 by mempool policy - public setRelatives( - tx: AuditTransaction, - mempool: { [txid: string]: AuditTransaction }, - ): void { - for (const parent of tx.vin) { - const parentTx = mempool[parent.txid]; - if (parentTx && !tx.ancestorMap!.has(parent.txid)) { - tx.ancestorMap.set(parent.txid, parentTx); - parentTx.children.add(tx); - // visit each node only once - if (!parentTx.relativesSet) { - this.setRelatives(parentTx, mempool); - } - parentTx.ancestorMap.forEach((ancestor) => { - tx.ancestorMap.set(ancestor.txid, ancestor); - }); - } - }; - tx.ancestorFee = tx.fee || 0; - tx.ancestorWeight = tx.weight || 0; - tx.ancestorMap.forEach((ancestor) => { - tx.ancestorFee += ancestor.fee; - tx.ancestorWeight += ancestor.weight; - }); - tx.score = tx.ancestorFee / (tx.ancestorWeight || 1); - tx.relativesSet = true; - } - - // iterate over remaining descendants, removing the root as a valid ancestor & updating the ancestor score - // avoids recursion to limit call stack depth - private updateDescendants( - rootTx: AuditTransaction, - mempool: { [txid: string]: AuditTransaction }, - modified: PairingHeap, - ): void { - const descendantSet: Set = new Set(); - // stack of nodes left to visit - const descendants: AuditTransaction[] = []; - let descendantTx; - let ancestorIndex; - let tmpScore; - rootTx.children.forEach(childTx => { - if (!descendantSet.has(childTx)) { - descendants.push(childTx); - descendantSet.add(childTx); + // copy CPFP info across to main thread's mempool + Object.keys(newMempool).forEach((txid) => { + if (newMempool[txid] && mempool[txid]) { + newMempool[txid].effectiveFeePerVsize = mempool[txid].effectiveFeePerVsize; + newMempool[txid].ancestors = mempool[txid].ancestors; + newMempool[txid].bestDescendant = mempool[txid].bestDescendant; + newMempool[txid].cpfpChecked = mempool[txid].cpfpChecked; } }); - while (descendants.length) { - descendantTx = descendants.pop(); - if (descendantTx && descendantTx.ancestorMap && descendantTx.ancestorMap.has(rootTx.txid)) { - // remove tx as ancestor - descendantTx.ancestorMap.delete(rootTx.txid); - descendantTx.ancestorFee -= rootTx.fee; - descendantTx.ancestorWeight -= rootTx.weight; - tmpScore = descendantTx.score; - descendantTx.score = descendantTx.ancestorFee / descendantTx.ancestorWeight; - if (!descendantTx.modifiedNode) { - descendantTx.modified = true; - descendantTx.modifiedNode = modified.add(descendantTx); - } else { - // rebalance modified heap if score has changed - if (descendantTx.score < tmpScore) { - modified.decreasePriority(descendantTx.modifiedNode); - } else if (descendantTx.score > tmpScore) { - modified.increasePriority(descendantTx.modifiedNode); - } - } - - // add this node's children to the stack - descendantTx.children.forEach(childTx => { - // visit each node only once - if (!descendantSet.has(childTx)) { - descendants.push(childTx); - descendantSet.add(childTx); - } - }); - } - } + this.mempoolBlocks = blocks; + this.mempoolBlockDeltas = deltas; } private dataToMempoolBlocks(transactions: TransactionExtended[], diff --git a/backend/src/api/tx-selection-worker.ts b/backend/src/api/tx-selection-worker.ts new file mode 100644 index 000000000..09d9b9102 --- /dev/null +++ b/backend/src/api/tx-selection-worker.ts @@ -0,0 +1,336 @@ +import config from '../config'; +import logger from '../logger'; +import { TransactionExtended, MempoolBlockWithTransactions, AuditTransaction } from '../mempool.interfaces'; +import { PairingHeap } from '../utils/pairing-heap'; +import { Common } from './common'; +import { parentPort } from 'worker_threads'; + +if (parentPort) { + parentPort.on('message', (params: { mempool: { [txid: string]: TransactionExtended }, blockLimit: number, weightLimit: number | null, condenseRest: boolean}) => { + const { mempool, blocks } = makeBlockTemplates(params); + + // return the result to main thread. + if (parentPort) { + parentPort.postMessage({ mempool, blocks }); + } + }); +} + +/* +* Build projected mempool blocks using an approximation of the transaction selection algorithm from Bitcoin Core +* (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp) +* +* blockLimit: number of blocks to build in total. +* weightLimit: maximum weight of transactions to consider using the selection algorithm. +* if weightLimit is significantly lower than the mempool size, results may start to diverge from getBlockTemplate +* condenseRest: whether to ignore excess transactions or append them to the final block. +*/ +function makeBlockTemplates({ mempool, blockLimit, weightLimit, condenseRest }: { mempool: { [txid: string]: TransactionExtended }, blockLimit: number, weightLimit?: number | null, condenseRest?: boolean | null }) + : { mempool: { [txid: string]: TransactionExtended }, blocks: MempoolBlockWithTransactions[] } { + const start = Date.now(); + const auditPool: { [txid: string]: AuditTransaction } = {}; + const mempoolArray: AuditTransaction[] = []; + const restOfArray: TransactionExtended[] = []; + + let weight = 0; + const maxWeight = weightLimit ? Math.max(4_000_000 * blockLimit, weightLimit) : Infinity; + // grab the top feerate txs up to maxWeight + Object.values(mempool).sort((a, b) => b.feePerVsize - a.feePerVsize).forEach(tx => { + weight += tx.weight; + if (weight >= maxWeight) { + restOfArray.push(tx); + return; + } + // initializing everything up front helps V8 optimize property access later + auditPool[tx.txid] = { + txid: tx.txid, + fee: tx.fee, + size: tx.size, + weight: tx.weight, + feePerVsize: tx.feePerVsize, + vin: tx.vin, + relativesSet: false, + ancestorMap: new Map(), + children: new Set(), + ancestorFee: 0, + ancestorWeight: 0, + score: 0, + used: false, + modified: false, + modifiedNode: null, + }; + mempoolArray.push(auditPool[tx.txid]); + }); + + // Build relatives graph & calculate ancestor scores + for (const tx of mempoolArray) { + if (!tx.relativesSet) { + setRelatives(tx, auditPool); + } + } + + // Sort by descending ancestor score + mempoolArray.sort((a, b) => (b.score || 0) - (a.score || 0)); + + // Build blocks by greedily choosing the highest feerate package + // (i.e. the package rooted in the transaction with the best ancestor score) + const blocks: MempoolBlockWithTransactions[] = []; + let blockWeight = 4000; + let blockSize = 0; + let transactions: AuditTransaction[] = []; + const modified: PairingHeap = new PairingHeap((a, b): boolean => (a.score || 0) > (b.score || 0)); + let overflow: AuditTransaction[] = []; + let failures = 0; + let top = 0; + while ((top < mempoolArray.length || !modified.isEmpty()) && (condenseRest || blocks.length < blockLimit)) { + // skip invalid transactions + while (top < mempoolArray.length && (mempoolArray[top].used || mempoolArray[top].modified)) { + top++; + } + + // Select best next package + let nextTx; + const nextPoolTx = mempoolArray[top]; + const nextModifiedTx = modified.peek(); + if (nextPoolTx && (!nextModifiedTx || (nextPoolTx.score || 0) > (nextModifiedTx.score || 0))) { + nextTx = nextPoolTx; + top++; + } else { + modified.pop(); + if (nextModifiedTx) { + nextTx = nextModifiedTx; + nextTx.modifiedNode = undefined; + } + } + + if (nextTx && !nextTx?.used) { + // Check if the package fits into this block + if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) { + blockWeight += nextTx.ancestorWeight; + const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values()); + // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count) + const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx]; + const effectiveFeeRate = nextTx.ancestorFee / (nextTx.ancestorWeight / 4); + sortedTxSet.forEach((ancestor, i, arr) => { + const mempoolTx = mempool[ancestor.txid]; + if (ancestor && !ancestor?.used) { + ancestor.used = true; + // update original copy of this tx with effective fee rate & relatives data + mempoolTx.effectiveFeePerVsize = effectiveFeeRate; + mempoolTx.ancestors = (Array.from(ancestor.ancestorMap?.values()) as AuditTransaction[]).map((a) => { + return { + txid: a.txid, + fee: a.fee, + weight: a.weight, + }; + }); + mempoolTx.cpfpChecked = true; + if (i < arr.length - 1) { + mempoolTx.bestDescendant = { + txid: arr[arr.length - 1].txid, + fee: arr[arr.length - 1].fee, + weight: arr[arr.length - 1].weight, + }; + } else { + mempoolTx.bestDescendant = null; + } + transactions.push(ancestor); + blockSize += ancestor.size; + } + }); + + // remove these as valid package ancestors for any descendants remaining in the mempool + if (sortedTxSet.length) { + sortedTxSet.forEach(tx => { + updateDescendants(tx, auditPool, modified); + }); + } + + failures = 0; + } else { + // hold this package in an overflow list while we check for smaller options + overflow.push(nextTx); + failures++; + } + } + + // this block is full + const exceededPackageTries = failures > 1000 && blockWeight > (config.MEMPOOL.BLOCK_WEIGHT_UNITS - 4000); + if (exceededPackageTries && (!condenseRest || blocks.length < blockLimit - 1)) { + // construct this block + if (transactions.length) { + blocks.push(dataToMempoolBlocks(transactions.map(t => mempool[t.txid]), blockSize, blockWeight, blocks.length)); + } + // reset for the next block + transactions = []; + blockSize = 0; + blockWeight = 4000; + + // 'overflow' packages didn't fit in this block, but are valid candidates for the next + for (const overflowTx of overflow.reverse()) { + if (overflowTx.modified) { + overflowTx.modifiedNode = modified.add(overflowTx); + } else { + top--; + mempoolArray[top] = overflowTx; + } + } + overflow = []; + } + } + if (condenseRest) { + // pack any leftover transactions into the last block + for (const tx of overflow) { + if (!tx || tx?.used) { + continue; + } + blockWeight += tx.weight; + blockSize += tx.size; + const mempoolTx = mempool[tx.txid]; + // update original copy of this tx with effective fee rate & relatives data + mempoolTx.effectiveFeePerVsize = tx.score; + mempoolTx.ancestors = (Array.from(tx.ancestorMap?.values()) as AuditTransaction[]).map((a) => { + return { + txid: a.txid, + fee: a.fee, + weight: a.weight, + }; + }); + mempoolTx.bestDescendant = null; + mempoolTx.cpfpChecked = true; + transactions.push(tx); + tx.used = true; + } + const blockTransactions = transactions.map(t => mempool[t.txid]); + restOfArray.forEach(tx => { + blockWeight += tx.weight; + blockSize += tx.size; + tx.effectiveFeePerVsize = tx.feePerVsize; + tx.cpfpChecked = false; + tx.ancestors = []; + tx.bestDescendant = null; + tx.ancestors + blockTransactions.push(tx); + }); + if (blockTransactions.length) { + blocks.push(dataToMempoolBlocks(blockTransactions, blockSize, blockWeight, blocks.length)); + } + transactions = []; + } else if (transactions.length) { + blocks.push(dataToMempoolBlocks(transactions.map(t => mempool[t.txid]), blockSize, blockWeight, blocks.length)); + } + + const end = Date.now(); + const time = end - start; + logger.debug('Mempool templates calculated in ' + time / 1000 + ' seconds'); + + return { + mempool, + blocks + }; +} + +// traverse in-mempool ancestors +// recursion unavoidable, but should be limited to depth < 25 by mempool policy +function setRelatives( + tx: AuditTransaction, + mempool: { [txid: string]: AuditTransaction }, +): void { + for (const parent of tx.vin) { + const parentTx = mempool[parent.txid]; + if (parentTx && !tx.ancestorMap?.has(parent.txid)) { + tx.ancestorMap.set(parent.txid, parentTx); + parentTx.children.add(tx); + // visit each node only once + if (!parentTx.relativesSet) { + setRelatives(parentTx, mempool); + } + parentTx.ancestorMap.forEach((ancestor) => { + tx.ancestorMap.set(ancestor.txid, ancestor); + }); + } + }; + tx.ancestorFee = tx.fee || 0; + tx.ancestorWeight = tx.weight || 0; + tx.ancestorMap.forEach((ancestor) => { + tx.ancestorFee += ancestor.fee; + tx.ancestorWeight += ancestor.weight; + }); + tx.score = tx.ancestorFee / ((tx.ancestorWeight / 4) || 1); + tx.relativesSet = true; +} + +// iterate over remaining descendants, removing the root as a valid ancestor & updating the ancestor score +// avoids recursion to limit call stack depth +function updateDescendants( + rootTx: AuditTransaction, + mempool: { [txid: string]: AuditTransaction }, + modified: PairingHeap, +): void { + const descendantSet: Set = new Set(); + // stack of nodes left to visit + const descendants: AuditTransaction[] = []; + let descendantTx; + let tmpScore; + rootTx.children.forEach(childTx => { + if (!descendantSet.has(childTx)) { + descendants.push(childTx); + descendantSet.add(childTx); + } + }); + while (descendants.length) { + descendantTx = descendants.pop(); + if (descendantTx && descendantTx.ancestorMap && descendantTx.ancestorMap.has(rootTx.txid)) { + // remove tx as ancestor + descendantTx.ancestorMap.delete(rootTx.txid); + descendantTx.ancestorFee -= rootTx.fee; + descendantTx.ancestorWeight -= rootTx.weight; + tmpScore = descendantTx.score; + descendantTx.score = descendantTx.ancestorFee / (descendantTx.ancestorWeight / 4); + + if (!descendantTx.modifiedNode) { + descendantTx.modified = true; + descendantTx.modifiedNode = modified.add(descendantTx); + } else { + // rebalance modified heap if score has changed + if (descendantTx.score < tmpScore) { + modified.decreasePriority(descendantTx.modifiedNode); + } else if (descendantTx.score > tmpScore) { + modified.increasePriority(descendantTx.modifiedNode); + } + } + + // add this node's children to the stack + descendantTx.children.forEach(childTx => { + // visit each node only once + if (!descendantSet.has(childTx)) { + descendants.push(childTx); + descendantSet.add(childTx); + } + }); + } + } +} + +function dataToMempoolBlocks(transactions: TransactionExtended[], + blockSize: number, blockWeight: number, blocksIndex: number): MempoolBlockWithTransactions { + let rangeLength = 4; + if (blocksIndex === 0) { + rangeLength = 8; + } + if (transactions.length > 4000) { + rangeLength = 6; + } else if (transactions.length > 10000) { + rangeLength = 8; + } + return { + blockSize: blockSize, + blockVSize: blockWeight / 4, + nTx: transactions.length, + totalFees: transactions.reduce((acc, cur) => acc + cur.fee, 0), + medianFee: Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE), + feeRange: Common.getFeesInRange(transactions, rangeLength), + transactionIds: transactions.map((tx) => tx.txid), + transactions: transactions.map((tx) => Common.stripTransaction(tx)), + }; +} \ No newline at end of file diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index ed29646ef..84fe50f36 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -417,10 +417,9 @@ class WebsocketHandler { const _memPool = memPool.getMempool(); if (Common.indexingEnabled() && memPool.isInSync()) { - const mempoolCopy = cloneMempool(_memPool); - const projectedBlocks = mempoolBlocks.makeBlockTemplates(mempoolCopy, 2); + const projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); - const { censored, added, score } = Audit.auditBlock(transactions, projectedBlocks, mempoolCopy); + const { censored, added, score } = Audit.auditBlock(transactions, projectedBlocks, _memPool); matchRate = Math.round(score * 100 * 100) / 100; const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => { @@ -569,14 +568,4 @@ class WebsocketHandler { } } -function cloneMempool(mempool: { [txid: string]: TransactionExtended }): { [txid: string]: TransactionExtended } { - const cloned = {}; - Object.keys(mempool).forEach(id => { - cloned[id] = { - ...mempool[id] - }; - }); - return cloned; -} - export default new WebsocketHandler(); From 786d73625a934b829701f2cfd8fc3b38fcc63956 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 16 Nov 2022 18:18:59 -0600 Subject: [PATCH 54/62] guard new tx selection algo behind config setting --- backend/mempool-config.sample.json | 3 +- backend/src/api/blocks.ts | 11 +++ backend/src/api/mempool.ts | 11 +++ backend/src/api/websocket-handler.ts | 111 ++++++++++++++++++++++++--- backend/src/config.ts | 2 + backend/src/index.ts | 2 + 6 files changed, 129 insertions(+), 11 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index c4227adce..fe5f2e213 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -24,7 +24,8 @@ "STDOUT_LOG_MIN_PRIORITY": "debug", "AUTOMATIC_BLOCK_REINDEXING": false, "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json", - "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master" + "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master", + "ADVANCED_TRANSACTION_SELECTION": false }, "CORE_RPC": { "HOST": "127.0.0.1", diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index eb8f75bf4..562f49de1 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -34,6 +34,7 @@ class Blocks { private lastDifficultyAdjustmentTime = 0; private previousDifficultyRetarget = 0; private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = []; + private newAsyncBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => Promise)[] = []; constructor() { } @@ -57,6 +58,10 @@ class Blocks { this.newBlockCallbacks.push(fn); } + public setNewAsyncBlockCallback(fn: (block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => Promise) { + this.newAsyncBlockCallbacks.push(fn); + } + /** * Return the list of transaction for a block * @param blockHash @@ -444,6 +449,9 @@ class Blocks { const blockExtended: BlockExtended = await this.$getBlockExtended(block, transactions); const blockSummary: BlockSummary = this.summarizeBlock(verboseBlock); + // start async callbacks + const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions)); + if (Common.indexingEnabled()) { if (!fastForwarded) { const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1); @@ -514,6 +522,9 @@ class Blocks { if (!memPool.hasPriority()) { diskCache.$saveCacheToDisk(); } + + // wait for pending async callbacks to finish + await Promise.all(callbackPromises); } } diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 76c8b169f..86538f51d 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -20,6 +20,7 @@ class Mempool { maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 }; private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) => void) | undefined; + private asyncMempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }) => void) | undefined; private txPerSecondArray: number[] = []; private txPerSecond: number = 0; @@ -63,6 +64,10 @@ class Mempool { this.mempoolChangedCallback = fn; } + public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: TransactionExtended; }) => void) { + this.asyncMempoolChangedCallback = fn; + } + public getMempool(): { [txid: string]: TransactionExtended } { return this.mempoolCache; } @@ -72,6 +77,9 @@ class Mempool { if (this.mempoolChangedCallback) { this.mempoolChangedCallback(this.mempoolCache, [], []); } + if (this.asyncMempoolChangedCallback) { + this.asyncMempoolChangedCallback(this.mempoolCache); + } } public async $updateMemPoolInfo() { @@ -187,6 +195,9 @@ class Mempool { if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) { this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions); } + if (this.asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) { + await this.asyncMempoolChangedCallback(this.mempoolCache); + } const end = new Date().getTime(); const time = end - start; diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 84fe50f36..73db85fe6 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -244,15 +244,59 @@ class WebsocketHandler { }); } - handleMempoolChange(newMempool: { [txid: string]: TransactionExtended }, - newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) { + async handleAsyncMempoolChange(newMempool: { [txid: string]: TransactionExtended }): Promise { if (!this.wss) { throw new Error('WebSocket.Server is not set'); } - mempoolBlocks.updateMempoolBlocks(newMempool); + if (!config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) { + return; + } + + await mempoolBlocks.makeBlockTemplates(newMempool, 8, null, true); const mBlocks = mempoolBlocks.getMempoolBlocks(); const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); + + this.wss.clients.forEach(async (client) => { + if (client.readyState !== WebSocket.OPEN) { + return; + } + + const response = {}; + + if (client['want-mempool-blocks']) { + response['mempool-blocks'] = mBlocks; + } + + if (client['track-mempool-block'] >= 0) { + const index = client['track-mempool-block']; + if (mBlockDeltas[index]) { + response['projected-block-transactions'] = { + index: index, + delta: mBlockDeltas[index], + }; + } + } + + if (Object.keys(response).length) { + client.send(JSON.stringify(response)); + } + }); + } + + handleMempoolChange(newMempool: { [txid: string]: TransactionExtended }, + newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]): void { + if (!this.wss) { + throw new Error('WebSocket.Server is not set'); + } + + let mBlocks; + let mBlockDeltas; + if (!config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) { + mempoolBlocks.updateMempoolBlocks(newMempool); + mBlocks = mempoolBlocks.getMempoolBlocks(); + mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); + } const mempoolInfo = memPool.getMempoolInfo(); const vBytesPerSecond = memPool.getVBytesPerSecond(); const rbfTransactions = Common.findRbfTransactions(newTransactions, deletedTransactions); @@ -275,7 +319,7 @@ class WebsocketHandler { response['fees'] = recommendedFees; } - if (client['want-mempool-blocks']) { + if (client['want-mempool-blocks'] && !config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) { response['mempool-blocks'] = mBlocks; } @@ -390,7 +434,7 @@ class WebsocketHandler { } } - if (client['track-mempool-block'] >= 0) { + if (client['track-mempool-block'] >= 0 && !config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) { const index = client['track-mempool-block']; if (mBlockDeltas[index]) { response['projected-block-transactions'] = { @@ -406,6 +450,51 @@ class WebsocketHandler { }); } + async handleNewAsyncBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]): Promise { + if (!this.wss) { + throw new Error('WebSocket.Server is not set'); + } + + if (!config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) { + return; + } + + const _memPool = memPool.getMempool(); + + await mempoolBlocks.makeBlockTemplates(_memPool, 2); + const mBlocks = mempoolBlocks.getMempoolBlocks(); + const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); + + this.wss.clients.forEach((client) => { + if (client.readyState !== WebSocket.OPEN) { + return; + } + + if (!client['want-blocks']) { + return; + } + + const response = {}; + + if (mBlocks && client['want-mempool-blocks']) { + response['mempool-blocks'] = mBlocks; + } + + if (client['track-mempool-block'] >= 0) { + const index = client['track-mempool-block']; + if (mBlockDeltas && mBlockDeltas[index]) { + response['projected-block-transactions'] = { + index: index, + delta: mBlockDeltas[index], + }; + } + } + + client.send(JSON.stringify(response)); + }); + } + + handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]): void { if (!this.wss) { throw new Error('WebSocket.Server is not set'); @@ -458,9 +547,11 @@ class WebsocketHandler { delete _memPool[txId]; } - mempoolBlocks.updateMempoolBlocks(_memPool); - mBlocks = mempoolBlocks.getMempoolBlocks(); - mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); + if (!config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) { + mempoolBlocks.updateMempoolBlocks(_memPool); + mBlocks = mempoolBlocks.getMempoolBlocks(); + mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); + } const da = difficultyAdjustment.getDifficultyAdjustment(); const fees = feeApi.getRecommendedFee(); @@ -481,7 +572,7 @@ class WebsocketHandler { 'fees': fees, }; - if (mBlocks && client['want-mempool-blocks']) { + if (mBlocks && client['want-mempool-blocks'] && !config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) { response['mempool-blocks'] = mBlocks; } @@ -553,7 +644,7 @@ class WebsocketHandler { } } - if (client['track-mempool-block'] >= 0) { + if (client['track-mempool-block'] >= 0 && !config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) { const index = client['track-mempool-block']; if (mBlockDeltas && mBlockDeltas[index]) { response['projected-block-transactions'] = { diff --git a/backend/src/config.ts b/backend/src/config.ts index 052affb45..4aab7a306 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -29,6 +29,7 @@ interface IConfig { AUTOMATIC_BLOCK_REINDEXING: boolean; POOLS_JSON_URL: string, POOLS_JSON_TREE_URL: string, + ADVANCED_TRANSACTION_SELECTION: boolean; }; ESPLORA: { REST_API_URL: string; @@ -145,6 +146,7 @@ const defaults: IConfig = { 'AUTOMATIC_BLOCK_REINDEXING': false, 'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json', 'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master', + 'ADVANCED_TRANSACTION_SELECTION': false, }, 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', diff --git a/backend/src/index.ts b/backend/src/index.ts index 2bcb98de1..2c15aa81a 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -217,7 +217,9 @@ class Server { if (config.MEMPOOL.ENABLED) { statistics.setNewStatisticsEntryCallback(websocketHandler.handleNewStatistic.bind(websocketHandler)); blocks.setNewBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler)); + blocks.setNewAsyncBlockCallback(websocketHandler.handleNewAsyncBlock.bind(websocketHandler)); memPool.setMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler)); + memPool.setAsyncMempoolChangedCallback(websocketHandler.handleAsyncMempoolChange.bind(websocketHandler)); } fiatConversion.setProgressChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler)); loadingIndicators.setProgressChangedCallback(websocketHandler.handleLoadingChanged.bind(websocketHandler)); From b1d490972bc321181ad6f9b6d3a443ac8c48a870 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 20 Nov 2022 16:12:39 +0900 Subject: [PATCH 55/62] refactor async mempool/block update callbacks --- backend/src/api/mempool.ts | 10 +- backend/src/api/tx-selection-worker.ts | 4 +- backend/src/api/websocket-handler.ts | 126 +++++-------------------- backend/src/index.ts | 6 +- 4 files changed, 34 insertions(+), 112 deletions(-) diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 86538f51d..584ddf816 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -20,7 +20,8 @@ class Mempool { maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 }; private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) => void) | undefined; - private asyncMempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }) => void) | undefined; + private asyncMempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[], + deletedTransactions: TransactionExtended[]) => void) | undefined; private txPerSecondArray: number[] = []; private txPerSecond: number = 0; @@ -64,7 +65,8 @@ class Mempool { this.mempoolChangedCallback = fn; } - public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: TransactionExtended; }) => void) { + public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: TransactionExtended; }, + newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]) => Promise) { this.asyncMempoolChangedCallback = fn; } @@ -78,7 +80,7 @@ class Mempool { this.mempoolChangedCallback(this.mempoolCache, [], []); } if (this.asyncMempoolChangedCallback) { - this.asyncMempoolChangedCallback(this.mempoolCache); + this.asyncMempoolChangedCallback(this.mempoolCache, [], []); } } @@ -196,7 +198,7 @@ class Mempool { this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions); } if (this.asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) { - await this.asyncMempoolChangedCallback(this.mempoolCache); + await this.asyncMempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions); } const end = new Date().getTime(); diff --git a/backend/src/api/tx-selection-worker.ts b/backend/src/api/tx-selection-worker.ts index 09d9b9102..10f65000b 100644 --- a/backend/src/api/tx-selection-worker.ts +++ b/backend/src/api/tx-selection-worker.ts @@ -156,7 +156,8 @@ function makeBlockTemplates({ mempool, blockLimit, weightLimit, condenseRest }: // this block is full const exceededPackageTries = failures > 1000 && blockWeight > (config.MEMPOOL.BLOCK_WEIGHT_UNITS - 4000); - if (exceededPackageTries && (!condenseRest || blocks.length < blockLimit - 1)) { + const queueEmpty = top >= mempoolArray.length && modified.isEmpty(); + if ((exceededPackageTries || queueEmpty) && (!condenseRest || blocks.length < blockLimit - 1)) { // construct this block if (transactions.length) { blocks.push(dataToMempoolBlocks(transactions.map(t => mempool[t.txid]), blockSize, blockWeight, blocks.length)); @@ -209,7 +210,6 @@ function makeBlockTemplates({ mempool, blockLimit, weightLimit, condenseRest }: tx.cpfpChecked = false; tx.ancestors = []; tx.bestDescendant = null; - tx.ancestors blockTransactions.push(tx); }); if (blockTransactions.length) { diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 73db85fe6..375869902 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -244,59 +244,20 @@ class WebsocketHandler { }); } - async handleAsyncMempoolChange(newMempool: { [txid: string]: TransactionExtended }): Promise { + async handleMempoolChange(newMempool: { [txid: string]: TransactionExtended }, + newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]): Promise { if (!this.wss) { throw new Error('WebSocket.Server is not set'); } - if (!config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) { - return; + if (config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) { + await mempoolBlocks.makeBlockTemplates(newMempool, 8, null, true); + } + else { + mempoolBlocks.updateMempoolBlocks(newMempool); } - - await mempoolBlocks.makeBlockTemplates(newMempool, 8, null, true); const mBlocks = mempoolBlocks.getMempoolBlocks(); const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); - - this.wss.clients.forEach(async (client) => { - if (client.readyState !== WebSocket.OPEN) { - return; - } - - const response = {}; - - if (client['want-mempool-blocks']) { - response['mempool-blocks'] = mBlocks; - } - - if (client['track-mempool-block'] >= 0) { - const index = client['track-mempool-block']; - if (mBlockDeltas[index]) { - response['projected-block-transactions'] = { - index: index, - delta: mBlockDeltas[index], - }; - } - } - - if (Object.keys(response).length) { - client.send(JSON.stringify(response)); - } - }); - } - - handleMempoolChange(newMempool: { [txid: string]: TransactionExtended }, - newTransactions: TransactionExtended[], deletedTransactions: TransactionExtended[]): void { - if (!this.wss) { - throw new Error('WebSocket.Server is not set'); - } - - let mBlocks; - let mBlockDeltas; - if (!config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) { - mempoolBlocks.updateMempoolBlocks(newMempool); - mBlocks = mempoolBlocks.getMempoolBlocks(); - mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); - } const mempoolInfo = memPool.getMempoolInfo(); const vBytesPerSecond = memPool.getVBytesPerSecond(); const rbfTransactions = Common.findRbfTransactions(newTransactions, deletedTransactions); @@ -319,7 +280,7 @@ class WebsocketHandler { response['fees'] = recommendedFees; } - if (client['want-mempool-blocks'] && !config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) { + if (client['want-mempool-blocks']) { response['mempool-blocks'] = mBlocks; } @@ -434,7 +395,7 @@ class WebsocketHandler { } } - if (client['track-mempool-block'] >= 0 && !config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) { + if (client['track-mempool-block'] >= 0) { const index = client['track-mempool-block']; if (mBlockDeltas[index]) { response['projected-block-transactions'] = { @@ -449,61 +410,20 @@ class WebsocketHandler { } }); } - - async handleNewAsyncBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]): Promise { - if (!this.wss) { - throw new Error('WebSocket.Server is not set'); - } - - if (!config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) { - return; - } - - const _memPool = memPool.getMempool(); - - await mempoolBlocks.makeBlockTemplates(_memPool, 2); - const mBlocks = mempoolBlocks.getMempoolBlocks(); - const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); - - this.wss.clients.forEach((client) => { - if (client.readyState !== WebSocket.OPEN) { - return; - } - - if (!client['want-blocks']) { - return; - } - - const response = {}; - - if (mBlocks && client['want-mempool-blocks']) { - response['mempool-blocks'] = mBlocks; - } - - if (client['track-mempool-block'] >= 0) { - const index = client['track-mempool-block']; - if (mBlockDeltas && mBlockDeltas[index]) { - response['projected-block-transactions'] = { - index: index, - delta: mBlockDeltas[index], - }; - } - } - - client.send(JSON.stringify(response)); - }); - } - - handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]): void { + async handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]): Promise { if (!this.wss) { throw new Error('WebSocket.Server is not set'); } - let mBlocks: undefined | MempoolBlock[]; - let mBlockDeltas: undefined | MempoolBlockDelta[]; - let matchRate; const _memPool = memPool.getMempool(); + let matchRate; + + if (config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) { + await mempoolBlocks.makeBlockTemplates(_memPool, 2); + } else { + mempoolBlocks.updateMempoolBlocks(_memPool); + } if (Common.indexingEnabled() && memPool.isInSync()) { const projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); @@ -547,11 +467,13 @@ class WebsocketHandler { delete _memPool[txId]; } - if (!config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) { + if (config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) { + await mempoolBlocks.makeBlockTemplates(_memPool, 2); + } else { mempoolBlocks.updateMempoolBlocks(_memPool); - mBlocks = mempoolBlocks.getMempoolBlocks(); - mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); } + const mBlocks = mempoolBlocks.getMempoolBlocks(); + const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); const da = difficultyAdjustment.getDifficultyAdjustment(); const fees = feeApi.getRecommendedFee(); @@ -572,7 +494,7 @@ class WebsocketHandler { 'fees': fees, }; - if (mBlocks && client['want-mempool-blocks'] && !config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) { + if (mBlocks && client['want-mempool-blocks']) { response['mempool-blocks'] = mBlocks; } @@ -644,7 +566,7 @@ class WebsocketHandler { } } - if (client['track-mempool-block'] >= 0 && !config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) { + if (client['track-mempool-block'] >= 0) { const index = client['track-mempool-block']; if (mBlockDeltas && mBlockDeltas[index]) { response['projected-block-transactions'] = { diff --git a/backend/src/index.ts b/backend/src/index.ts index 2c15aa81a..09a12e200 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -216,10 +216,8 @@ class Server { websocketHandler.setupConnectionHandling(); if (config.MEMPOOL.ENABLED) { statistics.setNewStatisticsEntryCallback(websocketHandler.handleNewStatistic.bind(websocketHandler)); - blocks.setNewBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler)); - blocks.setNewAsyncBlockCallback(websocketHandler.handleNewAsyncBlock.bind(websocketHandler)); - memPool.setMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler)); - memPool.setAsyncMempoolChangedCallback(websocketHandler.handleAsyncMempoolChange.bind(websocketHandler)); + memPool.setAsyncMempoolChangedCallback(websocketHandler.handleMempoolChange.bind(websocketHandler)); + blocks.setNewAsyncBlockCallback(websocketHandler.handleNewBlock.bind(websocketHandler)); } fiatConversion.setProgressChangedCallback(websocketHandler.handleNewConversionRates.bind(websocketHandler)); loadingIndicators.setProgressChangedCallback(websocketHandler.handleLoadingChanged.bind(websocketHandler)); From b9a761fb88452f69cf03f51127514dceadf58972 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 22 Nov 2022 14:50:08 +0900 Subject: [PATCH 56/62] add ADVANCED_TRANSACTION_SELECTION default to config test --- backend/src/__fixtures__/mempool-config.template.json | 3 ++- backend/src/__tests__/config.test.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index e6c5428e5..d54365cda 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -25,7 +25,8 @@ "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__", "INDEXING_BLOCKS_AMOUNT": 14, "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__", - "POOLS_JSON_URL": "__POOLS_JSON_URL__" + "POOLS_JSON_URL": "__POOLS_JSON_URL__", + "ADVANCED_TRANSACTION_SELECTION": "__ADVANCED_TRANSACTION_SELECTION__" }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 650beaac1..9bb06c58a 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -37,7 +37,8 @@ describe('Mempool Backend Config', () => { USER_AGENT: 'mempool', STDOUT_LOG_MIN_PRIORITY: 'debug', POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master', - POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json' + POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json', + ADVANCED_TRANSACTION_SELECTION: false, }); expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true }); From cb7e25d6461461accaccdf1746e6ec818d50b0bf Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 20 Nov 2022 11:32:03 +0900 Subject: [PATCH 57/62] disconnect zero value outputs from flow diagram --- .../tx-bowtie-graph.component.html | 23 ++++++++- .../tx-bowtie-graph.component.scss | 10 ++++ .../tx-bowtie-graph.component.ts | 47 +++++++++++++++---- 3 files changed, 69 insertions(+), 11 deletions(-) diff --git a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html index 23346f405..18b2b0eb2 100644 --- a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html +++ b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html @@ -21,6 +21,15 @@ markerWidth="1.5" markerHeight="1" orient="auto"> + + + + + + + + + @@ -117,7 +126,7 @@ (pointerout)="onBlur($event, 'output-connector', i);" (click)="onClick($event, 'output-connector', outputData[i].index);" /> - - + diff --git a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.scss b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.scss index 7c9ecf0ce..6ba76d5de 100644 --- a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.scss +++ b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.scss @@ -11,6 +11,10 @@ &.fee { stroke: url(#fee-gradient); } + &.zerovalue { + stroke: url(#gradient0); + stroke-linecap: round; + } &.highlight { z-index: 8; @@ -21,6 +25,9 @@ &.output { stroke: url(#output-highlight-gradient); } + &.zerovalue { + stroke: #1bd8f4; + } } } @@ -36,6 +43,9 @@ &.fee { stroke: url(#fee-hover-gradient); } + &.zerovalue { + stroke: white; + } } .connector { diff --git a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts index 39164314a..2802b4a45 100644 --- a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts +++ b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts @@ -13,6 +13,7 @@ interface SvgLine { class?: string; connectorPath?: string; markerPath?: string; + zeroValue?: boolean; } interface Xput { @@ -63,6 +64,8 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { hoverConnector: boolean = false; tooltipPosition = { x: 0, y: 0 }; outspends: Outspend[] = []; + zeroValueWidth = 60; + zeroValueThickness = 20; outspendsSubscription: Subscription; refreshOutspends$: ReplaySubject = new ReplaySubject(); @@ -130,6 +133,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { this.txWidth = this.connectors ? Math.max(this.width - 200, this.width * 0.8) : this.width - 20; this.combinedWeight = Math.min(this.maxCombinedWeight, Math.floor((this.txWidth - (2 * this.midWidth)) / 6)); this.connectorWidth = (this.width - this.txWidth) / 2; + this.zeroValueWidth = Math.max(20, Math.min((this.txWidth / 2) - this.midWidth - 110, 60)); const totalValue = this.calcTotalValue(this.tx); let voutWithFee = this.tx.vout.map((v, i) => { @@ -236,10 +240,10 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { } linesFromWeights(side: 'in' | 'out', xputs: Xput[], weights: number[], maxVisibleStrands: number): SvgLine[] { - const lineParams = weights.map((w) => { + const lineParams = weights.map((w, i) => { return { weight: w, - thickness: Math.max(this.minWeight - 1, w) + 1, + thickness: xputs[i].value === 0 ? this.zeroValueThickness : Math.max(this.minWeight - 1, w) + 1, offset: 0, innerY: 0, outerY: 0, @@ -265,6 +269,12 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { let lastWeight = 0; let pad = 0; lineParams.forEach((line, i) => { + if (xputs[i].value === 0) { + line.outerY = lastOuter + this.zeroValueThickness / 2; + lastOuter += this.zeroValueThickness + spacing; + return; + } + // set the vertical position of the (center of the) outer side of the line line.outerY = lastOuter + (line.thickness / 2); line.innerY = Math.min(innerBottom + (line.thickness / 2), Math.max(innerTop + (line.thickness / 2), lastInner + (line.weight / 2))); @@ -318,13 +328,22 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { maxOffset -= minOffset; return lineParams.map((line, i) => { - return { - path: this.makePath(side, line.outerY, line.innerY, line.thickness, line.offset, pad + maxOffset), - style: this.makeStyle(line.thickness, xputs[i].type), - class: xputs[i].type, - connectorPath: this.connectors ? this.makeConnectorPath(side, line.outerY, line.innerY, line.thickness): null, - markerPath: this.makeMarkerPath(side, line.outerY, line.innerY, line.thickness), - }; + if (xputs[i].value === 0) { + return { + path: this.makeZeroValuePath(side, line.outerY), + style: this.makeStyle(this.zeroValueThickness, xputs[i].type), + class: xputs[i].type, + zeroValue: true, + }; + } else { + return { + path: this.makePath(side, line.outerY, line.innerY, line.thickness, line.offset, pad + maxOffset), + style: this.makeStyle(line.thickness, xputs[i].type), + class: xputs[i].type, + connectorPath: this.connectors ? this.makeConnectorPath(side, line.outerY, line.innerY, line.thickness): null, + markerPath: this.makeMarkerPath(side, line.outerY, line.innerY, line.thickness), + }; + } }); } @@ -347,6 +366,16 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { } } + makeZeroValuePath(side: 'in' | 'out', y: number): string { + const offset = this.zeroValueThickness / 2; + const start = this.connectorWidth + 10; + if (side === 'in') { + return `M ${start + offset} ${y} L ${start + this.zeroValueWidth + offset} ${y + 1}`; + } else { // mirrored in y-axis for the right hand side + return `M ${this.width - start - offset} ${y} L ${this.width - start - this.zeroValueWidth - offset} ${y + 1}`; + } + } + makeConnectorPath(side: 'in' | 'out', y: number, inner, weight: number): string { const halfWidth = weight * 0.5; const offset = 10; //Math.max(2, halfWidth * 0.2); From 7e01a22265f6157782acc2ea02f226c96ff09987 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 22 Nov 2022 15:54:26 +0900 Subject: [PATCH 58/62] fix rendering of many zero value outputs --- .../tx-bowtie-graph/tx-bowtie-graph.component.html | 6 +++--- .../tx-bowtie-graph/tx-bowtie-graph.component.ts | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html index 18b2b0eb2..c58e77335 100644 --- a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html +++ b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html @@ -21,13 +21,13 @@ markerWidth="1.5" markerHeight="1" orient="auto"> - + - + - + diff --git a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts index 2802b4a45..1f9602512 100644 --- a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts +++ b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts @@ -260,7 +260,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { let lastOuter = 0; let lastInner = innerTop; // gap between strands - const spacing = (this.height - visibleWeight) / gaps; + const spacing = Math.max(4, (this.height - visibleWeight) / gaps); // curve adjustments to prevent overlaps let offset = 0; @@ -270,7 +270,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { let pad = 0; lineParams.forEach((line, i) => { if (xputs[i].value === 0) { - line.outerY = lastOuter + this.zeroValueThickness / 2; + line.outerY = lastOuter + (this.zeroValueThickness / 2); lastOuter += this.zeroValueThickness + spacing; return; } @@ -349,7 +349,7 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { makePath(side: 'in' | 'out', outer: number, inner: number, weight: number, offset: number, pad: number): string { const start = (weight * 0.5) + this.connectorWidth; - const curveStart = Math.max(start + 1, pad - offset); + const curveStart = Math.max(start + 5, pad - offset); const end = this.width / 2 - (this.midWidth * 0.9) + 1; const curveEnd = end - offset - 10; const midpoint = (curveStart + curveEnd) / 2; @@ -368,11 +368,11 @@ export class TxBowtieGraphComponent implements OnInit, OnChanges { makeZeroValuePath(side: 'in' | 'out', y: number): string { const offset = this.zeroValueThickness / 2; - const start = this.connectorWidth + 10; + const start = (this.connectorWidth / 2) + 10; if (side === 'in') { - return `M ${start + offset} ${y} L ${start + this.zeroValueWidth + offset} ${y + 1}`; + return `M ${start + offset} ${y} L ${start + this.zeroValueWidth + offset} ${y}`; } else { // mirrored in y-axis for the right hand side - return `M ${this.width - start - offset} ${y} L ${this.width - start - this.zeroValueWidth - offset} ${y + 1}`; + return `M ${this.width - start - offset} ${y} L ${this.width - start - this.zeroValueWidth - offset} ${y}`; } } From 6c1457e2574fda7ad7f75e7be333e8f5686a4e2e Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 21 Nov 2022 11:59:45 +0900 Subject: [PATCH 59/62] Reverse tx flow diagram for RTL locales --- .../tx-bowtie-graph.component.html | 2 +- .../tx-bowtie-graph.component.scss | 4 ++++ .../tx-bowtie-graph.component.ts | 17 ++++++++++++++--- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html index c58e77335..f7484fb70 100644 --- a/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html +++ b/frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html @@ -1,5 +1,5 @@
- + Date: Sun, 20 Nov 2022 15:35:03 +0900 Subject: [PATCH 60/62] Support Maxmind Lite fixes #2553 --- .../src/tasks/lightning/sync-tasks/node-locations.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/backend/src/tasks/lightning/sync-tasks/node-locations.ts b/backend/src/tasks/lightning/sync-tasks/node-locations.ts index ba59e9e48..afd280ec7 100644 --- a/backend/src/tasks/lightning/sync-tasks/node-locations.ts +++ b/backend/src/tasks/lightning/sync-tasks/node-locations.ts @@ -6,6 +6,7 @@ import DB from '../../../database'; import logger from '../../../logger'; import { ResultSetHeader } from 'mysql2'; import * as IPCheck from '../../../utils/ipcheck.js'; +import { Reader } from 'mmdb-lib'; export async function $lookupNodeLocation(): Promise { let loggerTimer = new Date().getTime() / 1000; @@ -18,7 +19,10 @@ export async function $lookupNodeLocation(): Promise { const nodes = await nodesApi.$getAllNodes(); const lookupCity = await maxmind.open(config.MAXMIND.GEOLITE2_CITY); const lookupAsn = await maxmind.open(config.MAXMIND.GEOLITE2_ASN); - const lookupIsp = await maxmind.open(config.MAXMIND.GEOIP2_ISP); + let lookupIsp: Reader | null = null; + try { + lookupIsp = await maxmind.open(config.MAXMIND.GEOIP2_ISP); + } catch (e) { } for (const node of nodes) { const sockets: string[] = node.sockets.split(','); @@ -29,7 +33,10 @@ export async function $lookupNodeLocation(): Promise { if (hasClearnet && ip !== '127.0.1.1' && ip !== '127.0.0.1') { const city = lookupCity.get(ip); const asn = lookupAsn.get(ip); - const isp = lookupIsp.get(ip); + let isp: IspResponse | null = null; + if (lookupIsp) { + isp = lookupIsp.get(ip); + } let asOverwrite: any | undefined; if (asn && (IPCheck.match(ip, '170.75.160.0/20') || IPCheck.match(ip, '172.81.176.0/21'))) { From 5198cc51dce519d22fe4803d92db622439f709bf Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 19 Nov 2022 20:27:39 +0900 Subject: [PATCH 61/62] ellipsis for long op_return messages in tx preview --- .../transaction-preview.component.html | 28 +++++++---------- .../transaction-preview.component.scss | 31 +++++++++++++------ 2 files changed, 33 insertions(+), 26 deletions(-) diff --git a/frontend/src/app/components/transaction/transaction-preview.component.html b/frontend/src/app/components/transaction/transaction-preview.component.html index cb273b16c..40ef94dde 100644 --- a/frontend/src/app/components/transaction/transaction-preview.component.html +++ b/frontend/src/app/components/transaction/transaction-preview.component.html @@ -41,24 +41,20 @@
- - - - - - - -
Coinbase{{ tx.vin[0].scriptsig | hex2ascii }}
- - +
+
+ Coinbase + {{ tx.vin[0].scriptsig | hex2ascii }} +
+
+
-
- - - +
+ OP_RETURN + {{ vout.scriptpubkey_asm | hex2ascii }} +
- -
OP_RETURN{{ vout.scriptpubkey_asm | hex2ascii }}
+
diff --git a/frontend/src/app/components/transaction/transaction-preview.component.scss b/frontend/src/app/components/transaction/transaction-preview.component.scss index 75eceb99e..9c0d75c2a 100644 --- a/frontend/src/app/components/transaction/transaction-preview.component.scss +++ b/frontend/src/app/components/transaction/transaction-preview.component.scss @@ -92,26 +92,37 @@ max-width: 90%; margin: auto; overflow: hidden; + display: flex; + flex-direction: row; + justify-content: center; .opreturns { + display: inline-block; width: auto; + max-width: 100%; margin: auto; table-layout: auto; background: #2d3348af; border-top-left-radius: 5px; border-top-right-radius: 5px; - td { - padding: 10px 10px; + .opreturn-row { + width: 100%; + display: flex; + flex-direction: row; + justify-content: flex-start; + padding: 0 10px; + } - &.message { - overflow: hidden; - display: inline-block; - vertical-align: bottom; - text-overflow: ellipsis; - white-space: nowrap; - text-align: left; - } + .label { + margin-right: 1em; + } + + .message { + flex-shrink: 1; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; } } } From 43bb3aa50be27e4d4998b641f21ccf60c5bace56 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 19 Nov 2022 20:27:44 +0900 Subject: [PATCH 62/62] align elements of tx preview --- .../transaction/transaction-preview.component.scss | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/frontend/src/app/components/transaction/transaction-preview.component.scss b/frontend/src/app/components/transaction/transaction-preview.component.scss index 9c0d75c2a..4fa8b661a 100644 --- a/frontend/src/app/components/transaction/transaction-preview.component.scss +++ b/frontend/src/app/components/transaction/transaction-preview.component.scss @@ -29,6 +29,8 @@ .features { font-size: 24px; margin-left: 1em; + margin-top: 0.5em; + margin-right: -4px; } .top-data { @@ -60,6 +62,15 @@ } } +.top-data .field { + &:first-child { + padding-left: 0; + } + &:last-child { + padding-right: 0; + } +} + .tx-link { display: inline; font-size: 28px;