From 2f27d9279d4551576343b6c62cc981804a48b01e Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 24 Feb 2023 20:21:57 -0600 Subject: [PATCH 001/205] extend unfurler to dynamically render search crawler requests --- frontend/src/app/services/seo.service.ts | 2 +- unfurler/src/concurrency/ReusableSSRPage.ts | 65 ++++++++++++ unfurler/src/index.ts | 107 ++++++++++++++++++-- 3 files changed, 167 insertions(+), 7 deletions(-) create mode 100644 unfurler/src/concurrency/ReusableSSRPage.ts diff --git a/frontend/src/app/services/seo.service.ts b/frontend/src/app/services/seo.service.ts index 5f5d15c89..78c7afc4c 100644 --- a/frontend/src/app/services/seo.service.ts +++ b/frontend/src/app/services/seo.service.ts @@ -7,7 +7,7 @@ import { StateService } from './state.service'; }) export class SeoService { network = ''; - baseTitle = 'mempool'; + baseTitle = 'Mempool'; constructor( private titleService: Title, diff --git a/unfurler/src/concurrency/ReusableSSRPage.ts b/unfurler/src/concurrency/ReusableSSRPage.ts new file mode 100644 index 000000000..c68514a16 --- /dev/null +++ b/unfurler/src/concurrency/ReusableSSRPage.ts @@ -0,0 +1,65 @@ +import * as puppeteer from 'puppeteer'; +import { timeoutExecute } from 'puppeteer-cluster/dist/util'; +import logger from '../logger'; +import config from '../config'; +import ReusablePage from './ReusablePage'; +const mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : ''); + +const mockImageBuffer = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVQYV2NgYAAAAAMAAWgmWQ0AAAAASUVORK5CYII=", 'base64'); + +interface RepairablePage extends puppeteer.Page { + repairRequested?: boolean; + language?: string | null; + createdAt?: number; + free?: boolean; + index?: number; +} + +export default class ReusableSSRPage extends ReusablePage { + + public constructor(options: puppeteer.LaunchOptions, puppeteer: any) { + super(options, puppeteer); + } + + public async close() { + await (this.browser as puppeteer.Browser).close(); + } + + protected async initPage(): Promise { + const page = await (this.browser as puppeteer.Browser).newPage() as RepairablePage; + page.language = null; + page.createdAt = Date.now(); + const defaultUrl = mempoolHost + '/about'; + + page.on('pageerror', (err) => { + console.log(err); + // page.repairRequested = true; + }); + await page.setRequestInterception(true); + page.on('request', req => { + if (req.isInterceptResolutionHandled()) { + return req.continue(); + } + if (req.resourceType() === 'image') { + return req.respond({ + contentType: 'image/png', + headers: {"Access-Control-Allow-Origin": "*"}, + body: mockImageBuffer + }); + } else if (!['document', 'script', 'xhr', 'fetch'].includes(req.resourceType())) { + return req.abort(); + } else { + return req.continue(); + } + }); + try { + await page.goto(defaultUrl, { waitUntil: "networkidle0" }); + await page.waitForSelector('meta[property="og:meta:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }); + } catch (e) { + logger.err(`failed to load frontend during ssr page initialization: ` + (e instanceof Error ? e.message : `${e}`)); + page.repairRequested = true; + } + page.free = true; + return page + } +} diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index 0b423ff92..c24cd65af 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -5,9 +5,11 @@ import * as https from 'https'; import config from './config'; import { Cluster } from 'puppeteer-cluster'; import ReusablePage from './concurrency/ReusablePage'; +import ReusableSSRPage from './concurrency/ReusablePage'; import { parseLanguageUrl } from './language/lang'; import { matchRoute } from './routes'; import logger from './logger'; +import { TimeoutError } from "puppeteer"; const puppeteerConfig = require('../puppeteer.config.json'); if (config.PUPPETEER.EXEC_PATH) { @@ -20,13 +22,16 @@ class Server { private server: http.Server | undefined; private app: Application; cluster?: Cluster; + ssrCluster?: Cluster; mempoolHost: string; + mempoolUrl: URL; network: string; secureHost = true; constructor() { this.app = express(); this.mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : ''); + this.mempoolUrl = new URL(this.mempoolHost); this.secureHost = config.SERVER.HOST.startsWith('https'); this.network = config.MEMPOOL.NETWORK || 'bitcoin'; this.startServer(); @@ -49,6 +54,12 @@ class Server { puppeteerOptions: puppeteerConfig, }); await this.cluster?.task(async (args) => { return this.clusterTask(args) }); + this.ssrCluster = await Cluster.launch({ + concurrency: ReusableSSRPage, + maxConcurrency: config.PUPPETEER.CLUSTER_SIZE, + puppeteerOptions: puppeteerConfig, + }); + await this.ssrCluster?.task(async (args) => { return this.ssrClusterTask(args) }); } this.setUpRoutes(); @@ -65,6 +76,10 @@ class Server { await this.cluster.idle(); await this.cluster.close(); } + if (this.ssrCluster) { + await this.ssrCluster.idle(); + await this.ssrCluster.close(); + } if (this.server) { await this.server.close(); } @@ -102,8 +117,8 @@ class Server { } // wait for preview component to initialize - await page.waitForSelector('meta[property="og:preview:loading"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }) let success; + await page.waitForSelector('meta[property="og:preview:loading"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }) success = await Promise.race([ page.waitForSelector('meta[property="og:preview:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => true), page.waitForSelector('meta[property="og:preview:fail"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => false) @@ -124,6 +139,44 @@ class Server { } } + async ssrClusterTask({ page, data: { url, path, action } }) { + try { + const urlParts = parseLanguageUrl(path); + if (page.language !== urlParts.lang) { + // switch language + page.language = urlParts.lang; + const localizedUrl = urlParts.lang ? `${this.mempoolHost}/${urlParts.lang}${urlParts.path}` : `${this.mempoolHost}${urlParts.path}` ; + await page.goto(localizedUrl, { waitUntil: "load" }); + } else { + const loaded = await page.evaluate(async (path) => { + if (window['ogService']) { + window['ogService'].loadPage(path); + return true; + } else { + return false; + } + }, urlParts.path); + if (!loaded) { + throw new Error('failed to access open graph service'); + } + } + + await page.waitForNetworkIdle({ + timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000, + }); + let html = await page.content(); + return html; + } catch (e) { + if (e instanceof TimeoutError) { + let html = await page.content(); + return html; + } else { + logger.err(`failed to render ${path} for ${action}: ` + (e instanceof Error ? e.message : `${e}`)); + page.repairRequested = true; + } + } + } + async renderDisabled(req, res) { res.status(500).send("preview rendering disabled"); } @@ -163,11 +216,44 @@ class Server { // drop requests for static files const rawPath = req.params[0]; const match = rawPath.match(/\.[\w]+$/); - if (match?.length && match[0] !== '.html') { - res.status(404).send(); - return; + if (match?.length && match[0] !== '.html' + || rawPath.startsWith('/api/v1/donations/images') + || rawPath.startsWith('/api/v1/contributors/images') + || rawPath.startsWith('/api/v1/translators/images') + || rawPath.startsWith('/resources/profile') + ) { + if (req.headers['user-agent'] === 'googlebot') { + if (this.secureHost) { + https.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res)); + } else { + http.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res)); + } + return; + } else { + res.status(404).send(); + return; + } } + let result = ''; + try { + if (req.headers['user-agent'] === 'googlebot') { + result = await this.renderSEOPage(rawPath); + } else { + result = await this.renderUnfurlMeta(rawPath); + } + if (result && result.length) { + res.send(result); + } else { + res.status(500).send(); + } + } catch (e) { + logger.err(e instanceof Error ? e.message : `${e} ${req.params[0]}`); + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + async renderUnfurlMeta(rawPath: string): Promise { const { lang, path } = parseLanguageUrl(rawPath); const matchedRoute = matchRoute(this.network, path); let ogImageUrl = config.SERVER.HOST + (matchedRoute.staticImg || matchedRoute.fallbackImg); @@ -178,7 +264,7 @@ class Server { ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${matchedRoute.networkMode !== 'mainnet' ? capitalize(matchedRoute.networkMode) + ' ' : ''}${matchedRoute.title}`; } - res.send(` + return ` @@ -199,7 +285,16 @@ class Server { - `); + `; + } + + async renderSEOPage(rawPath: string): Promise { + let html = await this.ssrCluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'ssr' }); + // remove javascript to prevent double hydration + if (html && html.length) { + html = html.replace(//g, ""); + } + return html; } } From b1e32ed55f413741ae95244bac8741766d78fa52 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 27 Feb 2023 10:48:13 -0600 Subject: [PATCH 002/205] Fix googlebot user-agent detection --- unfurler/src/index.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index c24cd65af..0bbcb32bc 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -222,7 +222,7 @@ class Server { || rawPath.startsWith('/api/v1/translators/images') || rawPath.startsWith('/resources/profile') ) { - if (req.headers['user-agent'] === 'googlebot') { + if (isSearchCrawler(req.headers['user-agent'])) { if (this.secureHost) { https.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res)); } else { @@ -237,7 +237,7 @@ class Server { let result = ''; try { - if (req.headers['user-agent'] === 'googlebot') { + if (isSearchCrawler(req.headers['user-agent'])) { result = await this.renderSEOPage(rawPath); } else { result = await this.renderUnfurlMeta(rawPath); @@ -313,3 +313,7 @@ function capitalize(str) { return str; } } + +function isSearchCrawler(useragent: string): boolean { + return /googlebot/i.test(useragent); +} From f6cae729a76f2166f563369a3190b765bdbf7930 Mon Sep 17 00:00:00 2001 From: mononaut <83316221+mononaut@users.noreply.github.com> Date: Tue, 28 Feb 2023 18:40:59 -0600 Subject: [PATCH 003/205] revert capitalization in title tag Co-authored-by: wiz --- frontend/src/app/services/seo.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/services/seo.service.ts b/frontend/src/app/services/seo.service.ts index 78c7afc4c..5f5d15c89 100644 --- a/frontend/src/app/services/seo.service.ts +++ b/frontend/src/app/services/seo.service.ts @@ -7,7 +7,7 @@ import { StateService } from './state.service'; }) export class SeoService { network = ''; - baseTitle = 'Mempool'; + baseTitle = 'mempool'; constructor( private titleService: Title, From 477f3bd70a7583cc80b62b73239baeba6ea4c268 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 8 Mar 2023 01:11:54 -0600 Subject: [PATCH 004/205] add applebot & bingbot to seo user-agent detection --- unfurler/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index 0bbcb32bc..6d209b140 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -315,5 +315,5 @@ function capitalize(str) { } function isSearchCrawler(useragent: string): boolean { - return /googlebot/i.test(useragent); + return /googlebot|applebot|bingbot/i.test(useragent); } From 82a808529b58172585fd196bd8eecdf7545321c4 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 8 Mar 2023 01:47:35 -0600 Subject: [PATCH 005/205] clean up unfurler meta html template --- unfurler/src/index.ts | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index 6d209b140..90938a5a8 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -264,28 +264,27 @@ class Server { ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${matchedRoute.networkMode !== 'mainnet' ? capitalize(matchedRoute.networkMode) + ' ' : ''}${matchedRoute.title}`; } - return ` - - - - - ${ogTitle} - - - - - - - - - - - - - - - - `; + return ` + + + + ${ogTitle} + + + + + + + + + + + + + + + +`; } async renderSEOPage(rawPath: string): Promise { From 2f3e49890628318f5f9261a2bbd12bda63f1789f Mon Sep 17 00:00:00 2001 From: Mononaut Date: Thu, 9 Mar 2023 00:26:28 -0600 Subject: [PATCH 006/205] fix canonical/meta tags in unfurler --- unfurler/src/index.ts | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index 90938a5a8..412dd0f4c 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -27,6 +27,7 @@ class Server { mempoolUrl: URL; network: string; secureHost = true; + canonicalHost: string; constructor() { this.app = express(); @@ -34,6 +35,20 @@ class Server { this.mempoolUrl = new URL(this.mempoolHost); this.secureHost = config.SERVER.HOST.startsWith('https'); this.network = config.MEMPOOL.NETWORK || 'bitcoin'; + + let canonical; + switch(config.MEMPOOL.NETWORK) { + case "liquid": + canonical = "https://liquid.network" + break; + case "bisq": + canonical = "https://bisq.markets" + break; + default: + canonical = "https://mempool.space" + } + this.canonicalHost = canonical; + this.startServer(); } @@ -259,6 +274,8 @@ class Server { let ogImageUrl = config.SERVER.HOST + (matchedRoute.staticImg || matchedRoute.fallbackImg); let ogTitle = 'The Mempool Open Source Project™'; + const canonical = this.canonicalHost + rawPath; + if (matchedRoute.render) { ogImageUrl = `${config.SERVER.HOST}/render/${lang || 'en'}/preview${path}`; ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${matchedRoute.networkMode !== 'mainnet' ? capitalize(matchedRoute.networkMode) + ' ' : ''}${matchedRoute.title}`; @@ -269,6 +286,7 @@ class Server { ${ogTitle} + @@ -291,7 +309,8 @@ class Server { let html = await this.ssrCluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'ssr' }); // remove javascript to prevent double hydration if (html && html.length) { - html = html.replace(//g, ""); + html = html.replaceAll(//g, ""); + html = html.replaceAll(this.mempoolHost, this.canonicalHost); } return html; } From 105cccf9b0fd6b2dc2b4229eb2c51d148f6467c7 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Thu, 9 Mar 2023 02:34:21 -0600 Subject: [PATCH 007/205] convert soft 404s to hard 404s in unfurler ssr --- .../bisq-address/bisq-address.component.ts | 2 ++ .../bisq/bisq-block/bisq-block.component.ts | 2 ++ .../bisq-transaction.component.ts | 3 +++ .../components/address/address.component.ts | 2 ++ .../app/components/asset/asset.component.ts | 2 ++ .../block/block-preview.component.ts | 2 ++ .../app/components/block/block.component.ts | 3 +++ .../components/pool/pool-preview.component.ts | 2 ++ .../src/app/components/pool/pool.component.ts | 17 +++++++++--- .../transaction-preview.component.ts | 2 ++ .../transaction/transaction.component.ts | 12 ++++++--- .../channel/channel-preview.component.ts | 1 + .../lightning/channel/channel.component.ts | 1 + .../group/group-preview.component.ts | 2 ++ .../lightning/node/node-preview.component.ts | 1 + .../src/app/lightning/node/node.component.ts | 1 + .../nodes-per-isp-preview.component.ts | 1 + frontend/src/app/services/seo.service.ts | 26 +++++++++++++++++++ unfurler/src/index.ts | 17 +++++++++--- 19 files changed, 90 insertions(+), 9 deletions(-) diff --git a/frontend/src/app/bisq/bisq-address/bisq-address.component.ts b/frontend/src/app/bisq/bisq-address/bisq-address.component.ts index eccc88bc7..7711c4d3c 100644 --- a/frontend/src/app/bisq/bisq-address/bisq-address.component.ts +++ b/frontend/src/app/bisq/bisq-address/bisq-address.component.ts @@ -47,6 +47,7 @@ export class BisqAddressComponent implements OnInit, OnDestroy { catchError((err) => { this.isLoadingAddress = false; this.error = err; + this.seoService.logSoft404(); console.log(err); return of(null); }) @@ -62,6 +63,7 @@ export class BisqAddressComponent implements OnInit, OnDestroy { (error) => { console.log(error); this.error = error; + this.seoService.logSoft404(); this.isLoadingAddress = false; }); } diff --git a/frontend/src/app/bisq/bisq-block/bisq-block.component.ts b/frontend/src/app/bisq/bisq-block/bisq-block.component.ts index 1bb3a24ab..206a18031 100644 --- a/frontend/src/app/bisq/bisq-block/bisq-block.component.ts +++ b/frontend/src/app/bisq/bisq-block/bisq-block.component.ts @@ -82,6 +82,7 @@ export class BisqBlockComponent implements OnInit, OnDestroy { ) .subscribe((block: BisqBlock) => { if (!block) { + this.seoService.logSoft404(); return; } this.isLoading = false; @@ -97,6 +98,7 @@ export class BisqBlockComponent implements OnInit, OnDestroy { caughtHttpError(err: HttpErrorResponse){ this.error = err; + this.seoService.logSoft404(); return of(null); } } diff --git a/frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.ts b/frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.ts index fb30fc59f..78e2c0bb0 100644 --- a/frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.ts +++ b/frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.ts @@ -70,11 +70,13 @@ export class BisqTransactionComponent implements OnInit, OnDestroy { catchError((txError: HttpErrorResponse) => { console.log(txError); this.error = txError; + this.seoService.logSoft404(); return of(null); }) ); } this.error = bisqTxError; + this.seoService.logSoft404(); return of(null); }) ); @@ -103,6 +105,7 @@ export class BisqTransactionComponent implements OnInit, OnDestroy { this.isLoadingTx = false; if (!tx) { + this.seoService.logSoft404(); return; } diff --git a/frontend/src/app/components/address/address.component.ts b/frontend/src/app/components/address/address.component.ts index 2ae9a962b..b7752228d 100644 --- a/frontend/src/app/components/address/address.component.ts +++ b/frontend/src/app/components/address/address.component.ts @@ -88,6 +88,7 @@ export class AddressComponent implements OnInit, OnDestroy { catchError((err) => { this.isLoadingAddress = false; this.error = err; + this.seoService.logSoft404(); console.log(err); return of(null); }) @@ -157,6 +158,7 @@ export class AddressComponent implements OnInit, OnDestroy { (error) => { console.log(error); this.error = error; + this.seoService.logSoft404(); this.isLoadingAddress = false; }); diff --git a/frontend/src/app/components/asset/asset.component.ts b/frontend/src/app/components/asset/asset.component.ts index 0e642063a..562ebff53 100644 --- a/frontend/src/app/components/asset/asset.component.ts +++ b/frontend/src/app/components/asset/asset.component.ts @@ -86,6 +86,7 @@ export class AssetComponent implements OnInit, OnDestroy { catchError((err) => { this.isLoadingAsset = false; this.error = err; + this.seoService.logSoft404(); console.log(err); return of(null); }) @@ -153,6 +154,7 @@ export class AssetComponent implements OnInit, OnDestroy { (error) => { console.log(error); this.error = error; + this.seoService.logSoft404(); this.isLoadingAsset = false; }); diff --git a/frontend/src/app/components/block/block-preview.component.ts b/frontend/src/app/components/block/block-preview.component.ts index d0fec960a..7c10dab6f 100644 --- a/frontend/src/app/components/block/block-preview.component.ts +++ b/frontend/src/app/components/block/block-preview.component.ts @@ -82,6 +82,7 @@ export class BlockPreviewComponent implements OnInit, OnDestroy { }), catchError((err) => { this.error = err; + this.seoService.logSoft404(); this.openGraphService.fail('block-data-' + this.rawId); this.openGraphService.fail('block-viz-' + this.rawId); return of(null); @@ -138,6 +139,7 @@ export class BlockPreviewComponent implements OnInit, OnDestroy { (error) => { this.error = error; this.isLoadingOverview = false; + this.seoService.logSoft404(); this.openGraphService.fail('block-viz-' + this.rawId); this.openGraphService.fail('block-data-' + this.rawId); if (this.blockGraph) { diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index f5a0c93b0..6494d2f70 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -192,6 +192,7 @@ export class BlockComponent implements OnInit, OnDestroy { this.error = err; this.isLoadingBlock = false; this.isLoadingOverview = false; + this.seoService.logSoft404(); return EMPTY; }) ); @@ -200,6 +201,7 @@ export class BlockComponent implements OnInit, OnDestroy { this.error = err; this.isLoadingBlock = false; this.isLoadingOverview = false; + this.seoService.logSoft404(); return EMPTY; }), ); @@ -215,6 +217,7 @@ export class BlockComponent implements OnInit, OnDestroy { this.error = err; this.isLoadingBlock = false; this.isLoadingOverview = false; + this.seoService.logSoft404(); return EMPTY; }) ); diff --git a/frontend/src/app/components/pool/pool-preview.component.ts b/frontend/src/app/components/pool/pool-preview.component.ts index 277bacb33..aa83537fe 100644 --- a/frontend/src/app/components/pool/pool-preview.component.ts +++ b/frontend/src/app/components/pool/pool-preview.component.ts @@ -61,6 +61,7 @@ export class PoolPreviewComponent implements OnInit { }), catchError(() => { this.isLoading = false; + this.seoService.logSoft404(); this.openGraphService.fail('pool-hash-' + this.slug); return of([slug]); }) @@ -70,6 +71,7 @@ export class PoolPreviewComponent implements OnInit { return this.apiService.getPoolStats$(slug).pipe( catchError(() => { this.isLoading = false; + this.seoService.logSoft404(); this.openGraphService.fail('pool-stats-' + this.slug); return of(null); }) diff --git a/frontend/src/app/components/pool/pool.component.ts b/frontend/src/app/components/pool/pool.component.ts index 56b8bd392..1c2ce5c1f 100644 --- a/frontend/src/app/components/pool/pool.component.ts +++ b/frontend/src/app/components/pool/pool.component.ts @@ -1,8 +1,8 @@ import { ChangeDetectionStrategy, Component, Inject, Input, LOCALE_ID, OnInit } from '@angular/core'; import { ActivatedRoute } from '@angular/router'; import { EChartsOption, graphic } from 'echarts'; -import { BehaviorSubject, Observable, timer } from 'rxjs'; -import { distinctUntilChanged, map, share, switchMap, tap } from 'rxjs/operators'; +import { BehaviorSubject, Observable, of, timer } from 'rxjs'; +import { catchError, distinctUntilChanged, map, share, switchMap, tap } from 'rxjs/operators'; import { BlockExtended, PoolStat } from '../../interfaces/node-api.interface'; import { ApiService } from '../../services/api.service'; import { StateService } from '../../services/state.service'; @@ -59,10 +59,21 @@ export class PoolComponent implements OnInit { this.prepareChartOptions(data.map(val => [val.timestamp * 1000, val.avgHashrate])); return [slug]; }), + catchError(() => { + this.isLoading = false; + this.seoService.logSoft404(); + return of([slug]); + }) ); }), switchMap((slug) => { - return this.apiService.getPoolStats$(slug); + return this.apiService.getPoolStats$(slug).pipe( + catchError(() => { + this.isLoading = false; + this.seoService.logSoft404(); + return of(null); + }) + ); }), tap(() => { this.loadMoreSubject.next(this.blocks[this.blocks.length - 1]?.height); diff --git a/frontend/src/app/components/transaction/transaction-preview.component.ts b/frontend/src/app/components/transaction/transaction-preview.component.ts index 6db0e588c..07843cc57 100644 --- a/frontend/src/app/components/transaction/transaction-preview.component.ts +++ b/frontend/src/app/components/transaction/transaction-preview.component.ts @@ -133,6 +133,7 @@ export class TransactionPreviewComponent implements OnInit, OnDestroy { ) .subscribe((tx: Transaction) => { if (!tx) { + this.seoService.logSoft404(); this.openGraphService.fail('tx-data-' + this.txId); return; } @@ -182,6 +183,7 @@ export class TransactionPreviewComponent implements OnInit, OnDestroy { this.openGraphService.waitOver('tx-data-' + this.txId); }, (error) => { + this.seoService.logSoft404(); this.openGraphService.fail('tx-data-' + this.txId); this.error = error; this.isLoadingTx = false; diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index 673743344..0754f6d18 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -193,8 +193,10 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { }) ).subscribe((tx) => { if (!tx) { + this.seoService.logSoft404(); return; } + this.seoService.clearSoft404(); this.tx = tx; this.isCached = true; @@ -286,9 +288,11 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { }) ) .subscribe((tx: Transaction) => { - if (!tx) { - return; - } + if (!tx) { + this.seoService.logSoft404(); + return; + } + this.seoService.clearSoft404(); this.tx = tx; this.isCached = false; @@ -340,6 +344,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { }, (error) => { this.error = error; + this.seoService.logSoft404(); this.isLoadingTx = false; } ); @@ -394,6 +399,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { this.waitingForTransaction = true; } this.error = error; + this.seoService.logSoft404(); this.isLoadingTx = false; return of(false); } diff --git a/frontend/src/app/lightning/channel/channel-preview.component.ts b/frontend/src/app/lightning/channel/channel-preview.component.ts index 9f1bea4b8..9c7fdc1d6 100644 --- a/frontend/src/app/lightning/channel/channel-preview.component.ts +++ b/frontend/src/app/lightning/channel/channel-preview.component.ts @@ -54,6 +54,7 @@ export class ChannelPreviewComponent implements OnInit { }), catchError((err) => { this.error = err; + this.seoService.logSoft404(); this.openGraphService.fail('channel-map-' + this.shortId); this.openGraphService.fail('channel-data-' + this.shortId); return of(null); diff --git a/frontend/src/app/lightning/channel/channel.component.ts b/frontend/src/app/lightning/channel/channel.component.ts index d57aa3f01..052225cc3 100644 --- a/frontend/src/app/lightning/channel/channel.component.ts +++ b/frontend/src/app/lightning/channel/channel.component.ts @@ -38,6 +38,7 @@ export class ChannelComponent implements OnInit { }), catchError((err) => { this.error = err; + this.seoService.logSoft404(); return [{ short_id: params.get('short_id') }]; diff --git a/frontend/src/app/lightning/group/group-preview.component.ts b/frontend/src/app/lightning/group/group-preview.component.ts index be23e6178..5fd730931 100644 --- a/frontend/src/app/lightning/group/group-preview.component.ts +++ b/frontend/src/app/lightning/group/group-preview.component.ts @@ -50,6 +50,7 @@ export class GroupPreviewComponent implements OnInit { name: this.slug.replace(/-/gi, ' '), description: '', }; + this.seoService.logSoft404(); this.openGraphService.fail('ln-group-map-' + this.slug); this.openGraphService.fail('ln-group-data-' + this.slug); return of(null); @@ -106,6 +107,7 @@ export class GroupPreviewComponent implements OnInit { }; }), catchError(() => { + this.seoService.logSoft404(); this.openGraphService.fail('ln-group-map-' + this.slug); this.openGraphService.fail('ln-group-data-' + this.slug); return of({ diff --git a/frontend/src/app/lightning/node/node-preview.component.ts b/frontend/src/app/lightning/node/node-preview.component.ts index 4c2782c2d..56753b18b 100644 --- a/frontend/src/app/lightning/node/node-preview.component.ts +++ b/frontend/src/app/lightning/node/node-preview.component.ts @@ -81,6 +81,7 @@ export class NodePreviewComponent implements OnInit { }), catchError(err => { this.error = err; + this.seoService.logSoft404(); this.openGraphService.fail('node-map-' + this.publicKey); this.openGraphService.fail('node-data-' + this.publicKey); return [{ diff --git a/frontend/src/app/lightning/node/node.component.ts b/frontend/src/app/lightning/node/node.component.ts index 47f65007f..6f222688d 100644 --- a/frontend/src/app/lightning/node/node.component.ts +++ b/frontend/src/app/lightning/node/node.component.ts @@ -121,6 +121,7 @@ export class NodeComponent implements OnInit { }), catchError(err => { this.error = err; + this.seoService.logSoft404(); return [{ alias: this.publicKey, public_key: this.publicKey, diff --git a/frontend/src/app/lightning/nodes-per-isp/nodes-per-isp-preview.component.ts b/frontend/src/app/lightning/nodes-per-isp/nodes-per-isp-preview.component.ts index 759606372..b823a5188 100644 --- a/frontend/src/app/lightning/nodes-per-isp/nodes-per-isp-preview.component.ts +++ b/frontend/src/app/lightning/nodes-per-isp/nodes-per-isp-preview.component.ts @@ -85,6 +85,7 @@ export class NodesPerISPPreview implements OnInit { }), catchError(err => { this.error = err; + this.seoService.logSoft404(); this.openGraphService.fail('isp-map-' + this.id); this.openGraphService.fail('isp-data-' + this.id); return of({ diff --git a/frontend/src/app/services/seo.service.ts b/frontend/src/app/services/seo.service.ts index 5f5d15c89..d536b4938 100644 --- a/frontend/src/app/services/seo.service.ts +++ b/frontend/src/app/services/seo.service.ts @@ -1,5 +1,7 @@ import { Injectable } from '@angular/core'; import { Title, Meta } from '@angular/platform-browser'; +import { ActivatedRoute, NavigationEnd, Router } from '@angular/router'; +import { filter, map, switchMap } from 'rxjs'; import { StateService } from './state.service'; @Injectable({ @@ -13,8 +15,22 @@ export class SeoService { private titleService: Title, private metaService: Meta, private stateService: StateService, + private router: Router, + private activatedRoute: ActivatedRoute, ) { this.stateService.networkChanged$.subscribe((network) => this.network = network); + this.router.events.pipe( + filter(event => event instanceof NavigationEnd), + map(() => this.activatedRoute), + map(route => { + while (route.firstChild) route = route.firstChild; + return route; + }), + filter(route => route.outlet === 'primary'), + switchMap(route => route.data), + ).subscribe((data) => { + this.clearSoft404(); + }); } setTitle(newTitle: string): void { @@ -53,4 +69,14 @@ export class SeoService { ucfirst(str: string) { return str.charAt(0).toUpperCase() + str.slice(1); } + + clearSoft404() { + window['soft404'] = false; + console.log('cleared soft 404'); + } + + logSoft404() { + window['soft404'] = true; + console.log('set soft 404'); + } } diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index 412dd0f4c..d7589797d 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -179,8 +179,15 @@ class Server { await page.waitForNetworkIdle({ timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000, }); - let html = await page.content(); - return html; + const is404 = await page.evaluate(async () => { + return !!window['soft404']; + }); + if (is404) { + return '404'; + } else { + let html = await page.content(); + return html; + } } catch (e) { if (e instanceof TimeoutError) { let html = await page.content(); @@ -258,7 +265,11 @@ class Server { result = await this.renderUnfurlMeta(rawPath); } if (result && result.length) { - res.send(result); + if (result === '404') { + res.status(404).send(); + } else { + res.send(result); + } } else { res.status(500).send(); } From e489f713eb46f8ece0d4190f1b49d72920d8ffdd Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 22 May 2023 18:16:58 -0400 Subject: [PATCH 008/205] include accelerated tx data in block audits --- backend/src/api/audit.ts | 11 ++++++++--- backend/src/api/database-migration.ts | 7 ++++++- backend/src/api/mempool-blocks.ts | 2 +- backend/src/api/websocket-handler.ts | 3 ++- backend/src/mempool.interfaces.ts | 1 + backend/src/repositories/BlocksAuditsRepository.ts | 9 ++++++--- .../app/components/block-overview-graph/tx-view.ts | 5 ++++- .../block-overview-tooltip.component.html | 1 + frontend/src/app/components/block/block.component.ts | 10 ++++++++++ frontend/src/app/interfaces/node-api.interface.ts | 3 ++- frontend/src/app/interfaces/websocket.interface.ts | 2 +- 11 files changed, 42 insertions(+), 12 deletions(-) diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index a909fc2b6..63da288a1 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -6,16 +6,17 @@ import rbfCache from './rbf-cache'; const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners class Audit { - auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }) - : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], score: number, similarity: number } { + auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }, accelerations: { [txid: string]: number }) + : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } { if (!projectedBlocks?.[0]?.transactionIds || !mempool) { - return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], score: 0, similarity: 1 }; + return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 0, similarity: 1 }; } const matches: string[] = []; // present in both mined block and template const added: string[] = []; // present in mined block, not in template const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN const rbf: string[] = []; // either missing or present, and either part of a full-rbf replacement, or a conflict with the mined block + const accelerated: string[] = []; // prioritized by the mempool accelerator const isCensored = {}; // missing, without excuse const isDisplaced = {}; let displacedWeight = 0; @@ -28,6 +29,9 @@ class Audit { const now = Math.round((Date.now() / 1000)); for (const tx of transactions) { inBlock[tx.txid] = tx; + if (accelerations[tx.txid]) { + accelerated.push(tx.txid); + } } // coinbase is always expected if (transactions[0]) { @@ -149,6 +153,7 @@ class Audit { fresh, sigop: [], fullrbf: rbf, + accelerated, score, similarity, }; diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 7c7608aff..b7dc39493 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository'; import { RowDataPacket } from 'mysql2'; class DatabaseMigration { - private static currentVersion = 64; + private static currentVersion = 65; private queryTimeout = 3600_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -548,6 +548,11 @@ class DatabaseMigration { await this.$executeQuery('ALTER TABLE `nodes` ADD features text NULL'); await this.updateToSchemaVersion(64); } + + if (databaseSchemaVersion < 65 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `blocks_audits` ADD accelerated_txs JSON DEFAULT "[]"'); + await this.updateToSchemaVersion(65); + } } /** diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 08508310d..5ca5cff09 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -206,7 +206,7 @@ class MempoolBlocks { return mempoolBlockDeltas; } - public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): Promise { + public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, accelerations: { [txid: string]: number } = {}): Promise { const start = Date.now(); // reset mempool short ids diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 56c8513cd..91004e292 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -666,7 +666,7 @@ class WebsocketHandler { } if (Common.indexingEnabled()) { - const { censored, added, fresh, sigop, fullrbf, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool); + const { censored, added, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool, accelerations); const matchRate = Math.round(score * 100 * 100) / 100; const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : []; @@ -695,6 +695,7 @@ class WebsocketHandler { freshTxs: fresh, sigopTxs: sigop, fullrbfTxs: fullrbf, + acceleratedTxs: accelerated, matchRate: matchRate, expectedFees: totalFees, expectedWeight: totalWeight, diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 25e7f0387..d0549cd97 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -36,6 +36,7 @@ export interface BlockAudit { sigopTxs: string[], fullrbfTxs: string[], addedTxs: string[], + acceleratedTxs: string[], matchRate: number, expectedFees?: number, expectedWeight?: number, diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index f7a2a59b5..9c7568567 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -6,9 +6,9 @@ import { BlockAudit, AuditScore } from '../mempool.interfaces'; class BlocksAuditRepositories { public async $saveAudit(audit: BlockAudit): Promise { try { - await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, fullrbf_txs, match_rate, expected_fees, expected_weight) - VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs), - JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]); + await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, fullrbf_txs, accelerated_txs, match_rate, expected_fees, expected_weight) + VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs), + JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), JSON.stringify(audit.acceleratedTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]); } catch (e: any) { if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`); @@ -69,6 +69,7 @@ class BlocksAuditRepositories { fresh_txs as freshTxs, sigop_txs as sigopTxs, fullrbf_txs as fullrbfTxs, + accelerated_txs as acceleratedTxs, match_rate as matchRate, expected_fees as expectedFees, expected_weight as expectedWeight @@ -83,6 +84,8 @@ class BlocksAuditRepositories { rows[0].freshTxs = JSON.parse(rows[0].freshTxs); rows[0].sigopTxs = JSON.parse(rows[0].sigopTxs); rows[0].fullrbfTxs = JSON.parse(rows[0].fullrbfTxs); + rows[0].acceleratedTxs = JSON.parse(rows[0].acceleratedTxs); + rows[0].transactions = JSON.parse(rows[0].transactions); rows[0].template = JSON.parse(rows[0].template); return rows[0]; diff --git a/frontend/src/app/components/block-overview-graph/tx-view.ts b/frontend/src/app/components/block-overview-graph/tx-view.ts index 1b8c88704..ac93eee0c 100644 --- a/frontend/src/app/components/block-overview-graph/tx-view.ts +++ b/frontend/src/app/components/block-overview-graph/tx-view.ts @@ -17,6 +17,7 @@ const auditColors = { missing: darken(desaturate(hexToColor('f344df'), 0.3), 0.7), added: hexToColor('0099ff'), selected: darken(desaturate(hexToColor('0099ff'), 0.3), 0.7), + accelerated: hexToColor('8F5FF6'), }; // convert from this class's update format to TxSprite's update format @@ -38,7 +39,7 @@ export default class TxView implements TransactionStripped { value: number; feerate: number; rate?: number; - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; scene?: BlockScene; @@ -216,6 +217,8 @@ export default class TxView implements TransactionStripped { return auditColors.added; case 'selected': return marginalFeeColors[feeLevelIndex] || marginalFeeColors[mempoolFeeColors.length - 1]; + case 'accelerated': + return auditColors.accelerated; case 'found': if (this.context === 'projected') { return auditFeeColors[feeLevelIndex] || auditFeeColors[mempoolFeeColors.length - 1]; diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html index c62779b69..8a410f3df 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html @@ -54,6 +54,7 @@ Added Marginal fee rate Conflicting + Accelerated diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index ec9a49504..1e94b1a9c 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -340,6 +340,7 @@ export class BlockComponent implements OnInit, OnDestroy { const isFresh = {}; const isSigop = {}; const isRbf = {}; + const isAccelerated = {}; this.numMissing = 0; this.numUnexpected = 0; @@ -365,6 +366,9 @@ export class BlockComponent implements OnInit, OnDestroy { for (const txid of blockAudit.fullrbfTxs || []) { isRbf[txid] = true; } + for (const txid of blockAudit.acceleratedTxs || []) { + isAccelerated[txid] = true; + } // set transaction statuses for (const tx of blockAudit.template) { tx.context = 'projected'; @@ -389,6 +393,9 @@ export class BlockComponent implements OnInit, OnDestroy { isMissing[tx.txid] = true; this.numMissing++; } + if (isAccelerated[tx.txid]) { + tx.status = 'accelerated'; + } } for (const [index, tx] of transactions.entries()) { tx.context = 'actual'; @@ -405,6 +412,9 @@ export class BlockComponent implements OnInit, OnDestroy { isSelected[tx.txid] = true; this.numUnexpected++; } + if (isAccelerated[tx.txid]) { + tx.status = 'accelerated'; + } } for (const tx of transactions) { inBlock[tx.txid] = true; diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 4249fd9db..7e7acfcf3 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -158,6 +158,7 @@ export interface BlockAudit extends BlockExtended { freshTxs: string[], sigopTxs: string[], fullrbfTxs: string[], + acceleratedTxs: string[], matchRate: number, expectedFees: number, expectedWeight: number, @@ -174,7 +175,7 @@ export interface TransactionStripped { vsize: number; value: number; rate?: number; // effective fee rate - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; } diff --git a/frontend/src/app/interfaces/websocket.interface.ts b/frontend/src/app/interfaces/websocket.interface.ts index e0ecdfeda..fb3c6d0c2 100644 --- a/frontend/src/app/interfaces/websocket.interface.ts +++ b/frontend/src/app/interfaces/websocket.interface.ts @@ -89,7 +89,7 @@ export interface TransactionStripped { vsize: number; value: number; rate?: number; // effective fee rate - status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf'; + status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; } From aa24f6a84dbb5af04d2886462f2c752b056f7da6 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 26 May 2023 21:10:32 -0400 Subject: [PATCH 009/205] use accelerated rates for block templates & show in viz --- backend/src/api/audit.ts | 4 +-- backend/src/api/bitcoin/bitcoin.routes.ts | 1 + backend/src/api/common.ts | 1 + backend/src/api/mempool-blocks.ts | 15 +++++----- backend/src/api/mempool.ts | 29 +++++++++++++++++++ backend/src/api/websocket-handler.ts | 5 +++- backend/src/mempool.interfaces.ts | 2 ++ .../block-overview-graph.component.ts | 2 +- .../block-overview-graph/block-scene.ts | 3 +- .../block-overview-graph/tx-view.ts | 13 ++++++++- .../block-overview-tooltip.component.html | 3 +- .../block-overview-tooltip.component.ts | 2 ++ .../mempool-block-overview.component.ts | 1 - .../transaction/transaction.component.html | 3 +- .../transaction/transaction.component.ts | 3 ++ .../src/app/interfaces/electrs.interface.ts | 1 + .../src/app/interfaces/node-api.interface.ts | 1 + .../src/app/interfaces/websocket.interface.ts | 3 +- 18 files changed, 75 insertions(+), 17 deletions(-) diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index 63da288a1..9710d0362 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -6,7 +6,7 @@ import rbfCache from './rbf-cache'; const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners class Audit { - auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }, accelerations: { [txid: string]: number }) + auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }) : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } { if (!projectedBlocks?.[0]?.transactionIds || !mempool) { return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 0, similarity: 1 }; @@ -29,7 +29,7 @@ class Audit { const now = Math.round((Date.now() / 1000)); for (const tx of transactions) { inBlock[tx.txid] = tx; - if (accelerations[tx.txid]) { + if (tx.acceleration) { accelerated.push(tx.txid); } } diff --git a/backend/src/api/bitcoin/bitcoin.routes.ts b/backend/src/api/bitcoin/bitcoin.routes.ts index ffdb2e629..e2887b706 100644 --- a/backend/src/api/bitcoin/bitcoin.routes.ts +++ b/backend/src/api/bitcoin/bitcoin.routes.ts @@ -214,6 +214,7 @@ class BitcoinRoutes { effectiveFeePerVsize: tx.effectiveFeePerVsize || null, sigops: tx.sigops, adjustedVsize: tx.adjustedVsize, + acceleration: tx.acceleration }); return; } diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index cd9da3d2a..775da2643 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -111,6 +111,7 @@ export class Common { fee: tx.fee, vsize: tx.weight / 4, value: tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0), + acc: tx.acceleration || undefined, rate: tx.effectiveFeePerVsize, }; } diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 5ca5cff09..81f2092a0 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -170,7 +170,7 @@ class MempoolBlocks { for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) { let added: TransactionStripped[] = []; let removed: string[] = []; - const changed: { txid: string, rate: number | undefined }[] = []; + const changed: { txid: string, rate: number | undefined, acc: number | undefined }[] = []; if (mempoolBlocks[i] && !prevBlocks[i]) { added = mempoolBlocks[i].transactions; } else if (!mempoolBlocks[i] && prevBlocks[i]) { @@ -192,8 +192,8 @@ class MempoolBlocks { mempoolBlocks[i].transactions.forEach(tx => { if (!prevIds[tx.txid]) { added.push(tx); - } else if (tx.rate !== prevIds[tx.txid].rate) { - changed.push({ txid: tx.txid, rate: tx.rate }); + } else if (tx.rate !== prevIds[tx.txid].rate || tx.acc !== prevIds[tx.txid].acc) { + changed.push({ txid: tx.txid, rate: tx.rate, acc: tx.acc }); } }); } @@ -206,7 +206,7 @@ class MempoolBlocks { return mempoolBlockDeltas; } - public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, accelerations: { [txid: string]: number } = {}): Promise { + public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): Promise { const start = Date.now(); // reset mempool short ids @@ -222,7 +222,7 @@ class MempoolBlocks { if (entry.uid !== null && entry.uid !== undefined) { const stripped = { uid: entry.uid, - fee: entry.fee, + fee: entry.fee + (entry.acceleration || 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, @@ -285,13 +285,14 @@ class MempoolBlocks { for (const tx of Object.values(added)) { this.setUid(tx, true); } - const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => (uid !== null && uid !== undefined)) as number[]; + + const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => uid != null) as number[]; // prepare a stripped down version of the mempool with only the minimum necessary data // to reduce the overhead of passing this data to the worker thread const addedStripped: CompactThreadTransaction[] = added.filter(entry => (entry.uid !== null && entry.uid !== undefined)).map(entry => { return { uid: entry.uid || 0, - fee: entry.fee, + fee: entry.fee + (entry.acceleration || 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 945b78738..8e350c4fe 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -23,6 +23,8 @@ class Mempool { private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]) => Promise) | undefined; + private accelerations: { [txId: string]: number } = {}; + private txPerSecondArray: number[] = []; private txPerSecond: number = 0; @@ -301,6 +303,17 @@ class Mempool { const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx)); this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6); + const newAccelerations: { txid: string, delta: number }[] = []; + newTransactions.forEach(tx => { + if (tx.txid.startsWith('00')) { + const delta = Math.floor(Math.random() * 100000) + 100000; + newAccelerations.push({ txid: tx.txid, delta }); + tx.acceleration = delta; + } + }); + this.addAccelerations(newAccelerations); + this.removeAccelerations(deletedTransactions.map(tx => tx.txid)); + this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize); if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) { @@ -325,6 +338,22 @@ class Mempool { this.clearTimer(timer); } + public getAccelerations(): { [txid: string]: number } { + return this.accelerations; + } + + public addAccelerations(newAccelerations: { txid: string, delta: number }[]): void { + for (const acceleration of newAccelerations) { + this.accelerations[acceleration.txid] = acceleration.delta; + } + } + + public removeAccelerations(txids: string[]): void { + for (const txid of txids) { + delete this.accelerations[txid]; + } + } + private startTimer() { const state: any = { start: Date.now(), diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 91004e292..a33a0f0fa 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -21,6 +21,7 @@ import Audit from './audit'; import { deepClone } from '../utils/clone'; import priceUpdater from '../tasks/price-updater'; import { ApiPrice } from '../repositories/PricesRepository'; +import mempool from './mempool'; // valid 'want' subscriptions const wantable = [ @@ -666,7 +667,7 @@ class WebsocketHandler { } if (Common.indexingEnabled()) { - const { censored, added, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool, accelerations); + const { censored, added, fresh, sigop, fullrbf, accelerated, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool); const matchRate = Math.round(score * 100 * 100) / 100; const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : []; @@ -737,6 +738,8 @@ class WebsocketHandler { const fees = feeApi.getRecommendedFee(); const mempoolInfo = memPool.getMempoolInfo(); + memPool.removeAccelerations(txIds); + // update init data this.updateSocketDataFields({ 'mempoolInfo': mempoolInfo, diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index d0549cd97..185256619 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -92,6 +92,7 @@ export interface TransactionExtended extends IEsploraApi.Transaction { block: number, vsize: number, }; + acceleration?: number; uid?: number; } @@ -183,6 +184,7 @@ export interface TransactionStripped { fee: number; vsize: number; value: number; + acc?: number; rate?: number; // effective fee rate } diff --git a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts index 49da16d55..fe847103f 100644 --- a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts +++ b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts @@ -147,7 +147,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On } } - update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { + update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { if (this.scene) { this.scene.update(add, remove, change, direction, resetLayout); this.start(); diff --git a/frontend/src/app/components/block-overview-graph/block-scene.ts b/frontend/src/app/components/block-overview-graph/block-scene.ts index 510803f03..94984bae2 100644 --- a/frontend/src/app/components/block-overview-graph/block-scene.ts +++ b/frontend/src/app/components/block-overview-graph/block-scene.ts @@ -150,7 +150,7 @@ export default class BlockScene { this.updateAll(startTime, 200, direction); } - update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { + update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { const startTime = performance.now(); const removed = this.removeBatch(remove, startTime, direction); @@ -175,6 +175,7 @@ export default class BlockScene { // update effective rates change.forEach(tx => { if (this.txs[tx.txid]) { + this.txs[tx.txid].acc = tx.acc; this.txs[tx.txid].feerate = tx.rate || (this.txs[tx.txid].fee / this.txs[tx.txid].vsize); this.txs[tx.txid].rate = tx.rate; this.txs[tx.txid].dirty = true; diff --git a/frontend/src/app/components/block-overview-graph/tx-view.ts b/frontend/src/app/components/block-overview-graph/tx-view.ts index ac93eee0c..690b974e3 100644 --- a/frontend/src/app/components/block-overview-graph/tx-view.ts +++ b/frontend/src/app/components/block-overview-graph/tx-view.ts @@ -38,6 +38,7 @@ export default class TxView implements TransactionStripped { vsize: number; value: number; feerate: number; + acc?: number; rate?: number; status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; @@ -64,6 +65,7 @@ export default class TxView implements TransactionStripped { this.vsize = tx.vsize; this.value = tx.value; this.feerate = tx.rate || (tx.fee / tx.vsize); // sort by effective fee rate where available + this.acc = tx.acc; this.rate = tx.rate; this.status = tx.status; this.initialised = false; @@ -200,6 +202,11 @@ export default class TxView implements TransactionStripped { const feeLevelColor = feeColors[feeLevelIndex] || feeColors[mempoolFeeColors.length - 1]; // Normal mode if (!this.scene?.highlightingEnabled) { + if (this.acc) { + return auditColors.accelerated; + } else { + return feeLevelColor; + } return feeLevelColor; } // Block audit @@ -226,7 +233,11 @@ export default class TxView implements TransactionStripped { return feeLevelColor; } default: - return feeLevelColor; + if (this.acc) { + return auditColors.accelerated; + } else { + return feeLevelColor; + } } } } diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html index 8a410f3df..a53cfdc9c 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html @@ -29,7 +29,8 @@ - Effective fee rate + Effective fee rate + Accelerated fee rate diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts index 61c294263..65d0f984c 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.ts @@ -21,6 +21,7 @@ export class BlockOverviewTooltipComponent implements OnChanges { vsize = 1; feeRate = 0; effectiveRate; + acceleration; tooltipPosition: Position = { x: 0, y: 0 }; @@ -53,6 +54,7 @@ export class BlockOverviewTooltipComponent implements OnChanges { this.vsize = tx.vsize || 1; this.feeRate = this.fee / this.vsize; this.effectiveRate = tx.rate; + this.acceleration = tx.acc; } } } diff --git a/frontend/src/app/components/mempool-block-overview/mempool-block-overview.component.ts b/frontend/src/app/components/mempool-block-overview/mempool-block-overview.component.ts index 30632a862..226be5210 100644 --- a/frontend/src/app/components/mempool-block-overview/mempool-block-overview.component.ts +++ b/frontend/src/app/components/mempool-block-overview/mempool-block-overview.component.ts @@ -94,7 +94,6 @@ export class MempoolBlockOverviewComponent implements OnInit, OnDestroy, OnChang updateBlock(delta: MempoolBlockDelta): void { const blockMined = (this.stateService.latestBlockHeight > this.lastBlockHeight); - if (this.blockIndex !== this.index) { const direction = (this.blockIndex == null || this.index < this.blockIndex) ? this.poolDirection : this.chainDirection; this.blockGraph.replace(delta.added, direction); diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index d4cd6913d..81a6106db 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -488,7 +488,8 @@ - Effective fee rate + Accelerated fee rate + Effective fee rate
diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index e856f34eb..f1d218a79 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -183,6 +183,9 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { } else { this.tx.effectiveFeePerVsize = cpfpInfo.effectiveFeePerVsize; } + if (cpfpInfo.acceleration) { + this.tx.acceleration = cpfpInfo.acceleration; + } this.cpfpInfo = cpfpInfo; this.hasEffectiveFeeRate = hasRelatives || (this.tx.effectiveFeePerVsize && (Math.abs(this.tx.effectiveFeePerVsize - this.tx.feePerVsize) > 0.01)); diff --git a/frontend/src/app/interfaces/electrs.interface.ts b/frontend/src/app/interfaces/electrs.interface.ts index df19f7491..5c15b0ae4 100644 --- a/frontend/src/app/interfaces/electrs.interface.ts +++ b/frontend/src/app/interfaces/electrs.interface.ts @@ -19,6 +19,7 @@ export interface Transaction { ancestors?: Ancestor[]; bestDescendant?: BestDescendant | null; cpfpChecked?: boolean; + acceleration?: number; deleteAfter?: number; _unblinded?: any; _deduced?: boolean; diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index 7e7acfcf3..fe6233866 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -27,6 +27,7 @@ export interface CpfpInfo { effectiveFeePerVsize?: number; sigops?: number; adjustedVsize?: number; + acceleration?: number; } export interface RbfInfo { diff --git a/frontend/src/app/interfaces/websocket.interface.ts b/frontend/src/app/interfaces/websocket.interface.ts index fb3c6d0c2..f8686042b 100644 --- a/frontend/src/app/interfaces/websocket.interface.ts +++ b/frontend/src/app/interfaces/websocket.interface.ts @@ -70,7 +70,7 @@ export interface MempoolBlockWithTransactions extends MempoolBlock { export interface MempoolBlockDelta { added: TransactionStripped[], removed: string[], - changed?: { txid: string, rate: number | undefined }[]; + changed?: { txid: string, rate: number | undefined, acc: number | undefined }[]; } export interface MempoolInfo { @@ -88,6 +88,7 @@ export interface TransactionStripped { fee: number; vsize: number; value: number; + acc?: number; // acceleration delta rate?: number; // effective fee rate status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; From c246db1cf9d116a3796d9a5e41c245e6880f9460 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 30 May 2023 19:35:39 -0400 Subject: [PATCH 010/205] Refactor acceleration tracking --- backend/src/api/mempool-blocks.ts | 20 ++++--- backend/src/api/mempool.ts | 72 +++++++++++++++--------- backend/src/api/services/acceleration.ts | 20 +++++++ backend/src/api/websocket-handler.ts | 6 +- 4 files changed, 79 insertions(+), 39 deletions(-) create mode 100644 backend/src/api/services/acceleration.ts diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 81f2092a0..bc0def7a8 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -5,6 +5,7 @@ import { Common, OnlineFeeStatsCalculator } from './common'; import config from '../config'; import { Worker } from 'worker_threads'; import path from 'path'; +import mempool from './mempool'; const MAX_UINT32 = Math.pow(2, 32) - 1; @@ -212,9 +213,11 @@ class MempoolBlocks { // reset mempool short ids this.resetUids(); for (const tx of Object.values(newMempool)) { - this.setUid(tx); + this.setUid(tx, true); } + const accelerations = mempool.getAccelerations(); + // prepare a stripped down version of the mempool with only the minimum necessary data // to reduce the overhead of passing this data to the worker thread const strippedMempool: Map = new Map(); @@ -222,7 +225,7 @@ class MempoolBlocks { if (entry.uid !== null && entry.uid !== undefined) { const stripped = { uid: entry.uid, - fee: entry.fee + (entry.acceleration || 0), + fee: entry.fee + (accelerations[entry.txid] || 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, @@ -273,7 +276,7 @@ class MempoolBlocks { return this.mempoolBlocks; } - public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], saveResults: boolean = false): Promise { + public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], accelerationDelta: string[] = [], saveResults: boolean = false): Promise { if (!this.txSelectionWorker) { // need to reset the worker await this.$makeBlockTemplates(newMempool, saveResults); @@ -282,17 +285,20 @@ class MempoolBlocks { const start = Date.now(); - for (const tx of Object.values(added)) { + const accelerations = mempool.getAccelerations(); + const addedAndChanged: MempoolTransactionExtended[] = accelerationDelta.map(txid => newMempool[txid]).filter(tx => tx != null).concat(added); + + for (const tx of addedAndChanged) { this.setUid(tx, true); } - const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => uid != null) as number[]; + // prepare a stripped down version of the mempool with only the minimum necessary data // to reduce the overhead of passing this data to the worker thread - const addedStripped: CompactThreadTransaction[] = added.filter(entry => (entry.uid !== null && entry.uid !== undefined)).map(entry => { + const addedStripped: CompactThreadTransaction[] = addedAndChanged.filter(entry => entry.uid != null).map(entry => { return { uid: entry.uid || 0, - fee: entry.fee + (entry.acceleration || 0), + fee: entry.fee + (accelerations[entry.txid] || 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 8e350c4fe..7bde83df9 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -1,5 +1,5 @@ import config from '../config'; -import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; +import bitcoinApi from './bitcoin/bitcoin-api-factory'; import { MempoolTransactionExtended, TransactionExtended, VbytesPerSecond } from '../mempool.interfaces'; import logger from '../logger'; import { Common } from './common'; @@ -9,7 +9,7 @@ import loadingIndicators from './loading-indicators'; import bitcoinClient from './bitcoin/bitcoin-client'; import bitcoinSecondClient from './bitcoin/bitcoin-second-client'; import rbfCache from './rbf-cache'; -import { IEsploraApi } from './bitcoin/esplora-api.interface'; +import accelerationApi from './services/acceleration'; class Mempool { private inSync: boolean = false; @@ -19,9 +19,9 @@ class Mempool { private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0, maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 }; private mempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, newTransactions: MempoolTransactionExtended[], - deletedTransactions: MempoolTransactionExtended[]) => void) | undefined; + deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => void) | undefined; private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[], - deletedTransactions: MempoolTransactionExtended[]) => Promise) | undefined; + deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise) | undefined; private accelerations: { [txId: string]: number } = {}; @@ -68,12 +68,12 @@ class Mempool { } public setMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; }, - newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]) => void): void { + newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => void): void { this.mempoolChangedCallback = fn; } public setAsyncMempoolChangedCallback(fn: (newMempool: { [txId: string]: MempoolTransactionExtended; }, mempoolSize: number, - newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]) => Promise): void { + newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise): void { this.$asyncMempoolChangedCallback = fn; } @@ -98,10 +98,10 @@ class Mempool { count++; } if (this.mempoolChangedCallback) { - this.mempoolChangedCallback(this.mempoolCache, [], []); + this.mempoolChangedCallback(this.mempoolCache, [], [], []); } if (this.$asyncMempoolChangedCallback) { - await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], []); + await this.$asyncMempoolChangedCallback(this.mempoolCache, count, [], [], []); } this.addToSpendMap(Object.values(this.mempoolCache)); } @@ -303,25 +303,19 @@ class Mempool { const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx)); this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6); - const newAccelerations: { txid: string, delta: number }[] = []; - newTransactions.forEach(tx => { - if (tx.txid.startsWith('00')) { - const delta = Math.floor(Math.random() * 100000) + 100000; - newAccelerations.push({ txid: tx.txid, delta }); - tx.acceleration = delta; - } - }); - this.addAccelerations(newAccelerations); - this.removeAccelerations(deletedTransactions.map(tx => tx.txid)); + const accelerationDelta = await this.$updateAccelerations(); + if (accelerationDelta.length) { + hasChange = true; + } this.mempoolCacheDelta = Math.abs(transactions.length - newMempoolSize); if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) { - this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions); + this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions, accelerationDelta); } if (this.$asyncMempoolChangedCallback && (hasChange || deletedTransactions.length)) { this.updateTimerProgress(timer, 'running async mempool callback'); - await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions); + await this.$asyncMempoolChangedCallback(this.mempoolCache, newMempoolSize, newTransactions, deletedTransactions, accelerationDelta); this.updateTimerProgress(timer, 'completed async mempool callback'); } @@ -342,15 +336,37 @@ class Mempool { return this.accelerations; } - public addAccelerations(newAccelerations: { txid: string, delta: number }[]): void { - for (const acceleration of newAccelerations) { - this.accelerations[acceleration.txid] = acceleration.delta; - } - } + public async $updateAccelerations(): Promise { + try { + const newAccelerations = await accelerationApi.fetchAccelerations$(); - public removeAccelerations(txids: string[]): void { - for (const txid of txids) { - delete this.accelerations[txid]; + const changed: string[] = []; + + const newAccelerationMap: { [txid: string]: number } = {}; + for (const acceleration of newAccelerations) { + newAccelerationMap[acceleration.txid] = acceleration.feeDelta; + if (this.accelerations[acceleration.txid] == null) { + // new acceleration + changed.push(acceleration.txid); + } else if (this.accelerations[acceleration.txid] !== acceleration.feeDelta) { + // feeDelta changed + changed.push(acceleration.txid); + } + } + + for (const oldTxid of Object.keys(this.accelerations)) { + if (!newAccelerationMap[oldTxid]) { + // removed + changed.push(oldTxid); + } + } + + this.accelerations = newAccelerationMap; + + return changed; + } catch (e: any) { + logger.debug(`Failed to update accelerations: ` + (e instanceof Error ? e.message : e)); + return []; } } diff --git a/backend/src/api/services/acceleration.ts b/backend/src/api/services/acceleration.ts new file mode 100644 index 000000000..efbf32f5d --- /dev/null +++ b/backend/src/api/services/acceleration.ts @@ -0,0 +1,20 @@ +import { query } from '../../utils/axios-query'; +import config from '../../config'; + +export interface Acceleration { + txid: string, + feeDelta: number, +} + +class AccelerationApi { + public async fetchAccelerations$(): Promise { + if (config.MEMPOOL_SERVICES.ACCELERATIONS) { + const response = await query(`${config.MEMPOOL_SERVICES.API}/accelerations`); + return (response as Acceleration[]) || []; + } else { + return []; + } + } +} + +export default new AccelerationApi(); \ No newline at end of file diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index a33a0f0fa..3a90b29f4 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -381,7 +381,7 @@ class WebsocketHandler { } async $handleMempoolChange(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, - newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[]): Promise { + newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]): Promise { if (!this.wss) { throw new Error('WebSocket.Server is not set'); } @@ -392,7 +392,7 @@ class WebsocketHandler { if (config.MEMPOOL.RUST_GBT) { await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions); } else { - await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, true); + await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true); } } else { mempoolBlocks.updateMempoolBlocks(newMempool, true); @@ -738,8 +738,6 @@ class WebsocketHandler { const fees = feeApi.getRecommendedFee(); const mempoolInfo = memPool.getMempoolInfo(); - memPool.removeAccelerations(txIds); - // update init data this.updateSocketDataFields({ 'mempoolInfo': mempoolInfo, From 20b3ceab1e318e088c8e2d9e266d367143c64bbc Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 30 May 2023 19:51:11 -0400 Subject: [PATCH 011/205] Implement accelerations API & config setting --- backend/mempool-config.sample.json | 4 ++++ .../src/__fixtures__/mempool-config.template.json | 4 ++++ backend/src/__tests__/config.test.ts | 7 +++++++ backend/src/api/mempool.ts | 4 ++++ backend/src/config.ts | 12 +++++++++++- docker/backend/mempool-config.json | 4 ++++ docker/backend/start.sh | 9 +++++++++ 7 files changed, 43 insertions(+), 1 deletion(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index e3df7d2fe..8f8b82475 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -136,5 +136,9 @@ "trusted", "servers" ] + }, + "MEMPOOL_SERVICES": { + "API": "https://mempool.space/api", + "ACCELERATIONS": false } } diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 4213f0ffb..d6754f966 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -127,5 +127,9 @@ "AUDIT": false, "AUDIT_START_HEIGHT": 774000, "SERVERS": [] + }, + "MEMPOOl_SERVICES": { + "API": "__MEMPOOL_SERVICES_API__", + "ACCELERATIONS": "__MEMPOOL_SERVICES_ACCELERATIONS__" } } diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index dc1beaa46..2c83d1af5 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -127,6 +127,11 @@ describe('Mempool Backend Config', () => { AUDIT_START_HEIGHT: 774000, SERVERS: [] }); + + expect(config.MEMPOOL_SERVICES).toStrictEqual({ + API: "", + ACCELERATIONS: false, + }); }); }); @@ -160,6 +165,8 @@ describe('Mempool Backend Config', () => { expect(config.PRICE_DATA_SERVER).toStrictEqual(fixture.PRICE_DATA_SERVER); expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER); + + expect(config.MEMPOOL_SERVICES).toStrictEqual(fixture.MEMPOOL_SERVICES); }); }); diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 7bde83df9..b114c89fc 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -337,6 +337,10 @@ class Mempool { } public async $updateAccelerations(): Promise { + if (!config.MEMPOOL_SERVICES.ACCELERATIONS) { + return []; + } + try { const newAccelerations = await accelerationApi.fetchAccelerations$(); diff --git a/backend/src/config.ts b/backend/src/config.ts index 09d279537..ceb569e06 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -137,7 +137,11 @@ interface IConfig { AUDIT: boolean; AUDIT_START_HEIGHT: number; SERVERS: string[]; - } + }, + MEMPOOL_SERVICES: { + API: string; + ACCELERATIONS: boolean; + }, } const defaults: IConfig = { @@ -275,6 +279,10 @@ const defaults: IConfig = { 'AUDIT': false, 'AUDIT_START_HEIGHT': 774000, 'SERVERS': [], + }, + 'MEMPOOL_SERVICES': { + 'API': '', + 'ACCELERATIONS': false, } }; @@ -296,6 +304,7 @@ class Config implements IConfig { EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER']; MAXMIND: IConfig['MAXMIND']; REPLICATION: IConfig['REPLICATION']; + MEMPOOL_SERVICES: IConfig['MEMPOOL_SERVICES']; constructor() { const configs = this.merge(configFromFile, defaults); @@ -316,6 +325,7 @@ class Config implements IConfig { this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER; this.MAXMIND = configs.MAXMIND; this.REPLICATION = configs.REPLICATION; + this.MEMPOOL_SERVICES = configs.MEMPOOL_SERVICES; } merge = (...objects: object[]): IConfig => { diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 2ff76d5dd..71fe3bd65 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -133,5 +133,9 @@ "AUDIT": __REPLICATION_AUDIT__, "AUDIT_START_HEIGHT": __REPLICATION_AUDIT_START_HEIGHT__, "SERVERS": __REPLICATION_SERVERS__ + }, + "MEMPOOL_SERVICES": { + "API": "__MEMPOOL_SERVICES_API__", + "ACCELERATIONS": __MEMPOOL_SERVICES_ACCELERATIONS__ } } diff --git a/docker/backend/start.sh b/docker/backend/start.sh index c34d804b4..66f873605 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -136,6 +136,10 @@ __REPLICATION_AUDIT__=${REPLICATION_AUDIT:=true} __REPLICATION_AUDIT_START_HEIGHT__=${REPLICATION_AUDIT_START_HEIGHT:=774000} __REPLICATION_SERVERS__=${REPLICATION_SERVERS:=[]} +# MEMPOOL_SERVICES +__MEMPOOL_SERVICES_API__==${MEMPOOL_SERVICES_API:=""} +__MEMPOOL_SERVICES_ACCELERATIONS__==${MEMPOOL_SERVICES_ACCELERATIONS:=false} + mkdir -p "${__MEMPOOL_CACHE_DIR__}" @@ -262,4 +266,9 @@ sed -i "s!__REPLICATION_AUDIT__!${__REPLICATION_AUDIT__}!g" mempool-config.json sed -i "s!__REPLICATION_AUDIT_START_HEIGHT__!${__REPLICATION_AUDIT_START_HEIGHT__}!g" mempool-config.json sed -i "s!__REPLICATION_SERVERS__!${__REPLICATION_SERVERS__}!g" mempool-config.json +# MEMPOOL_SERVICES +sed -i "s!__MEMPOOL_SERVICES_API__!${__MEMPOOL_SERVICES_API__}!g" mempool-config.json +sed -i "s!__MEMPOOL_SERVICES_ACCELERATIONS__!${__MEMPOOL_SERVICES_ACCELERATIONS__}!g" mempool-config.json + + node /backend/package/index.js From 083bfdba06875fd8d254b16c6db9a6070b6b2d0d Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sat, 3 Jun 2023 16:54:12 -0400 Subject: [PATCH 012/205] Refactor accelerated audits --- backend/src/api/audit.ts | 4 ++-- backend/src/api/mempool-blocks.ts | 28 +++++++++++++----------- backend/src/api/mempool.ts | 2 +- backend/src/api/services/acceleration.ts | 20 ++++++++++++++++- backend/src/api/websocket-handler.ts | 17 ++++++++++---- 5 files changed, 50 insertions(+), 21 deletions(-) diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index 9710d0362..e78b2796f 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -6,7 +6,7 @@ import rbfCache from './rbf-cache'; const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners class Audit { - auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }) + auditBlock(transactions: MempoolTransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: MempoolTransactionExtended }, useAccelerations: boolean = false) : { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], accelerated: string[], score: number, similarity: number } { if (!projectedBlocks?.[0]?.transactionIds || !mempool) { return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], accelerated: [], score: 0, similarity: 1 }; @@ -29,7 +29,7 @@ class Audit { const now = Math.round((Date.now() / 1000)); for (const tx of transactions) { inBlock[tx.txid] = tx; - if (tx.acceleration) { + if (mempool[tx.txid] && mempool[tx.txid].acceleration) { accelerated.push(tx.txid); } } diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index bc0def7a8..0ce0ea522 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -207,7 +207,7 @@ class MempoolBlocks { return mempoolBlockDeltas; } - public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): Promise { + public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false): Promise { const start = Date.now(); // reset mempool short ids @@ -216,7 +216,7 @@ class MempoolBlocks { this.setUid(tx, true); } - const accelerations = mempool.getAccelerations(); + const accelerations = useAccelerations ? mempool.getAccelerations() : {}; // prepare a stripped down version of the mempool with only the minimum necessary data // to reduce the overhead of passing this data to the worker thread @@ -225,7 +225,7 @@ class MempoolBlocks { if (entry.uid !== null && entry.uid !== undefined) { const stripped = { uid: entry.uid, - fee: entry.fee + (accelerations[entry.txid] || 0), + fee: entry.fee + (useAccelerations ? (accelerations[entry.txid] || 0) : 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, @@ -265,7 +265,7 @@ class MempoolBlocks { // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); - const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), saveResults); + const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, saveResults); logger.debug(`makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); @@ -276,17 +276,17 @@ class MempoolBlocks { return this.mempoolBlocks; } - public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], accelerationDelta: string[] = [], saveResults: boolean = false): Promise { + public async $updateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], accelerationDelta: string[] = [], saveResults: boolean = false, useAccelerations: boolean = false): Promise { if (!this.txSelectionWorker) { // need to reset the worker - await this.$makeBlockTemplates(newMempool, saveResults); + await this.$makeBlockTemplates(newMempool, saveResults, useAccelerations); return; } const start = Date.now(); - const accelerations = mempool.getAccelerations(); - const addedAndChanged: MempoolTransactionExtended[] = accelerationDelta.map(txid => newMempool[txid]).filter(tx => tx != null).concat(added); + const accelerations = useAccelerations ? mempool.getAccelerations() : {}; + const addedAndChanged: MempoolTransactionExtended[] = useAccelerations ? accelerationDelta.map(txid => newMempool[txid]).filter(tx => tx != null).concat(added) : added; for (const tx of addedAndChanged) { this.setUid(tx, true); @@ -298,7 +298,7 @@ class MempoolBlocks { const addedStripped: CompactThreadTransaction[] = addedAndChanged.filter(entry => entry.uid != null).map(entry => { return { uid: entry.uid || 0, - fee: entry.fee + (accelerations[entry.txid] || 0), + fee: entry.fee + (useAccelerations ? (accelerations[entry.txid] || 0) : 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, @@ -325,7 +325,7 @@ class MempoolBlocks { // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); - this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), saveResults); + this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, saveResults); logger.debug(`updateBlockTemplates completed in ${(Date.now() - start) / 1000} seconds`); } catch (e) { logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e)); @@ -362,7 +362,7 @@ class MempoolBlocks { if (saveResults) { this.rustInitialized = true; } - const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, saveResults); + const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, {}, saveResults); logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); return processed; } catch (e) { @@ -414,7 +414,7 @@ class MempoolBlocks { if (mempoolSize !== resultMempoolSize) { throw new Error('GBT returned wrong number of transactions, cache is probably out of sync'); } else { - this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, true); + this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, {}, true); } this.removeUids(removedUids); logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); @@ -424,7 +424,7 @@ class MempoolBlocks { } } - private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], saveResults): MempoolBlockWithTransactions[] { + private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], accelerations, saveResults): MempoolBlockWithTransactions[] { for (const [txid, rate] of rates) { if (txid in mempool) { mempool[txid].effectiveFeePerVsize = rate; @@ -503,6 +503,8 @@ class MempoolBlocks { mempoolTx.cpfpChecked = true; } + mempoolTx.acceleration = accelerations[txid]; + // online calculation of stack-of-blocks fee stats if (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) { feeStatsCalculator.processNext(mempoolTx); diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index b114c89fc..4efe17731 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -342,7 +342,7 @@ class Mempool { } try { - const newAccelerations = await accelerationApi.fetchAccelerations$(); + const newAccelerations = await accelerationApi.$fetchAccelerations(); const changed: string[] = []; diff --git a/backend/src/api/services/acceleration.ts b/backend/src/api/services/acceleration.ts index efbf32f5d..8c849cdd8 100644 --- a/backend/src/api/services/acceleration.ts +++ b/backend/src/api/services/acceleration.ts @@ -1,5 +1,6 @@ import { query } from '../../utils/axios-query'; import config from '../../config'; +import { BlockExtended, PoolTag } from '../../mempool.interfaces'; export interface Acceleration { txid: string, @@ -7,7 +8,7 @@ export interface Acceleration { } class AccelerationApi { - public async fetchAccelerations$(): Promise { + public async $fetchAccelerations(): Promise { if (config.MEMPOOL_SERVICES.ACCELERATIONS) { const response = await query(`${config.MEMPOOL_SERVICES.API}/accelerations`); return (response as Acceleration[]) || []; @@ -15,6 +16,23 @@ class AccelerationApi { return []; } } + + public async $fetchPools(): Promise { + if (config.MEMPOOL_SERVICES.ACCELERATIONS) { + const response = await query(`${config.MEMPOOL_SERVICES.API}/partners`); + return (response as PoolTag[]) || []; + } else { + return []; + } + } + + public async $isAcceleratedBlock(block: BlockExtended): Promise { + const pools = await this.$fetchPools(); + if (block?.extras?.pool?.id == null) { + return false; + } + return pools.reduce((match, tag) => match || tag.uniqueId === block.extras.pool.id, false); + } } export default new AccelerationApi(); \ No newline at end of file diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 3a90b29f4..51fccbcbb 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -21,7 +21,7 @@ import Audit from './audit'; import { deepClone } from '../utils/clone'; import priceUpdater from '../tasks/price-updater'; import { ApiPrice } from '../repositories/PricesRepository'; -import mempool from './mempool'; +import accelerationApi from './services/acceleration'; // valid 'want' subscriptions const wantable = [ @@ -392,7 +392,7 @@ class WebsocketHandler { if (config.MEMPOOL.RUST_GBT) { await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions); } else { - await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true); + await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS); } } else { mempoolBlocks.updateMempoolBlocks(newMempool, true); @@ -648,6 +648,7 @@ class WebsocketHandler { if (config.MEMPOOL.AUDIT && memPool.isInSync()) { let projectedBlocks; let auditMempool = _memPool; + const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && await accelerationApi.$isAcceleratedBlock(block); // template calculation functions have mempool side effects, so calculate audits using // a cloned copy of the mempool if we're running a different algorithm for mempool updates const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL; @@ -657,13 +658,17 @@ class WebsocketHandler { if (config.MEMPOOL.RUST_GBT) { projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool); } else { - projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false); + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated); } } else { projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false); } } else { - projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); + if ((config.MEMPOOL_SERVICES.ACCELERATIONS && !isAccelerated)) { + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated); + } else { + projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); + } } if (Common.indexingEnabled()) { @@ -723,11 +728,15 @@ class WebsocketHandler { } if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { +<<<<<<< HEAD if (config.MEMPOOL.RUST_GBT) { await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions); } else { await mempoolBlocks.$makeBlockTemplates(_memPool, true); } +======= + await mempoolBlocks.$makeBlockTemplates(_memPool, true, config.MEMPOOL_SERVICES.ACCELERATIONS); +>>>>>>> 77b0a8ecc (Refactor accelerated audits) } else { mempoolBlocks.updateMempoolBlocks(_memPool, true); } From ba54bc9d1593076a9ce380216580731df22518bb Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 13 Jun 2023 13:35:25 -0400 Subject: [PATCH 013/205] support for acceleration mempool blocks animation --- backend/src/api/websocket-handler.ts | 24 ++++++++++----- .../mempool-blocks.component.html | 2 +- .../mempool-blocks.component.scss | 30 +++++++++++++++++++ .../mempool-blocks.component.ts | 1 + .../transaction/transaction.component.html | 4 +-- .../src/app/interfaces/node-api.interface.ts | 1 + 6 files changed, 51 insertions(+), 11 deletions(-) diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 51fccbcbb..7716ed7b7 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -173,9 +173,15 @@ class WebsocketHandler { } const tx = memPool.getMempool()[trackTxid]; if (tx && tx.position) { + const position: { block: number, vsize: number, accelerated?: number } = { + ...tx.position + }; + if (tx.acceleration) { + position.accelerated = tx.acceleration; + } response['txPosition'] = JSON.stringify({ txid: trackTxid, - position: tx.position, + position }); } } else { @@ -600,7 +606,10 @@ class WebsocketHandler { if (mempoolTx && mempoolTx.position) { response['txPosition'] = JSON.stringify({ txid: trackTxid, - position: mempoolTx.position, + position: { + ...mempoolTx.position, + accelerated: mempoolTx.acceleration || undefined, + } }); } } @@ -728,15 +737,11 @@ class WebsocketHandler { } if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { -<<<<<<< HEAD if (config.MEMPOOL.RUST_GBT) { await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions); } else { - await mempoolBlocks.$makeBlockTemplates(_memPool, true); + await mempoolBlocks.$makeBlockTemplates(_memPool, true, config.MEMPOOL_SERVICES.ACCELERATIONS); } -======= - await mempoolBlocks.$makeBlockTemplates(_memPool, true, config.MEMPOOL_SERVICES.ACCELERATIONS); ->>>>>>> 77b0a8ecc (Refactor accelerated audits) } else { mempoolBlocks.updateMempoolBlocks(_memPool, true); } @@ -799,7 +804,10 @@ class WebsocketHandler { if (mempoolTx && mempoolTx.position) { response['txPosition'] = JSON.stringify({ txid: trackTxid, - position: mempoolTx.position, + position: { + ...mempoolTx.position, + accelerated: mempoolTx.acceleration || undefined, + } }); } } diff --git a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.html b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.html index 9c5c338c0..59d35c91e 100644 --- a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.html +++ b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.html @@ -49,7 +49,7 @@
-
+
diff --git a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.scss b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.scss index 40f43a015..606699d93 100644 --- a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.scss +++ b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.scss @@ -169,4 +169,34 @@ transform: translate(calc(-0.2 * var(--block-size)), calc(1.1 * var(--block-size))); border-radius: 2px; z-index: -1; +} + +.blink{ + width:400px; + height:400px; + border-bottom: 35px solid #FFF; + animation: blink 0.2s infinite; +} +@keyframes blink{ + 0% { + border-bottom: 35px solid green; + } + 50% { + border-bottom: 35px solid yellow; + } + 100% { + border-bottom: 35px solid orange; + } +} + +@-webkit-keyframes blink{ + 0% { + border-bottom: 35px solid green; + } + 50% { + border-bottom: 35px solid yellow; + } + 100% { + border-bottom: 35px solid orange; + } } \ No newline at end of file diff --git a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts index 71075b261..33db897a5 100644 --- a/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts +++ b/frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts @@ -26,6 +26,7 @@ import { animate, style, transition, trigger } from '@angular/animations'; export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy { @Input() minimal: boolean = false; @Input() blockWidth: number = 125; + @Input() containerWidth: number = null; @Input() count: number = null; @Input() spotlight: number = 0; @Input() getHref?: (index) => string = (index) => `/mempool-block/${index}`; diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index 81a6106db..7422c6894 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -488,8 +488,8 @@ - Accelerated fee rate - Effective fee rate + Accelerated fee rate + Effective fee rate
diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index fe6233866..a0a3da5e6 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -188,6 +188,7 @@ export interface RbfTransaction extends TransactionStripped { export interface MempoolPosition { block: number, vsize: number, + accelerated?: boolean } export interface RewardStats { From 6494f890fe587655e5ce3c6ef56633402f3823d4 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 13 Jun 2023 17:03:36 -0400 Subject: [PATCH 014/205] include per-tx pools in /accelerations endpoint --- backend/src/api/mempool-blocks.ts | 8 +++--- backend/src/api/mempool.ts | 36 +++++++++++++++++++----- backend/src/api/services/acceleration.ts | 20 ++++--------- backend/src/api/websocket-handler.ts | 5 ++-- 4 files changed, 42 insertions(+), 27 deletions(-) diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 0ce0ea522..606c98c83 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,6 +1,6 @@ import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction } from '../../rust-gbt'; import logger from '../logger'; -import { MempoolBlock, MempoolTransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats } from '../mempool.interfaces'; +import { MempoolBlock, MempoolTransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, PoolTag } from '../mempool.interfaces'; import { Common, OnlineFeeStatsCalculator } from './common'; import config from '../config'; import { Worker } from 'worker_threads'; @@ -207,7 +207,7 @@ class MempoolBlocks { return mempoolBlockDeltas; } - public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false): Promise { + public async $makeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { const start = Date.now(); // reset mempool short ids @@ -225,7 +225,7 @@ class MempoolBlocks { if (entry.uid !== null && entry.uid !== undefined) { const stripped = { uid: entry.uid, - fee: entry.fee + (useAccelerations ? (accelerations[entry.txid] || 0) : 0), + fee: entry.fee + (useAccelerations && (!accelerationPool || accelerations[entry.txid]?.pools?.includes(accelerationPool)) ? (accelerations[entry.txid]?.feeDelta || 0) : 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, @@ -298,7 +298,7 @@ class MempoolBlocks { const addedStripped: CompactThreadTransaction[] = addedAndChanged.filter(entry => entry.uid != null).map(entry => { return { uid: entry.uid || 0, - fee: entry.fee + (useAccelerations ? (accelerations[entry.txid] || 0) : 0), + fee: entry.fee + (useAccelerations ? (accelerations[entry.txid]?.feeDelta || 0) : 0), weight: (entry.adjustedVsize * 4), sigops: entry.sigops, feePerVsize: entry.adjustedFeePerVsize || entry.feePerVsize, diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 4efe17731..107aa41cb 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -23,7 +23,7 @@ class Mempool { private $asyncMempoolChangedCallback: ((newMempool: {[txId: string]: MempoolTransactionExtended; }, mempoolSize: number, newTransactions: MempoolTransactionExtended[], deletedTransactions: MempoolTransactionExtended[], accelerationDelta: string[]) => Promise) | undefined; - private accelerations: { [txId: string]: number } = {}; + private accelerations: { [txId: string]: Acceleration } = {}; private txPerSecondArray: number[] = []; private txPerSecond: number = 0; @@ -332,7 +332,7 @@ class Mempool { this.clearTimer(timer); } - public getAccelerations(): { [txid: string]: number } { + public getAccelerations(): { [txid: string]: Acceleration } { return this.accelerations; } @@ -346,15 +346,37 @@ class Mempool { const changed: string[] = []; - const newAccelerationMap: { [txid: string]: number } = {}; + const newAccelerationMap: { [txid: string]: Acceleration } = {}; for (const acceleration of newAccelerations) { - newAccelerationMap[acceleration.txid] = acceleration.feeDelta; + newAccelerationMap[acceleration.txid] = acceleration; if (this.accelerations[acceleration.txid] == null) { // new acceleration changed.push(acceleration.txid); - } else if (this.accelerations[acceleration.txid] !== acceleration.feeDelta) { - // feeDelta changed - changed.push(acceleration.txid); + } else { + if (this.accelerations[acceleration.txid].feeDelta !== acceleration.feeDelta) { + // feeDelta changed + changed.push(acceleration.txid); + } else if (this.accelerations[acceleration.txid].pools?.length) { + let poolsChanged = false; + const pools = new Set(); + this.accelerations[acceleration.txid].pools.forEach(pool => { + pools.add(pool); + }); + acceleration.pools.forEach(pool => { + if (!pools.has(pool)) { + poolsChanged = true; + } else { + pools.delete(pool); + } + }); + if (pools.size > 0) { + poolsChanged = true; + } + if (poolsChanged) { + // pools changed + changed.push(acceleration.txid); + } + } } } diff --git a/backend/src/api/services/acceleration.ts b/backend/src/api/services/acceleration.ts index 8c849cdd8..1c2fff7c6 100644 --- a/backend/src/api/services/acceleration.ts +++ b/backend/src/api/services/acceleration.ts @@ -5,6 +5,7 @@ import { BlockExtended, PoolTag } from '../../mempool.interfaces'; export interface Acceleration { txid: string, feeDelta: number, + pools: number[], } class AccelerationApi { @@ -17,21 +18,12 @@ class AccelerationApi { } } - public async $fetchPools(): Promise { - if (config.MEMPOOL_SERVICES.ACCELERATIONS) { - const response = await query(`${config.MEMPOOL_SERVICES.API}/partners`); - return (response as PoolTag[]) || []; - } else { - return []; + public isAcceleratedBlock(block: BlockExtended, accelerations: Acceleration[]): boolean { + let anyAccelerated = false; + for (let i = 0; i < accelerations.length && !anyAccelerated; i++) { + anyAccelerated = anyAccelerated || accelerations[i].pools?.includes(block.extras.pool.id); } - } - - public async $isAcceleratedBlock(block: BlockExtended): Promise { - const pools = await this.$fetchPools(); - if (block?.extras?.pool?.id == null) { - return false; - } - return pools.reduce((match, tag) => match || tag.uniqueId === block.extras.pool.id, false); + return anyAccelerated; } } diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 7716ed7b7..ced6642d8 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -22,6 +22,7 @@ import { deepClone } from '../utils/clone'; import priceUpdater from '../tasks/price-updater'; import { ApiPrice } from '../repositories/PricesRepository'; import accelerationApi from './services/acceleration'; +import mempool from './mempool'; // valid 'want' subscriptions const wantable = [ @@ -657,7 +658,7 @@ class WebsocketHandler { if (config.MEMPOOL.AUDIT && memPool.isInSync()) { let projectedBlocks; let auditMempool = _memPool; - const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && await accelerationApi.$isAcceleratedBlock(block); + const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations())); // template calculation functions have mempool side effects, so calculate audits using // a cloned copy of the mempool if we're running a different algorithm for mempool updates const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL; @@ -667,7 +668,7 @@ class WebsocketHandler { if (config.MEMPOOL.RUST_GBT) { projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool); } else { - projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated); + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id); } } else { projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false); From ffc2b6c53cecbd9f5c1b2e40732469ce05bf27f3 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 18 Jul 2023 15:05:44 +0900 Subject: [PATCH 015/205] Add acceleration support to rust gbt --- backend/rust-gbt/index.d.ts | 8 ++- backend/rust-gbt/src/audit_transaction.rs | 27 +++++---- backend/rust-gbt/src/gbt.rs | 13 ++++- backend/rust-gbt/src/lib.rs | 10 +++- backend/src/api/mempool-blocks.ts | 56 ++++++++++++++----- backend/src/api/mempool.ts | 2 +- backend/src/api/websocket-handler.ts | 8 +-- backend/src/mempool.interfaces.ts | 4 +- backend/src/replication/AuditReplication.ts | 1 + .../block-overview-graph.component.ts | 2 +- .../block-overview-graph/block-scene.ts | 2 +- .../block-overview-graph/tx-view.ts | 2 +- .../fee-distribution-graph.component.ts | 2 +- .../src/app/interfaces/websocket.interface.ts | 4 +- 14 files changed, 96 insertions(+), 45 deletions(-) diff --git a/backend/rust-gbt/index.d.ts b/backend/rust-gbt/index.d.ts index 33ae32bdf..2bd8a620a 100644 --- a/backend/rust-gbt/index.d.ts +++ b/backend/rust-gbt/index.d.ts @@ -12,6 +12,10 @@ export interface ThreadTransaction { effectiveFeePerVsize: number inputs: Array } +export interface ThreadAcceleration { + uid: number + delta: number +} export class GbtGenerator { constructor() /** @@ -19,13 +23,13 @@ export class GbtGenerator { * * Rejects if the thread panics or if the Mutex is poisoned. */ - make(mempool: Array, maxUid: number): Promise + make(mempool: Array, accelerations: Array, maxUid: number): Promise /** * # Errors * * Rejects if the thread panics or if the Mutex is poisoned. */ - update(newTxs: Array, removeTxs: Array, maxUid: number): Promise + update(newTxs: Array, removeTxs: Array, accelerations: Array, maxUid: number): Promise } /** * The result from calling the gbt function. diff --git a/backend/rust-gbt/src/audit_transaction.rs b/backend/rust-gbt/src/audit_transaction.rs index 3e25a18a0..9b7472c30 100644 --- a/backend/rust-gbt/src/audit_transaction.rs +++ b/backend/rust-gbt/src/audit_transaction.rs @@ -1,6 +1,6 @@ use crate::{ u32_hasher_types::{u32hashset_new, U32HasherState}, - ThreadTransaction, + ThreadTransaction, thread_acceleration::ThreadAcceleration, }; use std::{ cmp::Ordering, @@ -88,37 +88,42 @@ impl Ord for AuditTransaction { } #[inline] -fn calc_fee_rate(fee: f64, vsize: f64) -> f64 { - fee / (if vsize == 0.0 { 1.0 } else { vsize }) +fn calc_fee_rate(fee: u64, vsize: f64) -> f64 { + (fee as f64) / (if vsize == 0.0 { 1.0 } else { vsize }) } impl AuditTransaction { - pub fn from_thread_transaction(tx: &ThreadTransaction) -> Self { + pub fn from_thread_transaction(tx: &ThreadTransaction, maybe_acceleration: Option>) -> Self { + let fee_delta = match maybe_acceleration { + Some(Some(acceleration)) => acceleration.delta, + _ => 0.0 + }; + let fee = (tx.fee as u64) + (fee_delta as u64); // rounded up to the nearest integer let is_adjusted = tx.weight < (tx.sigops * 20); let sigop_adjusted_vsize = ((tx.weight + 3) / 4).max(tx.sigops * 5); let sigop_adjusted_weight = tx.weight.max(tx.sigops * 20); - let effective_fee_per_vsize = if is_adjusted { - calc_fee_rate(tx.fee, f64::from(sigop_adjusted_weight) / 4.0) + let effective_fee_per_vsize = if is_adjusted || fee_delta > 0.0 { + calc_fee_rate(fee, f64::from(sigop_adjusted_weight) / 4.0) } else { tx.effective_fee_per_vsize }; Self { uid: tx.uid, order: tx.order, - fee: tx.fee as u64, + fee, weight: tx.weight, sigop_adjusted_weight, sigop_adjusted_vsize, sigops: tx.sigops, - adjusted_fee_per_vsize: calc_fee_rate(tx.fee, f64::from(sigop_adjusted_vsize)), + adjusted_fee_per_vsize: calc_fee_rate(fee, f64::from(sigop_adjusted_vsize)), effective_fee_per_vsize, dependency_rate: f64::INFINITY, inputs: tx.inputs.clone(), relatives_set_flag: false, ancestors: u32hashset_new(), children: u32hashset_new(), - ancestor_fee: tx.fee as u64, + ancestor_fee: fee, ancestor_sigop_adjusted_weight: sigop_adjusted_weight, ancestor_sigop_adjusted_vsize: sigop_adjusted_vsize, ancestor_sigops: tx.sigops, @@ -156,7 +161,7 @@ impl AuditTransaction { // grows, so if we think of 0 as "grew infinitely" then dependency_rate would be // the smaller of the two. If either side is NaN, the other side is returned. self.dependency_rate.min(calc_fee_rate( - self.ancestor_fee as f64, + self.ancestor_fee, f64::from(self.ancestor_sigop_adjusted_weight) / 4.0, )) } @@ -172,7 +177,7 @@ impl AuditTransaction { #[inline] fn calc_new_score(&mut self) { self.score = self.adjusted_fee_per_vsize.min(calc_fee_rate( - self.ancestor_fee as f64, + self.ancestor_fee, f64::from(self.ancestor_sigop_adjusted_vsize), )); } diff --git a/backend/rust-gbt/src/gbt.rs b/backend/rust-gbt/src/gbt.rs index 09b6377e6..0bf7f9999 100644 --- a/backend/rust-gbt/src/gbt.rs +++ b/backend/rust-gbt/src/gbt.rs @@ -5,7 +5,7 @@ use tracing::{info, trace}; use crate::{ audit_transaction::{partial_cmp_uid_score, AuditTransaction}, u32_hasher_types::{u32hashset_new, u32priority_queue_with_capacity, U32HasherState}, - GbtResult, ThreadTransactionsMap, + GbtResult, ThreadTransactionsMap, thread_acceleration::ThreadAcceleration, }; const MAX_BLOCK_WEIGHT_UNITS: u32 = 4_000_000 - 4_000; @@ -53,7 +53,13 @@ impl Ord for TxPriority { // TODO: Make gbt smaller to fix these lints. #[allow(clippy::too_many_lines)] #[allow(clippy::cognitive_complexity)] -pub fn gbt(mempool: &mut ThreadTransactionsMap, max_uid: usize) -> GbtResult { +pub fn gbt(mempool: &mut ThreadTransactionsMap, accelerations: &[ThreadAcceleration], max_uid: usize) -> GbtResult { + let mut indexed_accelerations = Vec::with_capacity(max_uid + 1); + indexed_accelerations.resize(max_uid + 1, None); + for acceleration in accelerations { + indexed_accelerations[acceleration.uid as usize] = Some(acceleration); + } + let mempool_len = mempool.len(); let mut audit_pool: AuditPool = Vec::with_capacity(max_uid + 1); audit_pool.resize(max_uid + 1, None); @@ -63,7 +69,8 @@ pub fn gbt(mempool: &mut ThreadTransactionsMap, max_uid: usize) -> GbtResult { info!("Initializing working structs"); for (uid, tx) in &mut *mempool { - let audit_tx = AuditTransaction::from_thread_transaction(tx); + let acceleration = indexed_accelerations.get(*uid as usize); + let audit_tx = AuditTransaction::from_thread_transaction(tx, acceleration.copied()); // Safety: audit_pool and mempool_stack must always contain the same transactions audit_pool[*uid as usize] = Some(ManuallyDrop::new(audit_tx)); mempool_stack.push(*uid); diff --git a/backend/rust-gbt/src/lib.rs b/backend/rust-gbt/src/lib.rs index 516a26402..53db0ba21 100644 --- a/backend/rust-gbt/src/lib.rs +++ b/backend/rust-gbt/src/lib.rs @@ -9,6 +9,7 @@ use napi::bindgen_prelude::Result; use napi_derive::napi; use thread_transaction::ThreadTransaction; +use thread_acceleration::ThreadAcceleration; use tracing::{debug, info, trace}; use tracing_log::LogTracer; use tracing_subscriber::{EnvFilter, FmtSubscriber}; @@ -19,6 +20,7 @@ use std::sync::{Arc, Mutex}; mod audit_transaction; mod gbt; mod thread_transaction; +mod thread_acceleration; mod u32_hasher_types; use u32_hasher_types::{u32hashmap_with_capacity, U32HasherState}; @@ -74,10 +76,11 @@ impl GbtGenerator { /// /// Rejects if the thread panics or if the Mutex is poisoned. #[napi] - pub async fn make(&self, mempool: Vec, max_uid: u32) -> Result { + pub async fn make(&self, mempool: Vec, accelerations: Vec, max_uid: u32) -> Result { trace!("make: Current State {:#?}", self.thread_transactions); run_task( Arc::clone(&self.thread_transactions), + accelerations, max_uid as usize, move |map| { for tx in mempool { @@ -96,11 +99,13 @@ impl GbtGenerator { &self, new_txs: Vec, remove_txs: Vec, + accelerations: Vec, max_uid: u32, ) -> Result { trace!("update: Current State {:#?}", self.thread_transactions); run_task( Arc::clone(&self.thread_transactions), + accelerations, max_uid as usize, move |map| { for tx in new_txs { @@ -141,6 +146,7 @@ pub struct GbtResult { /// to the `HashMap` as the only argument. (A move closure is recommended to meet the bounds) async fn run_task( thread_transactions: Arc>, + accelerations: Vec, max_uid: usize, callback: F, ) -> Result @@ -159,7 +165,7 @@ where callback(&mut map); info!("Starting gbt algorithm for {} elements...", map.len()); - let result = gbt::gbt(&mut map, max_uid); + let result = gbt::gbt(&mut map, &accelerations, max_uid); info!("Finished gbt algorithm for {} elements...", map.len()); debug!( diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 606c98c83..16772b7cd 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,4 +1,4 @@ -import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction } from '../../rust-gbt'; +import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from '../../rust-gbt'; import logger from '../logger'; import { MempoolBlock, MempoolTransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, PoolTag } from '../mempool.interfaces'; import { Common, OnlineFeeStatsCalculator } from './common'; @@ -171,7 +171,7 @@ class MempoolBlocks { for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) { let added: TransactionStripped[] = []; let removed: string[] = []; - const changed: { txid: string, rate: number | undefined, acc: number | undefined }[] = []; + const changed: { txid: string, rate: number | undefined, acc: boolean | undefined }[] = []; if (mempoolBlocks[i] && !prevBlocks[i]) { added = mempoolBlocks[i].transactions; } else if (!mempoolBlocks[i] && prevBlocks[i]) { @@ -265,7 +265,7 @@ class MempoolBlocks { // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); - const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, saveResults); + const processed = this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, accelerationPool, saveResults); logger.debug(`makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); @@ -325,7 +325,7 @@ class MempoolBlocks { // clean up thread error listener this.txSelectionWorker?.removeListener('error', threadErrorListener); - this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, saveResults); + this.processBlockTemplates(newMempool, blocks, null, Object.entries(rates), Object.values(clusters), accelerations, null, saveResults); logger.debug(`updateBlockTemplates completed in ${(Date.now() - start) / 1000} seconds`); } catch (e) { logger.err('updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e)); @@ -337,7 +337,7 @@ class MempoolBlocks { this.rustGbtGenerator = new GbtGenerator(); } - private async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false): Promise { + private async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { const start = Date.now(); // reset mempool short ids @@ -353,16 +353,25 @@ class MempoolBlocks { tx.inputs = tx.vin.map(v => this.getUid(newMempool[v.txid])).filter(uid => (uid !== null && uid !== undefined)) as number[]; } + const accelerations = useAccelerations ? mempool.getAccelerations() : {}; + const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]); + const convertedAccelerations = acceleratedList.map(acc => { + return { + uid: this.getUid(newMempool[acc.txid]), + delta: acc.feeDelta, + }; + }); + // run the block construction algorithm in a separate thread, and wait for a result const rustGbt = saveResults ? this.rustGbtGenerator : new GbtGenerator(); try { const { blocks, blockWeights, rates, clusters } = this.convertNapiResultTxids( - await rustGbt.make(Object.values(newMempool) as RustThreadTransaction[], this.nextUid), + await rustGbt.make(Object.values(newMempool) as RustThreadTransaction[], convertedAccelerations as RustThreadAcceleration[], this.nextUid), ); if (saveResults) { this.rustInitialized = true; } - const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, {}, saveResults); + const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, saveResults); logger.debug(`RUST makeBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); return processed; } catch (e) { @@ -374,19 +383,20 @@ class MempoolBlocks { return this.mempoolBlocks; } - public async $oneOffRustBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }): Promise { - return this.$rustMakeBlockTemplates(newMempool, false); + public async $oneOffRustBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, useAccelerations: boolean, accelerationPool?: number): Promise { + return this.$rustMakeBlockTemplates(newMempool, false, useAccelerations, accelerationPool); } - public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[]): Promise { + public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], useAccelerations: boolean, accelerationPool?: number): Promise { // GBT optimization requires that uids never get too sparse // as a sanity check, we should also explicitly prevent uint32 uid overflow if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * mempoolSize), MAX_UINT32)) { this.resetRustGbt(); } + if (!this.rustInitialized) { // need to reset the worker - await this.$rustMakeBlockTemplates(newMempool, true); + await this.$rustMakeBlockTemplates(newMempool, true, useAccelerations, accelerationPool); return; } @@ -401,12 +411,22 @@ class MempoolBlocks { } const removedUids = removed.map(tx => this.getUid(tx)).filter(uid => (uid !== null && uid !== undefined)) as number[]; + const accelerations = useAccelerations ? mempool.getAccelerations() : {}; + const acceleratedList = accelerationPool ? Object.values(accelerations).filter(acc => newMempool[acc.txid] && acc.pools.includes(accelerationPool)) : Object.values(accelerations).filter(acc => newMempool[acc.txid]); + const convertedAccelerations = acceleratedList.map(acc => { + return { + uid: this.getUid(newMempool[acc.txid]), + delta: acc.feeDelta, + }; + }); + // run the block construction algorithm in a separate thread, and wait for a result try { const { blocks, blockWeights, rates, clusters } = this.convertNapiResultTxids( await this.rustGbtGenerator.update( added as RustThreadTransaction[], removedUids, + convertedAccelerations as RustThreadAcceleration[], this.nextUid, ), ); @@ -414,7 +434,7 @@ class MempoolBlocks { if (mempoolSize !== resultMempoolSize) { throw new Error('GBT returned wrong number of transactions, cache is probably out of sync'); } else { - this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, {}, true); + this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, true); } this.removeUids(removedUids); logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); @@ -424,7 +444,7 @@ class MempoolBlocks { } } - private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], accelerations, saveResults): MempoolBlockWithTransactions[] { + private processBlockTemplates(mempool: { [txid: string]: MempoolTransactionExtended }, blocks: string[][], blockWeights: number[] | null, rates: [string, number][], clusters: string[][], accelerations, accelerationPool, saveResults): MempoolBlockWithTransactions[] { for (const [txid, rate] of rates) { if (txid in mempool) { mempool[txid].effectiveFeePerVsize = rate; @@ -503,7 +523,15 @@ class MempoolBlocks { mempoolTx.cpfpChecked = true; } - mempoolTx.acceleration = accelerations[txid]; + const acceleration = accelerations[txid]; + if (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool))) { + mempoolTx.acceleration = true; + for (const ancestor of mempoolTx.ancestors || []) { + mempool[ancestor.txid].acceleration = true; + } + } else { + delete mempoolTx.acceleration; + } // online calculation of stack-of-blocks fee stats if (hasBlockStack && blockIndex === lastBlockIndex && feeStatsCalculator) { diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 107aa41cb..d1a2d70a9 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -9,7 +9,7 @@ import loadingIndicators from './loading-indicators'; import bitcoinClient from './bitcoin/bitcoin-client'; import bitcoinSecondClient from './bitcoin/bitcoin-second-client'; import rbfCache from './rbf-cache'; -import accelerationApi from './services/acceleration'; +import accelerationApi, { Acceleration } from './services/acceleration'; class Mempool { private inSync: boolean = false; diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index ced6642d8..e42e87acd 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -174,7 +174,7 @@ class WebsocketHandler { } const tx = memPool.getMempool()[trackTxid]; if (tx && tx.position) { - const position: { block: number, vsize: number, accelerated?: number } = { + const position: { block: number, vsize: number, accelerated?: boolean } = { ...tx.position }; if (tx.acceleration) { @@ -397,7 +397,7 @@ class WebsocketHandler { if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { if (config.MEMPOOL.RUST_GBT) { - await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions); + await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions, true,); } else { await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS); } @@ -666,7 +666,7 @@ class WebsocketHandler { auditMempool = deepClone(_memPool); if (config.MEMPOOL.ADVANCED_GBT_AUDIT) { if (config.MEMPOOL.RUST_GBT) { - projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool); + projectedBlocks = await mempoolBlocks.$oneOffRustBlockTemplates(auditMempool, isAccelerated, block.extras.pool.id); } else { projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id); } @@ -739,7 +739,7 @@ class WebsocketHandler { if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { if (config.MEMPOOL.RUST_GBT) { - await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions); + await mempoolBlocks.$rustUpdateBlockTemplates(_memPool, Object.keys(_memPool).length, [], transactions, true); } else { await mempoolBlocks.$makeBlockTemplates(_memPool, true, config.MEMPOOL_SERVICES.ACCELERATIONS); } diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 185256619..4715440e4 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -92,7 +92,7 @@ export interface TransactionExtended extends IEsploraApi.Transaction { block: number, vsize: number, }; - acceleration?: number; + acceleration?: boolean; uid?: number; } @@ -184,7 +184,7 @@ export interface TransactionStripped { fee: number; vsize: number; value: number; - acc?: number; + acc?: boolean; rate?: number; // effective fee rate } diff --git a/backend/src/replication/AuditReplication.ts b/backend/src/replication/AuditReplication.ts index 26bf6dad7..5de9de0da 100644 --- a/backend/src/replication/AuditReplication.ts +++ b/backend/src/replication/AuditReplication.ts @@ -116,6 +116,7 @@ class AuditReplication { freshTxs: auditSummary.freshTxs || [], sigopTxs: auditSummary.sigopTxs || [], fullrbfTxs: auditSummary.fullrbfTxs || [], + acceleratedTxs: auditSummary.acceleratedTxs || [], matchRate: auditSummary.matchRate, expectedFees: auditSummary.expectedFees, expectedWeight: auditSummary.expectedWeight, diff --git a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts index fe847103f..634d0f524 100644 --- a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts +++ b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts @@ -147,7 +147,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On } } - update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { + update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: boolean | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { if (this.scene) { this.scene.update(add, remove, change, direction, resetLayout); this.start(); diff --git a/frontend/src/app/components/block-overview-graph/block-scene.ts b/frontend/src/app/components/block-overview-graph/block-scene.ts index 94984bae2..cb0537e2a 100644 --- a/frontend/src/app/components/block-overview-graph/block-scene.ts +++ b/frontend/src/app/components/block-overview-graph/block-scene.ts @@ -150,7 +150,7 @@ export default class BlockScene { this.updateAll(startTime, 200, direction); } - update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: number | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { + update(add: TransactionStripped[], remove: string[], change: { txid: string, rate: number | undefined, acc: boolean | undefined }[], direction: string = 'left', resetLayout: boolean = false): void { const startTime = performance.now(); const removed = this.removeBatch(remove, startTime, direction); diff --git a/frontend/src/app/components/block-overview-graph/tx-view.ts b/frontend/src/app/components/block-overview-graph/tx-view.ts index 690b974e3..db2c4f6ae 100644 --- a/frontend/src/app/components/block-overview-graph/tx-view.ts +++ b/frontend/src/app/components/block-overview-graph/tx-view.ts @@ -38,7 +38,7 @@ export default class TxView implements TransactionStripped { vsize: number; value: number; feerate: number; - acc?: number; + acc?: boolean; rate?: number; status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; diff --git a/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts b/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts index f275588a1..1d9c289d8 100644 --- a/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts +++ b/frontend/src/app/components/fee-distribution-graph/fee-distribution-graph.component.ts @@ -64,7 +64,7 @@ export class FeeDistributionGraphComponent implements OnInit, OnChanges, OnDestr return; } const samples = []; - const txs = this.transactions.map(tx => { return { vsize: tx.vsize, rate: tx.rate || (tx.fee / tx.vsize) }; }).sort((a, b) => { return b.rate - a.rate; }); + const txs = this.transactions.filter(tx => !tx.acc).map(tx => { return { vsize: tx.vsize, rate: tx.rate || (tx.fee / tx.vsize) }; }).sort((a, b) => { return b.rate - a.rate; }); const maxBlockVSize = this.stateService.env.BLOCK_WEIGHT_UNITS / 4; const sampleInterval = maxBlockVSize / this.numSamples; let cumVSize = 0; diff --git a/frontend/src/app/interfaces/websocket.interface.ts b/frontend/src/app/interfaces/websocket.interface.ts index f8686042b..43ab1e5f4 100644 --- a/frontend/src/app/interfaces/websocket.interface.ts +++ b/frontend/src/app/interfaces/websocket.interface.ts @@ -70,7 +70,7 @@ export interface MempoolBlockWithTransactions extends MempoolBlock { export interface MempoolBlockDelta { added: TransactionStripped[], removed: string[], - changed?: { txid: string, rate: number | undefined, acc: number | undefined }[]; + changed?: { txid: string, rate: number | undefined, acc: boolean | undefined }[]; } export interface MempoolInfo { @@ -88,7 +88,7 @@ export interface TransactionStripped { fee: number; vsize: number; value: number; - acc?: number; // acceleration delta + acc?: boolean; // is accelerated? rate?: number; // effective fee rate status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; From 3838d947b13f24e1abf7ef2e03147e811a22922a Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 18 Jul 2023 15:14:33 +0900 Subject: [PATCH 016/205] fix tests --- backend/src/__fixtures__/mempool-config.template.json | 6 +++--- backend/src/__tests__/gbt/gbt-tests.ts | 2 +- docker/backend/start.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index d6754f966..21c8e51c0 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -128,8 +128,8 @@ "AUDIT_START_HEIGHT": 774000, "SERVERS": [] }, - "MEMPOOl_SERVICES": { - "API": "__MEMPOOL_SERVICES_API__", - "ACCELERATIONS": "__MEMPOOL_SERVICES_ACCELERATIONS__" + "MEMPOOL_SERVICES": { + "API": "", + "ACCELERATIONS": false } } diff --git a/backend/src/__tests__/gbt/gbt-tests.ts b/backend/src/__tests__/gbt/gbt-tests.ts index 0651faac4..0c2eb0176 100644 --- a/backend/src/__tests__/gbt/gbt-tests.ts +++ b/backend/src/__tests__/gbt/gbt-tests.ts @@ -15,7 +15,7 @@ describe('Rust GBT', () => { test('should produce the same template as getBlockTemplate from Bitcoin Core', async () => { const rustGbt = new GbtGenerator(); const { mempool, maxUid } = mempoolFromArrayBuffer(vectorBuffer.buffer); - const result = await rustGbt.make(mempool, maxUid); + const result = await rustGbt.make(mempool, [], maxUid); const blocks: [string, number][][] = result.blocks.map(block => { return block.map(uid => [vectorUidMap.get(uid) || 'missing', uid]); diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 66f873605..c9088c0a6 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -137,8 +137,8 @@ __REPLICATION_AUDIT_START_HEIGHT__=${REPLICATION_AUDIT_START_HEIGHT:=774000} __REPLICATION_SERVERS__=${REPLICATION_SERVERS:=[]} # MEMPOOL_SERVICES -__MEMPOOL_SERVICES_API__==${MEMPOOL_SERVICES_API:=""} -__MEMPOOL_SERVICES_ACCELERATIONS__==${MEMPOOL_SERVICES_ACCELERATIONS:=false} +__MEMPOOL_SERVICES_API__=${MEMPOOL_SERVICES_API:=""} +__MEMPOOL_SERVICES_ACCELERATIONS__=${MEMPOOL_SERVICES_ACCELERATIONS:=false} mkdir -p "${__MEMPOOL_CACHE_DIR__}" From 2a2aee21fb466128225d2f3791780152c9e5372f Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 18 Jul 2023 16:08:25 +0900 Subject: [PATCH 017/205] fix audit highlightning and fee ranges --- backend/src/api/common.ts | 2 +- backend/src/api/mempool-blocks.ts | 5 +++-- backend/src/api/websocket-handler.ts | 6 +++--- backend/src/repositories/BlocksAuditsRepository.ts | 1 - frontend/src/app/components/block/block.component.ts | 3 +++ frontend/src/app/interfaces/node-api.interface.ts | 1 + 6 files changed, 11 insertions(+), 7 deletions(-) diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 775da2643..b2f476da3 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -461,7 +461,7 @@ export class Common { }; } - static calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string }[]): EffectiveFeeStats { + static calcEffectiveFeeStatistics(transactions: { weight: number, fee: number, effectiveFeePerVsize?: number, txid: string, acceleration?: boolean }[]): EffectiveFeeStats { const sortedTxs = transactions.map(tx => { return { txid: tx.txid, weight: tx.weight, rate: tx.effectiveFeePerVsize || ((tx.fee || 0) / (tx.weight / 4)) }; }).sort((a, b) => a.rate - b.rate); let weightCount = 0; diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 16772b7cd..599a53c23 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -337,7 +337,8 @@ class MempoolBlocks { this.rustGbtGenerator = new GbtGenerator(); } - private async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { + public async $rustMakeBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, saveResults: boolean = false, useAccelerations: boolean = false, accelerationPool?: number): Promise { + console.log('$rustMakeBlockTemplates'); const start = Date.now(); // reset mempool short ids @@ -569,7 +570,7 @@ class MempoolBlocks { private dataToMempoolBlocks(transactionIds: string[], transactions: MempoolTransactionExtended[], totalSize: number, totalWeight: number, totalFees: number, feeStats?: EffectiveFeeStats ): MempoolBlockWithTransactions { if (!feeStats) { - feeStats = Common.calcEffectiveFeeStatistics(transactions); + feeStats = Common.calcEffectiveFeeStatistics(transactions.filter(tx => !tx.acceleration)); } return { blockSize: totalSize, diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index e42e87acd..9eb624b2e 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -397,7 +397,7 @@ class WebsocketHandler { if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { if (config.MEMPOOL.RUST_GBT) { - await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions, true,); + await mempoolBlocks.$rustUpdateBlockTemplates(newMempool, mempoolSize, newTransactions, deletedTransactions, config.MEMPOOL_SERVICES.ACCELERATIONS); } else { await mempoolBlocks.$updateBlockTemplates(newMempool, newTransactions, deletedTransactions, accelerationDelta, true, config.MEMPOOL_SERVICES.ACCELERATIONS); } @@ -661,7 +661,7 @@ class WebsocketHandler { const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations())); // template calculation functions have mempool side effects, so calculate audits using // a cloned copy of the mempool if we're running a different algorithm for mempool updates - const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL; + const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL || isAccelerated; if (separateAudit) { auditMempool = deepClone(_memPool); if (config.MEMPOOL.ADVANCED_GBT_AUDIT) { @@ -675,7 +675,7 @@ class WebsocketHandler { } } else { if ((config.MEMPOOL_SERVICES.ACCELERATIONS && !isAccelerated)) { - projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated); + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, false); } else { projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); } diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index 9c7568567..c17958d2b 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -85,7 +85,6 @@ class BlocksAuditRepositories { rows[0].sigopTxs = JSON.parse(rows[0].sigopTxs); rows[0].fullrbfTxs = JSON.parse(rows[0].fullrbfTxs); rows[0].acceleratedTxs = JSON.parse(rows[0].acceleratedTxs); - rows[0].transactions = JSON.parse(rows[0].transactions); rows[0].template = JSON.parse(rows[0].template); return rows[0]; diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 1e94b1a9c..1345717bd 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -347,6 +347,9 @@ export class BlockComponent implements OnInit, OnDestroy { if (blockAudit?.template) { for (const tx of blockAudit.template) { inTemplate[tx.txid] = true; + if (tx.acc) { + isAccelerated[tx.txid] = true; + } } for (const tx of transactions) { inBlock[tx.txid] = true; diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index a0a3da5e6..cb42eaed3 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -176,6 +176,7 @@ export interface TransactionStripped { vsize: number; value: number; rate?: number; // effective fee rate + acc?: boolean; status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'censored' | 'selected' | 'rbf' | 'accelerated'; context?: 'projected' | 'actual'; } From 7c641544b2fda1c48fe6d2df43bb343537ef98cc Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 18 Jul 2023 16:32:52 +0900 Subject: [PATCH 018/205] check in missing rust-gbt file --- backend/rust-gbt/src/thread_acceleration.rs | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 backend/rust-gbt/src/thread_acceleration.rs diff --git a/backend/rust-gbt/src/thread_acceleration.rs b/backend/rust-gbt/src/thread_acceleration.rs new file mode 100644 index 000000000..618cac3db --- /dev/null +++ b/backend/rust-gbt/src/thread_acceleration.rs @@ -0,0 +1,8 @@ +use napi_derive::napi; + +#[derive(Debug)] +#[napi(object)] +pub struct ThreadAcceleration { + pub uid: u32, + pub delta: f64, // fee delta +} From 928a8be846779820c4290aabd02dceae65983dd7 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 18 Jul 2023 17:30:51 +0900 Subject: [PATCH 019/205] fix pool-dependent accelerated audit handling --- backend/rust-gbt/src/audit_transaction.rs | 2 +- backend/src/api/mempool-blocks.ts | 13 +++++++------ backend/src/api/websocket-handler.ts | 6 +++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/backend/rust-gbt/src/audit_transaction.rs b/backend/rust-gbt/src/audit_transaction.rs index 9b7472c30..fe20e5a14 100644 --- a/backend/rust-gbt/src/audit_transaction.rs +++ b/backend/rust-gbt/src/audit_transaction.rs @@ -130,7 +130,7 @@ impl AuditTransaction { score: 0.0, used: false, modified: false, - dirty: effective_fee_per_vsize != tx.effective_fee_per_vsize, + dirty: effective_fee_per_vsize != tx.effective_fee_per_vsize || fee_delta > 0.0, } } diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 599a53c23..6d3fa0036 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -388,7 +388,7 @@ class MempoolBlocks { return this.$rustMakeBlockTemplates(newMempool, false, useAccelerations, accelerationPool); } - public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], useAccelerations: boolean, accelerationPool?: number): Promise { + public async $rustUpdateBlockTemplates(newMempool: { [txid: string]: MempoolTransactionExtended }, mempoolSize: number, added: MempoolTransactionExtended[], removed: MempoolTransactionExtended[], useAccelerations: boolean, accelerationPool?: number): Promise { // GBT optimization requires that uids never get too sparse // as a sanity check, we should also explicitly prevent uint32 uid overflow if (this.nextUid + added.length >= Math.min(Math.max(262144, 2 * mempoolSize), MAX_UINT32)) { @@ -397,8 +397,7 @@ class MempoolBlocks { if (!this.rustInitialized) { // need to reset the worker - await this.$rustMakeBlockTemplates(newMempool, true, useAccelerations, accelerationPool); - return; + return this.$rustMakeBlockTemplates(newMempool, true, useAccelerations, accelerationPool); } const start = Date.now(); @@ -435,13 +434,15 @@ class MempoolBlocks { if (mempoolSize !== resultMempoolSize) { throw new Error('GBT returned wrong number of transactions, cache is probably out of sync'); } else { - this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, true); + const processed = this.processBlockTemplates(newMempool, blocks, blockWeights, rates, clusters, accelerations, accelerationPool, true); + this.removeUids(removedUids); + logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); + return processed; } - this.removeUids(removedUids); - logger.debug(`RUST updateBlockTemplates completed in ${(Date.now() - start)/1000} seconds`); } catch (e) { logger.err('RUST updateBlockTemplates failed. ' + (e instanceof Error ? e.message : e)); this.resetRustGbt(); + return this.mempoolBlocks; } } diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 9eb624b2e..bed059ad2 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -661,7 +661,7 @@ class WebsocketHandler { const isAccelerated = config.MEMPOOL_SERVICES.ACCELERATIONS && accelerationApi.isAcceleratedBlock(block, Object.values(mempool.getAccelerations())); // template calculation functions have mempool side effects, so calculate audits using // a cloned copy of the mempool if we're running a different algorithm for mempool updates - const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL || isAccelerated; + const separateAudit = config.MEMPOOL.ADVANCED_GBT_AUDIT !== config.MEMPOOL.ADVANCED_GBT_MEMPOOL; if (separateAudit) { auditMempool = deepClone(_memPool); if (config.MEMPOOL.ADVANCED_GBT_AUDIT) { @@ -674,8 +674,8 @@ class WebsocketHandler { projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false); } } else { - if ((config.MEMPOOL_SERVICES.ACCELERATIONS && !isAccelerated)) { - projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, false); + if ((config.MEMPOOL_SERVICES.ACCELERATIONS)) { + projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(auditMempool, Object.keys(auditMempool).length, [], [], isAccelerated, block.extras.pool.id); } else { projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); } From cde4af59305e9feb99ad061b8645c676720bb156 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 19 Jul 2023 11:18:04 +0900 Subject: [PATCH 020/205] fix mismatched use of gbt implementations --- backend/src/api/mempool-blocks.ts | 12 +++++++++--- backend/src/api/websocket-handler.ts | 6 +++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 6d3fa0036..1ea4d2af8 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -211,9 +211,12 @@ class MempoolBlocks { const start = Date.now(); // reset mempool short ids - this.resetUids(); + if (saveResults) { + this.resetUids(); + } + // set missing short ids for (const tx of Object.values(newMempool)) { - this.setUid(tx, true); + this.setUid(tx, !saveResults); } const accelerations = useAccelerations ? mempool.getAccelerations() : {}; @@ -497,6 +500,8 @@ class MempoolBlocks { } } + const isAccelerated : { [txid: string]: boolean } = {}; + const sizeLimit = (config.MEMPOOL.BLOCK_WEIGHT_UNITS / 4) * 1.2; // update this thread's mempool with the results let mempoolTx: MempoolTransactionExtended; @@ -526,10 +531,11 @@ class MempoolBlocks { } const acceleration = accelerations[txid]; - if (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool))) { + if (isAccelerated[txid] || (acceleration && (!accelerationPool || acceleration.pools.includes(accelerationPool)))) { mempoolTx.acceleration = true; for (const ancestor of mempoolTx.ancestors || []) { mempool[ancestor.txid].acceleration = true; + isAccelerated[ancestor.txid] = true; } } else { delete mempoolTx.acceleration; diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index bed059ad2..f5c940218 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -675,7 +675,11 @@ class WebsocketHandler { } } else { if ((config.MEMPOOL_SERVICES.ACCELERATIONS)) { - projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(auditMempool, Object.keys(auditMempool).length, [], [], isAccelerated, block.extras.pool.id); + if (config.MEMPOOL.RUST_GBT) { + projectedBlocks = await mempoolBlocks.$rustUpdateBlockTemplates(auditMempool, Object.keys(auditMempool).length, [], [], isAccelerated, block.extras.pool.id); + } else { + projectedBlocks = await mempoolBlocks.$makeBlockTemplates(auditMempool, false, isAccelerated, block.extras.pool.id); + } } else { projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); } From 67cff804a6979ed6bf5220bdf829be8e394b82b6 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 22 Jul 2023 09:46:35 +0900 Subject: [PATCH 021/205] /accelerations -> /accelerator/accelerations --- backend/src/api/services/acceleration.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/api/services/acceleration.ts b/backend/src/api/services/acceleration.ts index 1c2fff7c6..635dc8300 100644 --- a/backend/src/api/services/acceleration.ts +++ b/backend/src/api/services/acceleration.ts @@ -11,7 +11,7 @@ export interface Acceleration { class AccelerationApi { public async $fetchAccelerations(): Promise { if (config.MEMPOOL_SERVICES.ACCELERATIONS) { - const response = await query(`${config.MEMPOOL_SERVICES.API}/accelerations`); + const response = await query(`${config.MEMPOOL_SERVICES.API}/accelerator/accelerations`); return (response as Acceleration[]) || []; } else { return []; From a23cd5ad2982fc8e25eb6567207f42569f511086 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Mon, 31 Jul 2023 14:44:59 -0700 Subject: [PATCH 022/205] Update Cypress deps --- frontend/package-lock.json | 34 +++++++++++++++++++--------------- frontend/package.json | 4 ++-- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 561b61096..d82eecd5b 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -58,9 +58,9 @@ }, "optionalDependencies": { "@cypress/schematic": "^2.5.0", - "cypress": "^12.17.1", + "cypress": "^12.17.2", "cypress-fail-on-console-error": "~4.0.3", - "cypress-wait-until": "^1.7.2", + "cypress-wait-until": "^2.0.0", "mock-socket": "~9.2.1", "start-server-and-test": "~2.0.0" } @@ -6641,9 +6641,9 @@ "peer": true }, "node_modules/cypress": { - "version": "12.17.1", - "resolved": "https://registry.npmjs.org/cypress/-/cypress-12.17.1.tgz", - "integrity": "sha512-eKfBgO6t8waEyhegL4gxD7tcI6uTCGttu+ZU7y9Hq8BlpMztd7iLeIF4AJFAnbZH1xjX+wwgg4cRKFNSvv3VWQ==", + "version": "12.17.2", + "resolved": "https://registry.npmjs.org/cypress/-/cypress-12.17.2.tgz", + "integrity": "sha512-hxWAaWbqQBzzMuadSGSuQg5PDvIGOovm6xm0hIfpCVcORsCAj/gF2p0EvfnJ4f+jK2PCiDgP6D2eeE9/FK4Mjg==", "hasInstallScript": true, "optional": true, "dependencies": { @@ -6710,10 +6710,14 @@ } }, "node_modules/cypress-wait-until": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/cypress-wait-until/-/cypress-wait-until-1.7.2.tgz", - "integrity": "sha512-uZ+M8/MqRcpf+FII/UZrU7g1qYZ4aVlHcgyVopnladyoBrpoaMJ4PKZDrdOJ05H5RHbr7s9Tid635X3E+ZLU/Q==", - "optional": true + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/cypress-wait-until/-/cypress-wait-until-2.0.0.tgz", + "integrity": "sha512-ulUZyrWBn+OuC8oiQuGKAScDYfpaWnE3dEE/raUo64w4RHQxZrQ/iMIWT4ZjGMMPr3P+BFEALCRnjQeRqzZj6g==", + "optional": true, + "engines": { + "node": ">=18.16.0", + "npm": ">=9.5.1" + } }, "node_modules/cypress/node_modules/@types/node": { "version": "14.18.53", @@ -20968,9 +20972,9 @@ "peer": true }, "cypress": { - "version": "12.17.1", - "resolved": "https://registry.npmjs.org/cypress/-/cypress-12.17.1.tgz", - "integrity": "sha512-eKfBgO6t8waEyhegL4gxD7tcI6uTCGttu+ZU7y9Hq8BlpMztd7iLeIF4AJFAnbZH1xjX+wwgg4cRKFNSvv3VWQ==", + "version": "12.17.2", + "resolved": "https://registry.npmjs.org/cypress/-/cypress-12.17.2.tgz", + "integrity": "sha512-hxWAaWbqQBzzMuadSGSuQg5PDvIGOovm6xm0hIfpCVcORsCAj/gF2p0EvfnJ4f+jK2PCiDgP6D2eeE9/FK4Mjg==", "optional": true, "requires": { "@cypress/request": "^2.88.11", @@ -21151,9 +21155,9 @@ } }, "cypress-wait-until": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/cypress-wait-until/-/cypress-wait-until-1.7.2.tgz", - "integrity": "sha512-uZ+M8/MqRcpf+FII/UZrU7g1qYZ4aVlHcgyVopnladyoBrpoaMJ4PKZDrdOJ05H5RHbr7s9Tid635X3E+ZLU/Q==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/cypress-wait-until/-/cypress-wait-until-2.0.0.tgz", + "integrity": "sha512-ulUZyrWBn+OuC8oiQuGKAScDYfpaWnE3dEE/raUo64w4RHQxZrQ/iMIWT4ZjGMMPr3P+BFEALCRnjQeRqzZj6g==", "optional": true }, "d": { diff --git a/frontend/package.json b/frontend/package.json index 29f57538e..db6ee0f3a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -110,9 +110,9 @@ }, "optionalDependencies": { "@cypress/schematic": "^2.5.0", - "cypress": "^12.17.1", + "cypress": "^12.17.2", "cypress-fail-on-console-error": "~4.0.3", - "cypress-wait-until": "^1.7.2", + "cypress-wait-until": "^2.0.0", "mock-socket": "~9.2.1", "start-server-and-test": "~2.0.0" }, From ca0c6b5e6e431ebfd27f1c31de61ffcaa0923c42 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 2 Aug 2023 13:24:56 +0900 Subject: [PATCH 023/205] use bulk mempool/txs post api to batch mempool update requests --- .../bitcoin/bitcoin-api-abstract-factory.ts | 3 +- backend/src/api/bitcoin/bitcoin-api.ts | 9 +++- backend/src/api/bitcoin/esplora-api.ts | 25 ++++++++- backend/src/api/mempool.ts | 54 ++++++++++--------- backend/src/api/transaction-utils.ts | 19 +++++++ 5 files changed, 82 insertions(+), 28 deletions(-) diff --git a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts index 7f4a5e53a..f14c5525d 100644 --- a/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts +++ b/backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts @@ -3,7 +3,8 @@ import { IEsploraApi } from './esplora-api.interface'; export interface AbstractBitcoinApi { $getRawMempool(): Promise; $getRawTransaction(txId: string, skipConversion?: boolean, addPrevout?: boolean, lazyPrevouts?: boolean): Promise; - $getMempoolTransactions(lastTxid: string); + $getMempoolTransactions(txids: string[]): Promise; + $getAllMempoolTransactions(lastTxid: string); $getTransactionHex(txId: string): Promise; $getBlockHeightTip(): Promise; $getBlockHashTip(): Promise; diff --git a/backend/src/api/bitcoin/bitcoin-api.ts b/backend/src/api/bitcoin/bitcoin-api.ts index 132cda91a..b315ed0f7 100644 --- a/backend/src/api/bitcoin/bitcoin-api.ts +++ b/backend/src/api/bitcoin/bitcoin-api.ts @@ -60,8 +60,13 @@ class BitcoinApi implements AbstractBitcoinApi { }); } - $getMempoolTransactions(lastTxid: string): Promise { - return Promise.resolve([]); + $getMempoolTransactions(txids: string[]): Promise { + throw new Error('Method getMempoolTransactions not supported by the Bitcoin RPC API.'); + } + + $getAllMempoolTransactions(lastTxid: string): Promise { + throw new Error('Method getAllMempoolTransactions not supported by the Bitcoin RPC API.'); + } async $getTransactionHex(txId: string): Promise { diff --git a/backend/src/api/bitcoin/esplora-api.ts b/backend/src/api/bitcoin/esplora-api.ts index ff10751e0..77c6d80fc 100644 --- a/backend/src/api/bitcoin/esplora-api.ts +++ b/backend/src/api/bitcoin/esplora-api.ts @@ -61,6 +61,25 @@ class ElectrsApi implements AbstractBitcoinApi { }); } + $postWrapper(url, body, responseType = 'json', params: any = undefined): Promise { + return axiosConnection.post(url, body, { ...this.activeAxiosConfig, responseType: responseType, params }) + .then((response) => response.data) + .catch((e) => { + if (e?.code === 'ECONNREFUSED') { + this.fallbackToTcpSocket(); + // Retry immediately + return axiosConnection.post(url, body, this.activeAxiosConfig) + .then((response) => response.data) + .catch((e) => { + logger.warn(`Cannot query esplora through the unix socket nor the tcp socket. Exception ${e}`); + throw e; + }); + } else { + throw e; + } + }); + } + $getRawMempool(): Promise { return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/mempool/txids'); } @@ -69,7 +88,11 @@ class ElectrsApi implements AbstractBitcoinApi { return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/tx/' + txId); } - async $getMempoolTransactions(lastSeenTxid?: string): Promise { + async $getMempoolTransactions(txids: string[]): Promise { + return this.$postWrapper(config.ESPLORA.REST_API_URL + '/mempool/txs', txids, 'json'); + } + + async $getAllMempoolTransactions(lastSeenTxid?: string): Promise { return this.$queryWrapper(config.ESPLORA.REST_API_URL + '/mempool/txs' + (lastSeenTxid ? '/' + lastSeenTxid : '')); } diff --git a/backend/src/api/mempool.ts b/backend/src/api/mempool.ts index 85b6e6101..ccdd307f8 100644 --- a/backend/src/api/mempool.ts +++ b/backend/src/api/mempool.ts @@ -123,7 +123,7 @@ class Mempool { loadingIndicators.setProgress('mempool', count / expectedCount * 100); while (!done) { try { - const result = await bitcoinApi.$getMempoolTransactions(last_txid); + const result = await bitcoinApi.$getAllMempoolTransactions(last_txid); if (result) { for (const tx of result) { const extendedTransaction = transactionUtils.extendMempoolTransaction(tx); @@ -231,31 +231,37 @@ class Mempool { } if (!loaded) { - for (const txid of transactions) { - if (!this.mempoolCache[txid]) { - try { - const transaction = await transactionUtils.$getMempoolTransactionExtended(txid, false, false, false); - this.updateTimerProgress(timer, 'fetched new transaction'); - this.mempoolCache[txid] = transaction; - if (this.inSync) { - this.txPerSecondArray.push(new Date().getTime()); - this.vBytesPerSecondArray.push({ - unixTime: new Date().getTime(), - vSize: transaction.vsize, - }); - } - hasChange = true; - newTransactions.push(transaction); + const remainingTxids = transactions.filter(txid => !this.mempoolCache[txid]); + const sliceLength = 10000; + for (let i = 0; i < Math.ceil(remainingTxids.length / sliceLength); i++) { + const slice = remainingTxids.slice(i * sliceLength, (i + 1) * sliceLength); + const txs = await transactionUtils.$getMempoolTransactionsExtended(slice, false, false, false); + logger.debug(`fetched ${txs.length} transactions`); + this.updateTimerProgress(timer, 'fetched new transactions'); - if (config.REDIS.ENABLED) { - await redisCache.$addTransaction(transaction); - } - } catch (e: any) { - if (config.MEMPOOL.BACKEND === 'esplora' && e.response?.status === 404) { - this.missingTxCount++; - } - logger.debug(`Error finding transaction '${txid}' in the mempool: ` + (e instanceof Error ? e.message : e)); + for (const transaction of txs) { + this.mempoolCache[transaction.txid] = transaction; + if (this.inSync) { + this.txPerSecondArray.push(new Date().getTime()); + this.vBytesPerSecondArray.push({ + unixTime: new Date().getTime(), + vSize: transaction.vsize, + }); } + hasChange = true; + newTransactions.push(transaction); + + if (config.REDIS.ENABLED) { + await redisCache.$addTransaction(transaction); + } + } + + if (txs.length < slice.length) { + const missing = slice.length - txs.length; + if (config.MEMPOOL.BACKEND === 'esplora') { + this.missingTxCount += missing; + } + logger.debug(`Error finding ${missing} transactions in the mempool: `); } if (Date.now() - intervalTimer > Math.max(pollRate * 2, 5_000)) { diff --git a/backend/src/api/transaction-utils.ts b/backend/src/api/transaction-utils.ts index e141a6076..02ee7c055 100644 --- a/backend/src/api/transaction-utils.ts +++ b/backend/src/api/transaction-utils.ts @@ -4,6 +4,7 @@ import { Common } from './common'; import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory'; import * as bitcoinjs from 'bitcoinjs-lib'; import logger from '../logger'; +import config from '../config'; class TransactionUtils { constructor() { } @@ -71,6 +72,24 @@ class TransactionUtils { return (await this.$getTransactionExtended(txId, addPrevouts, lazyPrevouts, forceCore, true)) as MempoolTransactionExtended; } + public async $getMempoolTransactionsExtended(txids: string[], addPrevouts = false, lazyPrevouts = false, forceCore = false): Promise { + if (forceCore || config.MEMPOOL.BACKEND !== 'esplora') { + const results = await Promise.allSettled(txids.map(txid => this.$getTransactionExtended(txid, addPrevouts, lazyPrevouts, forceCore, true))); + return (results.filter(r => r.status === 'fulfilled') as PromiseFulfilledResult[]).map(r => r.value); + } else { + const transactions = await bitcoinApi.$getMempoolTransactions(txids); + return transactions.map(transaction => { + if (Common.isLiquid()) { + if (!isFinite(Number(transaction.fee))) { + transaction.fee = Object.values(transaction.fee || {}).reduce((total, output) => total + output, 0); + } + } + + return this.extendMempoolTransaction(transaction); + }); + } + } + public extendTransaction(transaction: IEsploraApi.Transaction): TransactionExtended { // @ts-ignore if (transaction.vsize) { From d236d897178fcaf06270eff1de1a343d24229f30 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Mon, 31 Jul 2023 11:06:52 +0900 Subject: [PATCH 024/205] [mining] send pool unique id in /pools API --- backend/src/api/mining/mining.ts | 1 + backend/src/mempool.interfaces.ts | 1 + backend/src/repositories/PoolsRepository.ts | 3 ++- frontend/src/app/interfaces/node-api.interface.ts | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/src/api/mining/mining.ts b/backend/src/api/mining/mining.ts index e190492b8..79b255e7f 100644 --- a/backend/src/api/mining/mining.ts +++ b/backend/src/api/mining/mining.ts @@ -107,6 +107,7 @@ class Mining { slug: poolInfo.slug, avgMatchRate: poolInfo.avgMatchRate !== null ? Math.round(100 * poolInfo.avgMatchRate) / 100 : null, avgFeeDelta: poolInfo.avgFeeDelta, + poolUniqueId: poolInfo.poolUniqueId }; poolsStats.push(poolStat); }); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index 4715440e4..c08846191 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -20,6 +20,7 @@ export interface PoolInfo { slug: string; avgMatchRate: number | null; avgFeeDelta: number | null; + poolUniqueId: number; } export interface PoolStats extends PoolInfo { diff --git a/backend/src/repositories/PoolsRepository.ts b/backend/src/repositories/PoolsRepository.ts index 899712266..eda792bb3 100644 --- a/backend/src/repositories/PoolsRepository.ts +++ b/backend/src/repositories/PoolsRepository.ts @@ -40,7 +40,8 @@ class PoolsRepository { pools.link AS link, slug, AVG(blocks_audits.match_rate) AS avgMatchRate, - AVG((CAST(blocks.fees as SIGNED) - CAST(blocks_audits.expected_fees as SIGNED)) / NULLIF(CAST(blocks_audits.expected_fees as SIGNED), 0)) AS avgFeeDelta + AVG((CAST(blocks.fees as SIGNED) - CAST(blocks_audits.expected_fees as SIGNED)) / NULLIF(CAST(blocks_audits.expected_fees as SIGNED), 0)) AS avgFeeDelta, + unique_id as poolUniqueId FROM blocks JOIN pools on pools.id = pool_id LEFT JOIN blocks_audits ON blocks_audits.height = blocks.height diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index cb42eaed3..5316b1cdc 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -111,6 +111,7 @@ export interface PoolInfo { regexes: string; // JSON array addresses: string; // JSON array emptyBlocks: number; + poolUniqueId: number; } export interface PoolStat { pool: PoolInfo; From f2ae8580971abb8d62054b1c690a55408ab4fbe0 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Fri, 17 Feb 2023 18:54:48 +0900 Subject: [PATCH 025/205] [sponsors] show profile by tiers on about page --- .../app/components/about/about.component.html | 49 +++++++++++++++---- .../app/components/about/about.component.scss | 3 ++ .../app/components/about/about.component.ts | 24 ++++++--- frontend/src/app/services/api.service.ts | 14 ++---- frontend/src/resources/profile/unknown.svg | 1 + 5 files changed, 62 insertions(+), 29 deletions(-) create mode 100644 frontend/src/resources/profile/unknown.svg diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index 381353948..55f406d00 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -31,6 +31,12 @@ +

Support the Project

+ + +

Enterprise Sponsors 🚀

@@ -191,16 +197,39 @@
-
-

Community Sponsors ❤️

+
+
+

Whale Sponsors

+
+ + + + + + + +
+
- + +
+

OG Sponsors ❤️

+
+ + + +
@@ -340,7 +369,7 @@ @@ -354,7 +383,7 @@
- + {{ contributor.name }} @@ -366,7 +395,7 @@
- + {{ contributor.name }} diff --git a/frontend/src/app/components/about/about.component.scss b/frontend/src/app/components/about/about.component.scss index e1d6c829a..2a5710ca1 100644 --- a/frontend/src/app/components/about/about.component.scss +++ b/frontend/src/app/components/about/about.component.scss @@ -10,6 +10,9 @@ margin: 25px; line-height: 32px; } + .unknown { + border: 1px solid #b4b4b4; + } .image.not-rounded { border-radius: 0; diff --git a/frontend/src/app/components/about/about.component.ts b/frontend/src/app/components/about/about.component.ts index 176490add..4bf7869de 100644 --- a/frontend/src/app/components/about/about.component.ts +++ b/frontend/src/app/components/about/about.component.ts @@ -6,7 +6,7 @@ import { Observable } from 'rxjs'; import { ApiService } from '../../services/api.service'; import { IBackendInfo } from '../../interfaces/websocket.interface'; import { Router, ActivatedRoute } from '@angular/router'; -import { map, tap } from 'rxjs/operators'; +import { map, share, tap } from 'rxjs/operators'; import { ITranslators } from '../../interfaces/node-api.interface'; import { DOCUMENT } from '@angular/common'; @@ -19,14 +19,16 @@ import { DOCUMENT } from '@angular/common'; export class AboutComponent implements OnInit { @ViewChild('promoVideo') promoVideo: ElementRef; backendInfo$: Observable; - sponsors$: Observable; - translators$: Observable; - allContributors$: Observable; frontendGitCommitHash = this.stateService.env.GIT_COMMIT_HASH; packetJsonVersion = this.stateService.env.PACKAGE_JSON_VERSION; officialMempoolSpace = this.stateService.env.OFFICIAL_MEMPOOL_SPACE; showNavigateToSponsor = false; + profiles$: Observable; + translators$: Observable; + allContributors$: Observable; + ogs$: Observable; + constructor( private websocketService: WebsocketService, private seoService: SeoService, @@ -43,10 +45,13 @@ export class AboutComponent implements OnInit { this.seoService.setTitle($localize`:@@004b222ff9ef9dd4771b777950ca1d0e4cd4348a:About`); this.websocketService.want(['blocks']); - this.sponsors$ = this.apiService.getDonation$() - .pipe( - tap(() => this.goToAnchor()) - ); + this.profiles$ = this.apiService.getAboutPageProfiles$().pipe( + tap(() => { + this.goToAnchor() + }), + share(), + ) + this.translators$ = this.apiService.getTranslators$() .pipe( map((translators) => { @@ -59,6 +64,9 @@ export class AboutComponent implements OnInit { }), tap(() => this.goToAnchor()) ); + + this.ogs$ = this.apiService.getOgs$(); + this.allContributors$ = this.apiService.getContributor$().pipe( map((contributors) => { return { diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index e2d3be9be..24dbe734a 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -92,15 +92,11 @@ export class ApiService { return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/outspends', { params }); } - requestDonation$(amount: number, orderId: string): Observable { - const params = { - amount: amount, - orderId: orderId, - }; - return this.httpClient.post(this.apiBaseUrl + '/api/v1/donations', params); + getAboutPageProfiles$(): Observable { + return this.httpClient.get(this.apiBaseUrl + '/api/v1/about-page'); } - getDonation$(): Observable { + getOgs$(): Observable { return this.httpClient.get(this.apiBaseUrl + '/api/v1/donations'); } @@ -112,10 +108,6 @@ export class ApiService { return this.httpClient.get(this.apiBaseUrl + '/api/v1/contributors'); } - checkDonation$(orderId: string): Observable { - return this.httpClient.get(this.apiBaseUrl + '/api/v1/donations/check?order_id=' + orderId); - } - getInitData$(): Observable { return this.httpClient.get(this.apiBaseUrl + this.apiBasePath + '/api/v1/init-data'); } diff --git a/frontend/src/resources/profile/unknown.svg b/frontend/src/resources/profile/unknown.svg new file mode 100644 index 000000000..50a548c3c --- /dev/null +++ b/frontend/src/resources/profile/unknown.svg @@ -0,0 +1 @@ + \ No newline at end of file From 6c2c62ba2eb7d50f499cf71e457104adfccb0672 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Sat, 1 Apr 2023 11:40:04 +0900 Subject: [PATCH 026/205] [lightning] claim your node button --- .../src/app/lightning/lightning.module.ts | 3 +++ .../node-owner/node-owner.component.html | 17 ++++++++++++ .../node-owner/node-owner.component.scss | 4 +++ .../node-owner/node-owner.component.ts | 20 ++++++++++++++ .../app/lightning/node/node.component.html | 8 ++++-- .../app/lightning/node/node.component.scss | 14 ++++++++++ .../src/app/lightning/node/node.component.ts | 26 ++++++++++++++++--- frontend/src/app/services/api.service.ts | 11 ++++++++ 8 files changed, 98 insertions(+), 5 deletions(-) create mode 100644 frontend/src/app/lightning/node-owner/node-owner.component.html create mode 100644 frontend/src/app/lightning/node-owner/node-owner.component.scss create mode 100644 frontend/src/app/lightning/node-owner/node-owner.component.ts diff --git a/frontend/src/app/lightning/lightning.module.ts b/frontend/src/app/lightning/lightning.module.ts index 0b824ad78..f0154a15f 100644 --- a/frontend/src/app/lightning/lightning.module.ts +++ b/frontend/src/app/lightning/lightning.module.ts @@ -34,6 +34,7 @@ import { OldestNodes } from '../lightning/nodes-ranking/oldest-nodes/oldest-node import { NodesRankingsDashboard } from '../lightning/nodes-rankings-dashboard/nodes-rankings-dashboard.component'; import { NodeChannels } from '../lightning/nodes-channels/node-channels.component'; import { GroupComponent } from './group/group.component'; +import { NodeOwnerComponent } from './node-owner/node-owner.component'; @NgModule({ declarations: [ @@ -66,6 +67,7 @@ import { GroupComponent } from './group/group.component'; NodesRankingsDashboard, NodeChannels, GroupComponent, + NodeOwnerComponent, ], imports: [ CommonModule, @@ -103,6 +105,7 @@ import { GroupComponent } from './group/group.component'; OldestNodes, NodesRankingsDashboard, NodeChannels, + NodeOwnerComponent, ], providers: [ LightningApiService, diff --git a/frontend/src/app/lightning/node-owner/node-owner.component.html b/frontend/src/app/lightning/node-owner/node-owner.component.html new file mode 100644 index 000000000..e37b1e027 --- /dev/null +++ b/frontend/src/app/lightning/node-owner/node-owner.component.html @@ -0,0 +1,17 @@ +
+ +
+ +
+ + + +
+ +
+ Claim +
+ +
+ +
\ No newline at end of file diff --git a/frontend/src/app/lightning/node-owner/node-owner.component.scss b/frontend/src/app/lightning/node-owner/node-owner.component.scss new file mode 100644 index 000000000..6734168cf --- /dev/null +++ b/frontend/src/app/lightning/node-owner/node-owner.component.scss @@ -0,0 +1,4 @@ +.profile-photo { + width: 31px; + height: 31px; +} \ No newline at end of file diff --git a/frontend/src/app/lightning/node-owner/node-owner.component.ts b/frontend/src/app/lightning/node-owner/node-owner.component.ts new file mode 100644 index 000000000..a03c04901 --- /dev/null +++ b/frontend/src/app/lightning/node-owner/node-owner.component.ts @@ -0,0 +1,20 @@ +import { ChangeDetectionStrategy, Component, Input } from '@angular/core'; +import { Observable } from 'rxjs'; +import { StateService } from '../../services/state.service'; + +@Component({ + selector: 'app-node-owner', + templateUrl: './node-owner.component.html', + styleUrls: ['./node-owner.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class NodeOwnerComponent{ + @Input() publicKey: string = ''; + @Input() alias: string = ''; + @Input() nodeOwner$: Observable; + + constructor( + public stateService: StateService + ) { + } +} diff --git a/frontend/src/app/lightning/node/node.component.html b/frontend/src/app/lightning/node/node.component.html index c6c693a3a..11ddbc0eb 100644 --- a/frontend/src/app/lightning/node/node.component.html +++ b/frontend/src/app/lightning/node/node.component.html @@ -3,13 +3,17 @@
Lightning node
-

{{ node.alias }}

- +
+

{{ node.alias }}

+ +
+ +
diff --git a/frontend/src/app/lightning/node/node.component.scss b/frontend/src/app/lightning/node/node.component.scss index 272de4b09..117fc8a2c 100644 --- a/frontend/src/app/lightning/node/node.component.scss +++ b/frontend/src/app/lightning/node/node.component.scss @@ -111,3 +111,17 @@ app-fiat { margin: 0 0.25em; color: slategrey; } + +.claim-btn { + max-height: 32px; + @media (min-width: 850px) { + display: none; + } +} + +.claim-btn-mobile { + max-height: 32px; + @media (max-width: 850px) { + display: none; + } +} \ No newline at end of file diff --git a/frontend/src/app/lightning/node/node.component.ts b/frontend/src/app/lightning/node/node.component.ts index 719136d79..6447eb6bd 100644 --- a/frontend/src/app/lightning/node/node.component.ts +++ b/frontend/src/app/lightning/node/node.component.ts @@ -1,7 +1,7 @@ -import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core'; +import { ChangeDetectionStrategy, Component, OnInit, ChangeDetectorRef } from '@angular/core'; import { ActivatedRoute, ParamMap } from '@angular/router'; -import { Observable } from 'rxjs'; -import { catchError, map, switchMap, tap } from 'rxjs/operators'; +import { Observable, of, EMPTY } from 'rxjs'; +import { catchError, map, switchMap, tap, share } from 'rxjs/operators'; import { SeoService } from '../../services/seo.service'; import { ApiService } from '../../services/api.service'; import { LightningApiService } from '../lightning-api.service'; @@ -38,6 +38,7 @@ export class NodeComponent implements OnInit { tlvRecords: CustomRecord[]; avgChannelDistance$: Observable; showFeatures = false; + nodeOwner$: Observable; kmToMiles = kmToMiles; constructor( @@ -45,6 +46,7 @@ export class NodeComponent implements OnInit { private lightningApiService: LightningApiService, private activatedRoute: ActivatedRoute, private seoService: SeoService, + private cd: ChangeDetectorRef, ) { } ngOnInit(): void { @@ -147,6 +149,24 @@ export class NodeComponent implements OnInit { return null; }) ) as Observable; + + this.nodeOwner$ = this.activatedRoute.paramMap + .pipe( + switchMap((params: ParamMap) => { + return this.apiService.getNodeOwner$(params.get('public_key')).pipe( + switchMap((response) => { + if (response.status === 204) { + return of(false); + } + return of(response.body); + }), + catchError(() => { + return of(false); + }) + ) + }), + share(), + ); } toggleShowDetails(): void { diff --git a/frontend/src/app/services/api.service.ts b/frontend/src/app/services/api.service.ts index 24dbe734a..1ed9d2f5c 100644 --- a/frontend/src/app/services/api.service.ts +++ b/frontend/src/app/services/api.service.ts @@ -8,6 +8,8 @@ import { WebsocketResponse } from '../interfaces/websocket.interface'; import { Outspend, Transaction } from '../interfaces/electrs.interface'; import { Conversion } from './price.service'; +const SERVICES_API_PREFIX = `/api/v1/services`; + @Injectable({ providedIn: 'root' }) @@ -315,4 +317,13 @@ export class ApiService { (timestamp ? `?timestamp=${timestamp}` : '') ); } + + /** + * Services + */ + getNodeOwner$(publicKey: string) { + let params = new HttpParams() + .set('node_public_key', publicKey); + return this.httpClient.get(`${SERVICES_API_PREFIX}/lightning/claim/current`, { params, observe: 'response' }); + } } From 727d170c9c8df42919800e35ead381e428bf6b66 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Wed, 3 May 2023 16:32:00 +0200 Subject: [PATCH 027/205] [tx] add accel shortcut into transaction component --- .../transaction/transaction.component.html | 10 ++++++++-- .../transaction/transaction.component.scss | 19 +++++++++++++++++++ .../transaction/transaction.component.ts | 2 +- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index 7422c6894..59404bb77 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -99,14 +99,20 @@ - In several hours (or more) + + In several hours (or more) + Accelerate + - + + + Accelerate + diff --git a/frontend/src/app/components/transaction/transaction.component.scss b/frontend/src/app/components/transaction/transaction.component.scss index bea8e82bc..5bef401d7 100644 --- a/frontend/src/app/components/transaction/transaction.component.scss +++ b/frontend/src/app/components/transaction/transaction.component.scss @@ -216,4 +216,23 @@ .alert-link { display: block; } +} + +.eta { + display: flex; + justify-content: end; + flex-wrap: wrap; + align-content: center; + @media (min-width: 850px) { + justify-content: space-between; + } +} + +.accelerate { + align-self: auto; + margin-top: 3px; + @media (min-width: 850px) { + justify-self: start; + margin-left: 0px; + } } \ No newline at end of file diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index f1d218a79..26e39515e 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -97,7 +97,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { private router: Router, private relativeUrlPipe: RelativeUrlPipe, private electrsApiService: ElectrsApiService, - private stateService: StateService, + public stateService: StateService, private cacheService: CacheService, private websocketService: WebsocketService, private audioService: AudioService, From 5846862d550ec0a9d07c254ec13b98db6fb4f4b5 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 11 May 2023 18:00:52 +0200 Subject: [PATCH 028/205] [footer] dynamic CTA button based on login status --- .../app/components/about/about.component.html | 2 +- .../global-footer.component.html | 8 ++++-- .../global-footer/global-footer.component.ts | 26 +++++++++++++++++-- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index 55f406d00..b9309c916 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -34,7 +34,7 @@

Support the Project

diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index 8e5279a94..30727e662 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -14,8 +14,12 @@
- -

Sign In

+
+
Logged in as {{ username}}
+ + +
+

Sign In

Broadcast Transaction

Connect to our Nodes

diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.ts b/frontend/src/app/shared/components/global-footer/global-footer.component.ts index d4f1946ea..cdb0f1159 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.ts +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.ts @@ -1,10 +1,13 @@ -import { ChangeDetectionStrategy, Component, OnInit, Inject, LOCALE_ID } from '@angular/core'; -import { Observable, merge, of, Subject } from 'rxjs'; +import { ChangeDetectionStrategy, ChangeDetectorRef, Component, OnInit, Inject, LOCALE_ID } from '@angular/core'; +import { ActivatedRoute } from '@angular/router'; +import { Observable, merge, of, Subject, Subscription } from 'rxjs'; import { tap, takeUntil } from 'rxjs/operators'; import { Env, StateService } from '../../../services/state.service'; import { IBackendInfo } from '../../../interfaces/websocket.interface'; import { LanguageService } from '../../../services/language.service'; import { NavigationService } from '../../../services/navigation.service'; +import { StorageService } from '../../../services/storage.service'; +import { WebsocketService } from '../../../services/websocket.service'; @Component({ selector: 'app-global-footer', @@ -23,12 +26,19 @@ export class GlobalFooterComponent implements OnInit { network$: Observable; networkPaths: { [network: string]: string }; currentNetwork = ''; + loggedIn = false; + username = null; + urlSubscription: Subscription; constructor( public stateService: StateService, private languageService: LanguageService, private navigationService: NavigationService, @Inject(LOCALE_ID) public locale: string, + private storageService: StorageService, + private route: ActivatedRoute, + private cd: ChangeDetectorRef, + private websocketService: WebsocketService ) {} ngOnInit(): void { @@ -46,11 +56,23 @@ export class GlobalFooterComponent implements OnInit { this.network$.pipe(takeUntil(this.destroy$)).subscribe((network) => { this.currentNetwork = network; }); + + this.urlSubscription = this.route.url.subscribe((url) => { + this.loggedIn = JSON.parse(this.storageService.getValue('auth')) !== null; + const auth = JSON.parse(this.storageService.getValue('auth')); + if (auth?.user?.username) { + this.username = auth.user.username; + } else { + this.username = null; + } + this.cd.markForCheck(); + }) } ngOnDestroy(): void { this.destroy$.next(true); this.destroy$.complete(); + this.urlSubscription.unsubscribe(); } networkLink(network) { From 1458e89f3a67c3d9885f6accd69a96eff3a447c0 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Wed, 24 May 2023 21:49:08 -0700 Subject: [PATCH 029/205] [footer] main cta points to /accelerator --- .../src/app/components/about/about.component.html | 5 ++--- .../global-footer/global-footer.component.html | 14 ++++++-------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index b9309c916..93644a201 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -31,10 +31,9 @@ -

Support the Project

- +

Sponsor the project

diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index 30727e662..658e0ac72 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -13,14 +13,12 @@
- -
-
Logged in as {{ username}}
- - -
-

Sign In

-
+
+
Logged in as {{ username}}
+ + +
+

Sign In

Broadcast Transaction

Connect to our Nodes

About The Mempool Open Source Project™

From 7ec5d8265fcea3c0ae5f3e3a01bdafc9ab31e2a8 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 3 Aug 2023 14:29:10 +0900 Subject: [PATCH 030/205] toggle header visibility in master page component --- .../master-page/master-page.component.html | 2 +- .../master-page/master-page.component.ts | 15 +++++++++++---- frontend/src/app/shared/shared.module.ts | 1 + 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/frontend/src/app/components/master-page/master-page.component.html b/frontend/src/app/components/master-page/master-page.component.html index 79daf7de6..f12fbc960 100644 --- a/frontend/src/app/components/master-page/master-page.component.html +++ b/frontend/src/app/components/master-page/master-page.component.html @@ -1,5 +1,5 @@ -
+
-
-
-

Whale Sponsors

-
- - + +
+
+

Whale Sponsors

+
+ + + + + + + +
+
+ +
+

Chad Sponsors

+
+ - +
- -
-

Chad Sponsors

-
- - - - - -
-
-
+

OG Sponsors ❤️

diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index 59404bb77..9584cecfd 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -101,7 +101,7 @@ In several hours (or more) - Accelerate + Accelerate @@ -111,7 +111,7 @@ - Accelerate + Accelerate diff --git a/frontend/src/app/services/state.service.ts b/frontend/src/app/services/state.service.ts index 9ab8a7e93..675cf88d1 100644 --- a/frontend/src/app/services/state.service.ts +++ b/frontend/src/app/services/state.service.ts @@ -47,6 +47,7 @@ export interface Env { TESTNET_BLOCK_AUDIT_START_HEIGHT: number; SIGNET_BLOCK_AUDIT_START_HEIGHT: number; HISTORICAL_PRICE: boolean; + ACCELERATOR: boolean; } const defaultEnv: Env = { @@ -77,6 +78,7 @@ const defaultEnv: Env = { 'TESTNET_BLOCK_AUDIT_START_HEIGHT': 0, 'SIGNET_BLOCK_AUDIT_START_HEIGHT': 0, 'HISTORICAL_PRICE': true, + 'ACCELERATOR': false, }; @Injectable({ diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index 658e0ac72..c4436c4b6 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -13,7 +13,7 @@
-
+
Logged in as {{ username}}
From d06fe83bd996a77fc5d476b11bb6025ab47c423f Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 30 Jul 2023 17:59:35 +0900 Subject: [PATCH 032/205] Improve difficulty tooltip display on mobile --- .../difficulty-tooltip.component.html | 54 ++++++++++++------- .../difficulty-tooltip.component.ts | 12 ++++- .../difficulty/difficulty.component.ts | 1 + 3 files changed, 47 insertions(+), 20 deletions(-) diff --git a/frontend/src/app/components/difficulty/difficulty-tooltip.component.html b/frontend/src/app/components/difficulty/difficulty-tooltip.component.html index d06bb5e91..7e4b421e1 100644 --- a/frontend/src/app/components/difficulty/difficulty-tooltip.component.html +++ b/frontend/src/app/components/difficulty/difficulty-tooltip.component.html @@ -4,38 +4,56 @@ class="difficulty-tooltip" [style.visibility]="status ? 'visible' : 'hidden'" [style.left]="tooltipPosition.x + 'px'" - [style.top]="tooltipPosition.y + 'px'" + [style.top]="tooltipPosition.y + (isMobile ? -60 : 0) + 'px'" > - + - - {{ i }} blocks expected - {{ i }} block expected + - - {{ i }} blocks mined - {{ i }} block mined + - - {{ i }} blocks remaining - {{ i }} block remaining + - - {{ i }} blocks ahead - {{ i }} block ahead + - - {{ i }} blocks behind - {{ i }} block behind + Next Block -
\ No newline at end of file + + + + + + + +
+ +
+ + + + + + +
+
+ +{{ i }} blocks expected +{{ i }} block expected +{{ i }} blocks mined +{{ i }} block mined +{{ i }} blocks remaining +{{ i }} block remaining +{{ i }} blocks ahead +{{ i }} block ahead +{{ i }} blocks behind +{{ i }} block behind \ No newline at end of file diff --git a/frontend/src/app/components/difficulty/difficulty-tooltip.component.ts b/frontend/src/app/components/difficulty/difficulty-tooltip.component.ts index c7d26f61a..42d2a61b4 100644 --- a/frontend/src/app/components/difficulty/difficulty-tooltip.component.ts +++ b/frontend/src/app/components/difficulty/difficulty-tooltip.component.ts @@ -1,4 +1,4 @@ -import { Component, ElementRef, ViewChild, Input, OnChanges } from '@angular/core'; +import { Component, ElementRef, ViewChild, Input, OnChanges, HostListener } from '@angular/core'; interface EpochProgress { base: string; @@ -35,12 +35,15 @@ export class DifficultyTooltipComponent implements OnChanges { remaining: number; isAhead: boolean; isBehind: boolean; + isMobile: boolean; tooltipPosition = { x: 0, y: 0 }; @ViewChild('tooltip') tooltipElement: ElementRef; - constructor() {} + constructor() { + this.onResize(); + } ngOnChanges(changes): void { if (changes.cursorPosition && changes.cursorPosition.currentValue) { @@ -63,4 +66,9 @@ export class DifficultyTooltipComponent implements OnChanges { this.isBehind = this.behind > 0; } } + + @HostListener('window:resize', ['$event']) + onResize(): void { + this.isMobile = window.innerWidth <= 767.98; + } } diff --git a/frontend/src/app/components/difficulty/difficulty.component.ts b/frontend/src/app/components/difficulty/difficulty.component.ts index a2c03dc56..093dde79a 100644 --- a/frontend/src/app/components/difficulty/difficulty.component.ts +++ b/frontend/src/app/components/difficulty/difficulty.component.ts @@ -193,6 +193,7 @@ export class DifficultyComponent implements OnInit { @HostListener('pointerdown', ['$event']) onPointerDown(event) { this.onPointerMove(event); + event.preventDefault(); } @HostListener('pointermove', ['$event']) From 3d6a8a501da6b192634b00786eb1ed3ed229cc49 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Thu, 3 Aug 2023 15:49:15 +0900 Subject: [PATCH 033/205] limit mouse events to difficulty bar --- .../difficulty/difficulty.component.html | 4 +-- .../difficulty/difficulty.component.ts | 28 +++++++++++-------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/frontend/src/app/components/difficulty/difficulty.component.html b/frontend/src/app/components/difficulty/difficulty.component.html index 27cddc043..f08ea06f5 100644 --- a/frontend/src/app/components/difficulty/difficulty.component.html +++ b/frontend/src/app/components/difficulty/difficulty.component.html @@ -4,7 +4,7 @@
- + @@ -22,7 +22,7 @@ class="rect {{rect.status}}" [class.hover]="hoverSection && rect.status === hoverSection.status" (pointerover)="onHover($event, rect);" - (pointerout)="onBlur($event);" + (pointerout)="onBlur();" > ; isLoadingWebSocket$: Observable; difficultyEpoch$: Observable; @@ -191,22 +193,26 @@ export class DifficultyComponent implements OnInit { } @HostListener('pointerdown', ['$event']) - onPointerDown(event) { - this.onPointerMove(event); - event.preventDefault(); + onPointerDown(event): void { + if (this.epochSvgElement.nativeElement?.contains(event.target)) { + this.onPointerMove(event); + event.preventDefault(); + } } @HostListener('pointermove', ['$event']) - onPointerMove(event) { - this.tooltipPosition = { x: event.clientX, y: event.clientY }; - this.cd.markForCheck(); + onPointerMove(event): void { + if (this.epochSvgElement.nativeElement?.contains(event.target)) { + this.tooltipPosition = { x: event.clientX, y: event.clientY }; + this.cd.markForCheck(); + } } - onHover(event, rect): void { + onHover(_, rect): void { this.hoverSection = rect; } - onBlur(event): void { + onBlur(): void { this.hoverSection = null; } } From 4ee703325af4c9cc54f14514b5071d428dd7ea21 Mon Sep 17 00:00:00 2001 From: nymkappa <1612910616@pm.me> Date: Thu, 3 Aug 2023 16:12:59 +0900 Subject: [PATCH 034/205] [footer] polish cta using update footer with logo --- .../global-footer/global-footer.component.html | 13 +++++-------- .../global-footer/global-footer.component.scss | 8 ++------ 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/frontend/src/app/shared/components/global-footer/global-footer.component.html b/frontend/src/app/shared/components/global-footer/global-footer.component.html index 7dc98ca27..d4f303221 100644 --- a/frontend/src/app/shared/components/global-footer/global-footer.component.html +++ b/frontend/src/app/shared/components/global-footer/global-footer.component.html @@ -1,7 +1,7 @@
-
+