From 4154d3081ddc40bfc972bb183f7913008688d0ea Mon Sep 17 00:00:00 2001 From: softsimon Date: Thu, 29 Sep 2022 13:45:03 +0400 Subject: [PATCH 01/32] Handle network url ending matching better --- frontend/src/app/services/state.service.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/services/state.service.ts b/frontend/src/app/services/state.service.ts index b0e018941..7b513ee54 100644 --- a/frontend/src/app/services/state.service.ts +++ b/frontend/src/app/services/state.service.ts @@ -158,7 +158,8 @@ export class StateService { // (?:[a-z]{2}(?:-[A-Z]{2})?\/)? optional locale prefix (non-capturing) // (?:preview\/)? optional "preview" prefix (non-capturing) // (bisq|testnet|liquidtestnet|liquid|signet)/ network string (captured as networkMatches[1]) - const networkMatches = url.match(/^\/(?:[a-z]{2}(?:-[A-Z]{2})?\/)?(?:preview\/)?(bisq|testnet|liquidtestnet|liquid|signet)/); + // ($|\/) network string must end or end with a slash + const networkMatches = url.match(/^\/(?:[a-z]{2}(?:-[A-Z]{2})?\/)?(?:preview\/)?(bisq|testnet|liquidtestnet|liquid|signet)($|\/)/); switch (networkMatches && networkMatches[1]) { case 'liquid': if (this.network !== 'liquid') { From 0a4c1c24afa049f52edc2412a88fb627604b0cdd Mon Sep 17 00:00:00 2001 From: nymkappa Date: Fri, 30 Sep 2022 19:10:11 +0200 Subject: [PATCH 02/32] Fixes #2592 --- backend/src/api/lightning/clightning/clightning-convert.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/src/api/lightning/clightning/clightning-convert.ts b/backend/src/api/lightning/clightning/clightning-convert.ts index 121fb20ea..9b3c62f04 100644 --- a/backend/src/api/lightning/clightning/clightning-convert.ts +++ b/backend/src/api/lightning/clightning/clightning-convert.ts @@ -70,6 +70,8 @@ export async function convertAndmergeBidirectionalChannels(clChannels: any[]): P logger.info(`Building partial channels from clightning output. Channels processed: ${channelProcessed + 1} of ${keys.length}`); loggerTimer = new Date().getTime() / 1000; } + + channelProcessed++; } return consolidatedChannelList; From 39718147103b5fde0f46875fb77d5d34404e60a1 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 11 Oct 2022 17:01:23 +0000 Subject: [PATCH 03/32] Save flow diagram preference to localStorage --- .../transaction/transaction.component.html | 6 ++-- .../transaction/transaction.component.ts | 30 +++++++++++++++---- frontend/src/app/services/state.service.ts | 11 +++++++ frontend/src/app/services/storage.service.ts | 8 +++++ 4 files changed, 46 insertions(+), 9 deletions(-) diff --git a/frontend/src/app/components/transaction/transaction.component.html b/frontend/src/app/components/transaction/transaction.component.html index 360d3e34f..86930bcc7 100644 --- a/frontend/src/app/components/transaction/transaction.component.html +++ b/frontend/src/app/components/transaction/transaction.component.html @@ -190,7 +190,7 @@
- +

Flow

@@ -238,7 +238,7 @@
- +
@@ -329,7 +329,7 @@
- +

Flow

diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index c64c112b1..2be549569 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -49,12 +49,15 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { liquidUnblinding = new LiquidUnblinding(); inputIndex: number; outputIndex: number; - showFlow: boolean = true; graphExpanded: boolean = false; graphWidth: number = 1000; graphHeight: number = 360; inOutLimit: number = 150; maxInOut: number = 0; + flowPrefSubscription: Subscription; + hideFlow: boolean = this.stateService.hideFlow.value; + overrideFlowPreference: boolean = null; + flowEnabled: boolean; tooltipPosition: { x: number, y: number }; @@ -78,6 +81,12 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { (network) => (this.network = network) ); + this.setFlowEnabled(); + this.flowPrefSubscription = this.stateService.hideFlow.subscribe((hide) => { + this.hideFlow = !!hide; + this.setFlowEnabled(); + }); + this.timeAvg$ = timer(0, 1000) .pipe( switchMap(() => this.stateService.difficultyAdjustment$), @@ -245,11 +254,14 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { this.queryParamsSubscription = this.route.queryParams.subscribe((params) => { if (params.showFlow === 'false') { - this.showFlow = false; + this.overrideFlowPreference = false; + } else if (params.showFlow === 'true') { + this.overrideFlowPreference = true; } else { - this.showFlow = true; - this.setGraphSize(); + this.overrideFlowPreference = null; } + this.setFlowEnabled(); + this.setGraphSize(); }); } @@ -325,15 +337,20 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { } toggleGraph() { - this.showFlow = !this.showFlow; + const showFlow = !this.flowEnabled; + this.stateService.hideFlow.next(!showFlow); this.router.navigate([], { relativeTo: this.route, - queryParams: { showFlow: this.showFlow }, + queryParams: { showFlow: showFlow }, queryParamsHandling: 'merge', fragment: 'flow' }); } + setFlowEnabled() { + this.flowEnabled = (this.overrideFlowPreference != null ? this.overrideFlowPreference : !this.hideFlow); + } + expandGraph() { this.graphExpanded = true; } @@ -365,6 +382,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { this.txReplacedSubscription.unsubscribe(); this.blocksSubscription.unsubscribe(); this.queryParamsSubscription.unsubscribe(); + this.flowPrefSubscription.unsubscribe(); this.leaveTransaction(); } } diff --git a/frontend/src/app/services/state.service.ts b/frontend/src/app/services/state.service.ts index 9f7cc58a3..d7feb38bb 100644 --- a/frontend/src/app/services/state.service.ts +++ b/frontend/src/app/services/state.service.ts @@ -110,6 +110,7 @@ export class StateService { blockScrolling$: Subject = new Subject(); timeLtr: BehaviorSubject; + hideFlow: BehaviorSubject; constructor( @Inject(PLATFORM_ID) private platformId: any, @@ -159,6 +160,16 @@ export class StateService { this.timeLtr.subscribe((ltr) => { this.storageService.setValue('time-preference-ltr', ltr ? 'true' : 'false'); }); + + const savedFlowPreference = this.storageService.getValue('flow-preference'); + this.hideFlow = new BehaviorSubject(savedFlowPreference === 'hide'); + this.hideFlow.subscribe((hide) => { + if (hide) { + this.storageService.setValue('flow-preference', hide ? 'hide' : 'show'); + } else { + this.storageService.removeItem('flow-preference'); + } + }); } setNetworkBasedonUrl(url: string) { diff --git a/frontend/src/app/services/storage.service.ts b/frontend/src/app/services/storage.service.ts index f3ea694b2..73a013146 100644 --- a/frontend/src/app/services/storage.service.ts +++ b/frontend/src/app/services/storage.service.ts @@ -46,4 +46,12 @@ export class StorageService { console.log(e); } } + + removeItem(key: string): void { + try { + localStorage.removeItem(key); + } catch (e) { + console.log(e); + } + } } From 3e66e4d6dbf57acab80b628bb57d8b1cff8eb315 Mon Sep 17 00:00:00 2001 From: softsimon Date: Fri, 14 Oct 2022 05:09:25 +0400 Subject: [PATCH 04/32] Handle instant block, txid and address search fixes #2619 --- .../search-form/search-form.component.ts | 149 +++++++++++------- .../search-results.component.html | 28 +++- .../search-results.component.ts | 2 +- 3 files changed, 119 insertions(+), 60 deletions(-) diff --git a/frontend/src/app/components/search-form/search-form.component.ts b/frontend/src/app/components/search-form/search-form.component.ts index 3cff7c188..dc9ea8940 100644 --- a/frontend/src/app/components/search-form/search-form.component.ts +++ b/frontend/src/app/components/search-form/search-form.component.ts @@ -3,8 +3,8 @@ import { FormBuilder, FormGroup, Validators } from '@angular/forms'; import { Router } from '@angular/router'; import { AssetsService } from '../../services/assets.service'; import { StateService } from '../../services/state.service'; -import { Observable, of, Subject, zip, BehaviorSubject } from 'rxjs'; -import { debounceTime, distinctUntilChanged, switchMap, catchError, map } from 'rxjs/operators'; +import { Observable, of, Subject, zip, BehaviorSubject, combineLatest } from 'rxjs'; +import { debounceTime, distinctUntilChanged, switchMap, catchError, map, startWith, tap } from 'rxjs/operators'; import { ElectrsApiService } from '../../services/electrs-api.service'; import { RelativeUrlPipe } from '../../shared/pipes/relative-url/relative-url.pipe'; import { ApiService } from '../../services/api.service'; @@ -33,7 +33,7 @@ export class SearchFormComponent implements OnInit { @Output() searchTriggered = new EventEmitter(); @ViewChild('searchResults') searchResults: SearchResultsComponent; - @HostListener('keydown', ['$event']) keydown($event) { + @HostListener('keydown', ['$event']) keydown($event): void { this.handleKeyDown($event); } @@ -47,7 +47,7 @@ export class SearchFormComponent implements OnInit { private relativeUrlPipe: RelativeUrlPipe, ) { } - ngOnInit() { + ngOnInit(): void { this.stateService.networkChanged$.subscribe((network) => this.network = network); this.searchForm = this.formBuilder.group({ @@ -61,70 +61,111 @@ export class SearchFormComponent implements OnInit { }); } - this.typeAhead$ = this.searchForm.get('searchText').valueChanges - .pipe( - map((text) => { - if (this.network === 'bisq' && text.match(/^(b)[^c]/i)) { - return text.substr(1); - } - return text.trim(); - }), - debounceTime(200), - distinctUntilChanged(), - switchMap((text) => { - if (!text.length) { - return of([ - '', - [], - { - nodes: [], - channels: [], - } - ]); - } - this.isTypeaheading$.next(true); - if (!this.stateService.env.LIGHTNING) { - return zip( - of(text), - this.electrsApiService.getAddressesByPrefix$(text).pipe(catchError(() => of([]))), - [{ nodes: [], channels: [] }], - of(this.regexBlockheight.test(text)), - ); - } + const searchText$ = this.searchForm.get('searchText').valueChanges + .pipe( + map((text) => { + if (this.network === 'bisq' && text.match(/^(b)[^c]/i)) { + return text.substr(1); + } + return text.trim(); + }), + distinctUntilChanged(), + ); + + const searchResults$ = searchText$.pipe( + debounceTime(200), + switchMap((text) => { + if (!text.length) { + return of([ + [], + { nodes: [], channels: [] } + ]); + } + this.isTypeaheading$.next(true); + if (!this.stateService.env.LIGHTNING) { return zip( - of(text), this.electrsApiService.getAddressesByPrefix$(text).pipe(catchError(() => of([]))), - this.apiService.lightningSearch$(text).pipe(catchError(() => of({ + [{ nodes: [], channels: [] }], + ); + } + return zip( + this.electrsApiService.getAddressesByPrefix$(text).pipe(catchError(() => of([]))), + this.apiService.lightningSearch$(text).pipe(catchError(() => of({ + nodes: [], + channels: [], + }))), + ); + }), + tap((result: any[]) => { + this.isTypeaheading$.next(false); + }) + ); + + this.typeAhead$ = combineLatest( + [ + searchText$, + searchResults$.pipe( + startWith([ + [], + { + nodes: [], + channels: [], + } + ])) + ] + ).pipe( + map((latestData) => { + const searchText = latestData[0]; + if (!searchText.length) { + return { + searchText: '', + hashQuickMatch: false, + blockHeight: false, + txId: false, + address: false, + addresses: [], nodes: [], channels: [], - }))), - ); - }), - map((result: any[]) => { - this.isTypeaheading$.next(false); - if (this.network === 'bisq') { - return result[0].map((address: string) => 'B' + address); + }; } + + const result = latestData[1]; + const addressPrefixSearchResults = result[0]; + const lightningResults = result[1]; + + if (this.network === 'bisq') { + return searchText.map((address: string) => 'B' + address); + } + + const matchesBlockHeight = this.regexBlockheight.test(searchText); + const matchesTxId = this.regexTransaction.test(searchText) && !this.regexBlockhash.test(searchText); + const matchesBlockHash = this.regexBlockhash.test(searchText); + const matchesAddress = this.regexAddress.test(searchText); + return { - searchText: result[0], - blockHeight: this.regexBlockheight.test(result[0]) ? [parseInt(result[0], 10)] : [], - addresses: result[1], - nodes: result[2].nodes, - channels: result[2].channels, - totalResults: result[1].length + result[2].nodes.length + result[2].channels.length, + searchText: searchText, + hashQuickMatch: +(matchesBlockHeight || matchesBlockHash || matchesTxId || matchesAddress), + blockHeight: matchesBlockHeight, + txId: matchesTxId, + blockHash: matchesBlockHash, + address: matchesAddress, + addresses: addressPrefixSearchResults, + nodes: lightningResults.nodes, + channels: lightningResults.channels, }; }) ); } - handleKeyDown($event) { + + handleKeyDown($event): void { this.searchResults.handleKeyDown($event); } - itemSelected() { + itemSelected(): void { setTimeout(() => this.search()); } - selectedResult(result: any) { + selectedResult(result: any): void { if (typeof result === 'string') { this.search(result); } else if (typeof result === 'number') { @@ -136,7 +177,7 @@ export class SearchFormComponent implements OnInit { } } - search(result?: string) { + search(result?: string): void { const searchText = result || this.searchForm.value.searchText.trim(); if (searchText) { this.isSearching = true; @@ -170,7 +211,7 @@ export class SearchFormComponent implements OnInit { } } - navigate(url: string, searchText: string, extras?: any) { + navigate(url: string, searchText: string, extras?: any): void { this.router.navigate([this.relativeUrlPipe.transform(url), searchText], extras); this.searchTriggered.emit(); this.searchForm.setValue({ diff --git a/frontend/src/app/components/search-form/search-results/search-results.component.html b/frontend/src/app/components/search-form/search-results/search-results.component.html index 9ed829aff..4e8b9bb7c 100644 --- a/frontend/src/app/components/search-form/search-results/search-results.component.html +++ b/frontend/src/app/components/search-form/search-results/search-results.component.html @@ -1,14 +1,32 @@ - -
+
- - -
+ +
+ + +
+
+
diff --git a/frontend/src/app/components/television/television.component.scss b/frontend/src/app/components/television/television.component.scss index 50ee7b543..9a6cbcc24 100644 --- a/frontend/src/app/components/television/television.component.scss +++ b/frontend/src/app/components/television/television.component.scss @@ -31,8 +31,9 @@ .position-container { position: absolute; - left: 50%; + left: 0; bottom: 170px; + transform: translateX(50vw); } #divider { @@ -47,9 +48,33 @@ top: -28px; } } + + &.time-ltr { + .blocks-wrapper { + transform: scaleX(-1); + } + } } + +:host-context(.ltr-layout) { + .blockchain-wrapper.time-ltr .blocks-wrapper, + .blockchain-wrapper .blocks-wrapper { + direction: ltr; + } +} + +:host-context(.rtl-layout) { + .blockchain-wrapper.time-ltr .blocks-wrapper, + .blockchain-wrapper .blocks-wrapper { + direction: rtl; + } +} + .tv-container { display: flex; margin-top: 0px; flex-direction: column; } + + + diff --git a/frontend/src/app/components/television/television.component.ts b/frontend/src/app/components/television/television.component.ts index ab1770972..5e3888aa4 100644 --- a/frontend/src/app/components/television/television.component.ts +++ b/frontend/src/app/components/television/television.component.ts @@ -1,4 +1,4 @@ -import { Component, OnInit } from '@angular/core'; +import { Component, OnInit, OnDestroy } from '@angular/core'; import { WebsocketService } from '../../services/websocket.service'; import { OptimizedMempoolStats } from '../../interfaces/node-api.interface'; import { StateService } from '../../services/state.service'; @@ -6,7 +6,7 @@ import { ApiService } from '../../services/api.service'; import { SeoService } from '../../services/seo.service'; import { ActivatedRoute } from '@angular/router'; import { map, scan, startWith, switchMap, tap } from 'rxjs/operators'; -import { interval, merge, Observable } from 'rxjs'; +import { interval, merge, Observable, Subscription } from 'rxjs'; import { ChangeDetectionStrategy } from '@angular/core'; @Component({ @@ -15,11 +15,13 @@ import { ChangeDetectionStrategy } from '@angular/core'; styleUrls: ['./television.component.scss'], changeDetection: ChangeDetectionStrategy.OnPush }) -export class TelevisionComponent implements OnInit { +export class TelevisionComponent implements OnInit, OnDestroy { mempoolStats: OptimizedMempoolStats[] = []; statsSubscription$: Observable; fragment: string; + timeLtrSubscription: Subscription; + timeLtr: boolean = this.stateService.timeLtr.value; constructor( private websocketService: WebsocketService, @@ -37,6 +39,10 @@ export class TelevisionComponent implements OnInit { this.seoService.setTitle($localize`:@@46ce8155c9ab953edeec97e8950b5a21e67d7c4e:TV view`); this.websocketService.want(['blocks', 'live-2h-chart', 'mempool-blocks']); + this.timeLtrSubscription = this.stateService.timeLtr.subscribe((ltr) => { + this.timeLtr = !!ltr; + }); + this.statsSubscription$ = merge( this.stateService.live2Chart$.pipe(map(stats => [stats])), this.route.fragment @@ -70,4 +76,8 @@ export class TelevisionComponent implements OnInit { }) ); } + + ngOnDestroy() { + this.timeLtrSubscription.unsubscribe(); + } } From cb576ce601fabbfcfd717dd22b5a2ef70548162c Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Wed, 26 Oct 2022 12:33:13 -0400 Subject: [PATCH 17/32] Add electrum rpc doc tab for official instance --- .../src/app/docs/api-docs/api-docs.component.html | 13 +++++++++++++ .../src/app/docs/api-docs/api-docs.component.scss | 12 ++++++++++++ frontend/src/app/docs/docs/docs.component.html | 9 +++++++++ frontend/src/app/docs/docs/docs.component.ts | 7 ++++++- 4 files changed, 40 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/docs/api-docs/api-docs.component.html b/frontend/src/app/docs/api-docs/api-docs.component.html index f106c4bc5..90c35252a 100644 --- a/frontend/src/app/docs/api-docs/api-docs.component.html +++ b/frontend/src/app/docs/api-docs/api-docs.component.html @@ -106,6 +106,19 @@ +
+
+
+ +

This part of the API is available to sponsors only—whitelisting is required.

+
+ +

Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.

+
+
+
+
+
diff --git a/frontend/src/app/docs/api-docs/api-docs.component.scss b/frontend/src/app/docs/api-docs/api-docs.component.scss index 456983657..aebaafe6f 100644 --- a/frontend/src/app/docs/api-docs/api-docs.component.scss +++ b/frontend/src/app/docs/api-docs/api-docs.component.scss @@ -1,3 +1,11 @@ +.center { + text-align: center; +} + +.note { + font-style: italic; +} + .text-small { font-size: 12px; } @@ -116,6 +124,10 @@ li.nav-item { float: right; } +.doc-content.no-sidebar { + width: 100% +} + h3 { margin: 2rem 0 0 0; } diff --git a/frontend/src/app/docs/docs/docs.component.html b/frontend/src/app/docs/docs/docs.component.html index 04f9bb8cf..9e7f57c74 100644 --- a/frontend/src/app/docs/docs/docs.component.html +++ b/frontend/src/app/docs/docs/docs.component.html @@ -32,6 +32,15 @@ +
  • + API - Electrum RPC + + + + + +
  • +
    diff --git a/frontend/src/app/docs/docs/docs.component.ts b/frontend/src/app/docs/docs/docs.component.ts index 74cebc88f..c129cd21e 100644 --- a/frontend/src/app/docs/docs/docs.component.ts +++ b/frontend/src/app/docs/docs/docs.component.ts @@ -15,6 +15,7 @@ export class DocsComponent implements OnInit { env: Env; showWebSocketTab = true; showFaqTab = true; + showElectrsTab = true; @HostBinding('attr.dir') dir = 'ltr'; @@ -34,14 +35,18 @@ export class DocsComponent implements OnInit { } else if( url[1].path === "rest" ) { this.activeTab = 1; this.seoService.setTitle($localize`:@@e351b40b3869a5c7d19c3d4918cb1ac7aaab95c4:API`); - } else { + } else if( url[1].path === "websocket" ) { this.activeTab = 2; this.seoService.setTitle($localize`:@@e351b40b3869a5c7d19c3d4918cb1ac7aaab95c4:API`); + } else { + this.activeTab = 3; + this.seoService.setTitle($localize`:@@e351b40b3869a5c7d19c3d4918cb1ac7aaab95c4:API`); } this.env = this.stateService.env; this.showWebSocketTab = ( ! ( ( this.stateService.network === "bisq" ) || ( this.stateService.network === "liquidtestnet" ) ) ); this.showFaqTab = ( this.env.BASE_MODULE === 'mempool' ) ? true : false; + this.showElectrsTab = this.stateService.env.OFFICIAL_MEMPOOL_SPACE && ( this.stateService.network === "" || this.stateService.network === "mainnet" || this.stateService.network === "testnet" || this.stateService.network === "signet" ); document.querySelector( "html" ).style.scrollBehavior = "smooth"; } From 702ff2796acd688d12a08020e5492752243e3dc8 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 10 Oct 2022 22:13:04 +0000 Subject: [PATCH 18/32] New projected block transaction selection algo --- backend/src/api/mempool-blocks.ts | 229 ++++++++++++++++++++++++++- backend/src/api/websocket-handler.ts | 22 ++- backend/src/mempool.interfaces.ts | 11 +- 3 files changed, 254 insertions(+), 8 deletions(-) diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 5eb5aa9c8..9b58f4754 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,5 +1,5 @@ import logger from '../logger'; -import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta } from '../mempool.interfaces'; +import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, TransactionSet, Ancestor } from '../mempool.interfaces'; import { Common } from './common'; import config from '../config'; @@ -99,6 +99,7 @@ class MempoolBlocks { if (transactions.length) { mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length)); } + // Calculate change from previous block states for (let i = 0; i < Math.max(mempoolBlocks.length, prevBlocks.length); i++) { let added: TransactionStripped[] = []; @@ -132,12 +133,238 @@ class MempoolBlocks { removed }); } + return { blocks: mempoolBlocks, deltas: mempoolBlockDeltas }; } + /* + * Build projected mempool blocks using an approximation of the transaction selection algorithm from Bitcoin Core + * (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp) + * + * templateLimit: number of blocks to build using the full algo, + * remaining blocks up to blockLimit will skip the expensive updateDescendants step + * + * blockLimit: number of blocks to build in total. Excess transactions will be ignored. + */ + public makeBlockTemplates(mempool: { [txid: string]: TransactionExtended }, templateLimit: number = Infinity, blockLimit: number = Infinity): MempoolBlockWithTransactions[] { + const start = new Date().getTime(); + const txSets: { [txid: string]: TransactionSet } = {}; + const mempoolArray: TransactionExtended[] = Object.values(mempool); + + mempoolArray.forEach((tx) => { + tx.bestDescendant = null; + tx.ancestors = []; + tx.cpfpChecked = false; + tx.effectiveFeePerVsize = tx.feePerVsize; + txSets[tx.txid] = { + fee: 0, + weight: 1, + score: 0, + children: [], + available: true, + modified: false, + }; + }); + + // Build relatives graph & calculate ancestor scores + mempoolArray.forEach((tx) => { + this.setRelatives(tx, mempool, txSets); + }); + + // Sort by descending ancestor score + const byAncestor = (a, b): number => this.sortByAncestorScore(a, b, txSets); + mempoolArray.sort(byAncestor); + + // Build blocks by greedily choosing the highest feerate package + // (i.e. the package rooted in the transaction with the best ancestor score) + const blocks: MempoolBlockWithTransactions[] = []; + let blockWeight = 4000; + let blockSize = 0; + let transactions: TransactionExtended[] = []; + let modified: TransactionExtended[] = []; + let overflow: TransactionExtended[] = []; + let failures = 0; + while ((mempoolArray.length || modified.length) && blocks.length < blockLimit) { + const simpleMode = blocks.length >= templateLimit; + let anyModified = false; + // Select best next package + let nextTx; + if (mempoolArray.length && (!modified.length || txSets[mempoolArray[0].txid]?.score > txSets[modified[0].txid]?.score)) { + nextTx = mempoolArray.shift(); + if (txSets[nextTx?.txid]?.modified) { + nextTx = null; + } + } else { + nextTx = modified.shift(); + } + + if (nextTx && txSets[nextTx.txid]?.available) { + const nextTxSet = txSets[nextTx.txid]; + // Check if the package fits into this block + if (nextTxSet && blockWeight + nextTxSet.weight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) { + blockWeight += nextTxSet.weight; + // sort txSet by dependency graph (equivalent to sorting by ascending ancestor count) + const sortedTxSet = nextTx.ancestors.sort((a, b) => { + return (mempool[a.txid]?.ancestors?.length || 0) - (mempool[b.txid]?.ancestors?.length || 0); + }); + [...sortedTxSet, nextTx].forEach((ancestor, i, arr) => { + const tx = mempool[ancestor.txid]; + const txSet = txSets[ancestor.txid]; + if (txSet.available) { + txSet.available = false; + tx.effectiveFeePerVsize = nextTxSet.fee / (nextTxSet.weight / 4); + tx.cpfpChecked = true; + if (i < arr.length - 1) { + tx.bestDescendant = { + txid: arr[i + 1].txid, + fee: arr[i + 1].fee, + weight: arr[i + 1].weight, + }; + } + transactions.push(tx); + blockSize += tx.size; + } + }); + + // remove these as valid package ancestors for any remaining descendants + if (!simpleMode) { + sortedTxSet.forEach(tx => { + anyModified = this.updateDescendants(tx, tx, mempool, txSets, modified); + }); + } + + failures = 0; + } else { + // hold this package in an overflow list while we check for smaller options + txSets[nextTx.txid].modified = true; + overflow.push(nextTx); + failures++; + } + } + + // this block is full + const outOfTransactions = !mempoolArray.length && !modified.length; + const exceededPackageTries = failures > 1000 && blockWeight > (config.MEMPOOL.BLOCK_WEIGHT_UNITS - 4000); + const exceededSimpleTries = failures > 0 && simpleMode; + if (outOfTransactions || exceededPackageTries || exceededSimpleTries) { + // construct this block + blocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, blocks.length)); + // reset for the next block + transactions = []; + blockSize = 0; + blockWeight = 4000; + + // 'overflow' packages didn't fit in this block, but are valid candidates for the next + if (overflow.length) { + modified = modified.concat(overflow); + overflow = []; + anyModified = true; + } + } + + // re-sort modified list if necessary + if (anyModified) { + modified = modified.filter(tx => txSets[tx.txid]?.available).sort(byAncestor); + } + } + + const end = new Date().getTime(); + const time = end - start; + logger.debug('Mempool templates calculated in ' + time / 1000 + ' seconds'); + + return blocks; + } + + private sortByAncestorScore(a, b, txSets): number { + return txSets[b.txid]?.score - txSets[a.txid]?.score; + } + + private setRelatives(tx: TransactionExtended, mempool: { [txid: string]: TransactionExtended }, txSets: { [txid: string]: TransactionSet }): { [txid: string]: Ancestor } { + let ancestors: { [txid: string]: Ancestor } = {}; + tx.vin.forEach((parent) => { + const parentTx = mempool[parent.txid]; + const parentTxSet = txSets[parent.txid]; + if (parentTx && parentTxSet) { + ancestors[parentTx.txid] = parentTx; + if (!parentTxSet.children) { + parentTxSet.children = [tx.txid]; + } else { + parentTxSet.children.push(tx.txid); + } + if (!parentTxSet.score) { + ancestors = { + ...ancestors, + ...this.setRelatives(parentTx, mempool, txSets), + }; + } + } + }); + tx.ancestors = Object.values(ancestors).map(ancestor => { + return { + txid: ancestor.txid, + fee: ancestor.fee, + weight: ancestor.weight + }; + }); + let totalFees = tx.fee; + let totalWeight = tx.weight; + tx.ancestors.forEach(ancestor => { + totalFees += ancestor.fee; + totalWeight += ancestor.weight; + }); + txSets[tx.txid].fee = totalFees; + txSets[tx.txid].weight = totalWeight; + txSets[tx.txid].score = this.calcAncestorScore(tx, totalFees, totalWeight); + + return ancestors; + } + + private calcAncestorScore(tx: TransactionExtended, ancestorFees: number, ancestorWeight: number): number { + return Math.min(tx.fee / tx.weight, ancestorFees / ancestorWeight); + } + + // walk over remaining descendants, removing the root as a valid ancestor & updating the ancestor score + // returns whether any descendants were modified + private updateDescendants( + root: TransactionExtended, + tx: TransactionExtended, + mempool: { [txid: string]: TransactionExtended }, + txSets: { [txid: string]: TransactionSet }, + modified: TransactionExtended[], + ): boolean { + let anyModified = false; + const txSet = txSets[tx.txid]; + if (txSet.children) { + txSet.children.forEach(childId => { + const child = mempool[childId]; + if (child && child.ancestors && txSets[childId]?.available) { + const ancestorIndex = child.ancestors.findIndex(a => a.txid === root.txid); + if (ancestorIndex > -1) { + // remove tx as ancestor + child.ancestors.splice(ancestorIndex, 1); + const childTxSet = txSets[childId]; + childTxSet.fee -= root.fee; + childTxSet.weight -= root.weight; + childTxSet.score = this.calcAncestorScore(child, childTxSet.fee, childTxSet.weight); + anyModified = true; + + if (!childTxSet.modified) { + childTxSet.modified = true; + modified.push(child); + } + + // recursively update grandchildren + anyModified = this.updateDescendants(root, child, mempool, txSets, modified) || anyModified; + } + } + }); + } + return anyModified; + } + private dataToMempoolBlocks(transactions: TransactionExtended[], blockSize: number, blockWeight: number, blocksIndex: number): MempoolBlockWithTransactions { let rangeLength = 4; diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 4896ee058..f183a4799 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -414,15 +414,15 @@ class WebsocketHandler { let mBlockDeltas: undefined | MempoolBlockDelta[]; let matchRate = 0; const _memPool = memPool.getMempool(); - const _mempoolBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); + const projectedBlocks = mempoolBlocks.makeBlockTemplates(cloneMempool(_memPool), 1, 1); - if (_mempoolBlocks[0]) { + if (projectedBlocks[0]) { const matches: string[] = []; const added: string[] = []; const missing: string[] = []; for (const txId of txIds) { - if (_mempoolBlocks[0].transactionIds.indexOf(txId) > -1) { + if (projectedBlocks[0].transactionIds.indexOf(txId) > -1) { matches.push(txId); } else { added.push(txId); @@ -430,7 +430,7 @@ class WebsocketHandler { delete _memPool[txId]; } - for (const txId of _mempoolBlocks[0].transactionIds) { + for (const txId of projectedBlocks[0].transactionIds) { if (matches.includes(txId) || added.includes(txId)) { continue; } @@ -443,14 +443,14 @@ class WebsocketHandler { mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); if (Common.indexingEnabled()) { - const stripped = _mempoolBlocks[0].transactions.map((tx) => { + const stripped = projectedBlocks[0].transactions.map((tx) => { return { txid: tx.txid, vsize: tx.vsize, fee: tx.fee ? Math.round(tx.fee) : 0, value: tx.value, }; - }); + }); BlocksSummariesRepository.$saveSummary({ height: block.height, template: { @@ -580,4 +580,14 @@ class WebsocketHandler { } } +function cloneMempool(mempool: { [txid: string]: TransactionExtended }): { [txid: string]: TransactionExtended } { + const cloned = {}; + Object.keys(mempool).forEach(id => { + cloned[id] = { + ...mempool[id] + }; + }); + return cloned; +} + export default new WebsocketHandler(); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index d72b13576..fec26c0f3 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -70,12 +70,21 @@ export interface TransactionExtended extends IEsploraApi.Transaction { deleteAfter?: number; } -interface Ancestor { +export interface Ancestor { txid: string; weight: number; fee: number; } +export interface TransactionSet { + fee: number; + weight: number; + score: number; + children?: string[]; + available?: boolean; + modified?: boolean; +} + interface BestDescendant { txid: string; weight: number; From 39afa4cda13979f81f58248a324bddcfd9811548 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 18 Oct 2022 21:03:21 +0000 Subject: [PATCH 19/32] Fix errors in block audit tx selection algorithm --- backend/src/api/mempool-blocks.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 9b58f4754..2fb524b11 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -207,10 +207,10 @@ class MempoolBlocks { if (nextTxSet && blockWeight + nextTxSet.weight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) { blockWeight += nextTxSet.weight; // sort txSet by dependency graph (equivalent to sorting by ascending ancestor count) - const sortedTxSet = nextTx.ancestors.sort((a, b) => { + const sortedTxSet = [...nextTx.ancestors.sort((a, b) => { return (mempool[a.txid]?.ancestors?.length || 0) - (mempool[b.txid]?.ancestors?.length || 0); - }); - [...sortedTxSet, nextTx].forEach((ancestor, i, arr) => { + }), nextTx]; + sortedTxSet.forEach((ancestor, i, arr) => { const tx = mempool[ancestor.txid]; const txSet = txSets[ancestor.txid]; if (txSet.available) { @@ -340,7 +340,7 @@ class MempoolBlocks { if (txSet.children) { txSet.children.forEach(childId => { const child = mempool[childId]; - if (child && child.ancestors && txSets[childId]?.available) { + if (child && child.ancestors) { const ancestorIndex = child.ancestors.findIndex(a => a.txid === root.txid); if (ancestorIndex > -1) { // remove tx as ancestor @@ -355,11 +355,12 @@ class MempoolBlocks { childTxSet.modified = true; modified.push(child); } - - // recursively update grandchildren - anyModified = this.updateDescendants(root, child, mempool, txSets, modified) || anyModified; } } + // recursively update grandchildren + if (child) { + anyModified = this.updateDescendants(root, child, mempool, txSets, modified) || anyModified; + } }); } return anyModified; From 832ccdac46442c4c1c7b6e8d324a4c0a0d84a391 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 19 Oct 2022 17:10:45 +0000 Subject: [PATCH 20/32] improve audit analysis and scoring --- backend/src/api/audit.ts | 114 +++++++++++++++++++++++++++ backend/src/api/websocket-handler.ts | 28 +++---- 2 files changed, 123 insertions(+), 19 deletions(-) create mode 100644 backend/src/api/audit.ts diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts new file mode 100644 index 000000000..6efb50938 --- /dev/null +++ b/backend/src/api/audit.ts @@ -0,0 +1,114 @@ +import { BlockExtended, TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces'; + +const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners + +class Audit { + auditBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[], + projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended }, + ): { censored: string[], added: string[], score: number } { + const matches: string[] = []; // present in both mined block and template + const added: string[] = []; // present in mined block, not in template + const fresh: string[] = []; // missing, but firstSeen within PROPAGATION_MARGIN + const isCensored = {}; // missing, without excuse + const isDisplaced = {}; + let displacedWeight = 0; + + const inBlock = {}; + const inTemplate = {}; + + const now = Math.round((Date.now() / 1000)); + for (const tx of transactions) { + inBlock[tx.txid] = tx; + } + // coinbase is always expected + if (transactions[0]) { + inTemplate[transactions[0].txid] = true; + } + // look for transactions that were expected in the template, but missing from the mined block + for (const txid of projectedBlocks[0].transactionIds) { + if (!inBlock[txid]) { + // tx is recent, may have reached the miner too late for inclusion + if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) { + fresh.push(txid); + } else { + isCensored[txid] = true; + } + displacedWeight += mempool[txid].weight; + } + inTemplate[txid] = true; + } + + displacedWeight += (4000 - transactions[0].weight); + + logger.warn(`${fresh.length} fresh, ${Object.keys(isCensored).length} possibly censored, ${displacedWeight} displaced weight`); + + // we can expect an honest miner to include 'displaced' transactions in place of recent arrivals and censored txs + // these displaced transactions should occupy the first N weight units of the next projected block + let displacedWeightRemaining = displacedWeight; + let index = 0; + let lastFeeRate = Infinity; + let failures = 0; + while (projectedBlocks[1] && index < projectedBlocks[1].transactionIds.length && failures < 500) { + const txid = projectedBlocks[1].transactionIds[index]; + const fits = (mempool[txid].weight - displacedWeightRemaining) < 4000; + const feeMatches = mempool[txid].effectiveFeePerVsize >= lastFeeRate; + if (fits || feeMatches) { + isDisplaced[txid] = true; + if (fits) { + lastFeeRate = Math.min(lastFeeRate, mempool[txid].effectiveFeePerVsize); + } + if (mempool[txid].firstSeen == null || (now - (mempool[txid]?.firstSeen || 0)) > PROPAGATION_MARGIN) { + displacedWeightRemaining -= mempool[txid].weight; + } + failures = 0; + } else { + failures++; + } + index++; + } + + // mark unexpected transactions in the mined block as 'added' + let overflowWeight = 0; + for (const tx of transactions) { + if (inTemplate[tx.txid]) { + matches.push(tx.txid); + } else { + if (!isDisplaced[tx.txid]) { + added.push(tx.txid); + } + overflowWeight += tx.weight; + } + } + + // transactions missing from near the end of our template are probably not being censored + let overflowWeightRemaining = overflowWeight; + let lastOverflowRate = 1.00; + index = projectedBlocks[0].transactionIds.length - 1; + while (index >= 0) { + const txid = projectedBlocks[0].transactionIds[index]; + if (overflowWeightRemaining > 0) { + if (isCensored[txid]) { + delete isCensored[txid]; + } + lastOverflowRate = mempool[txid].effectiveFeePerVsize; + } else if (Math.floor(mempool[txid].effectiveFeePerVsize * 100) <= Math.ceil(lastOverflowRate * 100)) { // tolerance of 0.01 sat/vb + if (isCensored[txid]) { + delete isCensored[txid]; + } + } + overflowWeightRemaining -= (mempool[txid]?.weight || 0); + index--; + } + + const numCensored = Object.keys(isCensored).length; + const score = matches.length > 0 ? (matches.length / (matches.length + numCensored)) : 0; + + return { + censored: Object.keys(isCensored), + added, + score + }; + } +} + +export default new Audit(); \ No newline at end of file diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index f183a4799..c3d4dad9b 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -18,6 +18,7 @@ import difficultyAdjustment from './difficulty-adjustment'; import feeApi from './fee-api'; import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository'; import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; +import Audit from './audit'; class WebsocketHandler { private wss: WebSocket.Server | undefined; @@ -405,7 +406,7 @@ class WebsocketHandler { }); } - handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) { + handleNewBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[]): void { if (!this.wss) { throw new Error('WebSocket.Server is not set'); } @@ -414,30 +415,19 @@ class WebsocketHandler { let mBlockDeltas: undefined | MempoolBlockDelta[]; let matchRate = 0; const _memPool = memPool.getMempool(); - const projectedBlocks = mempoolBlocks.makeBlockTemplates(cloneMempool(_memPool), 1, 1); + const mempoolCopy = cloneMempool(_memPool); + + const projectedBlocks = mempoolBlocks.makeBlockTemplates(mempoolCopy, 2, 2); if (projectedBlocks[0]) { - const matches: string[] = []; - const added: string[] = []; - const missing: string[] = []; + const { censored, added, score } = Audit.auditBlock(block, txIds, transactions, projectedBlocks, mempoolCopy); + matchRate = Math.round(score * 100 * 100) / 100; + // Update mempool to remove transactions included in the new block for (const txId of txIds) { - if (projectedBlocks[0].transactionIds.indexOf(txId) > -1) { - matches.push(txId); - } else { - added.push(txId); - } delete _memPool[txId]; } - for (const txId of projectedBlocks[0].transactionIds) { - if (matches.includes(txId) || added.includes(txId)) { - continue; - } - missing.push(txId); - } - - matchRate = Math.round((Math.max(0, matches.length - missing.length - added.length) / txIds.length * 100) * 100) / 100; mempoolBlocks.updateMempoolBlocks(_memPool); mBlocks = mempoolBlocks.getMempoolBlocks(); mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); @@ -464,7 +454,7 @@ class WebsocketHandler { height: block.height, hash: block.id, addedTxs: added, - missingTxs: missing, + missingTxs: censored, matchRate: matchRate, }); } From 968d7b827b9208dcb62f7e1976732c18421e3a71 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Thu, 27 Oct 2022 10:21:39 -0600 Subject: [PATCH 21/32] Optimize makeBlockTemplates --- backend/src/api/audit.ts | 1 + backend/src/api/mempool-blocks.ts | 365 +++++++++++++++------------ backend/src/api/websocket-handler.ts | 4 +- backend/src/mempool.interfaces.ts | 22 +- backend/src/utils/pairing-heap.ts | 174 +++++++++++++ 5 files changed, 406 insertions(+), 160 deletions(-) create mode 100644 backend/src/utils/pairing-heap.ts diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index 6efb50938..2d9fbc430 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -1,3 +1,4 @@ +import logger from '../logger'; import { BlockExtended, TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces'; const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners diff --git a/backend/src/api/mempool-blocks.ts b/backend/src/api/mempool-blocks.ts index 2fb524b11..d0c2a4f63 100644 --- a/backend/src/api/mempool-blocks.ts +++ b/backend/src/api/mempool-blocks.ts @@ -1,7 +1,8 @@ import logger from '../logger'; -import { MempoolBlock, TransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, TransactionSet, Ancestor } from '../mempool.interfaces'; +import { MempoolBlock, TransactionExtended, AuditTransaction, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor } from '../mempool.interfaces'; import { Common } from './common'; import config from '../config'; +import { PairingHeap } from '../utils/pairing-heap'; class MempoolBlocks { private mempoolBlocks: MempoolBlockWithTransactions[] = []; @@ -72,6 +73,7 @@ class MempoolBlocks { logger.debug('Mempool blocks calculated in ' + time / 1000 + ' seconds'); const { blocks, deltas } = this.calculateMempoolBlocks(memPoolArray, this.mempoolBlocks); + this.mempoolBlocks = blocks; this.mempoolBlockDeltas = deltas; } @@ -144,226 +146,273 @@ class MempoolBlocks { * Build projected mempool blocks using an approximation of the transaction selection algorithm from Bitcoin Core * (see BlockAssembler in https://github.com/bitcoin/bitcoin/blob/master/src/node/miner.cpp) * - * templateLimit: number of blocks to build using the full algo, - * remaining blocks up to blockLimit will skip the expensive updateDescendants step - * - * blockLimit: number of blocks to build in total. Excess transactions will be ignored. + * blockLimit: number of blocks to build in total. + * weightLimit: maximum weight of transactions to consider using the selection algorithm. + * if weightLimit is significantly lower than the mempool size, results may start to diverge from getBlockTemplate + * condenseRest: whether to ignore excess transactions or append them to the final block. */ - public makeBlockTemplates(mempool: { [txid: string]: TransactionExtended }, templateLimit: number = Infinity, blockLimit: number = Infinity): MempoolBlockWithTransactions[] { - const start = new Date().getTime(); - const txSets: { [txid: string]: TransactionSet } = {}; - const mempoolArray: TransactionExtended[] = Object.values(mempool); - - mempoolArray.forEach((tx) => { - tx.bestDescendant = null; - tx.ancestors = []; - tx.cpfpChecked = false; - tx.effectiveFeePerVsize = tx.feePerVsize; - txSets[tx.txid] = { - fee: 0, - weight: 1, + public makeBlockTemplates(mempool: { [txid: string]: TransactionExtended }, blockLimit: number, weightLimit: number | null = null, condenseRest = false): MempoolBlockWithTransactions[] { + const start = Date.now(); + const auditPool: { [txid: string]: AuditTransaction } = {}; + const mempoolArray: AuditTransaction[] = []; + const restOfArray: TransactionExtended[] = []; + + let weight = 0; + const maxWeight = weightLimit ? Math.max(4_000_000 * blockLimit, weightLimit) : Infinity; + // grab the top feerate txs up to maxWeight + Object.values(mempool).sort((a, b) => b.feePerVsize - a.feePerVsize).forEach(tx => { + weight += tx.weight; + if (weight >= maxWeight) { + restOfArray.push(tx); + return; + } + // initializing everything up front helps V8 optimize property access later + auditPool[tx.txid] = { + txid: tx.txid, + fee: tx.fee, + size: tx.size, + weight: tx.weight, + feePerVsize: tx.feePerVsize, + vin: tx.vin, + relativesSet: false, + ancestorMap: new Map(), + children: new Set(), + ancestorFee: 0, + ancestorWeight: 0, score: 0, - children: [], - available: true, + used: false, modified: false, - }; - }); + modifiedNode: null, + } + mempoolArray.push(auditPool[tx.txid]); + }) // Build relatives graph & calculate ancestor scores - mempoolArray.forEach((tx) => { - this.setRelatives(tx, mempool, txSets); - }); + for (const tx of mempoolArray) { + if (!tx.relativesSet) { + this.setRelatives(tx, auditPool); + } + } // Sort by descending ancestor score - const byAncestor = (a, b): number => this.sortByAncestorScore(a, b, txSets); - mempoolArray.sort(byAncestor); + mempoolArray.sort((a, b) => (b.score || 0) - (a.score || 0)); // Build blocks by greedily choosing the highest feerate package // (i.e. the package rooted in the transaction with the best ancestor score) const blocks: MempoolBlockWithTransactions[] = []; let blockWeight = 4000; let blockSize = 0; - let transactions: TransactionExtended[] = []; - let modified: TransactionExtended[] = []; - let overflow: TransactionExtended[] = []; + let transactions: AuditTransaction[] = []; + const modified: PairingHeap = new PairingHeap((a, b): boolean => (a.score || 0) > (b.score || 0)); + let overflow: AuditTransaction[] = []; let failures = 0; - while ((mempoolArray.length || modified.length) && blocks.length < blockLimit) { - const simpleMode = blocks.length >= templateLimit; - let anyModified = false; - // Select best next package - let nextTx; - if (mempoolArray.length && (!modified.length || txSets[mempoolArray[0].txid]?.score > txSets[modified[0].txid]?.score)) { - nextTx = mempoolArray.shift(); - if (txSets[nextTx?.txid]?.modified) { - nextTx = null; - } - } else { - nextTx = modified.shift(); + let top = 0; + while ((top < mempoolArray.length || !modified.isEmpty()) && (condenseRest || blocks.length < blockLimit)) { + // skip invalid transactions + while (top < mempoolArray.length && (mempoolArray[top].used || mempoolArray[top].modified)) { + top++; } - if (nextTx && txSets[nextTx.txid]?.available) { - const nextTxSet = txSets[nextTx.txid]; + // Select best next package + let nextTx; + const nextPoolTx = mempoolArray[top]; + const nextModifiedTx = modified.peek(); + if (nextPoolTx && (!nextModifiedTx || (nextPoolTx.score || 0) > (nextModifiedTx.score || 0))) { + nextTx = nextPoolTx; + top++; + } else { + modified.pop(); + if (nextModifiedTx) { + nextTx = nextModifiedTx; + nextTx.modifiedNode = undefined; + } + } + + if (nextTx && !nextTx?.used) { // Check if the package fits into this block - if (nextTxSet && blockWeight + nextTxSet.weight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) { - blockWeight += nextTxSet.weight; - // sort txSet by dependency graph (equivalent to sorting by ascending ancestor count) - const sortedTxSet = [...nextTx.ancestors.sort((a, b) => { - return (mempool[a.txid]?.ancestors?.length || 0) - (mempool[b.txid]?.ancestors?.length || 0); - }), nextTx]; + if (blockWeight + nextTx.ancestorWeight < config.MEMPOOL.BLOCK_WEIGHT_UNITS) { + blockWeight += nextTx.ancestorWeight; + const ancestors: AuditTransaction[] = Array.from(nextTx.ancestorMap.values()); + // sort ancestors by dependency graph (equivalent to sorting by ascending ancestor count) + const sortedTxSet = [...ancestors.sort((a, b) => { return (a.ancestorMap.size || 0) - (b.ancestorMap.size || 0); }), nextTx]; + const effectiveFeeRate = nextTx.ancestorFee / (nextTx.ancestorWeight / 4); sortedTxSet.forEach((ancestor, i, arr) => { - const tx = mempool[ancestor.txid]; - const txSet = txSets[ancestor.txid]; - if (txSet.available) { - txSet.available = false; - tx.effectiveFeePerVsize = nextTxSet.fee / (nextTxSet.weight / 4); - tx.cpfpChecked = true; + const mempoolTx = mempool[ancestor.txid]; + if (ancestor && !ancestor?.used) { + ancestor.used = true; + // update original copy of this tx with effective fee rate & relatives data + mempoolTx.effectiveFeePerVsize = effectiveFeeRate; + mempoolTx.ancestors = (Array.from(ancestor.ancestorMap?.values()) as AuditTransaction[]).map((a) => { + return { + txid: a.txid, + fee: a.fee, + weight: a.weight, + } + }) if (i < arr.length - 1) { - tx.bestDescendant = { - txid: arr[i + 1].txid, - fee: arr[i + 1].fee, - weight: arr[i + 1].weight, + mempoolTx.bestDescendant = { + txid: arr[arr.length - 1].txid, + fee: arr[arr.length - 1].fee, + weight: arr[arr.length - 1].weight, }; } - transactions.push(tx); - blockSize += tx.size; + transactions.push(ancestor); + blockSize += ancestor.size; } }); - // remove these as valid package ancestors for any remaining descendants - if (!simpleMode) { + // remove these as valid package ancestors for any descendants remaining in the mempool + if (sortedTxSet.length) { sortedTxSet.forEach(tx => { - anyModified = this.updateDescendants(tx, tx, mempool, txSets, modified); + this.updateDescendants(tx, auditPool, modified); }); } failures = 0; } else { // hold this package in an overflow list while we check for smaller options - txSets[nextTx.txid].modified = true; overflow.push(nextTx); failures++; } } // this block is full - const outOfTransactions = !mempoolArray.length && !modified.length; const exceededPackageTries = failures > 1000 && blockWeight > (config.MEMPOOL.BLOCK_WEIGHT_UNITS - 4000); - const exceededSimpleTries = failures > 0 && simpleMode; - if (outOfTransactions || exceededPackageTries || exceededSimpleTries) { + if (exceededPackageTries && (!condenseRest || blocks.length < blockLimit - 1)) { // construct this block - blocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, blocks.length)); + if (transactions.length) { + blocks.push(this.dataToMempoolBlocks(transactions.map(t => mempool[t.txid]), blockSize, blockWeight, blocks.length)); + } // reset for the next block transactions = []; blockSize = 0; blockWeight = 4000; // 'overflow' packages didn't fit in this block, but are valid candidates for the next - if (overflow.length) { - modified = modified.concat(overflow); - overflow = []; - anyModified = true; + for (const overflowTx of overflow.reverse()) { + if (overflowTx.modified) { + overflowTx.modifiedNode = modified.add(overflowTx); + } else { + top--; + mempoolArray[top] = overflowTx; + } } - } - - // re-sort modified list if necessary - if (anyModified) { - modified = modified.filter(tx => txSets[tx.txid]?.available).sort(byAncestor); + overflow = []; } } + if (condenseRest) { + // pack any leftover transactions into the last block + for (const tx of overflow) { + if (!tx || tx?.used) { + continue; + } + blockWeight += tx.weight; + blockSize += tx.size; + transactions.push(tx); + tx.used = true; + } + const blockTransactions = transactions.map(t => mempool[t.txid]) + restOfArray.forEach(tx => { + blockWeight += tx.weight; + blockSize += tx.size; + blockTransactions.push(tx); + }); + if (blockTransactions.length) { + blocks.push(this.dataToMempoolBlocks(blockTransactions, blockSize, blockWeight, blocks.length)); + } + transactions = []; + } else if (transactions.length) { + blocks.push(this.dataToMempoolBlocks(transactions.map(t => mempool[t.txid]), blockSize, blockWeight, blocks.length)); + } - const end = new Date().getTime(); + const end = Date.now(); const time = end - start; logger.debug('Mempool templates calculated in ' + time / 1000 + ' seconds'); return blocks; } - private sortByAncestorScore(a, b, txSets): number { - return txSets[b.txid]?.score - txSets[a.txid]?.score; + // traverse in-mempool ancestors + // recursion unavoidable, but should be limited to depth < 25 by mempool policy + public setRelatives( + tx: AuditTransaction, + mempool: { [txid: string]: AuditTransaction }, + ): void { + for (const parent of tx.vin) { + const parentTx = mempool[parent.txid]; + if (parentTx && !tx.ancestorMap!.has(parent.txid)) { + tx.ancestorMap.set(parent.txid, parentTx); + parentTx.children.add(tx); + // visit each node only once + if (!parentTx.relativesSet) { + this.setRelatives(parentTx, mempool); + } + parentTx.ancestorMap.forEach((ancestor) => { + tx.ancestorMap.set(ancestor.txid, ancestor); + }); + } + }; + tx.ancestorFee = tx.fee || 0; + tx.ancestorWeight = tx.weight || 0; + tx.ancestorMap.forEach((ancestor) => { + tx.ancestorFee += ancestor.fee; + tx.ancestorWeight += ancestor.weight; + }); + tx.score = tx.ancestorFee / (tx.ancestorWeight || 1); + tx.relativesSet = true; } - private setRelatives(tx: TransactionExtended, mempool: { [txid: string]: TransactionExtended }, txSets: { [txid: string]: TransactionSet }): { [txid: string]: Ancestor } { - let ancestors: { [txid: string]: Ancestor } = {}; - tx.vin.forEach((parent) => { - const parentTx = mempool[parent.txid]; - const parentTxSet = txSets[parent.txid]; - if (parentTx && parentTxSet) { - ancestors[parentTx.txid] = parentTx; - if (!parentTxSet.children) { - parentTxSet.children = [tx.txid]; - } else { - parentTxSet.children.push(tx.txid); - } - if (!parentTxSet.score) { - ancestors = { - ...ancestors, - ...this.setRelatives(parentTx, mempool, txSets), - }; - } + // iterate over remaining descendants, removing the root as a valid ancestor & updating the ancestor score + // avoids recursion to limit call stack depth + private updateDescendants( + rootTx: AuditTransaction, + mempool: { [txid: string]: AuditTransaction }, + modified: PairingHeap, + ): void { + const descendantSet: Set = new Set(); + // stack of nodes left to visit + const descendants: AuditTransaction[] = []; + let descendantTx; + let ancestorIndex; + let tmpScore; + rootTx.children.forEach(childTx => { + if (!descendantSet.has(childTx)) { + descendants.push(childTx); + descendantSet.add(childTx); } }); - tx.ancestors = Object.values(ancestors).map(ancestor => { - return { - txid: ancestor.txid, - fee: ancestor.fee, - weight: ancestor.weight - }; - }); - let totalFees = tx.fee; - let totalWeight = tx.weight; - tx.ancestors.forEach(ancestor => { - totalFees += ancestor.fee; - totalWeight += ancestor.weight; - }); - txSets[tx.txid].fee = totalFees; - txSets[tx.txid].weight = totalWeight; - txSets[tx.txid].score = this.calcAncestorScore(tx, totalFees, totalWeight); + while (descendants.length) { + descendantTx = descendants.pop(); + if (descendantTx && descendantTx.ancestorMap && descendantTx.ancestorMap.has(rootTx.txid)) { + // remove tx as ancestor + descendantTx.ancestorMap.delete(rootTx.txid); + descendantTx.ancestorFee -= rootTx.fee; + descendantTx.ancestorWeight -= rootTx.weight; + tmpScore = descendantTx.score; + descendantTx.score = descendantTx.ancestorFee / descendantTx.ancestorWeight; - return ancestors; - } - - private calcAncestorScore(tx: TransactionExtended, ancestorFees: number, ancestorWeight: number): number { - return Math.min(tx.fee / tx.weight, ancestorFees / ancestorWeight); - } - - // walk over remaining descendants, removing the root as a valid ancestor & updating the ancestor score - // returns whether any descendants were modified - private updateDescendants( - root: TransactionExtended, - tx: TransactionExtended, - mempool: { [txid: string]: TransactionExtended }, - txSets: { [txid: string]: TransactionSet }, - modified: TransactionExtended[], - ): boolean { - let anyModified = false; - const txSet = txSets[tx.txid]; - if (txSet.children) { - txSet.children.forEach(childId => { - const child = mempool[childId]; - if (child && child.ancestors) { - const ancestorIndex = child.ancestors.findIndex(a => a.txid === root.txid); - if (ancestorIndex > -1) { - // remove tx as ancestor - child.ancestors.splice(ancestorIndex, 1); - const childTxSet = txSets[childId]; - childTxSet.fee -= root.fee; - childTxSet.weight -= root.weight; - childTxSet.score = this.calcAncestorScore(child, childTxSet.fee, childTxSet.weight); - anyModified = true; - - if (!childTxSet.modified) { - childTxSet.modified = true; - modified.push(child); - } + if (!descendantTx.modifiedNode) { + descendantTx.modified = true; + descendantTx.modifiedNode = modified.add(descendantTx); + } else { + // rebalance modified heap if score has changed + if (descendantTx.score < tmpScore) { + modified.decreasePriority(descendantTx.modifiedNode); + } else if (descendantTx.score > tmpScore) { + modified.increasePriority(descendantTx.modifiedNode); } } - // recursively update grandchildren - if (child) { - anyModified = this.updateDescendants(root, child, mempool, txSets, modified) || anyModified; - } - }); + + // add this node's children to the stack + descendantTx.children.forEach(childTx => { + // visit each node only once + if (!descendantSet.has(childTx)) { + descendants.push(childTx); + descendantSet.add(childTx); + } + }); + } } - return anyModified; } private dataToMempoolBlocks(transactions: TransactionExtended[], diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index c3d4dad9b..9daad3161 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -250,6 +250,8 @@ class WebsocketHandler { throw new Error('WebSocket.Server is not set'); } + logger.debug("mempool changed!"); + mempoolBlocks.updateMempoolBlocks(newMempool); const mBlocks = mempoolBlocks.getMempoolBlocks(); const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); @@ -417,7 +419,7 @@ class WebsocketHandler { const _memPool = memPool.getMempool(); const mempoolCopy = cloneMempool(_memPool); - const projectedBlocks = mempoolBlocks.makeBlockTemplates(mempoolCopy, 2, 2); + const projectedBlocks = mempoolBlocks.makeBlockTemplates(mempoolCopy, 2); if (projectedBlocks[0]) { const { censored, added, score } = Audit.auditBlock(block, txIds, transactions, projectedBlocks, mempoolCopy); diff --git a/backend/src/mempool.interfaces.ts b/backend/src/mempool.interfaces.ts index fec26c0f3..32d87f3dc 100644 --- a/backend/src/mempool.interfaces.ts +++ b/backend/src/mempool.interfaces.ts @@ -1,4 +1,5 @@ import { IEsploraApi } from './api/bitcoin/esplora-api.interface'; +import { HeapNode } from "./utils/pairing-heap"; export interface PoolTag { id: number; // mysql row id @@ -70,6 +71,24 @@ export interface TransactionExtended extends IEsploraApi.Transaction { deleteAfter?: number; } +export interface AuditTransaction { + txid: string; + fee: number; + size: number; + weight: number; + feePerVsize: number; + vin: IEsploraApi.Vin[]; + relativesSet: boolean; + ancestorMap: Map; + children: Set; + ancestorFee: number; + ancestorWeight: number; + score: number; + used: boolean; + modified: boolean; + modifiedNode: HeapNode; +} + export interface Ancestor { txid: string; weight: number; @@ -80,9 +99,10 @@ export interface TransactionSet { fee: number; weight: number; score: number; - children?: string[]; + children?: Set; available?: boolean; modified?: boolean; + modifiedNode?: HeapNode; } interface BestDescendant { diff --git a/backend/src/utils/pairing-heap.ts b/backend/src/utils/pairing-heap.ts new file mode 100644 index 000000000..876e056c4 --- /dev/null +++ b/backend/src/utils/pairing-heap.ts @@ -0,0 +1,174 @@ +export type HeapNode = { + element: T + child?: HeapNode + next?: HeapNode + prev?: HeapNode +} | null | undefined; + +// minimal pairing heap priority queue implementation +export class PairingHeap { + private root: HeapNode = null; + private comparator: (a: T, b: T) => boolean; + + // comparator function should return 'true' if a is higher priority than b + constructor(comparator: (a: T, b: T) => boolean) { + this.comparator = comparator; + } + + isEmpty(): boolean { + return !this.root; + } + + add(element: T): HeapNode { + const node: HeapNode = { + element + }; + + this.root = this.meld(this.root, node); + + return node; + } + + // returns the top priority element without modifying the queue + peek(): T | void { + return this.root?.element; + } + + // removes and returns the top priority element + pop(): T | void { + let element; + if (this.root) { + const node = this.root; + element = node.element; + this.root = this.mergePairs(node.child); + } + return element; + } + + deleteNode(node: HeapNode): void { + if (!node) { + return; + } + + if (node === this.root) { + this.root = this.mergePairs(node.child); + } + else { + if (node.prev) { + if (node.prev.child === node) { + node.prev.child = node.next; + } + else { + node.prev.next = node.next; + } + } + if (node.next) { + node.next.prev = node.prev; + } + this.root = this.meld(this.root, this.mergePairs(node.child)); + } + + node.child = null; + node.prev = null; + node.next = null; + } + + // fix the heap after increasing the priority of a given node + increasePriority(node: HeapNode): void { + // already the top priority element + if (!node || node === this.root) { + return; + } + // extract from siblings + if (node.prev) { + if (node.prev?.child === node) { + if (this.comparator(node.prev.element, node.element)) { + // already in a valid position + return; + } + node.prev.child = node.next; + } + else { + node.prev.next = node.next; + } + } + if (node.next) { + node.next.prev = node.prev; + } + + this.root = this.meld(this.root, node); + } + + decreasePriority(node: HeapNode): void { + this.deleteNode(node); + this.root = this.meld(this.root, node); + } + + meld(a: HeapNode, b: HeapNode): HeapNode { + if (!a) { + return b; + } + if (!b || a === b) { + return a; + } + + let parent: HeapNode = b; + let child: HeapNode = a; + if (this.comparator(a.element, b.element)) { + parent = a; + child = b; + } + + child.next = parent.child; + if (parent.child) { + parent.child.prev = child; + } + child.prev = parent; + parent.child = child; + + parent.next = null; + parent.prev = null; + + return parent; + } + + mergePairs(node: HeapNode): HeapNode { + if (!node) { + return null; + } + + let current: HeapNode = node; + let next: HeapNode; + let nextCurrent: HeapNode; + let pairs: HeapNode; + let melded: HeapNode; + while (current) { + next = current.next; + if (next) { + nextCurrent = next.next; + melded = this.meld(current, next); + if (melded) { + melded.prev = pairs; + } + pairs = melded; + } + else { + nextCurrent = null; + current.prev = pairs; + pairs = current; + break; + } + current = nextCurrent; + } + + melded = null; + let prev: HeapNode; + while (pairs) { + prev = pairs.prev; + melded = this.meld(melded, pairs); + pairs = prev; + } + + return melded; + } +} \ No newline at end of file From 6d282595153cd7b366d9398d5237ebb13b8e92ac Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 28 Oct 2022 15:16:03 -0600 Subject: [PATCH 22/32] disable block audits unless indexing is enabled --- backend/src/api/audit.ts | 9 ++-- backend/src/api/websocket-handler.ts | 75 +++++++++++++--------------- 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/backend/src/api/audit.ts b/backend/src/api/audit.ts index 2d9fbc430..77a6e7459 100644 --- a/backend/src/api/audit.ts +++ b/backend/src/api/audit.ts @@ -4,9 +4,12 @@ import { BlockExtended, TransactionExtended, MempoolBlockWithTransactions } from const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners class Audit { - auditBlock(block: BlockExtended, txIds: string[], transactions: TransactionExtended[], - projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended }, - ): { censored: string[], added: string[], score: number } { + auditBlock(transactions: TransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended }) + : { censored: string[], added: string[], score: number } { + if (!projectedBlocks?.[0]?.transactionIds || !mempool) { + return { censored: [], added: [], score: 0 }; + } + const matches: string[] = []; // present in both mined block and template const added: string[] = []; // present in mined block, not in template const fresh: string[] = []; // missing, but firstSeen within PROPAGATION_MARGIN diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 9daad3161..60560b93c 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -250,8 +250,6 @@ class WebsocketHandler { throw new Error('WebSocket.Server is not set'); } - logger.debug("mempool changed!"); - mempoolBlocks.updateMempoolBlocks(newMempool); const mBlocks = mempoolBlocks.getMempoolBlocks(); const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); @@ -417,55 +415,54 @@ class WebsocketHandler { let mBlockDeltas: undefined | MempoolBlockDelta[]; let matchRate = 0; const _memPool = memPool.getMempool(); - const mempoolCopy = cloneMempool(_memPool); - const projectedBlocks = mempoolBlocks.makeBlockTemplates(mempoolCopy, 2); + if (Common.indexingEnabled()) { + const mempoolCopy = cloneMempool(_memPool); + const projectedBlocks = mempoolBlocks.makeBlockTemplates(mempoolCopy, 2); - if (projectedBlocks[0]) { - const { censored, added, score } = Audit.auditBlock(block, txIds, transactions, projectedBlocks, mempoolCopy); + const { censored, added, score } = Audit.auditBlock(transactions, projectedBlocks, mempoolCopy); matchRate = Math.round(score * 100 * 100) / 100; - // Update mempool to remove transactions included in the new block - for (const txId of txIds) { - delete _memPool[txId]; - } + const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => { + return { + txid: tx.txid, + vsize: tx.vsize, + fee: tx.fee ? Math.round(tx.fee) : 0, + value: tx.value, + }; + }) : []; - mempoolBlocks.updateMempoolBlocks(_memPool); - mBlocks = mempoolBlocks.getMempoolBlocks(); - mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); + BlocksSummariesRepository.$saveSummary({ + height: block.height, + template: { + id: block.id, + transactions: stripped + } + }); - if (Common.indexingEnabled()) { - const stripped = projectedBlocks[0].transactions.map((tx) => { - return { - txid: tx.txid, - vsize: tx.vsize, - fee: tx.fee ? Math.round(tx.fee) : 0, - value: tx.value, - }; - }); - BlocksSummariesRepository.$saveSummary({ - height: block.height, - template: { - id: block.id, - transactions: stripped - } - }); + BlocksAuditsRepository.$saveAudit({ + time: block.timestamp, + height: block.height, + hash: block.id, + addedTxs: added, + missingTxs: censored, + matchRate: matchRate, + }); - BlocksAuditsRepository.$saveAudit({ - time: block.timestamp, - height: block.height, - hash: block.id, - addedTxs: added, - missingTxs: censored, - matchRate: matchRate, - }); + if (block.extras) { + block.extras.matchRate = matchRate; } } - if (block.extras) { - block.extras.matchRate = matchRate; + // Update mempool to remove transactions included in the new block + for (const txId of txIds) { + delete _memPool[txId]; } + mempoolBlocks.updateMempoolBlocks(_memPool); + mBlocks = mempoolBlocks.getMempoolBlocks(); + mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); + const da = difficultyAdjustment.getDifficultyAdjustment(); const fees = feeApi.getRecommendedFee(); From e2e50ac6bf5cd7d47107375d449b4c65f989d5d1 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Tue, 12 Jul 2022 16:13:41 +0000 Subject: [PATCH 23/32] Fix block audit mobile toggle buttons --- .../block-audit/block-audit.component.ts | 69 +++++++++++++------ 1 file changed, 49 insertions(+), 20 deletions(-) diff --git a/frontend/src/app/components/block-audit/block-audit.component.ts b/frontend/src/app/components/block-audit/block-audit.component.ts index ff6c0ea7f..8a43a32c1 100644 --- a/frontend/src/app/components/block-audit/block-audit.component.ts +++ b/frontend/src/app/components/block-audit/block-audit.component.ts @@ -1,12 +1,12 @@ -import { Component, OnDestroy, OnInit, ViewChild } from '@angular/core'; +import { Component, OnDestroy, OnInit, AfterViewInit, ViewChildren, QueryList } from '@angular/core'; import { ActivatedRoute, ParamMap, Router } from '@angular/router'; -import { Observable } from 'rxjs'; -import { map, share, switchMap, tap } from 'rxjs/operators'; -import { BlockAudit, TransactionStripped } from '../../interfaces/node-api.interface'; -import { ApiService } from '../../services/api.service'; -import { StateService } from '../../services/state.service'; -import { detectWebGL } from '../../shared/graphs.utils'; -import { RelativeUrlPipe } from '../../shared/pipes/relative-url/relative-url.pipe'; +import { Observable, Subscription, combineLatest } from 'rxjs'; +import { map, share, switchMap, tap, startWith } from 'rxjs/operators'; +import { BlockAudit, TransactionStripped } from 'src/app/interfaces/node-api.interface'; +import { ApiService } from 'src/app/services/api.service'; +import { StateService } from 'src/app/services/state.service'; +import { detectWebGL } from 'src/app/shared/graphs.utils'; +import { RelativeUrlPipe } from 'src/app/shared/pipes/relative-url/relative-url.pipe'; import { BlockOverviewGraphComponent } from '../block-overview-graph/block-overview-graph.component'; @Component({ @@ -22,7 +22,7 @@ import { BlockOverviewGraphComponent } from '../block-overview-graph/block-overv } `], }) -export class BlockAuditComponent implements OnInit, OnDestroy { +export class BlockAuditComponent implements OnInit, AfterViewInit, OnDestroy { blockAudit: BlockAudit = undefined; transactions: string[]; auditObservable$: Observable; @@ -36,8 +36,10 @@ export class BlockAuditComponent implements OnInit, OnDestroy { webGlEnabled = true; isMobile = window.innerWidth <= 767.98; - @ViewChild('blockGraphTemplate') blockGraphTemplate: BlockOverviewGraphComponent; - @ViewChild('blockGraphMined') blockGraphMined: BlockOverviewGraphComponent; + childChangeSubscription: Subscription; + + @ViewChildren('blockGraphTemplate') blockGraphTemplate: QueryList; + @ViewChildren('blockGraphMined') blockGraphMined: QueryList; constructor( private route: ActivatedRoute, @@ -49,6 +51,7 @@ export class BlockAuditComponent implements OnInit, OnDestroy { } ngOnDestroy(): void { + this.childChangeSubscription.unsubscribe(); } ngOnInit(): void { @@ -83,15 +86,8 @@ export class BlockAuditComponent implements OnInit, OnDestroy { return blockAudit; }), tap((blockAudit) => { + this.blockAudit = blockAudit; this.changeMode(this.mode); - if (this.blockGraphTemplate) { - this.blockGraphTemplate.destroy(); - this.blockGraphTemplate.setup(blockAudit.template); - } - if (this.blockGraphMined) { - this.blockGraphMined.destroy(); - this.blockGraphMined.setup(blockAudit.transactions); - } this.isLoading = false; }), ); @@ -100,14 +96,47 @@ export class BlockAuditComponent implements OnInit, OnDestroy { ); } + ngAfterViewInit() { + this.childChangeSubscription = combineLatest([this.blockGraphTemplate.changes.pipe(startWith(null)), this.blockGraphMined.changes.pipe(startWith(null))]).subscribe(() => { + console.log('changed!'); + this.setupBlockGraphs(); + }) + } + + setupBlockGraphs() { + console.log('setting up block graphs') + if (this.blockAudit) { + this.blockGraphTemplate.forEach(graph => { + graph.destroy(); + if (this.isMobile && this.mode === 'added') { + graph.setup(this.blockAudit.transactions); + } else { + graph.setup(this.blockAudit.template); + } + }) + this.blockGraphMined.forEach(graph => { + graph.destroy(); + graph.setup(this.blockAudit.transactions); + }) + } + } + onResize(event: any) { - this.isMobile = event.target.innerWidth <= 767.98; + const isMobile = event.target.innerWidth <= 767.98; + const changed = isMobile !== this.isMobile; + this.isMobile = isMobile; this.paginationMaxSize = event.target.innerWidth < 670 ? 3 : 5; + + if (changed) { + this.changeMode(this.mode); + } } changeMode(mode: 'missing' | 'added') { this.router.navigate([], { fragment: mode }); this.mode = mode; + + this.setupBlockGraphs(); } onTxClick(event: TransactionStripped): void { From d86f04515050288c750ea5d26645e26245feaadf Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 19 Oct 2022 00:23:45 +0000 Subject: [PATCH 24/32] differentiate censored/missing txs in block audit --- .../block-audit/block-audit.component.ts | 48 ++++++++++++++----- .../block-overview-graph/tx-view.ts | 25 ++++++---- .../block-overview-tooltip.component.html | 10 ++++ .../src/app/interfaces/node-api.interface.ts | 2 +- .../src/app/interfaces/websocket.interface.ts | 2 +- 5 files changed, 63 insertions(+), 24 deletions(-) diff --git a/frontend/src/app/components/block-audit/block-audit.component.ts b/frontend/src/app/components/block-audit/block-audit.component.ts index 8a43a32c1..ed884e728 100644 --- a/frontend/src/app/components/block-audit/block-audit.component.ts +++ b/frontend/src/app/components/block-audit/block-audit.component.ts @@ -65,24 +65,48 @@ export class BlockAuditComponent implements OnInit, AfterViewInit, OnDestroy { .pipe( map((response) => { const blockAudit = response.body; - for (let i = 0; i < blockAudit.template.length; ++i) { - if (blockAudit.missingTxs.includes(blockAudit.template[i].txid)) { - blockAudit.template[i].status = 'missing'; - } else if (blockAudit.addedTxs.includes(blockAudit.template[i].txid)) { - blockAudit.template[i].status = 'added'; + const inTemplate = {}; + const inBlock = {}; + const isAdded = {}; + const isCensored = {}; + const isMissing = {}; + const isSelected = {}; + for (const tx of blockAudit.template) { + inTemplate[tx.txid] = true; + } + for (const tx of blockAudit.transactions) { + inBlock[tx.txid] = true; + } + for (const txid of blockAudit.addedTxs) { + isAdded[txid] = true; + } + for (const txid of blockAudit.missingTxs) { + isCensored[txid] = true; + } + // set transaction statuses + for (const tx of blockAudit.template) { + if (isCensored[tx.txid]) { + tx.status = 'censored'; + } else if (inBlock[tx.txid]) { + tx.status = 'found'; } else { - blockAudit.template[i].status = 'found'; + tx.status = 'missing'; + isMissing[tx.txid] = true; } } - for (let i = 0; i < blockAudit.transactions.length; ++i) { - if (blockAudit.missingTxs.includes(blockAudit.transactions[i].txid)) { - blockAudit.transactions[i].status = 'missing'; - } else if (blockAudit.addedTxs.includes(blockAudit.transactions[i].txid)) { - blockAudit.transactions[i].status = 'added'; + for (const [index, tx] of blockAudit.transactions.entries()) { + if (isAdded[tx.txid]) { + tx.status = 'added'; + } else if (index === 0 || inTemplate[tx.txid]) { + tx.status = 'found'; } else { - blockAudit.transactions[i].status = 'found'; + tx.status = 'selected'; + isSelected[tx.txid] = true; } } + for (const tx of blockAudit.transactions) { + inBlock[tx.txid] = true; + } return blockAudit; }), tap((blockAudit) => { diff --git a/frontend/src/app/components/block-overview-graph/tx-view.ts b/frontend/src/app/components/block-overview-graph/tx-view.ts index 5f2ebf898..1ddc55630 100644 --- a/frontend/src/app/components/block-overview-graph/tx-view.ts +++ b/frontend/src/app/components/block-overview-graph/tx-view.ts @@ -25,7 +25,7 @@ export default class TxView implements TransactionStripped { vsize: number; value: number; feerate: number; - status?: 'found' | 'missing' | 'added'; + status?: 'found' | 'missing' | 'added' | 'censored' | 'selected'; initialised: boolean; vertexArray: FastVertexArray; @@ -142,16 +142,21 @@ export default class TxView implements TransactionStripped { } getColor(): Color { - // Block audit - if (this.status === 'missing') { - return hexToColor('039BE5'); - } else if (this.status === 'added') { - return hexToColor('D81B60'); - } - - // Block component const feeLevelIndex = feeLevels.findIndex((feeLvl) => Math.max(1, this.feerate) < feeLvl) - 1; - return hexToColor(mempoolFeeColors[feeLevelIndex] || mempoolFeeColors[mempoolFeeColors.length - 1]); + const feeLevelColor = hexToColor(mempoolFeeColors[feeLevelIndex] || mempoolFeeColors[mempoolFeeColors.length - 1]); + // Block audit + switch(this.status) { + case 'censored': + return hexToColor('D81BC2'); + case 'missing': + return hexToColor('8C1BD8'); + case 'added': + return hexToColor('03E1E5'); + case 'selected': + return hexToColor('039BE5'); + default: + return feeLevelColor; + } } } diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html index 03d7fc1e9..83fc627be 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html @@ -32,6 +32,16 @@ Virtual size + + Audit status + + match + censored + missing + prioritized + unexpected + + diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index d9670936d..8e04c8635 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -141,7 +141,7 @@ export interface TransactionStripped { fee: number; vsize: number; value: number; - status?: 'found' | 'missing' | 'added'; + status?: 'found' | 'missing' | 'added' | 'censored' | 'selected'; } export interface RewardStats { diff --git a/frontend/src/app/interfaces/websocket.interface.ts b/frontend/src/app/interfaces/websocket.interface.ts index e4ceefb44..67cc0ffc7 100644 --- a/frontend/src/app/interfaces/websocket.interface.ts +++ b/frontend/src/app/interfaces/websocket.interface.ts @@ -70,7 +70,7 @@ export interface TransactionStripped { fee: number; vsize: number; value: number; - status?: 'found' | 'missing' | 'added'; + status?: 'found' | 'missing' | 'added' | 'censored' | 'selected'; } export interface IBackendInfo { From b6343ddc2d22f184cc32f27edf105773e696be7c Mon Sep 17 00:00:00 2001 From: Mononaut Date: Thu, 27 Oct 2022 18:39:26 -0600 Subject: [PATCH 25/32] Clean up block audit page & tweak color scheme --- backend/src/api/mining/mining-routes.ts | 6 + backend/src/api/websocket-handler.ts | 2 +- .../repositories/BlocksAuditsRepository.ts | 10 +- .../block-audit/block-audit.component.html | 150 ++++++++++++++---- .../block-audit/block-audit.component.scss | 4 + .../block-audit/block-audit.component.ts | 79 +++++---- .../block-overview-graph/tx-view.ts | 40 ++++- .../block-overview-tooltip.component.html | 6 +- 8 files changed, 222 insertions(+), 75 deletions(-) diff --git a/backend/src/api/mining/mining-routes.ts b/backend/src/api/mining/mining-routes.ts index f52d42d1f..47704f993 100644 --- a/backend/src/api/mining/mining-routes.ts +++ b/backend/src/api/mining/mining-routes.ts @@ -238,6 +238,12 @@ class MiningRoutes { public async $getBlockAudit(req: Request, res: Response) { try { const audit = await BlocksAuditsRepository.$getBlockAudit(req.params.hash); + + if (!audit) { + res.status(404).send(`This block has not been audited.`); + return; + } + res.header('Pragma', 'public'); res.header('Cache-control', 'public'); res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString()); diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 60560b93c..4bd7cfc8d 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -413,7 +413,7 @@ class WebsocketHandler { let mBlocks: undefined | MempoolBlock[]; let mBlockDeltas: undefined | MempoolBlockDelta[]; - let matchRate = 0; + let matchRate; const _memPool = memPool.getMempool(); if (Common.indexingEnabled()) { diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index be85b22b9..4ddd7d761 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -58,10 +58,12 @@ class BlocksAuditRepositories { WHERE blocks_audits.hash = "${hash}" `); - rows[0].missingTxs = JSON.parse(rows[0].missingTxs); - rows[0].addedTxs = JSON.parse(rows[0].addedTxs); - rows[0].transactions = JSON.parse(rows[0].transactions); - rows[0].template = JSON.parse(rows[0].template); + if (rows.length) { + rows[0].missingTxs = JSON.parse(rows[0].missingTxs); + rows[0].addedTxs = JSON.parse(rows[0].addedTxs); + rows[0].transactions = JSON.parse(rows[0].transactions); + rows[0].template = JSON.parse(rows[0].template); + } return rows[0]; } catch (e: any) { diff --git a/frontend/src/app/components/block-audit/block-audit.component.html b/frontend/src/app/components/block-audit/block-audit.component.html index 0ee6bef44..543dbb705 100644 --- a/frontend/src/app/components/block-audit/block-audit.component.html +++ b/frontend/src/app/components/block-audit/block-audit.component.html @@ -1,21 +1,22 @@
    -
    -
    -

    - - Block -   - {{ blockAudit.height }} -   - Template vs Mined - -

    +
    +

    + + Block Audit +   + {{ blockAudit.height }} +   + +

    -
    +
    - -
    + +
    + +
    +
    @@ -26,8 +27,8 @@ Hash - {{ blockAudit.id | shortenString : 13 }} - + {{ blockHash | shortenString : 13 }} + @@ -40,6 +41,10 @@
    + + Transactions + {{ blockAudit.tx_count }} + Size @@ -57,21 +62,25 @@ - - - - - + - + + + + + + + + +
    Transactions{{ blockAudit.tx_count }}
    Match rateBlock health {{ blockAudit.matchRate }}%
    Missing txsRemoved txs {{ blockAudit.missingTxs.length }}
    Omitted txs{{ numMissing }}
    Added txs {{ blockAudit.addedTxs.length }}
    Included txs{{ numUnexpected }}
    @@ -79,33 +88,110 @@
    - + +
    +

    + + Block Audit +   + {{ blockAudit.height }} +   + +

    + +
    + + +
    + + +
    +
    + +
    + + + + + + + + +
    +
    + + +
    + + + + + + + + +
    +
    +
    +
    + + + +
    + + +
    +
    + audit unavailable +

    + {{ error.error }} +
    +
    +
    + +
    +
    + Error loading data. +

    + {{ error }} +
    +
    +
    +
    +
    + -
    +
    - Projected Block +
    - Actual Block +
    - -
    \ No newline at end of file diff --git a/frontend/src/app/components/block-audit/block-audit.component.scss b/frontend/src/app/components/block-audit/block-audit.component.scss index 7ec503891..1e35b7c63 100644 --- a/frontend/src/app/components/block-audit/block-audit.component.scss +++ b/frontend/src/app/components/block-audit/block-audit.component.scss @@ -37,4 +37,8 @@ @media (min-width: 768px) { max-width: 150px; } +} + +.block-subtitle { + text-align: center; } \ No newline at end of file diff --git a/frontend/src/app/components/block-audit/block-audit.component.ts b/frontend/src/app/components/block-audit/block-audit.component.ts index ed884e728..a7eb879b4 100644 --- a/frontend/src/app/components/block-audit/block-audit.component.ts +++ b/frontend/src/app/components/block-audit/block-audit.component.ts @@ -1,7 +1,7 @@ import { Component, OnDestroy, OnInit, AfterViewInit, ViewChildren, QueryList } from '@angular/core'; import { ActivatedRoute, ParamMap, Router } from '@angular/router'; import { Observable, Subscription, combineLatest } from 'rxjs'; -import { map, share, switchMap, tap, startWith } from 'rxjs/operators'; +import { map, switchMap, startWith, catchError } from 'rxjs/operators'; import { BlockAudit, TransactionStripped } from 'src/app/interfaces/node-api.interface'; import { ApiService } from 'src/app/services/api.service'; import { StateService } from 'src/app/services/state.service'; @@ -25,21 +25,27 @@ import { BlockOverviewGraphComponent } from '../block-overview-graph/block-overv export class BlockAuditComponent implements OnInit, AfterViewInit, OnDestroy { blockAudit: BlockAudit = undefined; transactions: string[]; - auditObservable$: Observable; + auditSubscription: Subscription; + urlFragmentSubscription: Subscription; paginationMaxSize: number; page = 1; itemsPerPage: number; - mode: 'missing' | 'added' = 'missing'; + mode: 'projected' | 'actual' = 'projected'; + error: any; isLoading = true; webGlEnabled = true; isMobile = window.innerWidth <= 767.98; childChangeSubscription: Subscription; - @ViewChildren('blockGraphTemplate') blockGraphTemplate: QueryList; - @ViewChildren('blockGraphMined') blockGraphMined: QueryList; + blockHash: string; + numMissing: number = 0; + numUnexpected: number = 0; + + @ViewChildren('blockGraphProjected') blockGraphProjected: QueryList; + @ViewChildren('blockGraphActual') blockGraphActual: QueryList; constructor( private route: ActivatedRoute, @@ -50,18 +56,31 @@ export class BlockAuditComponent implements OnInit, AfterViewInit, OnDestroy { this.webGlEnabled = detectWebGL(); } - ngOnDestroy(): void { + ngOnDestroy() { this.childChangeSubscription.unsubscribe(); + this.urlFragmentSubscription.unsubscribe(); } ngOnInit(): void { this.paginationMaxSize = window.matchMedia('(max-width: 670px)').matches ? 3 : 5; this.itemsPerPage = this.stateService.env.ITEMS_PER_PAGE; - this.auditObservable$ = this.route.paramMap.pipe( + this.urlFragmentSubscription = this.route.fragment.subscribe((fragment) => { + if (fragment === 'actual') { + this.mode = 'actual'; + } else { + this.mode = 'projected' + } + this.setupBlockGraphs(); + }); + + this.auditSubscription = this.route.paramMap.pipe( switchMap((params: ParamMap) => { - const blockHash: string = params.get('id') || ''; - return this.apiService.getBlockAudit$(blockHash) + this.blockHash = params.get('id') || null; + if (!this.blockHash) { + return null; + } + return this.apiService.getBlockAudit$(this.blockHash) .pipe( map((response) => { const blockAudit = response.body; @@ -71,6 +90,8 @@ export class BlockAuditComponent implements OnInit, AfterViewInit, OnDestroy { const isCensored = {}; const isMissing = {}; const isSelected = {}; + this.numMissing = 0; + this.numUnexpected = 0; for (const tx of blockAudit.template) { inTemplate[tx.txid] = true; } @@ -92,6 +113,7 @@ export class BlockAuditComponent implements OnInit, AfterViewInit, OnDestroy { } else { tx.status = 'missing'; isMissing[tx.txid] = true; + this.numMissing++; } } for (const [index, tx] of blockAudit.transactions.entries()) { @@ -102,43 +124,46 @@ export class BlockAuditComponent implements OnInit, AfterViewInit, OnDestroy { } else { tx.status = 'selected'; isSelected[tx.txid] = true; + this.numUnexpected++; } } for (const tx of blockAudit.transactions) { inBlock[tx.txid] = true; } return blockAudit; - }), - tap((blockAudit) => { - this.blockAudit = blockAudit; - this.changeMode(this.mode); - this.isLoading = false; - }), + }) ); }), - share() - ); + catchError((err) => { + console.log(err); + this.error = err; + this.isLoading = false; + return null; + }), + ).subscribe((blockAudit) => { + this.blockAudit = blockAudit; + this.setupBlockGraphs(); + this.isLoading = false; + }); } ngAfterViewInit() { - this.childChangeSubscription = combineLatest([this.blockGraphTemplate.changes.pipe(startWith(null)), this.blockGraphMined.changes.pipe(startWith(null))]).subscribe(() => { - console.log('changed!'); + this.childChangeSubscription = combineLatest([this.blockGraphProjected.changes.pipe(startWith(null)), this.blockGraphActual.changes.pipe(startWith(null))]).subscribe(() => { this.setupBlockGraphs(); }) } setupBlockGraphs() { - console.log('setting up block graphs') if (this.blockAudit) { - this.blockGraphTemplate.forEach(graph => { + this.blockGraphProjected.forEach(graph => { graph.destroy(); - if (this.isMobile && this.mode === 'added') { + if (this.isMobile && this.mode === 'actual') { graph.setup(this.blockAudit.transactions); } else { graph.setup(this.blockAudit.template); } }) - this.blockGraphMined.forEach(graph => { + this.blockGraphActual.forEach(graph => { graph.destroy(); graph.setup(this.blockAudit.transactions); }) @@ -156,18 +181,12 @@ export class BlockAuditComponent implements OnInit, AfterViewInit, OnDestroy { } } - changeMode(mode: 'missing' | 'added') { + changeMode(mode: 'projected' | 'actual') { this.router.navigate([], { fragment: mode }); - this.mode = mode; - - this.setupBlockGraphs(); } onTxClick(event: TransactionStripped): void { const url = new RelativeUrlPipe(this.stateService).transform(`/tx/${event.txid}`); this.router.navigate([url]); } - - pageChange(page: number, target: HTMLElement) { - } } diff --git a/frontend/src/app/components/block-overview-graph/tx-view.ts b/frontend/src/app/components/block-overview-graph/tx-view.ts index 1ddc55630..ac2a4655a 100644 --- a/frontend/src/app/components/block-overview-graph/tx-view.ts +++ b/frontend/src/app/components/block-overview-graph/tx-view.ts @@ -7,6 +7,15 @@ import { feeLevels, mempoolFeeColors } from '../../app.constants'; const hoverTransitionTime = 300; const defaultHoverColor = hexToColor('1bd8f4'); +const feeColors = mempoolFeeColors.map(hexToColor); +const auditFeeColors = feeColors.map((color) => desaturate(color, 0.3)); +const auditColors = { + censored: hexToColor('f344df'), + missing: darken(desaturate(hexToColor('f344df'), 0.3), 0.7), + added: hexToColor('03E1E5'), + selected: darken(desaturate(hexToColor('039BE5'), 0.3), 0.7), +} + // convert from this class's update format to TxSprite's update format function toSpriteUpdate(params: ViewUpdateParams): SpriteUpdateParams { return { @@ -143,17 +152,19 @@ export default class TxView implements TransactionStripped { getColor(): Color { const feeLevelIndex = feeLevels.findIndex((feeLvl) => Math.max(1, this.feerate) < feeLvl) - 1; - const feeLevelColor = hexToColor(mempoolFeeColors[feeLevelIndex] || mempoolFeeColors[mempoolFeeColors.length - 1]); + const feeLevelColor = feeColors[feeLevelIndex] || feeColors[mempoolFeeColors.length - 1]; // Block audit switch(this.status) { case 'censored': - return hexToColor('D81BC2'); + return auditColors.censored; case 'missing': - return hexToColor('8C1BD8'); + return auditColors.missing; case 'added': - return hexToColor('03E1E5'); + return auditColors.added; case 'selected': - return hexToColor('039BE5'); + return auditColors.selected; + case 'found': + return auditFeeColors[feeLevelIndex] || auditFeeColors[mempoolFeeColors.length - 1]; default: return feeLevelColor; } @@ -168,3 +179,22 @@ function hexToColor(hex: string): Color { a: 1 }; } + +function desaturate(color: Color, amount: number): Color { + const gray = (color.r + color.g + color.b) / 6; + return { + r: color.r + ((gray - color.r) * amount), + g: color.g + ((gray - color.g) * amount), + b: color.b + ((gray - color.b) * amount), + a: color.a, + }; +} + +function darken(color: Color, amount: number): Color { + return { + r: color.r * amount, + g: color.g * amount, + b: color.b * amount, + a: color.a, + } +} diff --git a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html index 83fc627be..b19b67b06 100644 --- a/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html +++ b/frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html @@ -36,10 +36,10 @@ Audit status match - censored + removed missing - prioritized - unexpected + added + included From f3eb403c174628a5426ab1752a19901aaff108b6 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 28 Oct 2022 10:31:55 -0600 Subject: [PATCH 26/32] Add match rate to block page --- backend/src/api/blocks.ts | 17 +++++++++++------ .../src/repositories/BlocksAuditsRepository.ts | 14 ++++++++++++++ .../app/components/block/block.component.html | 7 +++++++ .../src/app/components/block/block.component.ts | 4 ++++ 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index d702b4927..f536ce3d5 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -20,6 +20,7 @@ import indexer from '../indexer'; import fiatConversion from './fiat-conversion'; import poolsParser from './pools-parser'; import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; +import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository'; import mining from './mining/mining'; import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository'; import PricesRepository from '../repositories/PricesRepository'; @@ -186,14 +187,18 @@ class Blocks { if (!pool) { // We should never have this situation in practise logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` + `Check your "pools" table entries`); - return blockExtended; + } else { + blockExtended.extras.pool = { + id: pool.id, + name: pool.name, + slug: pool.slug, + }; } - blockExtended.extras.pool = { - id: pool.id, - name: pool.name, - slug: pool.slug, - }; + const auditSummary = await BlocksAuditsRepository.$getShortBlockAudit(block.id); + if (auditSummary) { + blockExtended.extras.matchRate = auditSummary.matchRate; + } } return blockExtended; diff --git a/backend/src/repositories/BlocksAuditsRepository.ts b/backend/src/repositories/BlocksAuditsRepository.ts index 4ddd7d761..188cf4c38 100644 --- a/backend/src/repositories/BlocksAuditsRepository.ts +++ b/backend/src/repositories/BlocksAuditsRepository.ts @@ -71,6 +71,20 @@ class BlocksAuditRepositories { throw e; } } + + public async $getShortBlockAudit(hash: string): Promise { + try { + const [rows]: any[] = await DB.query( + `SELECT hash as id, match_rate as matchRate + FROM blocks_audits + WHERE blocks_audits.hash = "${hash}" + `); + return rows[0]; + } catch (e: any) { + logger.err(`Cannot fetch block audit from db. Reason: ` + (e instanceof Error ? e.message : e)); + throw e; + } + } } export default new BlocksAuditRepositories(); diff --git a/frontend/src/app/components/block/block.component.html b/frontend/src/app/components/block/block.component.html index a44abe3a0..819b05c81 100644 --- a/frontend/src/app/components/block/block.component.html +++ b/frontend/src/app/components/block/block.component.html @@ -110,6 +110,13 @@ + + Block health + + {{ block.extras.matchRate }}% + Unknown + + diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index 2e6a73c62..8f977b81d 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -47,6 +47,7 @@ export class BlockComponent implements OnInit, OnDestroy { transactionsError: any = null; overviewError: any = null; webGlEnabled = true; + indexingAvailable = false; transactionSubscription: Subscription; overviewSubscription: Subscription; @@ -86,6 +87,9 @@ export class BlockComponent implements OnInit, OnDestroy { this.timeLtr = !!ltr; }); + this.indexingAvailable = (this.stateService.env.BASE_MODULE === 'mempool' && + this.stateService.env.MINING_DASHBOARD === true); + this.txsLoadingStatus$ = this.route.paramMap .pipe( switchMap(() => this.stateService.loadingIndicators$), From b657eb4e7d3491ea48d7b9e0aa35897c65e5fa5f Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 28 Oct 2022 10:33:15 -0600 Subject: [PATCH 27/32] Add match rate to blocks list page --- .../blocks-list/blocks-list.component.html | 25 ++++++++- .../blocks-list/blocks-list.component.scss | 56 ++++++++++++++++--- frontend/src/styles.scss | 9 +++ 3 files changed, 81 insertions(+), 9 deletions(-) diff --git a/frontend/src/app/components/blocks-list/blocks-list.component.html b/frontend/src/app/components/blocks-list/blocks-list.component.html index 660481ecd..68acf71ea 100644 --- a/frontend/src/app/components/blocks-list/blocks-list.component.html +++ b/frontend/src/app/components/blocks-list/blocks-list.component.html @@ -14,6 +14,8 @@ i18n-ngbTooltip="mining.pool-name" ngbTooltip="Pool" placement="bottom" #miningpool [disableTooltip]="!isEllipsisActive(miningpool)">Pool Timestamp Mined + Health Reward Fees @@ -37,12 +39,30 @@ {{ block.extras.coinbaseRaw | hex2ascii }}
    - + ‎{{ block.timestamp * 1000 | date:'yyyy-MM-dd HH:mm' }} + + +
    +
    +
    + {{ block.extras.matchRate }}% +
    +
    +
    +
    +
    +
    + ~ +
    +
    + @@ -77,6 +97,9 @@ + + + diff --git a/frontend/src/app/components/blocks-list/blocks-list.component.scss b/frontend/src/app/components/blocks-list/blocks-list.component.scss index 5dc265017..6617cec58 100644 --- a/frontend/src/app/components/blocks-list/blocks-list.component.scss +++ b/frontend/src/app/components/blocks-list/blocks-list.component.scss @@ -63,7 +63,7 @@ tr, td, th { } .height { - width: 10%; + width: 8%; } .height.widget { width: 15%; @@ -77,12 +77,18 @@ tr, td, th { .timestamp { width: 18%; - @media (max-width: 900px) { + @media (max-width: 1100px) { display: none; } } .timestamp.legacy { width: 20%; + @media (max-width: 1100px) { + display: table-cell; + } + @media (max-width: 850px) { + display: none; + } } .mined { @@ -93,6 +99,10 @@ tr, td, th { } .mined.legacy { width: 15%; + @media (max-width: 1000px) { + padding-right: 20px; + width: 20%; + } @media (max-width: 576px) { display: table-cell; } @@ -100,6 +110,7 @@ tr, td, th { .txs { padding-right: 40px; + width: 8%; @media (max-width: 1100px) { padding-right: 10px; } @@ -113,17 +124,21 @@ tr, td, th { } .txs.widget { padding-right: 0; + display: none; @media (max-width: 650px) { display: none; } } .txs.legacy { - padding-right: 80px; - width: 10%; + width: 18%; + display: table-cell; + @media (max-width: 1000px) { + padding-right: 20px; + } } .fees { - width: 10%; + width: 8%; @media (max-width: 650px) { display: none; } @@ -133,7 +148,7 @@ tr, td, th { } .reward { - width: 10%; + width: 8%; @media (max-width: 576px) { width: 7%; padding-right: 30px; @@ -152,8 +167,11 @@ tr, td, th { } .size { - width: 12%; + width: 10%; @media (max-width: 1000px) { + width: 13%; + } + @media (max-width: 950px) { width: 15%; } @media (max-width: 650px) { @@ -164,12 +182,34 @@ tr, td, th { } } .size.legacy { - width: 20%; + width: 30%; @media (max-width: 576px) { display: table-cell; } } +.health { + width: 10%; + @media (max-width: 1000px) { + width: 13%; + } + @media (max-width: 950px) { + display: none; + } +} +.health.widget { + width: 25%; + @media (max-width: 1000px) { + display: none; + } + @media (max-width: 767px) { + display: table-cell; + } + @media (max-width: 500px) { + display: none; + } +} + /* Tooltip text */ .tooltip-custom { position: relative; diff --git a/frontend/src/styles.scss b/frontend/src/styles.scss index 65293098d..dd9de6aae 100644 --- a/frontend/src/styles.scss +++ b/frontend/src/styles.scss @@ -668,6 +668,15 @@ h1, h2, h3 { background-color: #2e324e; } +.progress.progress-health { + background: repeating-linear-gradient(to right, #2d3348, #2d3348 0%, #105fb0 0%, #1a9436 100%); + justify-content: flex-end; +} + +.progress-bar.progress-bar-health { + background: #2d3348; +} + .mt-2-5, .my-2-5 { margin-top: 0.75rem !important; } From dbb6f267f4945981dc3cef8a6d0e32333586dbe3 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Sun, 30 Oct 2022 12:39:20 -0400 Subject: [PATCH 28/32] Add electrum rpc port numbers and update note --- .../app/docs/api-docs/api-docs.component.html | 13 +++++++------ .../src/app/docs/api-docs/api-docs.component.ts | 16 ++++++++++++++++ frontend/src/app/docs/docs/docs.component.ts | 2 +- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/frontend/src/app/docs/api-docs/api-docs.component.html b/frontend/src/app/docs/api-docs/api-docs.component.html index 90c35252a..e2524a27d 100644 --- a/frontend/src/app/docs/api-docs/api-docs.component.html +++ b/frontend/src/app/docs/api-docs/api-docs.component.html @@ -109,12 +109,13 @@
    - -

    This part of the API is available to sponsors only—whitelisting is required.

    -
    - -

    Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.

    -
    +

    Hostname

    +

    {{plainHostname}}

    +

    Port

    +

    {{electrsPort}}

    +

    SSL

    +

    Enabled

    +

    Electrum RPC interface for Bitcoin Signet is publicly available. Electrum RPC interface for all other networks is available to sponsors only—whitelisting is required.

    diff --git a/frontend/src/app/docs/api-docs/api-docs.component.ts b/frontend/src/app/docs/api-docs/api-docs.component.ts index ed0ecb0a2..7b78d187b 100644 --- a/frontend/src/app/docs/api-docs/api-docs.component.ts +++ b/frontend/src/app/docs/api-docs/api-docs.component.ts @@ -12,6 +12,8 @@ import { FaqTemplateDirective } from '../faq-template/faq-template.component'; styleUrls: ['./api-docs.component.scss'] }) export class ApiDocsComponent implements OnInit, AfterViewInit { + plainHostname = document.location.hostname; + electrsPort = 0; hostname = document.location.hostname; network$: Observable; active = 0; @@ -82,6 +84,20 @@ export class ApiDocsComponent implements OnInit, AfterViewInit { this.network$.subscribe((network) => { this.active = (network === 'liquid' || network === 'liquidtestnet') ? 2 : 0; + switch( network ) { + case "": + this.electrsPort = 50002; break; + case "mainnet": + this.electrsPort = 50002; break; + case "testnet": + this.electrsPort = 60002; break; + case "signet": + this.electrsPort = 60602; break; + case "liquid": + this.electrsPort = 51002; break; + case "liquidtestnet": + this.electrsPort = 51302; break; + } }); } diff --git a/frontend/src/app/docs/docs/docs.component.ts b/frontend/src/app/docs/docs/docs.component.ts index c129cd21e..3e74ba959 100644 --- a/frontend/src/app/docs/docs/docs.component.ts +++ b/frontend/src/app/docs/docs/docs.component.ts @@ -46,7 +46,7 @@ export class DocsComponent implements OnInit { this.env = this.stateService.env; this.showWebSocketTab = ( ! ( ( this.stateService.network === "bisq" ) || ( this.stateService.network === "liquidtestnet" ) ) ); this.showFaqTab = ( this.env.BASE_MODULE === 'mempool' ) ? true : false; - this.showElectrsTab = this.stateService.env.OFFICIAL_MEMPOOL_SPACE && ( this.stateService.network === "" || this.stateService.network === "mainnet" || this.stateService.network === "testnet" || this.stateService.network === "signet" ); + this.showElectrsTab = this.stateService.env.OFFICIAL_MEMPOOL_SPACE && ( this.stateService.network !== "bisq" ); document.querySelector( "html" ).style.scrollBehavior = "smooth"; } From 38ec5ef957dfc795db3d44efa0550b3b42aa61f0 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Sun, 30 Oct 2022 13:08:26 -0400 Subject: [PATCH 29/32] Position docs footer on bottom For short docs pages (like electrum rpc). --- frontend/src/app/docs/api-docs/api-docs.component.scss | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frontend/src/app/docs/api-docs/api-docs.component.scss b/frontend/src/app/docs/api-docs/api-docs.component.scss index aebaafe6f..acfc209e5 100644 --- a/frontend/src/app/docs/api-docs/api-docs.component.scss +++ b/frontend/src/app/docs/api-docs/api-docs.component.scss @@ -10,6 +10,12 @@ font-size: 12px; } +.container-xl { + display: flex; + min-height: 75vh; + flex-direction: column; +} + code { background-color: #1d1f31; font-family: Consolas,Monaco,Lucida Console,Liberation Mono,DejaVu Sans Mono,Bitstream Vera Sans Mono,Courier New; From 387a51d87eec7a0b2a846a2f15c9294853ce3d39 Mon Sep 17 00:00:00 2001 From: softsimon Date: Mon, 7 Nov 2022 04:28:23 +0400 Subject: [PATCH 30/32] Use relative import paths in the frontend --- .../components/block-audit/block-audit.component.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/frontend/src/app/components/block-audit/block-audit.component.ts b/frontend/src/app/components/block-audit/block-audit.component.ts index a7eb879b4..f8ce8d9bb 100644 --- a/frontend/src/app/components/block-audit/block-audit.component.ts +++ b/frontend/src/app/components/block-audit/block-audit.component.ts @@ -1,12 +1,12 @@ import { Component, OnDestroy, OnInit, AfterViewInit, ViewChildren, QueryList } from '@angular/core'; import { ActivatedRoute, ParamMap, Router } from '@angular/router'; -import { Observable, Subscription, combineLatest } from 'rxjs'; +import { Subscription, combineLatest } from 'rxjs'; import { map, switchMap, startWith, catchError } from 'rxjs/operators'; -import { BlockAudit, TransactionStripped } from 'src/app/interfaces/node-api.interface'; -import { ApiService } from 'src/app/services/api.service'; -import { StateService } from 'src/app/services/state.service'; -import { detectWebGL } from 'src/app/shared/graphs.utils'; -import { RelativeUrlPipe } from 'src/app/shared/pipes/relative-url/relative-url.pipe'; +import { BlockAudit, TransactionStripped } from '../../interfaces/node-api.interface'; +import { ApiService } from '../../services/api.service'; +import { StateService } from '../../services/state.service'; +import { detectWebGL } from '../../shared/graphs.utils'; +import { RelativeUrlPipe } from '../../shared/pipes/relative-url/relative-url.pipe'; import { BlockOverviewGraphComponent } from '../block-overview-graph/block-overview-graph.component'; @Component({ From 69a36e17a87fa0dcd81c873e72addab8a1860ded Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sun, 6 Nov 2022 20:24:12 -0800 Subject: [PATCH 31/32] Update staging hosts for testing --- frontend/proxy.conf.staging.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/proxy.conf.staging.js b/frontend/proxy.conf.staging.js index 098edb619..0cf366ca7 100644 --- a/frontend/proxy.conf.staging.js +++ b/frontend/proxy.conf.staging.js @@ -3,9 +3,9 @@ const fs = require('fs'); let PROXY_CONFIG = require('./proxy.conf'); PROXY_CONFIG.forEach(entry => { - entry.target = entry.target.replace("mempool.space", "mempool.ninja"); - entry.target = entry.target.replace("liquid.network", "liquid.place"); - entry.target = entry.target.replace("bisq.markets", "bisq.ninja"); + entry.target = entry.target.replace("mempool.space", "mempool-staging.fra.mempool.space"); + entry.target = entry.target.replace("liquid.network", "liquid-staging.fra.mempool.space"); + entry.target = entry.target.replace("bisq.markets", "bisq-staging.fra.mempool.space"); }); module.exports = PROXY_CONFIG; From b6d4e6b9939dd62773a4dfad6bb2308532a22dfa Mon Sep 17 00:00:00 2001 From: wiz Date: Mon, 7 Nov 2022 15:44:25 +0900 Subject: [PATCH 32/32] [ops] Fix nvidia-driver package name --- production/install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/install b/production/install index 96bb4b623..9bab3a418 100755 --- a/production/install +++ b/production/install @@ -401,7 +401,7 @@ FREEBSD_PKG+=(nginx rsync py39-certbot-nginx mariadb105-server keybase) FREEBSD_PKG+=(geoipupdate) FREEBSD_UNFURL_PKG=() -FREEBSD_UNFURL_PKG+=(nvidia-driver-470-470.129.06 chromium xinit xterm twm ja-sourcehansans-otf) +FREEBSD_UNFURL_PKG+=(nvidia-driver-470 chromium xinit xterm twm ja-sourcehansans-otf) FREEBSD_UNFURL_PKG+=(zh-sourcehansans-sc-otf ko-aleefonts-ttf lohit tlwg-ttf) #############################