machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add audit / block health config feature flag

Browse Source
This commit is contained in:
Mononaut 2023-02-12 21:43:12 -06:00
parent a7b6b43bf4
commit 240b4cb6e2
19 changed files with 88 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
backend/mempool-config.sample.json
View File

@ -25,6 +25,7 @@
"AUTOMATIC_BLOCK_REINDEXING": false,
"POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json",
"POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
"AUDIT": false,
"ADVANCED_GBT_AUDIT": false,
"ADVANCED_GBT_MEMPOOL": false,
"CPFP_INDEXING": false

1
backend/src/__fixtures__/mempool-config.template.json
View File

@ -26,6 +26,7 @@
"INDEXING_BLOCKS_AMOUNT": 14,
"POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__",
"POOLS_JSON_URL": "__POOLS_JSON_URL__",
"AUDIT": "__MEMPOOL_AUDIT__",
"ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__",
"ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__",
"CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__"

1
backend/src/__tests__/config.test.ts
View File

@ -38,6 +38,7 @@ describe('Mempool Backend Config', () => {
STDOUT_LOG_MIN_PRIORITY: 'debug',
POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
AUDIT: false,
ADVANCED_GBT_AUDIT: false,
ADVANCED_GBT_MEMPOOL: false,
CPFP_INDEXING: false,

8
backend/src/api/blocks.ts
View File

@ -212,9 +212,11 @@ class Blocks {
};
}
const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
if (auditScore != null) {
blockExtended.extras.matchRate = auditScore.matchRate;
if (config.MEMPOOL.AUDIT) {
const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
if (auditScore != null) {
blockExtended.extras.matchRate = auditScore.matchRate;
}
}
}

80
backend/src/api/websocket-handler.ts
View File

@ -419,49 +419,51 @@ class WebsocketHandler {
const _memPool = memPool.getMempool();
let projectedBlocks;
// template calculation functions have mempool side effects, so calculate audits using
// a cloned copy of the mempool if we're running a different algorithm for mempool updates
const auditMempool = (config.MEMPOOL.ADVANCED_GBT_AUDIT === config.MEMPOOL.ADVANCED_GBT_MEMPOOL) ? _memPool : deepClone(_memPool);
if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
projectedBlocks = await mempoolBlocks.makeBlockTemplates(auditMempool, false);
} else {
projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false);
}
if (config.MEMPOOL.AUDIT) {
let projectedBlocks;
// template calculation functions have mempool side effects, so calculate audits using
// a cloned copy of the mempool if we're running a different algorithm for mempool updates
const auditMempool = (config.MEMPOOL.ADVANCED_GBT_AUDIT === config.MEMPOOL.ADVANCED_GBT_MEMPOOL) ? _memPool : deepClone(_memPool);
if (config.MEMPOOL.ADVANCED_GBT_AUDIT) {
projectedBlocks = await mempoolBlocks.makeBlockTemplates(auditMempool, false);
} else {
projectedBlocks = mempoolBlocks.updateMempoolBlocks(auditMempool, false);
}
if (Common.indexingEnabled() && memPool.isInSync()) {
const { censored, added, fresh, score } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
const matchRate = Math.round(score * 100 * 100) / 100;
if (Common.indexingEnabled() && memPool.isInSync()) {
const { censored, added, fresh, score } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
const matchRate = Math.round(score * 100 * 100) / 100;
const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
return {
txid: tx.txid,
vsize: tx.vsize,
fee: tx.fee ? Math.round(tx.fee) : 0,
value: tx.value,
};
}) : [];
const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => {
return {
txid: tx.txid,
vsize: tx.vsize,
fee: tx.fee ? Math.round(tx.fee) : 0,
value: tx.value,
};
}) : [];
BlocksSummariesRepository.$saveTemplate({
height: block.height,
template: {
id: block.id,
transactions: stripped
BlocksSummariesRepository.$saveTemplate({
height: block.height,
template: {
id: block.id,
transactions: stripped
}
});
BlocksAuditsRepository.$saveAudit({
time: block.timestamp,
height: block.height,
hash: block.id,
addedTxs: added,
missingTxs: censored,
freshTxs: fresh,
matchRate: matchRate,
});
if (block.extras) {
block.extras.matchRate = matchRate;
}
});
BlocksAuditsRepository.$saveAudit({
time: block.timestamp,
height: block.height,
hash: block.id,
addedTxs: added,
missingTxs: censored,
freshTxs: fresh,
matchRate: matchRate,
});
if (block.extras) {
block.extras.matchRate = matchRate;
}
}

2
backend/src/config.ts
View File

@ -29,6 +29,7 @@ interface IConfig {
AUTOMATIC_BLOCK_REINDEXING: boolean;
POOLS_JSON_URL: string,
POOLS_JSON_TREE_URL: string,
AUDIT: boolean;
ADVANCED_GBT_AUDIT: boolean;
ADVANCED_GBT_MEMPOOL: boolean;
CPFP_INDEXING: boolean;
@ -150,6 +151,7 @@ const defaults: IConfig = {
'AUTOMATIC_BLOCK_REINDEXING': false,
'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json',
'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
'AUDIT': false,
'ADVANCED_GBT_AUDIT': false,
'ADVANCED_GBT_MEMPOOL': false,
'CPFP_INDEXING': false,

1
docker/backend/mempool-config.json
View File

@ -23,6 +23,7 @@
"INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__,
"BLOCKS_SUMMARIES_INDEXING": __MEMPOOL_BLOCKS_SUMMARIES_INDEXING__,
"AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__,
"AUDIT": __MEMPOOL_AUDIT__,
"ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__,
"ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__,
"CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__

2
docker/backend/start.sh
View File

@ -27,6 +27,7 @@ __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=false}
__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false}
__MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json}
__MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master}
__MEMPOOL_AUDIT__=${MEMPOOL_AUDIT:=false}
__MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false}
__MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false}
__MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false}
@ -139,6 +140,7 @@ sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT_
sed -i "s/__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__/${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}/g" mempool-config.json
sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-config.json
sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json
sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json
sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json
sed -i "s!__MEMPOOL_ADVANCED_GBT_AUDIT__!${__MEMPOOL_ADVANCED_GBT_AUDIT__}!g" mempool-config.json
sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json

2
docker/frontend/entrypoint.sh
View File

@ -31,6 +31,7 @@ __LIQUID_WEBSITE_URL__=${LIQUID_WEBSITE_URL:=https://liquid.network}
__BISQ_WEBSITE_URL__=${BISQ_WEBSITE_URL:=https://bisq.markets}
__MINING_DASHBOARD__=${MINING_DASHBOARD:=true}
__LIGHTNING__=${LIGHTNING:=false}
__AUDIT__=${AUDIT:=false}
__MAINNET_BLOCK_AUDIT_START_HEIGHT__=${MAINNET_BLOCK_AUDIT_START_HEIGHT:=0}
__TESTNET_BLOCK_AUDIT_START_HEIGHT__=${TESTNET_BLOCK_AUDIT_START_HEIGHT:=0}
__SIGNET_BLOCK_AUDIT_START_HEIGHT__=${SIGNET_BLOCK_AUDIT_START_HEIGHT:=0}
@ -55,6 +56,7 @@ export __LIQUID_WEBSITE_URL__
export __BISQ_WEBSITE_URL__
export __MINING_DASHBOARD__
export __LIGHTNING__
export __AUDIT__
export __MAINNET_BLOCK_AUDIT_START_HEIGHT__
export __TESTNET_BLOCK_AUDIT_START_HEIGHT__
export __SIGNET_BLOCK_AUDIT_START_HEIGHT__

1
frontend/mempool-frontend-config.sample.json
View File

@ -17,6 +17,7 @@
"LIQUID_WEBSITE_URL": "https://liquid.network",
"BISQ_WEBSITE_URL": "https://bisq.markets",
"MINING_DASHBOARD": true,
"AUDIT": false,
"MAINNET_BLOCK_AUDIT_START_HEIGHT": 0,
"TESTNET_BLOCK_AUDIT_START_HEIGHT": 0,
"SIGNET_BLOCK_AUDIT_START_HEIGHT": 0,

26
frontend/src/app/components/block/block.component.ts
View File

@ -58,7 +58,7 @@ export class BlockComponent implements OnInit, OnDestroy {
overviewError: any = null;
webGlEnabled = true;
indexingAvailable = false;
auditModeEnabled: boolean = !this.stateService.hideAudit.value;
auditModeEnabled: boolean = this.stateService.env.AUDIT && !this.stateService.hideAudit.value;
auditAvailable = true;
showAudit: boolean;
isMobile = window.innerWidth <= 767.98;
@ -110,12 +110,15 @@ export class BlockComponent implements OnInit, OnDestroy {
});
this.indexingAvailable = (this.stateService.env.BASE_MODULE === 'mempool' && this.stateService.env.MINING_DASHBOARD === true);
this.setAuditAvailable(this.indexingAvailable);
this.auditPrefSubscription = this.stateService.hideAudit.subscribe((hide) => {
this.auditModeEnabled = !hide;
this.showAudit = this.auditAvailable && this.auditModeEnabled;
});
this.setAuditAvailable(this.stateService.env.AUDIT && this.indexingAvailable);
if (this.stateService.env.AUDIT) {
this.auditPrefSubscription = this.stateService.hideAudit.subscribe((hide) => {
this.auditModeEnabled = !hide;
this.showAudit = this.auditAvailable && this.auditModeEnabled;
});
}
this.txsLoadingStatus$ = this.route.paramMap
.pipe(
@ -221,7 +224,9 @@ export class BlockComponent implements OnInit, OnDestroy {
setTimeout(() => {
this.nextBlockSubscription = this.apiService.getBlock$(block.previousblockhash).subscribe();
this.nextBlockTxListSubscription = this.electrsApiService.getBlockTransactions$(block.previousblockhash).subscribe();
this.apiService.getBlockAudit$(block.previousblockhash);
if (this.stateService.env.AUDIT) {
this.apiService.getBlockAudit$(block.previousblockhash);
}
}, 100);
}
this.updateAuditAvailableFromBlockHeight(block.height);
@ -269,7 +274,7 @@ export class BlockComponent implements OnInit, OnDestroy {
this.isLoadingOverview = false;
});
if (!this.indexingAvailable) {
if (!this.indexingAvailable || !this.stateService.env.AUDIT) {
this.overviewSubscription = block$.pipe(
startWith(null),
pairwise(),
@ -300,7 +305,7 @@ export class BlockComponent implements OnInit, OnDestroy {
});
}
if (this.indexingAvailable) {
if (this.indexingAvailable && this.stateService.env.AUDIT) {
this.auditSubscription = block$.pipe(
startWith(null),
pairwise(),
@ -613,6 +618,9 @@ export class BlockComponent implements OnInit, OnDestroy {
}
updateAuditAvailableFromBlockHeight(blockHeight: number): void {
if (!this.stateService.env.AUDIT) {
this.setAuditAvailable(false);
}
switch (this.stateService.network) {
case 'testnet':
if (blockHeight < this.stateService.env.TESTNET_BLOCK_AUDIT_START_HEIGHT) {

6
frontend/src/app/components/blocks-list/blocks-list.component.html
View File

@ -14,7 +14,7 @@
i18n-ngbTooltip="mining.pool-name" ngbTooltip="Pool" placement="bottom" #miningpool [disableTooltip]="!isEllipsisActive(miningpool)">Pool</th>
<th class="timestamp" i18n="latest-blocks.timestamp" *ngIf="!widget" [class]="indexingAvailable ? '' : 'legacy'">Timestamp</th>
<th class="mined" i18n="latest-blocks.mined" *ngIf="!widget" [class]="indexingAvailable ? '' : 'legacy'">Mined</th>
<th *ngIf="indexingAvailable" class="health text-right" i18n="latest-blocks.health" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}"
<th *ngIf="indexingAvailable && auditAvailable" class="health text-right" i18n="latest-blocks.health" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}"
i18n-ngbTooltip="latest-blocks.health" ngbTooltip="Health" placement="bottom" #health [disableTooltip]="!isEllipsisActive(health)">Health</th>
<th *ngIf="indexingAvailable" class="reward text-right" i18n="latest-blocks.reward" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}"
i18n-ngbTooltip="latest-blocks.reward" ngbTooltip="Reward" placement="bottom" #reward [disableTooltip]="!isEllipsisActive(reward)">Reward</th>
@ -45,7 +45,7 @@
<td class="mined" *ngIf="!widget" [class]="indexingAvailable ? '' : 'legacy'">
<app-time-since [time]="block.timestamp" [fastRender]="true"></app-time-since>
</td>
<td *ngIf="indexingAvailable" class="health text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
<td *ngIf="indexingAvailable && auditAvailable" class="health text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
<a
class="health-badge badge"
[class.badge-success]="auditScores[block.id] >= 99"
@ -98,7 +98,7 @@
<td class="mined" *ngIf="!widget" [class]="indexingAvailable ? '' : 'legacy'">
<span class="skeleton-loader" style="max-width: 125px"></span>
</td>
<td *ngIf="indexingAvailable" class="health text-left" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
<td *ngIf="indexingAvailable && auditAvailable" class="health text-left" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
<span class="skeleton-loader" style="max-width: 75px"></span>
</td>
<td *ngIf="indexingAvailable" class="reward text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">

4
frontend/src/app/components/blocks-list/blocks-list.component.ts
View File

@ -22,6 +22,7 @@ export class BlocksList implements OnInit, OnDestroy {
latestScoreSubscription: Subscription;
indexingAvailable = false;
auditAvailable = false;
isLoading = true;
loadingScores = true;
fromBlockHeight = undefined;
@ -44,6 +45,7 @@ export class BlocksList implements OnInit, OnDestroy {
ngOnInit(): void {
this.indexingAvailable = (this.stateService.env.BASE_MODULE === 'mempool' &&
this.stateService.env.MINING_DASHBOARD === true);
this.auditAvailable = this.stateService.env.AUDIT;
if (!this.widget) {
this.websocketService.want(['blocks']);
@ -111,7 +113,7 @@ export class BlocksList implements OnInit, OnDestroy {
}, [])
);
if (this.indexingAvailable) {
if (this.indexingAvailable && this.auditAvailable) {
this.auditScoreSubscription = this.fromHeightSubject.pipe(
switchMap((fromBlockHeight) => {
this.loadingScores = true;

2
frontend/src/app/components/graphs/graphs.component.html
View File

@ -22,7 +22,7 @@
i18n="mining.block-rewards">Block Rewards</a>
<a class="dropdown-item" routerLinkActive="active"
[routerLink]="['/graphs/mining/block-sizes-weights' | relativeUrl]" i18n="mining.block-sizes-weights">Block Sizes and Weights</a>
<a class="dropdown-item" routerLinkActive="active"
<a *ngIf="stateService.env.AUDIT" class="dropdown-item" routerLinkActive="active"
[routerLink]="['/graphs/mining/block-prediction' | relativeUrl]" i18n="mining.block-prediction-accuracy">Block Prediction Accuracy</a>
</div>
</div>

2
frontend/src/app/services/state.service.ts
View File

@ -39,6 +39,7 @@ export interface Env {
BISQ_WEBSITE_URL: string;
MINING_DASHBOARD: boolean;
LIGHTNING: boolean;
AUDIT: boolean;
MAINNET_BLOCK_AUDIT_START_HEIGHT: number;
TESTNET_BLOCK_AUDIT_START_HEIGHT: number;
SIGNET_BLOCK_AUDIT_START_HEIGHT: number;
@ -67,6 +68,7 @@ const defaultEnv: Env = {
'BISQ_WEBSITE_URL': 'https://bisq.markets',
'MINING_DASHBOARD': true,
'LIGHTNING': false,
'AUDIT': false,
'MAINNET_BLOCK_AUDIT_START_HEIGHT': 0,
'TESTNET_BLOCK_AUDIT_START_HEIGHT': 0,
'SIGNET_BLOCK_AUDIT_START_HEIGHT': 0,

1
production/mempool-config.mainnet.json
View File

@ -10,6 +10,7 @@
"POLL_RATE_MS": 1000,
"INDEXING_BLOCKS_AMOUNT": -1,
"BLOCKS_SUMMARIES_INDEXING": true,
"AUDIT": true,
"ADVANCED_GBT_AUDIT": true,
"ADVANCED_GBT_MEMPOOL": false,
"USE_SECOND_NODE_FOR_MINFEE": true

1
production/mempool-config.signet.json
View File

@ -7,6 +7,7 @@
"SPAWN_CLUSTER_PROCS": 0,
"API_URL_PREFIX": "/api/v1/",
"INDEXING_BLOCKS_AMOUNT": -1,
"AUDIT": true,
"ADVANCED_GBT_AUDIT": true,
"ADVANCED_GBT_MEMPOOL": false,
"POLL_RATE_MS": 1000

1
production/mempool-config.testnet.json
View File

@ -7,6 +7,7 @@
"SPAWN_CLUSTER_PROCS": 0,
"API_URL_PREFIX": "/api/v1/",
"INDEXING_BLOCKS_AMOUNT": -1,
"AUDIT": true,
"ADVANCED_GBT_AUDIT": true,
"ADVANCED_GBT_MEMPOOL": false,
"POLL_RATE_MS": 1000

3
production/mempool-frontend-config.mainnet.json
View File

@ -10,5 +10,6 @@
"LIQUID_WEBSITE_URL": "https://liquid.network",
"BISQ_WEBSITE_URL": "https://bisq.markets",
"ITEMS_PER_PAGE": 25,
"LIGHTNING": true
"LIGHTNING": true,
"AUDIT": true
}