machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into mononaut/fix-mined-rbf-conflicts

Browse Source
This commit is contained in:
wiz 2023-07-12 16:17:47 +09:00 committed by GitHub
commit f871300bfb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 144 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
backend/src/api/audit.ts
View File

@ -1,19 +1,21 @@
import config from '../config';
import logger from '../logger';
import { TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
import rbfCache from './rbf-cache';
const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
class Audit {
auditBlock(transactions: TransactionExtended[], projectedBlocks: MempoolBlockWithTransactions[], mempool: { [txId: string]: TransactionExtended })
: { censored: string[], added: string[], fresh: string[], sigop: string[], score: number, similarity: number } {
: { censored: string[], added: string[], fresh: string[], sigop: string[], fullrbf: string[], score: number, similarity: number } {
if (!projectedBlocks?.[0]?.transactionIds || !mempool) {
return { censored: [], added: [], fresh: [], sigop: [], score: 0, similarity: 1 };
return { censored: [], added: [], fresh: [], sigop: [], fullrbf: [], score: 0, similarity: 1 };
}
const matches: string[] = []; // present in both mined block and template
const added: string[] = []; // present in mined block, not in template
const fresh: string[] = []; // missing, but firstSeen within PROPAGATION_MARGIN
const fullrbf: string[] = []; // either missing or present, and part of a fullrbf replacement
const isCensored = {}; // missing, without excuse
const isDisplaced = {};
let displacedWeight = 0;
@ -35,7 +37,9 @@ class Audit {
for (const txid of projectedBlocks[0].transactionIds) {
if (!inBlock[txid]) {
// tx is recent, may have reached the miner too late for inclusion
if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) {
if (rbfCache.isFullRbf(txid)) {
fullrbf.push(txid);
} else if (mempool[txid]?.firstSeen != null && (now - (mempool[txid]?.firstSeen || 0)) <= PROPAGATION_MARGIN) {
fresh.push(txid);
} else {
isCensored[txid] = true;
@ -91,7 +95,9 @@ class Audit {
if (inTemplate[tx.txid]) {
matches.push(tx.txid);
} else {
if (!isDisplaced[tx.txid]) {
if (rbfCache.isFullRbf(tx.txid)) {
fullrbf.push(tx.txid);
} else if (!isDisplaced[tx.txid]) {
added.push(tx.txid);
}
overflowWeight += tx.weight;
@ -138,6 +144,7 @@ class Audit {
added,
fresh,
sigop: [],
fullrbf,
score,
similarity,
};

1
backend/src/api/bitcoin/bitcoin-api.ts
View File

@ -29,6 +29,7 @@ class BitcoinApi implements AbstractBitcoinApi {
weight: block.weight,
previousblockhash: block.previousblockhash,
mediantime: block.mediantime,
stale: block.confirmations === -1,
};
}

1
backend/src/api/bitcoin/esplora-api.interface.ts
View File

@ -89,6 +89,7 @@ export namespace IEsploraApi {
weight: number;
previousblockhash: string;
mediantime: number;
stale: boolean;
}
export interface Address {

26
backend/src/api/blocks.ts
View File

@ -656,10 +656,6 @@ class Blocks {
const blockSummary: BlockSummary = this.summarizeBlockTransactions(block.id, cpfpSummary.transactions);
this.updateTimerProgress(timer, `got block data for ${this.currentBlockHeight}`);
// start async callbacks
this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`);
const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions));
if (Common.indexingEnabled()) {
if (!fastForwarded) {
const lastBlock = await blocksRepository.$getBlockByHeight(blockExtended.height - 1);
@ -671,9 +667,11 @@ class Blocks {
await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10);
await HashratesRepository.$deleteLastEntries();
await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10);
this.blocks = this.blocks.slice(0, -10);
this.updateTimerProgress(timer, `rolled back chain divergence from ${this.currentBlockHeight}`);
for (let i = 10; i >= 0; --i) {
const newBlock = await this.$indexBlock(lastBlock.height - i);
this.blocks.push(newBlock);
this.updateTimerProgress(timer, `reindexed block`);
let cpfpSummary;
if (config.MEMPOOL.CPFP_INDEXING) {
@ -722,6 +720,10 @@ class Blocks {
}
}
// start async callbacks
this.updateTimerProgress(timer, `starting async callbacks for ${this.currentBlockHeight}`);
const callbackPromises = this.newAsyncBlockCallbacks.map((cb) => cb(blockExtended, txIds, transactions));
if (block.height % 2016 === 0) {
if (Common.indexingEnabled()) {
await DifficultyAdjustmentsRepository.$saveAdjustments({
@ -814,6 +816,16 @@ class Blocks {
return blockExtended;
}
public async $indexStaleBlock(hash: string): Promise<BlockExtended> {
const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash);
const transactions = await this.$getTransactionsExtended(hash, block.height, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
blockExtended.canonical = await bitcoinApi.$getBlockHash(block.height);
return blockExtended;
}
/**
* Get one block by its hash
*/
@ -831,7 +843,11 @@ class Blocks {
// Bitcoin network, add our custom data on top
const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash);
return await this.$indexBlock(block.height);
if (block.stale) {
return await this.$indexStaleBlock(hash);
} else {
return await this.$indexBlock(block.height);
}
}
public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false,

6
backend/src/api/database-migration.ts
View File

@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
import { RowDataPacket } from 'mysql2';
class DatabaseMigration {
private static currentVersion = 62;
private static currentVersion = 63;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@ -539,6 +539,10 @@ class DatabaseMigration {
await this.updateToSchemaVersion(62);
}
if (databaseSchemaVersion < 63 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD fullrbf_txs JSON DEFAULT "[]"');
await this.updateToSchemaVersion(63);
}
}
/**

13
backend/src/api/rbf-cache.ts
View File

@ -169,6 +169,19 @@ class RbfCache {
}
}
// is the transaction involved in a full rbf replacement?
public isFullRbf(txid: string): boolean {
const treeId = this.treeMap.get(txid);
if (!treeId) {
return false;
}
const tree = this.rbfTrees.get(treeId);
if (!tree) {
return false;
}
return tree?.fullRbf;
}
private cleanup(): void {
const now = Date.now();
for (const txid of this.expiring.keys()) {

11
backend/src/api/websocket-handler.ts
View File

@ -583,6 +583,10 @@ class WebsocketHandler {
const _memPool = memPool.getMempool();
const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
memPool.handleMinedRbfTransactions(rbfTransactions);
memPool.removeFromSpendMap(transactions);
if (config.MEMPOOL.AUDIT) {
let projectedBlocks;
let auditMempool = _memPool;
@ -605,7 +609,7 @@ class WebsocketHandler {
}
if (Common.indexingEnabled() && memPool.isInSync()) {
const { censored, added, fresh, sigop, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
const { censored, added, fresh, sigop, fullrbf, score, similarity } = Audit.auditBlock(transactions, projectedBlocks, auditMempool);
const matchRate = Math.round(score * 100 * 100) / 100;
const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions : [];
@ -633,6 +637,7 @@ class WebsocketHandler {
missingTxs: censored,
freshTxs: fresh,
sigopTxs: sigop,
fullrbfTxs: fullrbf,
matchRate: matchRate,
expectedFees: totalFees,
expectedWeight: totalWeight,
@ -652,10 +657,6 @@ class WebsocketHandler {
}
}
const rbfTransactions = Common.findMinedRbfTransactions(transactions, memPool.getSpendMap());
memPool.handleMinedRbfTransactions(rbfTransactions);
memPool.removeFromSpendMap(transactions);
// Update mempool to remove transactions included in the new block
for (const txId of txIds) {
delete _memPool[txId];

3
backend/src/indexer.ts
View File

@ -6,6 +6,7 @@ import logger from './logger';
import bitcoinClient from './api/bitcoin/bitcoin-client';
import priceUpdater from './tasks/price-updater';
import PricesRepository from './repositories/PricesRepository';
import config from './config';
export interface CoreIndex {
name: string;
@ -72,7 +73,7 @@ class Indexer {
return;
}
if (task === 'blocksPrices' && !this.tasksRunning.includes(task)) {
if (task === 'blocksPrices' && !this.tasksRunning.includes(task) && !['testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
this.tasksRunning.push(task);
const lastestPriceId = await PricesRepository.$getLatestPriceId();
if (priceUpdater.historyInserted === false || lastestPriceId === null) {

2
backend/src/mempool.interfaces.ts
View File

@ -34,6 +34,7 @@ export interface BlockAudit {
missingTxs: string[],
freshTxs: string[],
sigopTxs: string[],
fullrbfTxs: string[],
addedTxs: string[],
matchRate: number,
expectedFees?: number,
@ -227,6 +228,7 @@ export interface BlockExtension {
*/
export interface BlockExtended extends IEsploraApi.Block {
extras: BlockExtension;
canonical?: string;
}
export interface BlockSummary {

12
backend/src/repositories/BlocksAuditsRepository.ts
View File

@ -6,9 +6,9 @@ import { BlockAudit, AuditScore } from '../mempool.interfaces';
class BlocksAuditRepositories {
public async $saveAudit(audit: BlockAudit): Promise<void> {
try {
await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, match_rate, expected_fees, expected_weight)
VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]);
await DB.query(`INSERT INTO blocks_audits(time, height, hash, missing_txs, added_txs, fresh_txs, sigop_txs, fullrbf_txs, match_rate, expected_fees, expected_weight)
VALUE (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, [audit.time, audit.height, audit.hash, JSON.stringify(audit.missingTxs),
JSON.stringify(audit.addedTxs), JSON.stringify(audit.freshTxs), JSON.stringify(audit.sigopTxs), JSON.stringify(audit.fullrbfTxs), audit.matchRate, audit.expectedFees, audit.expectedWeight]);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`Cannot save block audit for block ${audit.hash} because it has already been indexed, ignoring`);
@ -62,18 +62,17 @@ class BlocksAuditRepositories {
public async $getBlockAudit(hash: string): Promise<any> {
try {
const [rows]: any[] = await DB.query(
`SELECT blocks.height, blocks.hash as id, UNIX_TIMESTAMP(blocks.blockTimestamp) as timestamp, blocks.size,
blocks.weight, blocks.tx_count,
`SELECT blocks_audits.height, blocks_audits.hash as id, UNIX_TIMESTAMP(blocks_audits.time) as timestamp,
template,
missing_txs as missingTxs,
added_txs as addedTxs,
fresh_txs as freshTxs,
sigop_txs as sigopTxs,
fullrbf_txs as fullrbfTxs,
match_rate as matchRate,
expected_fees as expectedFees,
expected_weight as expectedWeight
FROM blocks_audits
JOIN blocks ON blocks.hash = blocks_audits.hash
JOIN blocks_templates ON blocks_templates.id = blocks_audits.hash
WHERE blocks_audits.hash = "${hash}"
`);
@ -83,6 +82,7 @@ class BlocksAuditRepositories {
rows[0].addedTxs = JSON.parse(rows[0].addedTxs);
rows[0].freshTxs = JSON.parse(rows[0].freshTxs);
rows[0].sigopTxs = JSON.parse(rows[0].sigopTxs);
rows[0].fullrbfTxs = JSON.parse(rows[0].fullrbfTxs);
rows[0].template = JSON.parse(rows[0].template);
return rows[0];

8
backend/src/tasks/lightning/network-sync.service.ts
View File

@ -269,7 +269,11 @@ class NetworkSyncService {
}
private async $scanForClosedChannels(): Promise<void> {
if (this.closedChannelsScanBlock === blocks.getCurrentBlockHeight()) {
let currentBlockHeight = blocks.getCurrentBlockHeight();
if (config.MEMPOOL.ENABLED === false) { // https://github.com/mempool/mempool/issues/3582
currentBlockHeight = await bitcoinApi.$getBlockHeightTip();
}
if (this.closedChannelsScanBlock === currentBlockHeight) {
logger.debug(`We've already scan closed channels for this block, skipping.`);
return;
}
@ -305,7 +309,7 @@ class NetworkSyncService {
}
}
this.closedChannelsScanBlock = blocks.getCurrentBlockHeight();
this.closedChannelsScanBlock = currentBlockHeight;
logger.debug(`Closed channels scan completed at block ${this.closedChannelsScanBlock}`, logger.tags.ln);
} catch (e) {
logger.err(`$scanForClosedChannels() error: ${e instanceof Error ? e.message : e}`, logger.tags.ln);

3
contributors/pfoytik.txt Normal file
View File

@ -0,0 +1,3 @@
I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of June 15, 2023.
Signed pfoytik

3
contributors/secondl1ght.txt Normal file
View File

@ -0,0 +1,3 @@
I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of June 14, 2023.
Signed: secondl1ght

4
docker/README.md
View File

@ -144,8 +144,8 @@ Corresponding `docker-compose.yml` overrides:
MEMPOOL_ADVANCED_GBT_AUDIT: ""
MEMPOOL_ADVANCED_GBT_MEMPOOL: ""
MEMPOOL_CPFP_INDEXING: ""
MAX_BLOCKS_BULK_QUERY: ""
DISK_CACHE_BLOCK_INTERVAL: ""
MEMPOOL_MAX_BLOCKS_BULK_QUERY: ""
MEMPOOL_DISK_CACHE_BLOCK_INTERVAL: ""
...
```

2
frontend/README.md
View File

@ -17,7 +17,7 @@ Get the latest Mempool code:
```
git clone https://github.com/mempool/mempool
cd mempool
cd mempool/frontend
```
### 2. Specify Website

3
frontend/src/app/components/block-overview-graph/tx-view.ts
View File

@ -37,7 +37,7 @@ export default class TxView implements TransactionStripped {
value: number;
feerate: number;
rate?: number;
status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected';
status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf';
context?: 'projected' | 'actual';
scene?: BlockScene;
@ -172,6 +172,7 @@ export default class TxView implements TransactionStripped {
return auditColors.censored;
case 'missing':
case 'sigop':
case 'fullrbf':
return marginalFeeColors[feeLevelIndex] || marginalFeeColors[mempoolFeeColors.length - 1];
case 'fresh':
return auditColors.missing;

1
frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html
View File

@ -52,6 +52,7 @@
<td *ngSwitchCase="'fresh'"><span class="badge badge-warning" i18n="transaction.audit.recently-broadcasted">Recently broadcasted</span></td>
<td *ngSwitchCase="'added'"><span class="badge badge-warning" i18n="transaction.audit.added">Added</span></td>
<td *ngSwitchCase="'selected'"><span class="badge badge-warning" i18n="transaction.audit.marginal">Marginal fee rate</span></td>
<td *ngSwitchCase="'fullrbf'"><span class="badge badge-warning" i18n="transaction.audit.fullrbf">Full RBF</span></td>
</ng-container>
</tr>
</tbody>

6
frontend/src/app/components/block/block.component.html
View File

@ -1,6 +1,10 @@
<div class="container-xl" (window:resize)="onResize($event)">
<div class="title-block" [class.time-ltr]="timeLtr" id="block">
<div *ngIf="block?.stale" class="alert alert-mempool" role="alert">
<span i18n="block.reorged|Block reorg" class="alert-text">This block does not belong to the main chain, it has been replaced by:</span>
<app-truncate [text]="block.canonical" [lastChars]="12" [link]="['/block/' | relativeUrl, block.canonical]" [maxWidth]="480"></app-truncate>
</div>
<h1>
<ng-container *ngIf="blockHeight == null || blockHeight > 0; else genesis" i18n="shared.block-title">Block</ng-container>
<ng-template #genesis i18n="@@2303359202781425764">Genesis</ng-template>
@ -23,6 +27,8 @@
<div class="grow"></div>
<button *ngIf="block?.stale" type="button" class="btn btn-sm btn-danger container-button" i18n="block.stale|Stale block state">Stale</button>
<button [routerLink]="['/' | relativeUrl]" class="btn btn-sm">&#10005;</button>
</div>

23
frontend/src/app/components/block/block.component.scss
View File

@ -1,3 +1,26 @@
.title-block {
flex-wrap: wrap;
align-items: baseline;
@media (min-width: 650px) {
flex-direction: row;
}
h1 {
margin: 0rem;
margin-right: 15px;
line-height: 1;
}
.alert-mempool {
flex-direction: row;
flex-wrap: wrap;
}
.container-button {
align-self: center;
margin-right: 1em;
}
}
.qr-wrapper {
background-color: #FFF;
padding: 10px;

16
frontend/src/app/components/block/block.component.ts
View File

@ -317,6 +317,7 @@ export class BlockComponent implements OnInit, OnDestroy {
const isSelected = {};
const isFresh = {};
const isSigop = {};
const isFullRbf = {};
this.numMissing = 0;
this.numUnexpected = 0;
@ -339,6 +340,9 @@ export class BlockComponent implements OnInit, OnDestroy {
for (const txid of blockAudit.sigopTxs || []) {
isSigop[txid] = true;
}
for (const txid of blockAudit.fullrbfTxs || []) {
isFullRbf[txid] = true;
}
// set transaction statuses
for (const tx of blockAudit.template) {
tx.context = 'projected';
@ -347,7 +351,15 @@ export class BlockComponent implements OnInit, OnDestroy {
} else if (inBlock[tx.txid]) {
tx.status = 'found';
} else {
tx.status = isFresh[tx.txid] ? 'fresh' : (isSigop[tx.txid] ? 'sigop' : 'missing');
if (isFresh[tx.txid]) {
tx.status = 'fresh';
} else if (isSigop[tx.txid]) {
tx.status = 'sigop';
} else if (isFullRbf[tx.txid]) {
tx.status = 'fullrbf';
} else {
tx.status = 'missing';
}
isMissing[tx.txid] = true;
this.numMissing++;
}
@ -360,6 +372,8 @@ export class BlockComponent implements OnInit, OnDestroy {
tx.status = 'added';
} else if (inTemplate[tx.txid]) {
tx.status = 'found';
} else if (isFullRbf[tx.txid]) {
tx.status = 'fullrbf';
} else {
tx.status = 'selected';
isSelected[tx.txid] = true;

2
frontend/src/app/components/change/change.component.html
View File

@ -1,3 +1,3 @@
<span [style]="change >= 0 ? 'color: #42B747' : 'color: #B74242'">
{{ change >= 0 ? '+' : '' }}{{ change | amountShortener }}%
&lrm;{{ change >= 0 ? '+' : '' }}{{ change | amountShortener }}%
</span>

2
frontend/src/app/interfaces/electrs.interface.ts
View File

@ -120,6 +120,8 @@ export interface Block {
size: number;
weight: number;
previousblockhash: string;
stale?: boolean;
canonical?: string;
}
export interface Address {

3
frontend/src/app/interfaces/node-api.interface.ts
View File

@ -156,6 +156,7 @@ export interface BlockAudit extends BlockExtended {
addedTxs: string[],
freshTxs: string[],
sigopTxs: string[],
fullrbfTxs: string[],
matchRate: number,
expectedFees: number,
expectedWeight: number,
@ -171,7 +172,7 @@ export interface TransactionStripped {
fee: number;
vsize: number;
value: number;
status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected';
status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf';
context?: 'projected' | 'actual';
}

2
frontend/src/app/interfaces/websocket.interface.ts
View File

@ -77,7 +77,7 @@ export interface TransactionStripped {
vsize: number;
value: number;
rate?: number; // effective fee rate
status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected';
status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'added' | 'censored' | 'selected' | 'fullrbf';
context?: 'projected' | 'actual';
}

4
frontend/src/styles.scss
View File

@ -993,6 +993,10 @@ th {
margin-right: 10px;
}
}
.btn-audit {
margin-left: .5em;
}
}
.scriptmessage {

2
production/install
View File

@ -332,7 +332,7 @@ BITCOIN_REPO_URL=https://github.com/bitcoin/bitcoin
BITCOIN_REPO_NAME=bitcoin
BITCOIN_REPO_BRANCH=master
#BITCOIN_LATEST_RELEASE=$(curl -s https://api.github.com/repos/bitcoin/bitcoin/releases/latest|grep tag_name|head -1|cut -d '"' -f4)
BITCOIN_LATEST_RELEASE=v23.0
BITCOIN_LATEST_RELEASE=v25.0
echo -n '.'
BISQ_REPO_URL=https://github.com/bisq-network/bisq

1
production/mempool-config.mainnet-lightning.json
View File

@ -1,5 +1,6 @@
{
"MEMPOOL": {
"ENABLED": false,
"NETWORK": "mainnet",
"BACKEND": "esplora",
"HTTP_PORT": 8993,

1
production/mempool-config.signet-lightning.json
View File

@ -1,5 +1,6 @@
{
"MEMPOOL": {
"ENABLED": false,
"NETWORK": "signet",
"BACKEND": "esplora",
"HTTP_PORT": 8991,

1
production/mempool-config.testnet-lightning.json
View File

@ -1,5 +1,6 @@
{
"MEMPOOL": {
"ENABLED": false,
"NETWORK": "testnet",
"BACKEND": "esplora",
"HTTP_PORT": 8992,