machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve prioritized transaction detection algorithm

Browse Source
This commit is contained in:
Mononaut 2024-08-17 00:14:33 +00:00
parent db10ab9aae
commit 248cef7718
No known key found for this signature in database
GPG Key ID: A3F058E41374C04E
8 changed files with 201 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
backend/src/api/audit.ts
View File

@ -2,6 +2,7 @@ import config from '../config';
import logger from '../logger';
import { MempoolTransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
import rbfCache from './rbf-cache';
import transactionUtils from './transaction-utils';
const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
@ -15,7 +16,8 @@ class Audit {
const matches: string[] = []; // present in both mined block and template
const added: string[] = []; // present in mined block, not in template
const unseen: string[] = []; // present in the mined block, not in our mempool
const prioritized: string[] = []; // higher in the block than would be expected by in-band feerate alone
let prioritized: string[] = []; // higher in the block than would be expected by in-band feerate alone
let deprioritized: string[] = []; // lower in the block than would be expected by in-band feerate alone
const fresh: string[] = []; // missing, but firstSeen or lastBoosted within PROPAGATION_MARGIN
const rbf: string[] = []; // either missing or present, and either part of a full-rbf replacement, or a conflict with the mined block
const accelerated: string[] = []; // prioritized by the mempool accelerator
@ -133,23 +135,7 @@ class Audit {
totalWeight += tx.weight;
}
// identify "prioritized" transactions
let lastEffectiveRate = 0;
// Iterate over the mined template from bottom to top (excluding the coinbase)
// Transactions should appear in ascending order of mining priority.
for (let i = transactions.length - 1; i > 0; i--) {
const blockTx = transactions[i];
// If a tx has a lower in-band effective fee rate than the previous tx,
// it must have been prioritized out-of-band (in order to have a higher mining priority)
// so exclude from the analysis.
if ((blockTx.effectiveFeePerVsize || 0) < lastEffectiveRate) {
prioritized.push(blockTx.txid);
// accelerated txs may or may not have their prioritized fee rate applied, so don't use them as a reference
} else if (!isAccelerated[blockTx.txid]) {
lastEffectiveRate = blockTx.effectiveFeePerVsize || 0;
}
}
({ prioritized, deprioritized } = transactionUtils.identifyPrioritizedTransactions(transactions, 'effectiveFeePerVsize'));
// transactions missing from near the end of our template are probably not being censored
let overflowWeightRemaining = overflowWeight - (config.MEMPOOL.BLOCK_WEIGHT_UNITS - totalWeight);

81
backend/src/api/transaction-utils.ts
View File

@ -338,6 +338,87 @@ class TransactionUtils {
const positionOfScript = hasAnnex ? witness.length - 3 : witness.length - 2;
return witness[positionOfScript];
}
// calculate the most parsimonious set of prioritizations given a list of block transactions
// (i.e. the most likely prioritizations and deprioritizations)
public identifyPrioritizedTransactions(transactions: any[], rateKey: string): { prioritized: string[], deprioritized: string[] } {
// find the longest increasing subsequence of transactions
// (adapted from https://en.wikipedia.org/wiki/Longest_increasing_subsequence#Efficient_algorithms)
// should be O(n log n)
const X = transactions.slice(1).reverse().map((tx) => ({ txid: tx.txid, rate: tx[rateKey] })); // standard block order is by *decreasing* effective fee rate, but we want to iterate in increasing order (and skip the coinbase)
if (X.length < 2) {
return { prioritized: [], deprioritized: [] };
}
const N = X.length;
const P: number[] = new Array(N);
const M: number[] = new Array(N + 1);
M[0] = -1; // undefined so can be set to any value
let L = 0;
for (let i = 0; i < N; i++) {
// Binary search for the smallest positive l ≤ L
// such that X[M[l]].effectiveFeePerVsize > X[i].effectiveFeePerVsize
let lo = 1;
let hi = L + 1;
while (lo < hi) {
const mid = lo + Math.floor((hi - lo) / 2); // lo <= mid < hi
if (X[M[mid]].rate > X[i].rate) {
hi = mid;
} else { // if X[M[mid]].effectiveFeePerVsize < X[i].effectiveFeePerVsize
lo = mid + 1;
}
}
// After searching, lo == hi is 1 greater than the
// length of the longest prefix of X[i]
const newL = lo;
// The predecessor of X[i] is the last index of
// the subsequence of length newL-1
P[i] = M[newL - 1];
M[newL] = i;
if (newL > L) {
// If we found a subsequence longer than any we've
// found yet, update L
L = newL;
}
}
// Reconstruct the longest increasing subsequence
// It consists of the values of X at the L indices:
// ..., P[P[M[L]]], P[M[L]], M[L]
const LIS: any[] = new Array(L);
let k = M[L];
for (let j = L - 1; j >= 0; j--) {
LIS[j] = X[k];
k = P[k];
}
const lisMap = new Map<string, number>();
LIS.forEach((tx, index) => lisMap.set(tx.txid, index));
const prioritized: string[] = [];
const deprioritized: string[] = [];
let lastRate = X[0].rate;
for (const tx of X) {
if (lisMap.has(tx.txid)) {
lastRate = tx.rate;
} else {
if (Math.abs(tx.rate - lastRate) < 0.1) {
// skip if the rate is almost the same as the previous transaction
} else if (tx.rate <= lastRate) {
prioritized.push(tx.txid);
} else {
deprioritized.push(tx.txid);
}
}
}
return { prioritized, deprioritized };
}
}
export default new TransactionUtils();

2
frontend/src/app/components/block-overview-graph/tx-view.ts
View File

@ -33,7 +33,7 @@ export default class TxView implements TransactionStripped {
flags: number;
bigintFlags?: bigint | null = 0b00000100_00000000_00000000_00000000n;
time?: number;
status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'added_prioritized' | 'prioritized' | 'censored' | 'selected' | 'rbf' | 'accelerated';
status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'added_prioritized' | 'prioritized' | 'added_deprioritized' | 'deprioritized' | 'censored' | 'selected' | 'rbf' | 'accelerated';
context?: 'projected' | 'actual';
scene?: BlockScene;

4
frontend/src/app/components/block-overview-graph/utils.ts
View File

@ -142,6 +142,10 @@ export function defaultColorFunction(
return auditColors.added_prioritized;
case 'prioritized':
return auditColors.prioritized;
case 'added_deprioritized':
return auditColors.added_prioritized;
case 'deprioritized':
return auditColors.prioritized;
case 'selected':
return colors.marginal[levelIndex] || colors.marginal[defaultMempoolFeeColors.length - 1];
case 'accelerated':

5
frontend/src/app/components/block-overview-tooltip/block-overview-tooltip.component.html
View File

@ -79,6 +79,11 @@
<span class="badge badge-warning" i18n="tx-features.tag.added|Added">Added</span>
<span class="badge badge-warning ml-1" i18n="tx-features.tag.prioritized|Prioritized">Prioritized</span>
</ng-container>
<span *ngSwitchCase="'deprioritized'" class="badge badge-warning" i18n="tx-features.tag.prioritized|Deprioritized">Deprioritized</span>
<ng-container *ngSwitchCase="'added_deprioritized'">
<span class="badge badge-warning" i18n="tx-features.tag.added|Added">Added</span>
<span class="badge badge-warning ml-1" i18n="tx-features.tag.prioritized|Deprioritized">Deprioritized</span>
</ng-container>
<span *ngSwitchCase="'selected'" class="badge badge-warning" i18n="transaction.audit.marginal">Marginal fee rate</span>
<span *ngSwitchCase="'rbf'" class="badge badge-warning" i18n="tx-features.tag.conflict|Conflict">Conflict</span>
<span *ngSwitchCase="'accelerated'" class="badge badge-accelerated" i18n="transaction.audit.accelerated">Accelerated</span>

26
frontend/src/app/components/block/block.component.ts
View File

@ -17,6 +17,7 @@ import { PriceService, Price } from '../../services/price.service';
import { CacheService } from '../../services/cache.service';
import { ServicesApiServices } from '../../services/services-api.service';
import { PreloadService } from '../../services/preload.service';
import { identifyPrioritizedTransactions } from '../../shared/transaction.utils';
@Component({
selector: 'app-block',
@ -524,6 +525,7 @@ export class BlockComponent implements OnInit, OnDestroy {
const isUnseen = {};
const isAdded = {};
const isPrioritized = {};
const isDeprioritized = {};
const isCensored = {};
const isMissing = {};
const isSelected = {};
@ -535,6 +537,17 @@ export class BlockComponent implements OnInit, OnDestroy {
this.numUnexpected = 0;
if (blockAudit?.template) {
// augment with locally calculated *de*prioritized transactions if possible
const { prioritized, deprioritized } = identifyPrioritizedTransactions(transactions);
// but if the local calculation produces returns unexpected results, don't use it
let useLocalDeprioritized = deprioritized.length < (transactions.length * 0.1);
for (const tx of prioritized) {
if (!isPrioritized[tx] && !isAccelerated[tx]) {
useLocalDeprioritized = false;
break;
}
}
for (const tx of blockAudit.template) {
inTemplate[tx.txid] = true;
if (tx.acc) {
@ -550,9 +563,14 @@ export class BlockComponent implements OnInit, OnDestroy {
for (const txid of blockAudit.addedTxs) {
isAdded[txid] = true;
}
for (const txid of blockAudit.prioritizedTxs || []) {
for (const txid of blockAudit.prioritizedTxs) {
isPrioritized[txid] = true;
}
if (useLocalDeprioritized) {
for (const txid of deprioritized || []) {
isDeprioritized[txid] = true;
}
}
for (const txid of blockAudit.missingTxs) {
isCensored[txid] = true;
}
@ -608,6 +626,12 @@ export class BlockComponent implements OnInit, OnDestroy {
} else {
tx.status = 'prioritized';
}
} else if (isDeprioritized[tx.txid]) {
if (isAdded[tx.txid] || (blockAudit.version > 0 && isUnseen[tx.txid])) {
tx.status = 'added_deprioritized';
} else {
tx.status = 'deprioritized';
}
} else if (isAdded[tx.txid] && (blockAudit.version === 0 || isUnseen[tx.txid])) {
tx.status = 'added';
} else if (inTemplate[tx.txid]) {

2
frontend/src/app/interfaces/node-api.interface.ts
View File

@ -239,7 +239,7 @@ export interface TransactionStripped {
acc?: boolean;
flags?: number | null;
time?: number;
status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'added_prioritized' | 'prioritized' | 'censored' | 'selected' | 'rbf' | 'accelerated';
status?: 'found' | 'missing' | 'sigop' | 'fresh' | 'freshcpfp' | 'added' | 'added_prioritized' | 'prioritized' | 'added_deprioritized' | 'deprioritized' | 'censored' | 'selected' | 'rbf' | 'accelerated';
context?: 'projected' | 'actual';
}

81
frontend/src/app/shared/transaction.utils.ts
View File

@ -1,7 +1,7 @@
import { TransactionFlags } from './filters.utils';
import { getVarIntLength, opcodes, parseMultisigScript, isPoint } from './script.utils';
import { Transaction } from '../interfaces/electrs.interface';
import { CpfpInfo, RbfInfo } from '../interfaces/node-api.interface';
import { CpfpInfo, RbfInfo, TransactionStripped } from '../interfaces/node-api.interface';
// Bitcoin Core default policy settings
const TX_MAX_STANDARD_VERSION = 2;
@ -458,4 +458,83 @@ export function getUnacceleratedFeeRate(tx: Transaction, accelerated: boolean):
} else {
return tx.effectiveFeePerVsize;
}
}
export function identifyPrioritizedTransactions(transactions: TransactionStripped[]): { prioritized: string[], deprioritized: string[] } {
// find the longest increasing subsequence of transactions
// (adapted from https://en.wikipedia.org/wiki/Longest_increasing_subsequence#Efficient_algorithms)
// should be O(n log n)
const X = transactions.slice(1).reverse(); // standard block order is by *decreasing* effective fee rate, but we want to iterate in increasing order (and skip the coinbase)
if (X.length < 2) {
return { prioritized: [], deprioritized: [] };
}
const N = X.length;
const P: number[] = new Array(N);
const M: number[] = new Array(N + 1);
M[0] = -1; // undefined so can be set to any value
let L = 0;
for (let i = 0; i < N; i++) {
// Binary search for the smallest positive l ≤ L
// such that X[M[l]].effectiveFeePerVsize > X[i].effectiveFeePerVsize
let lo = 1;
let hi = L + 1;
while (lo < hi) {
const mid = lo + Math.floor((hi - lo) / 2); // lo <= mid < hi
if (X[M[mid]].rate > X[i].rate) {
hi = mid;
} else { // if X[M[mid]].effectiveFeePerVsize < X[i].effectiveFeePerVsize
lo = mid + 1;
}
}
// After searching, lo == hi is 1 greater than the
// length of the longest prefix of X[i]
const newL = lo;
// The predecessor of X[i] is the last index of
// the subsequence of length newL-1
P[i] = M[newL - 1];
M[newL] = i;
if (newL > L) {
// If we found a subsequence longer than any we've
// found yet, update L
L = newL;
}
}
// Reconstruct the longest increasing subsequence
// It consists of the values of X at the L indices:
// ..., P[P[M[L]]], P[M[L]], M[L]
const LIS: TransactionStripped[] = new Array(L);
let k = M[L];
for (let j = L - 1; j >= 0; j--) {
LIS[j] = X[k];
k = P[k];
}
const lisMap = new Map<string, number>();
LIS.forEach((tx, index) => lisMap.set(tx.txid, index));
const prioritized: string[] = [];
const deprioritized: string[] = [];
let lastRate = 0;
for (const tx of X) {
if (lisMap.has(tx.txid)) {
lastRate = tx.rate;
} else {
if (Math.abs(tx.rate - lastRate) < 0.1) {
// skip if the rate is almost the same as the previous transaction
} else if (tx.rate <= lastRate) {
prioritized.push(tx.txid);
} else {
deprioritized.push(tx.txid);
}
}
}
return { prioritized, deprioritized };
}