machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into natsee/liquid-federation-audit

Browse Source
This commit is contained in:
natsee
2024-01-25 15:17:51 +01:00
committed by GitHub
parent cef060be2d e81d1e273a
commit b6d2008e97
83 changed files with 23197 additions and 9944 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
backend/src/__fixtures__/mempool-config.template.json
View File

@@ -34,7 +34,8 @@
"DISK_CACHE_BLOCK_INTERVAL": 999,
"MAX_PUSH_TX_SIZE_WEIGHT": 4000000,
"ALLOW_UNREACHABLE": true,
"PRICE_UPDATES_PER_HOUR": 1
"PRICE_UPDATES_PER_HOUR": 1,
"MAX_TRACKED_ADDRESSES": 1
},
"CORE_RPC": {
"HOST": "__CORE_RPC_HOST__",

1
backend/src/__tests__/config.test.ts
View File

@@ -48,6 +48,7 @@ describe('Mempool Backend Config', () => {
MAX_PUSH_TX_SIZE_WEIGHT: 400000,
ALLOW_UNREACHABLE: true,
PRICE_UPDATES_PER_HOUR: 1,
MAX_TRACKED_ADDRESSES: 1,
});
expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });

1
backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts
View File

@@ -1,3 +1,4 @@
import { IBitcoinApi } from './bitcoin-api.interface';
import { IEsploraApi } from './esplora-api.interface';
export interface AbstractBitcoinApi {

136
backend/src/api/blocks.ts
View File

@@ -2,7 +2,7 @@ import config from '../config';
import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
import logger from '../logger';
import memPool from './mempool';
import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo, CpfpSummary, MempoolTransactionExtended } from '../mempool.interfaces';
import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionMinerInfo, CpfpSummary, MempoolTransactionExtended, TransactionClassified } from '../mempool.interfaces';
import { Common } from './common';
import diskCache from './disk-cache';
import transactionUtils from './transaction-utils';
@@ -201,7 +201,8 @@ class Blocks {
txid: tx.txid,
vsize: tx.weight / 4,
fee: tx.fee ? Math.round(tx.fee * 100000000) : 0,
value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0) * 100000000)
value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0) * 100000000),
flags: 0,
};
});
@@ -214,7 +215,7 @@ class Blocks {
public summarizeBlockTransactions(hash: string, transactions: TransactionExtended[]): BlockSummary {
return {
id: hash,
transactions: Common.stripTransactions(transactions),
transactions: Common.classifyTransactions(transactions),
};
}
@@ -560,6 +561,121 @@ class Blocks {
logger.debug(`Indexing block audit details completed`);
}
/**
* [INDEXING] Index transaction classification flags for Goggles
*/
public async $classifyBlocks(): Promise<void> {
// classification requires an esplora backend
if (!Common.blocksSummariesIndexingEnabled() || config.MEMPOOL.BACKEND !== 'esplora') {
return;
}
const blockchainInfo = await bitcoinClient.getBlockchainInfo();
const currentBlockHeight = blockchainInfo.blocks;
const unclassifiedBlocksList = await BlocksSummariesRepository.$getSummariesWithVersion(0);
const unclassifiedTemplatesList = await BlocksSummariesRepository.$getTemplatesWithVersion(0);
// nothing to do
if (!unclassifiedBlocksList?.length && !unclassifiedTemplatesList?.length) {
return;
}
let timer = Date.now();
let indexedThisRun = 0;
let indexedTotal = 0;
const minHeight = Math.min(
unclassifiedBlocksList[unclassifiedBlocksList.length - 1]?.height ?? Infinity,
unclassifiedTemplatesList[unclassifiedTemplatesList.length - 1]?.height ?? Infinity,
);
const numToIndex = Math.max(
unclassifiedBlocksList.length,
unclassifiedTemplatesList.length,
);
const unclassifiedBlocks = {};
const unclassifiedTemplates = {};
for (const block of unclassifiedBlocksList) {
unclassifiedBlocks[block.height] = block.id;
}
for (const template of unclassifiedTemplatesList) {
unclassifiedTemplates[template.height] = template.id;
}
logger.debug(`Classifying blocks and templates from #${currentBlockHeight} to #${minHeight}`, logger.tags.goggles);
for (let height = currentBlockHeight; height >= 0; height--) {
try {
let txs: TransactionExtended[] | null = null;
if (unclassifiedBlocks[height]) {
const blockHash = unclassifiedBlocks[height];
// fetch transactions
txs = (await bitcoinApi.$getTxsForBlock(blockHash)).map(tx => transactionUtils.extendTransaction(tx)) || [];
// add CPFP
const cpfpSummary = Common.calculateCpfp(height, txs, true);
// classify
const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions);
await BlocksSummariesRepository.$saveTransactions(height, blockHash, classifiedTxs, 1);
}
if (unclassifiedTemplates[height]) {
// classify template
const blockHash = unclassifiedTemplates[height];
const template = await BlocksSummariesRepository.$getTemplate(blockHash);
const alreadyClassified = template?.transactions?.reduce((classified, tx) => (classified || tx.flags > 0), false);
let classifiedTemplate = template?.transactions || [];
if (!alreadyClassified) {
const templateTxs: (TransactionExtended | TransactionClassified)[] = [];
const blockTxMap: { [txid: string]: TransactionExtended } = {};
for (const tx of (txs || [])) {
blockTxMap[tx.txid] = tx;
}
for (const templateTx of (template?.transactions || [])) {
let tx: TransactionExtended | null = blockTxMap[templateTx.txid];
if (!tx) {
try {
tx = await transactionUtils.$getTransactionExtended(templateTx.txid, false, true, false);
} catch (e) {
// transaction probably not found
}
}
templateTxs.push(tx || templateTx);
}
const cpfpSummary = Common.calculateCpfp(height, templateTxs?.filter(tx => tx['effectiveFeePerVsize'] != null) as TransactionExtended[], true);
// classify
const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions);
const classifiedTxMap: { [txid: string]: TransactionClassified } = {};
for (const tx of classifiedTxs) {
classifiedTxMap[tx.txid] = tx;
}
classifiedTemplate = classifiedTemplate.map(tx => {
if (classifiedTxMap[tx.txid]) {
tx.flags = classifiedTxMap[tx.txid].flags || 0;
}
return tx;
});
}
await BlocksSummariesRepository.$saveTemplate({ height, template: { id: blockHash, transactions: classifiedTemplate }, version: 1 });
}
} catch (e) {
logger.warn(`Failed to classify template or block summary at ${height}`, logger.tags.goggles);
}
// timing & logging
if (unclassifiedBlocks[height] || unclassifiedTemplates[height]) {
indexedThisRun++;
indexedTotal++;
}
const elapsedSeconds = (Date.now() - timer) / 1000;
if (elapsedSeconds > 5) {
const perSecond = indexedThisRun / elapsedSeconds;
logger.debug(`Classified #${height}: ${indexedTotal} / ${numToIndex} blocks (${perSecond.toFixed(1)}/s)`);
timer = Date.now();
indexedThisRun = 0;
}
}
}
/**
* [INDEXING] Index all blocks metadata for the mining dashboard
*/
@@ -945,7 +1061,7 @@ class Blocks {
}
public async $getStrippedBlockTransactions(hash: string, skipMemoryCache = false,
skipDBLookup = false, cpfpSummary?: CpfpSummary, blockHeight?: number): Promise<TransactionStripped[]>
skipDBLookup = false, cpfpSummary?: CpfpSummary, blockHeight?: number): Promise<TransactionClassified[]>
{
if (skipMemoryCache === false) {
// Check the memory cache
@@ -965,6 +1081,7 @@ class Blocks {
let height = blockHeight;
let summary: BlockSummary;
let summaryVersion = 0;
if (cpfpSummary && !Common.isLiquid()) {
summary = {
id: hash,
@@ -974,14 +1091,17 @@ class Blocks {
fee: tx.fee || 0,
vsize: tx.vsize,
value: Math.round(tx.vout.reduce((acc, vout) => acc + (vout.value ? vout.value : 0), 0)),
rate: tx.effectiveFeePerVsize
rate: tx.effectiveFeePerVsize,
flags: tx.flags || Common.getTransactionFlags(tx),
};
}),
};
summaryVersion = 1;
} else {
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx));
summary = this.summarizeBlockTransactions(hash, txs);
summaryVersion = 1;
} else {
// Call Core RPC
const block = await bitcoinClient.getBlock(hash, 2);
@@ -996,7 +1116,7 @@ class Blocks {
// Index the response if needed
if (Common.blocksSummariesIndexingEnabled() === true) {
await BlocksSummariesRepository.$saveTransactions(height, hash, summary.transactions);
await BlocksSummariesRepository.$saveTransactions(height, hash, summary.transactions, summaryVersion);
}
return summary.transactions;
@@ -1112,16 +1232,18 @@ class Blocks {
if (cleanBlock.fee_amt_percentiles === null) {
let summary;
let summaryVersion = 0;
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(cleanBlock.hash)).map(tx => transactionUtils.extendTransaction(tx));
summary = this.summarizeBlockTransactions(cleanBlock.hash, txs);
summaryVersion = 1;
} else {
// Call Core RPC
const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
summary = this.summarizeBlock(block);
}
await BlocksSummariesRepository.$saveTransactions(cleanBlock.height, cleanBlock.hash, summary.transactions);
await BlocksSummariesRepository.$saveTransactions(cleanBlock.height, cleanBlock.hash, summary.transactions, summaryVersion);
cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
}
if (cleanBlock.fee_amt_percentiles !== null) {

26
backend/src/api/common.ts
View File

@@ -1,10 +1,9 @@
import * as bitcoinjs from 'bitcoinjs-lib';
import { Request } from 'express';
import { Ancestor, CpfpInfo, CpfpSummary, CpfpCluster, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, MempoolTransactionExtended, TransactionStripped, WorkingEffectiveFeeStats, TransactionClassified, TransactionFlags } from '../mempool.interfaces';
import { CpfpInfo, CpfpSummary, CpfpCluster, EffectiveFeeStats, MempoolBlockWithTransactions, TransactionExtended, MempoolTransactionExtended, TransactionStripped, WorkingEffectiveFeeStats, TransactionClassified, TransactionFlags } from '../mempool.interfaces';
import config from '../config';
import { NodeSocket } from '../repositories/NodesSocketsRepository';
import { isIP } from 'net';
import rbfCache from './rbf-cache';
import transactionUtils from './transaction-utils';
import { isPoint } from '../utils/secp256k1';
export class Common {
@@ -349,14 +348,18 @@ export class Common {
}
static classifyTransaction(tx: TransactionExtended): TransactionClassified {
const flags = this.getTransactionFlags(tx);
const flags = Common.getTransactionFlags(tx);
tx.flags = flags;
return {
...this.stripTransaction(tx),
...Common.stripTransaction(tx),
flags,
};
}
static classifyTransactions(txs: TransactionExtended[]): TransactionClassified[] {
return txs.map(Common.classifyTransaction);
}
static stripTransaction(tx: TransactionExtended): TransactionStripped {
return {
txid: tx.txid,
@@ -369,7 +372,7 @@ export class Common {
}
static stripTransactions(txs: TransactionExtended[]): TransactionStripped[] {
return txs.map(this.stripTransaction);
return txs.map(Common.stripTransaction);
}
static sleep$(ms: number): Promise<void> {
@@ -632,12 +635,12 @@ export class Common {
}
}
static calculateCpfp(height: number, transactions: TransactionExtended[]): CpfpSummary {
static calculateCpfp(height: number, transactions: TransactionExtended[], saveRelatives: boolean = false): CpfpSummary {
const clusters: CpfpCluster[] = []; // list of all cpfp clusters in this block
const clusterMap: { [txid: string]: CpfpCluster } = {}; // map transactions to their cpfp cluster
let clusterTxs: TransactionExtended[] = []; // working list of elements of the current cluster
let ancestors: { [txid: string]: boolean } = {}; // working set of ancestors of the current cluster root
const txMap = {};
const txMap: { [txid: string]: TransactionExtended } = {};
// initialize the txMap
for (const tx of transactions) {
txMap[tx.txid] = tx;
@@ -707,6 +710,15 @@ export class Common {
}
}
}
if (saveRelatives) {
for (const cluster of clusters) {
cluster.txs.forEach((member, index) => {
txMap[member.txid].descendants = cluster.txs.slice(0, index).reverse();
txMap[member.txid].ancestors = cluster.txs.slice(index + 1).reverse();
txMap[member.txid].effectiveFeePerVsize = cluster.effectiveFeePerVsize;
});
}
}
return {
transactions,
clusters,

15
backend/src/api/database-migration.ts
View File

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
import { RowDataPacket } from 'mysql2';
class DatabaseMigration {
private static currentVersion = 67;
private static currentVersion = 68;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -559,8 +559,15 @@ class DatabaseMigration {
await this.updateToSchemaVersion(66);
}
if (databaseSchemaVersion < 67 && config.MEMPOOL.NETWORK === "liquid") {
// Drop and re-create the elements_pegs table
if (databaseSchemaVersion < 67 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks_summaries` ADD version INT NOT NULL DEFAULT 0');
await this.$executeQuery('ALTER TABLE `blocks_summaries` ADD INDEX `version` (`version`)');
await this.$executeQuery('ALTER TABLE `blocks_templates` ADD version INT NOT NULL DEFAULT 0');
await this.$executeQuery('ALTER TABLE `blocks_templates` ADD INDEX `version` (`version`)');
await this.updateToSchemaVersion(67);
}
if (databaseSchemaVersion < 68 && config.MEMPOOL.NETWORK === "liquid") {
await this.$executeQuery('TRUNCATE TABLE elements_pegs');
await this.$executeQuery('ALTER TABLE elements_pegs ADD PRIMARY KEY (txid, txindex);');
await this.$executeQuery(`UPDATE state SET number = 0 WHERE name = 'last_elements_block';`);
@@ -571,8 +578,6 @@ class DatabaseMigration {
// Create the federation_txos table that uses the federation_addresses table as a foreign key
await this.$executeQuery(this.getCreateFederationTxosTableQuery(), await this.$checkIfTableExists('federation_txos'));
await this.$executeQuery(`INSERT INTO state VALUES('last_bitcoin_block_audit', 0, NULL);`);
await this.updateToSchemaVersion(67);
}
}
/**

2
backend/src/api/mempool-blocks.ts
View File

@@ -1,6 +1,6 @@
import { GbtGenerator, GbtResult, ThreadTransaction as RustThreadTransaction, ThreadAcceleration as RustThreadAcceleration } from 'rust-gbt';
import logger from '../logger';
import { MempoolBlock, MempoolTransactionExtended, TransactionStripped, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, PoolTag, TransactionClassified } from '../mempool.interfaces';
import { MempoolBlock, MempoolTransactionExtended, MempoolBlockWithTransactions, MempoolBlockDelta, Ancestor, CompactThreadTransaction, EffectiveFeeStats, PoolTag, TransactionClassified } from '../mempool.interfaces';
import { Common, OnlineFeeStatsCalculator } from './common';
import config from '../config';
import { Worker } from 'worker_threads';

162
backend/src/api/websocket-handler.ts
View File

@@ -24,6 +24,12 @@ import { ApiPrice } from '../repositories/PricesRepository';
import accelerationApi from './services/acceleration';
import mempool from './mempool';
interface AddressTransactions {
mempool: MempoolTransactionExtended[],
confirmed: MempoolTransactionExtended[],
removed: MempoolTransactionExtended[],
}
// valid 'want' subscriptions
const wantable = [
'blocks',
@@ -195,24 +201,49 @@ class WebsocketHandler {
}
if (parsedMessage && parsedMessage['track-address']) {
if (/^([a-km-zA-HJ-NP-Z1-9]{26,35}|[a-km-zA-HJ-NP-Z1-9]{80}|[a-z]{2,5}1[ac-hj-np-z02-9]{8,100}|[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}|04[a-fA-F0-9]{128}|(02|03)[a-fA-F0-9]{64})$/
.test(parsedMessage['track-address'])) {
let matchedAddress = parsedMessage['track-address'];
if (/^[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}$/.test(parsedMessage['track-address'])) {
matchedAddress = matchedAddress.toLowerCase();
}
if (/^04[a-fA-F0-9]{128}$/.test(parsedMessage['track-address'])) {
client['track-address'] = '41' + matchedAddress + 'ac';
} else if (/^(02|03)[a-fA-F0-9]{64}$/.test(parsedMessage['track-address'])) {
client['track-address'] = '21' + matchedAddress + 'ac';
} else {
client['track-address'] = matchedAddress;
}
const validAddress = this.testAddress(parsedMessage['track-address']);
if (validAddress) {
client['track-address'] = validAddress;
} else {
client['track-address'] = null;
}
}
if (parsedMessage && parsedMessage['track-addresses'] && Array.isArray(parsedMessage['track-addresses'])) {
const addressMap: { [address: string]: string } = {};
for (const address of parsedMessage['track-addresses']) {
const validAddress = this.testAddress(address);
if (validAddress) {
addressMap[address] = validAddress;
}
}
if (Object.keys(addressMap).length > config.MEMPOOL.MAX_TRACKED_ADDRESSES) {
response['track-addresses-error'] = `"too many addresses requested, this connection supports tracking a maximum of ${config.MEMPOOL.MAX_TRACKED_ADDRESSES} addresses"`;
client['track-addresses'] = null;
} else if (Object.keys(addressMap).length > 0) {
client['track-addresses'] = addressMap;
} else {
client['track-addresses'] = null;
}
}
if (parsedMessage && parsedMessage['track-scriptpubkeys'] && Array.isArray(parsedMessage['track-scriptpubkeys'])) {
const spks: string[] = [];
for (const spk of parsedMessage['track-scriptpubkeys']) {
if (/^[a-fA-F0-9]+$/.test(spk)) {
spks.push(spk.toLowerCase());
}
}
if (spks.length > config.MEMPOOL.MAX_TRACKED_ADDRESSES) {
response['track-scriptpubkeys-error'] = `"too many scriptpubkeys requested, this connection supports tracking a maximum of ${config.MEMPOOL.MAX_TRACKED_ADDRESSES} scriptpubkeys"`;
client['track-scriptpubkeys'] = null;
} else if (spks.length) {
client['track-scriptpubkeys'] = spks;
} else {
client['track-scriptpubkeys'] = null;
}
}
if (parsedMessage && parsedMessage['track-asset']) {
if (/^[a-fA-F0-9]{64}$/.test(parsedMessage['track-asset'])) {
client['track-asset'] = parsedMessage['track-asset'];
@@ -544,6 +575,50 @@ class WebsocketHandler {
}
}
if (client['track-addresses']) {
const addressMap: { [address: string]: AddressTransactions } = {};
for (const [address, key] of Object.entries(client['track-addresses'] || {})) {
const newTransactions = Array.from(addressCache[key as string]?.values() || []);
const removedTransactions = Array.from(removedAddressCache[key as string]?.values() || []);
// txs may be missing prevouts in non-esplora backends
// so fetch the full transactions now
const fullTransactions = (config.MEMPOOL.BACKEND !== 'esplora') ? await this.getFullTransactions(newTransactions) : newTransactions;
if (fullTransactions?.length) {
addressMap[address] = {
mempool: fullTransactions,
confirmed: [],
removed: removedTransactions,
};
}
}
if (Object.keys(addressMap).length > 0) {
response['multi-address-transactions'] = JSON.stringify(addressMap);
}
}
if (client['track-scriptpubkeys']) {
const spkMap: { [spk: string]: AddressTransactions } = {};
for (const spk of client['track-scriptpubkeys'] || []) {
const newTransactions = Array.from(addressCache[spk as string]?.values() || []);
const removedTransactions = Array.from(removedAddressCache[spk as string]?.values() || []);
// txs may be missing prevouts in non-esplora backends
// so fetch the full transactions now
const fullTransactions = (config.MEMPOOL.BACKEND !== 'esplora') ? await this.getFullTransactions(newTransactions) : newTransactions;
if (fullTransactions?.length) {
spkMap[spk] = {
mempool: fullTransactions,
confirmed: [],
removed: removedTransactions,
};
}
}
if (Object.keys(spkMap).length > 0) {
response['multi-scriptpubkey-transactions'] = JSON.stringify(spkMap);
}
}
if (client['track-asset']) {
const foundTransactions: TransactionExtended[] = [];
@@ -703,7 +778,8 @@ class WebsocketHandler {
template: {
id: block.id,
transactions: stripped,
}
},
version: 1,
});
BlocksAuditsRepository.$saveAudit({
@@ -843,6 +919,42 @@ class WebsocketHandler {
}
}
if (client['track-addresses']) {
const addressMap: { [address: string]: AddressTransactions } = {};
for (const [address, key] of Object.entries(client['track-addresses'] || {})) {
const fullTransactions = Array.from(addressCache[key as string]?.values() || []);
if (fullTransactions?.length) {
addressMap[address] = {
mempool: [],
confirmed: fullTransactions,
removed: [],
};
}
}
if (Object.keys(addressMap).length > 0) {
response['multi-address-transactions'] = JSON.stringify(addressMap);
}
}
if (client['track-scriptpubkeys']) {
const spkMap: { [spk: string]: AddressTransactions } = {};
for (const spk of client['track-scriptpubkeys'] || []) {
const fullTransactions = Array.from(addressCache[spk as string]?.values() || []);
if (fullTransactions?.length) {
spkMap[spk] = {
mempool: [],
confirmed: fullTransactions,
removed: [],
};
}
}
if (Object.keys(spkMap).length > 0) {
response['multi-scriptpubkey-transactions'] = JSON.stringify(spkMap);
}
}
if (client['track-asset']) {
const foundTransactions: TransactionExtended[] = [];
@@ -912,6 +1024,28 @@ class WebsocketHandler {
+ '}';
}
// checks if an address conforms to a valid format
// returns the canonical form:
// - lowercase for bech32(m)
// - lowercase scriptpubkey for P2PK
// or false if invalid
private testAddress(address): string | false {
if (/^([a-km-zA-HJ-NP-Z1-9]{26,35}|[a-km-zA-HJ-NP-Z1-9]{80}|[a-z]{2,5}1[ac-hj-np-z02-9]{8,100}|[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}|04[a-fA-F0-9]{128}|(02|03)[a-fA-F0-9]{64})$/.test(address)) {
if (/^[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}|04[a-fA-F0-9]{128}|(02|03)[a-fA-F0-9]{64}$/.test(address)) {
address = address.toLowerCase();
}
if (/^04[a-fA-F0-9]{128}$/.test(address)) {
return '41' + address + 'ac';
} else if (/^(02|03)[a-fA-F0-9]{64}$/.test(address)) {
return '21' + address + 'ac';
} else {
return address;
}
} else {
return false;
}
}
private makeAddressCache(transactions: MempoolTransactionExtended[]): { [address: string]: Set<MempoolTransactionExtended> } {
const addressCache: { [address: string]: Set<MempoolTransactionExtended> } = {};
for (const tx of transactions) {

2
backend/src/config.ts
View File

@@ -39,6 +39,7 @@ interface IConfig {
MAX_PUSH_TX_SIZE_WEIGHT: number;
ALLOW_UNREACHABLE: boolean;
PRICE_UPDATES_PER_HOUR: number;
MAX_TRACKED_ADDRESSES: number;
};
ESPLORA: {
REST_API_URL: string;
@@ -193,6 +194,7 @@ const defaults: IConfig = {
'MAX_PUSH_TX_SIZE_WEIGHT': 400000,
'ALLOW_UNREACHABLE': true,
'PRICE_UPDATES_PER_HOUR': 1,
'MAX_TRACKED_ADDRESSES': 1,
},
'ESPLORA': {
'REST_API_URL': 'http://127.0.0.1:3000',

1
backend/src/indexer.ts
View File

@@ -185,6 +185,7 @@ class Indexer {
await blocks.$generateCPFPDatabase();
await blocks.$generateAuditStats();
await auditReplicator.$sync();
await blocks.$classifyBlocks();
} catch (e) {
this.indexerRunning = false;
logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e));

1
backend/src/logger.ts
View File

@@ -35,6 +35,7 @@ class Logger {
public tags = {
mining: 'Mining',
ln: 'Lightning',
goggles: 'Goggles',
};
// @ts-ignore

7
backend/src/mempool.interfaces.ts
View File

@@ -280,7 +280,8 @@ export interface BlockExtended extends IEsploraApi.Block {
export interface BlockSummary {
id: string;
transactions: TransactionStripped[];
transactions: TransactionClassified[];
version?: number;
}
export interface AuditSummary extends BlockAudit {
@@ -288,8 +289,8 @@ export interface AuditSummary extends BlockAudit {
size?: number,
weight?: number,
tx_count?: number,
transactions: TransactionStripped[];
template?: TransactionStripped[];
transactions: TransactionClassified[];
template?: TransactionClassified[];
}
export interface BlockPrice {

3
backend/src/replication/AuditReplication.ts
View File

@@ -105,7 +105,8 @@ class AuditReplication {
template: {
id: blockHash,
transactions: auditSummary.template || []
}
},
version: 1,
});
await blocksAuditsRepository.$saveAudit({
hash: blockHash,

4
backend/src/repositories/BlocksRepository.ts
View File

@@ -1040,16 +1040,18 @@ class BlocksRepository {
if (extras.feePercentiles === null) {
let summary;
let summaryVersion = 0;
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(dbBlk.id)).map(tx => transactionUtils.extendTransaction(tx));
summary = blocks.summarizeBlockTransactions(dbBlk.id, txs);
summaryVersion = 1;
} else {
// Call Core RPC
const block = await bitcoinClient.getBlock(dbBlk.id, 2);
summary = blocks.summarizeBlock(block);
}
await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.id, summary.transactions);
await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.id, summary.transactions, summaryVersion);
extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
}
if (extras.feePercentiles !== null) {

57
backend/src/repositories/BlocksSummariesRepository.ts
View File

@@ -1,6 +1,6 @@
import DB from '../database';
import logger from '../logger';
import { BlockSummary, TransactionStripped } from '../mempool.interfaces';
import { BlockSummary, TransactionClassified } from '../mempool.interfaces';
class BlocksSummariesRepository {
public async $getByBlockId(id: string): Promise<BlockSummary | undefined> {
@@ -17,30 +17,31 @@ class BlocksSummariesRepository {
return undefined;
}
public async $saveTransactions(blockHeight: number, blockId: string, transactions: TransactionStripped[]): Promise<void> {
public async $saveTransactions(blockHeight: number, blockId: string, transactions: TransactionClassified[], version: number): Promise<void> {
try {
const transactionsStr = JSON.stringify(transactions);
await DB.query(`
INSERT INTO blocks_summaries
SET height = ?, transactions = ?, id = ?
ON DUPLICATE KEY UPDATE transactions = ?`,
[blockHeight, transactionsStr, blockId, transactionsStr]);
SET height = ?, transactions = ?, id = ?, version = ?
ON DUPLICATE KEY UPDATE transactions = ?, version = ?`,
[blockHeight, transactionsStr, blockId, version, transactionsStr, version]);
} catch (e: any) {
logger.debug(`Cannot save block summary transactions for ${blockId}. Reason: ${e instanceof Error ? e.message : e}`);
throw e;
}
}
public async $saveTemplate(params: { height: number, template: BlockSummary}) {
public async $saveTemplate(params: { height: number, template: BlockSummary, version: number}): Promise<void> {
const blockId = params.template?.id;
try {
const transactions = JSON.stringify(params.template?.transactions || []);
await DB.query(`
INSERT INTO blocks_templates (id, template)
VALUE (?, ?)
INSERT INTO blocks_templates (id, template, version)
VALUE (?, ?, ?)
ON DUPLICATE KEY UPDATE
template = ?
`, [blockId, transactions, transactions]);
template = ?,
version = ?
`, [blockId, transactions, params.version, transactions, params.version]);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`Cannot save block template for ${blockId} because it has already been indexed, ignoring`);
@@ -57,6 +58,7 @@ class BlocksSummariesRepository {
return {
id: templates[0].id,
transactions: JSON.parse(templates[0].template),
version: templates[0].version,
};
}
} catch (e) {
@@ -76,6 +78,41 @@ class BlocksSummariesRepository {
return [];
}
public async $getSummariesWithVersion(version: number): Promise<{ height: number, id: string }[]> {
try {
const [rows]: any[] = await DB.query(`
SELECT
height,
id
FROM blocks_summaries
WHERE version = ?
ORDER BY height DESC;`, [version]);
return rows;
} catch (e) {
logger.err(`Cannot get block summaries with version. Reason: ` + (e instanceof Error ? e.message : e));
}
return [];
}
public async $getTemplatesWithVersion(version: number): Promise<{ height: number, id: string }[]> {
try {
const [rows]: any[] = await DB.query(`
SELECT
blocks_summaries.height as height,
blocks_templates.id as id
FROM blocks_templates
JOIN blocks_summaries ON blocks_templates.id = blocks_summaries.id
WHERE blocks_templates.version = ?
ORDER BY height DESC;`, [version]);
return rows;
} catch (e) {
logger.err(`Cannot get block summaries with version. Reason: ` + (e instanceof Error ? e.message : e));
}
return [];
}
/**
* Get the fee percentiles if the block has already been indexed, [] otherwise
*