machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into mononaut/electrs-blocks

Browse Source
This commit is contained in:
wiz 2023-08-01 13:54:05 +09:00 committed by GitHub
commit 8c396978a8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 81 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
backend/src/api/blocks.ts
View File

@ -419,8 +419,8 @@ class Blocks {
let newlyIndexed = 0; let newlyIndexed = 0;
let totalIndexed = indexedBlockSummariesHashesArray.length; let totalIndexed = indexedBlockSummariesHashesArray.length;
let indexedThisRun = 0; let indexedThisRun = 0;
let timer = new Date().getTime() / 1000; let timer = Date.now() / 1000;
const startedAt = new Date().getTime() / 1000; const startedAt = Date.now() / 1000;
for (const block of indexedBlocks) { for (const block of indexedBlocks) {
if (indexedBlockSummariesHashes[block.hash] === true) { if (indexedBlockSummariesHashes[block.hash] === true) {
@ -428,17 +428,24 @@ class Blocks {
} }
// Logging // Logging
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - timer)); const elapsedSeconds = (Date.now() / 1000) - timer;
if (elapsedSeconds > 5) { if (elapsedSeconds > 5) {
const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); const runningFor = (Date.now() / 1000) - startedAt;
const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds); const blockPerSeconds = indexedThisRun / elapsedSeconds;
const progress = Math.round(totalIndexed / indexedBlocks.length * 10000) / 100; const progress = Math.round(totalIndexed / indexedBlocks.length * 10000) / 100;
logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining); logger.debug(`Indexing block summary for #${block.height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexedBlocks.length} (${progress}%) | elapsed: ${runningFor.toFixed(2)} seconds`, logger.tags.mining);
timer = new Date().getTime() / 1000; timer = Date.now() / 1000;
indexedThisRun = 0; indexedThisRun = 0;
} }
await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(block.hash)).map(tx => transactionUtils.extendTransaction(tx));
const cpfpSummary = await this.$indexCPFP(block.hash, block.height, txs);
await this.$getStrippedBlockTransactions(block.hash, true, true, cpfpSummary, block.height); // This will index the block summary
} else {
await this.$getStrippedBlockTransactions(block.hash, true, true); // This will index the block summary
}
// Logging // Logging
indexedThisRun++; indexedThisRun++;
@ -477,18 +484,18 @@ class Blocks {
// Logging // Logging
let count = 0; let count = 0;
let countThisRun = 0; let countThisRun = 0;
let timer = new Date().getTime() / 1000; let timer = Date.now() / 1000;
const startedAt = new Date().getTime() / 1000; const startedAt = Date.now() / 1000;
for (const height of unindexedBlockHeights) { for (const height of unindexedBlockHeights) {
// Logging // Logging
const hash = await bitcoinApi.$getBlockHash(height); const hash = await bitcoinApi.$getBlockHash(height);
const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer); const elapsedSeconds = (Date.now() / 1000) - timer;
if (elapsedSeconds > 5) { if (elapsedSeconds > 5) {
const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); const runningFor = (Date.now() / 1000) - startedAt;
const blockPerSeconds = (countThisRun / elapsedSeconds); const blockPerSeconds = countThisRun / elapsedSeconds;
const progress = Math.round(count / unindexedBlockHeights.length * 10000) / 100; const progress = Math.round(count / unindexedBlockHeights.length * 10000) / 100;
logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor} seconds`); logger.debug(`Indexing cpfp clusters for #${height} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${count}/${unindexedBlockHeights.length} (${progress}%) | elapsed: ${runningFor.toFixed(2)} seconds`);
timer = new Date().getTime() / 1000; timer = Date.now() / 1000;
countThisRun = 0; countThisRun = 0;
} }
@ -567,8 +574,8 @@ class Blocks {
let totalIndexed = await blocksRepository.$blockCountBetweenHeight(currentBlockHeight, lastBlockToIndex); let totalIndexed = await blocksRepository.$blockCountBetweenHeight(currentBlockHeight, lastBlockToIndex);
let indexedThisRun = 0; let indexedThisRun = 0;
let newlyIndexed = 0; let newlyIndexed = 0;
const startedAt = new Date().getTime() / 1000; const startedAt = Date.now() / 1000;
let timer = new Date().getTime() / 1000; let timer = Date.now() / 1000;
while (currentBlockHeight >= lastBlockToIndex) { while (currentBlockHeight >= lastBlockToIndex) {
const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1); const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1);
@ -588,13 +595,13 @@ class Blocks {
} }
++indexedThisRun; ++indexedThisRun;
++totalIndexed; ++totalIndexed;
const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer); const elapsedSeconds = (Date.now() / 1000) - timer;
if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) { if (elapsedSeconds > 5 || blockHeight === lastBlockToIndex) {
const runningFor = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt)); const runningFor = (Date.now() / 1000) - startedAt;
const blockPerSeconds = Math.max(1, indexedThisRun / elapsedSeconds); const blockPerSeconds = indexedThisRun / elapsedSeconds;
const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100; const progress = Math.round(totalIndexed / indexingBlockAmount * 10000) / 100;
logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${runningFor} seconds`, logger.tags.mining); logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds.toFixed(2)} blocks/sec | total: ${totalIndexed}/${indexingBlockAmount} (${progress.toFixed(2)}%) | elapsed: ${runningFor.toFixed(2)} seconds`, logger.tags.mining);
timer = new Date().getTime() / 1000; timer = Date.now() / 1000;
indexedThisRun = 0; indexedThisRun = 0;
loadingIndicators.setProgress('block-indexing', progress, false); loadingIndicators.setProgress('block-indexing', progress, false);
} }
@ -942,10 +949,15 @@ class Blocks {
}), }),
}; };
} else { } else {
// Call Core RPC if (config.MEMPOOL.BACKEND === 'esplora') {
const block = await bitcoinClient.getBlock(hash, 2); const txs = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx));
summary = this.summarizeBlock(block); summary = this.summarizeBlockTransactions(hash, txs);
height = block.height; } else {
// Call Core RPC
const block = await bitcoinClient.getBlock(hash, 2);
summary = this.summarizeBlock(block);
height = block.height;
}
} }
if (height == null) { if (height == null) {
const block = await bitcoinApi.$getBlock(hash); const block = await bitcoinApi.$getBlock(hash);
@ -1068,8 +1080,17 @@ class Blocks {
if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) { if (Common.blocksSummariesIndexingEnabled() && cleanBlock.fee_amt_percentiles === null) {
cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash); cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
if (cleanBlock.fee_amt_percentiles === null) { if (cleanBlock.fee_amt_percentiles === null) {
const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
const summary = this.summarizeBlock(block); let summary;
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(cleanBlock.hash)).map(tx => transactionUtils.extendTransaction(tx));
summary = this.summarizeBlockTransactions(cleanBlock.hash, txs);
} else {
// Call Core RPC
const block = await bitcoinClient.getBlock(cleanBlock.hash, 2);
summary = this.summarizeBlock(block);
}
await BlocksSummariesRepository.$saveTransactions(cleanBlock.height, cleanBlock.hash, summary.transactions); await BlocksSummariesRepository.$saveTransactions(cleanBlock.height, cleanBlock.hash, summary.transactions);
cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash); cleanBlock.fee_amt_percentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(cleanBlock.hash);
} }
@ -1129,19 +1150,29 @@ class Blocks {
return this.currentBlockHeight; return this.currentBlockHeight;
} }
public async $indexCPFP(hash: string, height: number): Promise<void> { public async $indexCPFP(hash: string, height: number, txs?: TransactionExtended[]): Promise<CpfpSummary> {
const block = await bitcoinClient.getBlock(hash, 2); let transactions = txs;
const transactions = block.tx.map(tx => { if (!transactions) {
tx.fee *= 100_000_000; if (config.MEMPOOL.BACKEND === 'esplora') {
return tx; transactions = (await bitcoinApi.$getTxsForBlock(hash)).map(tx => transactionUtils.extendTransaction(tx));
}); }
if (!transactions) {
const block = await bitcoinClient.getBlock(hash, 2);
transactions = block.tx.map(tx => {
tx.fee *= 100_000_000;
return tx;
});
}
}
const summary = Common.calculateCpfp(height, transactions); const summary = Common.calculateCpfp(height, transactions as TransactionExtended[]);
await this.$saveCpfp(hash, height, summary); await this.$saveCpfp(hash, height, summary);
const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions); const effectiveFeeStats = Common.calcEffectiveFeeStatistics(summary.transactions);
await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats); await blocksRepository.$saveEffectiveFeeStats(hash, effectiveFeeStats);
return summary;
} }
public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise<void> { public async $saveCpfp(hash: string, height: number, cpfpSummary: CpfpSummary): Promise<void> {

3
backend/src/index.ts
View File

@ -195,7 +195,8 @@ class Server {
// rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS // rerun immediately if we skipped the mempool update, otherwise wait POLL_RATE_MS
const elapsed = Date.now() - start; const elapsed = Date.now() - start;
const remainingTime = Math.max(0, config.MEMPOOL.POLL_RATE_MS - elapsed) const pollRate = config.MEMPOOL.POLL_RATE_MS * (indexer.indexerRunning ? 10 : 1);
const remainingTime = Math.max(0, pollRate - elapsed);
setTimeout(this.runMainUpdateLoop.bind(this), numHandledBlocks > 0 ? 0 : remainingTime); setTimeout(this.runMainUpdateLoop.bind(this), numHandledBlocks > 0 ? 0 : remainingTime);
this.backendRetryCount = 0; this.backendRetryCount = 0;
} catch (e: any) { } catch (e: any) {

15
backend/src/repositories/BlocksRepository.ts
View File

@ -1,3 +1,4 @@
import bitcoinApi from '../api/bitcoin/bitcoin-api-factory';
import { BlockExtended, BlockExtension, BlockPrice, EffectiveFeeStats } from '../mempool.interfaces'; import { BlockExtended, BlockExtension, BlockPrice, EffectiveFeeStats } from '../mempool.interfaces';
import DB from '../database'; import DB from '../database';
import logger from '../logger'; import logger from '../logger';
@ -12,6 +13,7 @@ import config from '../config';
import chainTips from '../api/chain-tips'; import chainTips from '../api/chain-tips';
import blocks from '../api/blocks'; import blocks from '../api/blocks';
import BlocksAuditsRepository from './BlocksAuditsRepository'; import BlocksAuditsRepository from './BlocksAuditsRepository';
import transactionUtils from '../api/transaction-utils';
interface DatabaseBlock { interface DatabaseBlock {
id: string; id: string;
@ -1036,8 +1038,17 @@ class BlocksRepository {
{ {
extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id); extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
if (extras.feePercentiles === null) { if (extras.feePercentiles === null) {
const block = await bitcoinClient.getBlock(dbBlk.id, 2);
const summary = blocks.summarizeBlock(block); let summary;
if (config.MEMPOOL.BACKEND === 'esplora') {
const txs = (await bitcoinApi.$getTxsForBlock(dbBlk.id)).map(tx => transactionUtils.extendTransaction(tx));
summary = blocks.summarizeBlockTransactions(dbBlk.id, txs);
} else {
// Call Core RPC
const block = await bitcoinClient.getBlock(dbBlk.id, 2);
summary = blocks.summarizeBlock(block);
}
await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.id, summary.transactions); await BlocksSummariesRepository.$saveTransactions(dbBlk.height, dbBlk.id, summary.transactions);
extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id); extras.feePercentiles = await BlocksSummariesRepository.$getFeePercentilesByBlockId(dbBlk.id);
} }