machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1243 from nymkappa/feature/start-indexing-before-mempool-sync

Browse Source 
Don't wait for 100% mempool sync before starting block indexing
This commit is contained in:
softsimon 2022-02-15 12:07:30 +04:00 committed by GitHub
commit 93f872efe5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 20 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
backend/src/api/blocks.ts
View File

@ -61,7 +61,7 @@ class Blocks {
// optimize here by directly fetching txs in the "outdated" mempool // optimize here by directly fetching txs in the "outdated" mempool
transactions.push(mempool[txIds[i]]); transactions.push(mempool[txIds[i]]);
transactionsFound++; transactionsFound++;
} else if (config.MEMPOOL.BACKEND === 'esplora' || memPool.isInSync() || i === 0) { } else if (config.MEMPOOL.BACKEND === 'esplora' || !memPool.hasPriority() || i === 0) {
// Otherwise we fetch the tx data through backend services (esplora, electrum, core rpc...) // Otherwise we fetch the tx data through backend services (esplora, electrum, core rpc...)
if (!quiet && (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length)) { // Avoid log spam if (!quiet && (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length)) { // Avoid log spam
logger.debug(`Indexing tx ${i + 1} of ${txIds.length} in block #${blockHeight}`); logger.debug(`Indexing tx ${i + 1} of ${txIds.length} in block #${blockHeight}`);
@ -175,7 +175,7 @@ class Blocks {
public async $generateBlockDatabase() { public async $generateBlockDatabase() {
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false || // Bitcoin only if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false || // Bitcoin only
config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === 0 || // Indexing of older blocks must be enabled config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === 0 || // Indexing of older blocks must be enabled
!memPool.isInSync() || // We sync the mempool first memPool.hasPriority() || // We sync the mempool first
this.blockIndexingStarted === true || // Indexing must not already be in progress this.blockIndexingStarted === true || // Indexing must not already be in progress
config.DATABASE.ENABLED === false config.DATABASE.ENABLED === false
) { ) {
@ -314,7 +314,7 @@ class Blocks {
if (this.newBlockCallbacks.length) { if (this.newBlockCallbacks.length) {
this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions)); this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
} }
if (memPool.isInSync()) { if (!memPool.hasPriority()) {
diskCache.$saveCacheToDisk(); diskCache.$saveCacheToDisk();
} }
} }

19
backend/src/api/mempool.ts
View File

@ -13,6 +13,7 @@ class Mempool {
private static WEBSOCKET_REFRESH_RATE_MS = 10000; private static WEBSOCKET_REFRESH_RATE_MS = 10000;
private static LAZY_DELETE_AFTER_SECONDS = 30; private static LAZY_DELETE_AFTER_SECONDS = 30;
private inSync: boolean = false; private inSync: boolean = false;
private mempoolCacheDelta: number = -1;
private mempoolCache: { [txId: string]: TransactionExtended } = {}; private mempoolCache: { [txId: string]: TransactionExtended } = {};
private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0, private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 }; maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 };
@ -32,6 +33,17 @@ class Mempool {
setInterval(this.deleteExpiredTransactions.bind(this), 20000); setInterval(this.deleteExpiredTransactions.bind(this), 20000);
} }
/**
* Return true if we should leave resources available for mempool tx caching
*/
public hasPriority(): boolean {
if (this.inSync) {
return false;
} else {
return this.mempoolCacheDelta == -1 || this.mempoolCacheDelta > 25;
}
}
public isInSync(): boolean { public isInSync(): boolean {
return this.inSync; return this.inSync;
} }
@ -100,6 +112,8 @@ class Mempool {
const diff = transactions.length - currentMempoolSize; const diff = transactions.length - currentMempoolSize;
const newTransactions: TransactionExtended[] = []; const newTransactions: TransactionExtended[] = [];
this.mempoolCacheDelta = Math.abs(diff);
if (!this.inSync) { if (!this.inSync) {
loadingIndicators.setProgress('mempool', Object.keys(this.mempoolCache).length / transactions.length * 100); loadingIndicators.setProgress('mempool', Object.keys(this.mempoolCache).length / transactions.length * 100);
} }
@ -168,13 +182,14 @@ class Mempool {
const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx)); const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx));
this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6); this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6);
const syncedThreshold = 0.99; // If we synced 99% of the mempool tx count, consider we're synced if (!this.inSync && transactions.length === Object.keys(this.mempoolCache).length) {
if (!this.inSync && Object.keys(this.mempoolCache).length >= transactions.length * syncedThreshold) {
this.inSync = true; this.inSync = true;
logger.notice('The mempool is now in sync!'); logger.notice('The mempool is now in sync!');
loadingIndicators.setProgress('mempool', 100); loadingIndicators.setProgress('mempool', 100);
} }
this.mempoolCacheDelta = Math.abs(transactions.length - Object.keys(this.mempoolCache).length);
if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) { if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions); this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
} }