machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into regtest-1

Browse Source
This commit is contained in:
Antoni Spaanderman
2022-03-08 19:45:03 +01:00
parent 82462d47e7 0e9f8773c5
commit 27f6654299
42 changed files with 1109 additions and 391 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
backend/src/api/bitcoin/bitcoin-api.ts
View File

@@ -89,19 +89,19 @@ class BitcoinApi implements AbstractBitcoinApi {
}
$getAddressPrefix(prefix: string): string[] {
const found: string[] = [];
const found: { [address: string]: string } = {};
const mp = mempool.getMempool();
for (const tx in mp) {
for (const vout of mp[tx].vout) {
if (vout.scriptpubkey_address.indexOf(prefix) === 0) {
found.push(vout.scriptpubkey_address);
if (found.length >= 10) {
return found;
found[vout.scriptpubkey_address] = '';
if (Object.keys(found).length >= 10) {
return Object.keys(found);
}
}
}
}
return found;
return Object.keys(found);
}
$sendRawTransaction(rawTransaction: string): Promise<string> {

35
backend/src/api/blocks.ts
View File

@@ -171,7 +171,7 @@ class Blocks {
}
/**
* Index all blocks metadata for the mining dashboard
* [INDEXING] Index all blocks metadata for the mining dashboard
*/
public async $generateBlockDatabase() {
if (this.blockIndexingStarted) {
@@ -218,31 +218,28 @@ class Blocks {
if (blockHeight < lastBlockToIndex) {
break;
}
try {
++indexedThisRun;
if (++totaIndexed % 100 === 0 || blockHeight === lastBlockToIndex) {
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
const blockPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
const progress = Math.round(totaIndexed / indexingBlockAmount * 100);
const timeLeft = Math.round((indexingBlockAmount - totaIndexed) / blockPerSeconds);
logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds} blocks/sec | total: ${totaIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${elapsedSeconds} seconds | left: ~${timeLeft} seconds`);
}
const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
const block = await bitcoinApi.$getBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
await blocksRepository.$saveBlockInDatabase(blockExtended);
} catch (e) {
logger.err(`Something went wrong while indexing blocks.` + e);
++indexedThisRun;
if (++totaIndexed % 100 === 0 || blockHeight === lastBlockToIndex) {
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
const blockPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
const progress = Math.round(totaIndexed / indexingBlockAmount * 100);
const timeLeft = Math.round((indexingBlockAmount - totaIndexed) / blockPerSeconds);
logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds} blocks/sec | total: ${totaIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${elapsedSeconds} seconds | left: ~${timeLeft} seconds`);
}
const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
const block = await bitcoinApi.$getBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
await blocksRepository.$saveBlockInDatabase(blockExtended);
}
currentBlockHeight -= chunkSize;
}
logger.info('Block indexing completed');
} catch (e) {
logger.err('An error occured in $generateBlockDatabase(). Skipping block indexing. ' + e);
console.log(e);
logger.err('An error occured in $generateBlockDatabase(). Trying again later. ' + e);
this.blockIndexingStarted = false;
return;
}
this.blockIndexingCompleted = true;

17
backend/src/api/database-migration.ts
View File

@@ -6,7 +6,7 @@ import logger from '../logger';
const sleep = (ms: number) => new Promise(res => setTimeout(res, ms));
class DatabaseMigration {
private static currentVersion = 8;
private static currentVersion = 9;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
@@ -133,6 +133,17 @@ class DatabaseMigration {
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"');
}
if (databaseSchemaVersion < 9 && isBitcoin === true) {
logger.warn(`'hashrates' table has been truncated. Re-indexing from scratch.'`);
await this.$executeQuery(connection, 'TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery(connection, 'ALTER TABLE `state` CHANGE `name` `name` varchar(100)');
await this.$executeQuery(connection, 'ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)');
}
if (databaseSchemaVersion < 10 && isBitcoin === true) {
await this.$executeQuery(connection, 'ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)');
}
connection.release();
} catch (e) {
connection.release();
@@ -276,6 +287,10 @@ class DatabaseMigration {
queries.push(`INSERT INTO state(name, number, string) VALUES ('last_hashrates_indexing', 0, NULL)`);
}
if (version < 9) {
queries.push(`INSERT INTO state(name, number, string) VALUES ('last_weekly_hashrates_indexing', 0, NULL)`);
}
return queries;
}

12
backend/src/api/mempool.ts
View File

@@ -8,6 +8,7 @@ import { IBitcoinApi } from './bitcoin/bitcoin-api.interface';
import loadingIndicators from './loading-indicators';
import bitcoinClient from './bitcoin/bitcoin-client';
import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
import rbfCache from './rbf-cache';
class Mempool {
private static WEBSOCKET_REFRESH_RATE_MS = 10000;
@@ -200,6 +201,17 @@ class Mempool {
logger.debug('Mempool updated in ' + time / 1000 + ' seconds');
}
public handleRbfTransactions(rbfTransactions: { [txid: string]: TransactionExtended; }) {
for (const rbfTransaction in rbfTransactions) {
if (this.mempoolCache[rbfTransaction]) {
// Store replaced transactions
rbfCache.add(rbfTransaction, rbfTransactions[rbfTransaction].txid);
// Erase the replaced transactions from the local mempool
delete this.mempoolCache[rbfTransaction];
}
}
}
private updateTxPerSecond() {
const nowMinusTimeSpan = new Date().getTime() - (1000 * config.STATISTICS.TX_PER_SECOND_SAMPLE_PERIOD);
this.txPerSecondArray = this.txPerSecondArray.filter((unixTime) => unixTime > nowMinusTimeSpan);

258
backend/src/api/mining.ts
View File

@@ -8,6 +8,7 @@ import blocks from './blocks';
class Mining {
hashrateIndexingStarted = false;
weeklyHashrateIndexingStarted = false;
constructor() {
}
@@ -74,140 +75,205 @@ class Mining {
}
/**
* Return the historical difficulty adjustments and oldest indexed block timestamp
* [INDEXING] Generate weekly mining pool hashrate history
*/
public async $getHistoricalDifficulty(interval: string | null): Promise<object> {
return await BlocksRepository.$getBlocksDifficulty(interval);
}
public async $generatePoolHashrateHistory(): Promise<void> {
if (!blocks.blockIndexingCompleted || this.weeklyHashrateIndexingStarted) {
return;
}
/**
* Return the historical hashrates and oldest indexed block timestamp
*/
public async $getNetworkHistoricalHashrates(interval: string | null): Promise<object> {
return await HashratesRepository.$getNetworkDailyHashrate(interval);
}
/**
* Return the historical hashrates and oldest indexed block timestamp for one or all pools
*/
public async $getPoolsHistoricalHashrates(interval: string | null, poolId: number): Promise<object> {
return await HashratesRepository.$getPoolsWeeklyHashrate(interval);
}
/**
* Generate daily hashrate data
*/
public async $generateNetworkHashrateHistory(): Promise<void> {
// We only run this once a day
const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp();
// We only run this once a week
const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp('last_weekly_hashrates_indexing');
const now = new Date().getTime() / 1000;
if (now - latestTimestamp < 86400) {
if (now - latestTimestamp < 604800) {
return;
}
if (!blocks.blockIndexingCompleted || this.hashrateIndexingStarted) {
return;
}
this.hashrateIndexingStarted = true;
try {
this.weeklyHashrateIndexingStarted = true;
logger.info(`Indexing hashrates`);
logger.info(`Indexing mining pools weekly hashrates`);
const totalDayIndexed = (await BlocksRepository.$blockCount(null, null)) / 144;
const indexedTimestamp = (await HashratesRepository.$getNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
let startedAt = new Date().getTime() / 1000;
const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
const lastMidnight = new Date();
lastMidnight.setUTCHours(0); lastMidnight.setUTCMinutes(0); lastMidnight.setUTCSeconds(0); lastMidnight.setUTCMilliseconds(0);
let toTimestamp = Math.round(lastMidnight.getTime() / 1000);
let indexedThisRun = 0;
let totalIndexed = 0;
const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
const hashrates: any[] = [];
const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
const lastMidnight = this.getDateMidnight(new Date());
let toTimestamp = Math.round((lastMidnight.getTime() - 604800) / 1000);
const hashrates: any[] = [];
const totalWeekIndexed = (await BlocksRepository.$blockCount(null, null)) / 1008;
let indexedThisRun = 0;
let totalIndexed = 0;
let startedAt = new Date().getTime() / 1000;
while (toTimestamp > genesisTimestamp) {
const fromTimestamp = toTimestamp - 86400;
if (indexedTimestamp.includes(fromTimestamp)) {
toTimestamp -= 86400;
++totalIndexed;
continue;
}
while (toTimestamp > genesisTimestamp) {
const fromTimestamp = toTimestamp - 604800;
const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
null, fromTimestamp, toTimestamp);
if (blockStats.blockCount === 0) { // We are done indexing, no blocks left
break;
}
// Skip already indexed weeks
if (indexedTimestamp.includes(toTimestamp + 1)) {
toTimestamp -= 604800;
++totalIndexed;
continue;
}
let lastBlockHashrate = 0;
lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
blockStats.lastBlockHeight);
const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
null, fromTimestamp, toTimestamp);
if (blockStats.blockCount === 0) { // We are done indexing, no blocks left
break;
}
if (totalIndexed % 7 === 0 && !indexedTimestamp.includes(fromTimestamp + 1)) { // Save weekly pools hashrate
logger.debug("Indexing weekly hashrates for mining pools");
let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp - 604800, fromTimestamp);
const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
blockStats.lastBlockHeight);
let pools = await PoolsRepository.$getPoolsInfoBetween(fromTimestamp, toTimestamp);
const totalBlocks = pools.reduce((acc, pool) => acc + pool.blockCount, 0);
pools = pools.map((pool: any) => {
pool.hashrate = (pool.blockCount / totalBlocks) * lastBlockHashrate;
pool.share = (pool.blockCount / totalBlocks);
return pool;
});
for (const pool of pools) {
hashrates.push({
hashrateTimestamp: fromTimestamp + 1,
hashrateTimestamp: toTimestamp + 1,
avgHashrate: pool['hashrate'],
poolId: pool.poolId,
share: pool['share'],
type: 'weekly',
});
}
}
hashrates.push({
hashrateTimestamp: fromTimestamp,
avgHashrate: lastBlockHashrate,
poolId: null,
share: 1,
type: 'daily',
});
if (hashrates.length > 10) {
await HashratesRepository.$saveHashrates(hashrates);
hashrates.length = 0;
}
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
if (elapsedSeconds > 5) {
const daysPerSeconds = (indexedThisRun / elapsedSeconds).toFixed(2);
const formattedDate = new Date(fromTimestamp * 1000).toUTCString();
const daysLeft = Math.round(totalDayIndexed - totalIndexed);
logger.debug(`Getting hashrate for ${formattedDate} | ~${daysPerSeconds} days/sec | ~${daysLeft} days left to index`);
startedAt = new Date().getTime() / 1000;
indexedThisRun = 0;
}
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
if (elapsedSeconds > 5) {
const weeksPerSeconds = (indexedThisRun / elapsedSeconds).toFixed(2);
const formattedDate = new Date(fromTimestamp * 1000).toUTCString();
const weeksLeft = Math.round(totalWeekIndexed - totalIndexed);
logger.debug(`Getting weekly pool hashrate for ${formattedDate} | ~${weeksPerSeconds} weeks/sec | ~${weeksLeft} weeks left to index`);
startedAt = new Date().getTime() / 1000;
indexedThisRun = 0;
}
toTimestamp -= 86400;
++indexedThisRun;
++totalIndexed;
toTimestamp -= 604800;
++indexedThisRun;
++totalIndexed;
}
this.weeklyHashrateIndexingStarted = false;
await HashratesRepository.$setLatestRunTimestamp('last_weekly_hashrates_indexing');
logger.info(`Weekly pools hashrate indexing completed`);
} catch (e) {
this.weeklyHashrateIndexingStarted = false;
throw e;
}
}
/**
* [INDEXING] Generate daily hashrate data
*/
public async $generateNetworkHashrateHistory(): Promise<void> {
if (!blocks.blockIndexingCompleted || this.hashrateIndexingStarted) {
return;
}
// Add genesis block manually
if (toTimestamp <= genesisTimestamp && !indexedTimestamp.includes(genesisTimestamp)) {
hashrates.push({
hashrateTimestamp: genesisTimestamp,
avgHashrate: await bitcoinClient.getNetworkHashPs(1, 1),
poolId: null,
type: 'daily',
});
// We only run this once a day
const latestTimestamp = await HashratesRepository.$getLatestRunTimestamp('last_hashrates_indexing');
const now = new Date().getTime() / 1000;
if (now - latestTimestamp < 86400) {
return;
}
if (hashrates.length > 0) {
try {
this.hashrateIndexingStarted = true;
logger.info(`Indexing network daily hashrate`);
const indexedTimestamp = (await HashratesRepository.$getNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
const genesisTimestamp = 1231006505; // bitcoin-cli getblock 000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f
const lastMidnight = this.getDateMidnight(new Date());
let toTimestamp = Math.round(lastMidnight.getTime() / 1000);
const hashrates: any[] = [];
const totalDayIndexed = (await BlocksRepository.$blockCount(null, null)) / 144;
let indexedThisRun = 0;
let totalIndexed = 0;
let startedAt = new Date().getTime() / 1000;
while (toTimestamp > genesisTimestamp) {
const fromTimestamp = toTimestamp - 86400;
// Skip already indexed weeks
if (indexedTimestamp.includes(fromTimestamp)) {
toTimestamp -= 86400;
++totalIndexed;
continue;
}
const blockStats: any = await BlocksRepository.$blockCountBetweenTimestamp(
null, fromTimestamp, toTimestamp);
if (blockStats.blockCount === 0) { // We are done indexing, no blocks left
break;
}
const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(blockStats.blockCount,
blockStats.lastBlockHeight);
hashrates.push({
hashrateTimestamp: toTimestamp,
avgHashrate: lastBlockHashrate,
poolId: null,
share: 1,
type: 'daily',
});
if (hashrates.length > 10) {
await HashratesRepository.$saveHashrates(hashrates);
hashrates.length = 0;
}
const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
if (elapsedSeconds > 5) {
const daysPerSeconds = (indexedThisRun / elapsedSeconds).toFixed(2);
const formattedDate = new Date(fromTimestamp * 1000).toUTCString();
const daysLeft = Math.round(totalDayIndexed - totalIndexed);
logger.debug(`Getting network daily hashrate for ${formattedDate} | ~${daysPerSeconds} days/sec | ~${daysLeft} days left to index`);
startedAt = new Date().getTime() / 1000;
indexedThisRun = 0;
}
toTimestamp -= 86400;
++indexedThisRun;
++totalIndexed;
}
// Add genesis block manually
if (toTimestamp <= genesisTimestamp && !indexedTimestamp.includes(genesisTimestamp)) {
hashrates.push({
hashrateTimestamp: genesisTimestamp,
avgHashrate: await bitcoinClient.getNetworkHashPs(1, 1),
poolId: null,
type: 'daily',
});
}
await HashratesRepository.$saveHashrates(hashrates);
}
await HashratesRepository.$setLatestRunTimestamp();
this.hashrateIndexingStarted = false;
logger.info(`Hashrates indexing completed`);
await HashratesRepository.$setLatestRunTimestamp('last_hashrates_indexing');
this.hashrateIndexingStarted = false;
logger.info(`Daily network hashrate indexing completed`);
} catch (e) {
this.hashrateIndexingStarted = false;
throw e;
}
}
private getDateMidnight(date: Date): Date {
date.setUTCHours(0);
date.setUTCMinutes(0);
date.setUTCSeconds(0);
date.setUTCMilliseconds(0);
return date;
}
}

34
backend/src/api/rbf-cache.ts Normal file
View File

@@ -0,0 +1,34 @@
export interface CachedRbf {
txid: string;
expires: Date;
}
class RbfCache {
private cache: { [txid: string]: CachedRbf; } = {};
constructor() {
setInterval(this.cleanup.bind(this), 1000 * 60 * 60);
}
public add(replacedTxId: string, newTxId: string): void {
this.cache[replacedTxId] = {
expires: new Date(Date.now() + 1000 * 604800), // 1 week
txid: newTxId,
};
}
public get(txId: string): CachedRbf | undefined {
return this.cache[txId];
}
private cleanup(): void {
const currentDate = new Date();
for (const c in this.cache) {
if (this.cache[c].expires < currentDate) {
delete this.cache[c];
}
}
}
}
export default new RbfCache();

85
backend/src/api/websocket-handler.ts
View File

@@ -11,6 +11,7 @@ import { Common } from './common';
import loadingIndicators from './loading-indicators';
import config from '../config';
import transactionUtils from './transaction-utils';
import rbfCache from './rbf-cache';
class WebsocketHandler {
private wss: WebSocket.Server | undefined;
@@ -48,29 +49,38 @@ class WebsocketHandler {
if (parsedMessage && parsedMessage['track-tx']) {
if (/^[a-fA-F0-9]{64}$/.test(parsedMessage['track-tx'])) {
client['track-tx'] = parsedMessage['track-tx'];
// Client is telling the transaction wasn't found but it might have appeared before we had the time to start watching for it
// Client is telling the transaction wasn't found
if (parsedMessage['watch-mempool']) {
const tx = memPool.getMempool()[client['track-tx']];
if (tx) {
if (config.MEMPOOL.BACKEND === 'esplora') {
response['tx'] = tx;
const rbfCacheTx = rbfCache.get(client['track-tx']);
if (rbfCacheTx) {
response['txReplaced'] = {
txid: rbfCacheTx.txid,
};
client['track-tx'] = null;
} else {
// It might have appeared before we had the time to start watching for it
const tx = memPool.getMempool()[client['track-tx']];
if (tx) {
if (config.MEMPOOL.BACKEND === 'esplora') {
response['tx'] = tx;
} else {
// tx.prevout is missing from transactions when in bitcoind mode
try {
const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, true);
response['tx'] = fullTx;
} catch (e) {
logger.debug('Error finding transaction: ' + (e instanceof Error ? e.message : e));
}
}
} else {
// tx.prevouts is missing from transactions when in bitcoind mode
try {
const fullTx = await transactionUtils.$getTransactionExtended(tx.txid, true);
const fullTx = await transactionUtils.$getTransactionExtended(client['track-tx'], true);
response['tx'] = fullTx;
} catch (e) {
logger.debug('Error finding transaction: ' + (e instanceof Error ? e.message : e));
logger.debug('Error finding transaction. ' + (e instanceof Error ? e.message : e));
client['track-mempool-tx'] = parsedMessage['track-tx'];
}
}
} else {
try {
const fullTx = await transactionUtils.$getTransactionExtended(client['track-tx'], true);
response['tx'] = fullTx;
} catch (e) {
logger.debug('Error finding transaction. ' + (e instanceof Error ? e.message : e));
client['track-mempool-tx'] = parsedMessage['track-tx'];
}
}
}
} else {
@@ -221,14 +231,10 @@ class WebsocketHandler {
mempoolBlocks.updateMempoolBlocks(newMempool);
const mBlocks = mempoolBlocks.getMempoolBlocks();
const mempool = memPool.getMempool();
const mempoolInfo = memPool.getMempoolInfo();
const vBytesPerSecond = memPool.getVBytesPerSecond();
const rbfTransactions = Common.findRbfTransactions(newTransactions, deletedTransactions);
for (const rbfTransaction in rbfTransactions) {
delete mempool[rbfTransaction];
}
memPool.handleRbfTransactions(rbfTransactions);
this.wss.clients.forEach(async (client: WebSocket) => {
if (client.readyState !== WebSocket.OPEN) {
@@ -331,21 +337,29 @@ class WebsocketHandler {
}
}
if (client['track-tx'] && rbfTransactions[client['track-tx']]) {
for (const rbfTransaction in rbfTransactions) {
if (client['track-tx'] === rbfTransaction) {
const rbfTx = rbfTransactions[rbfTransaction];
if (config.MEMPOOL.BACKEND !== 'esplora') {
try {
const fullTx = await transactionUtils.$getTransactionExtended(rbfTransaction, true);
response['rbfTransaction'] = fullTx;
} catch (e) {
logger.debug('Error finding transaction in mempool: ' + (e instanceof Error ? e.message : e));
}
} else {
response['rbfTransaction'] = rbfTx;
if (client['track-tx']) {
const outspends: object = {};
newTransactions.forEach((tx) => tx.vin.forEach((vin, i) => {
if (vin.txid === client['track-tx']) {
outspends[vin.vout] = {
vin: i,
txid: tx.txid,
};
}
}));
if (Object.keys(outspends).length) {
response['utxoSpent'] = outspends;
}
if (rbfTransactions[client['track-tx']]) {
for (const rbfTransaction in rbfTransactions) {
if (client['track-tx'] === rbfTransaction) {
response['rbfTransaction'] = {
txid: rbfTransactions[rbfTransaction].txid,
};
break;
}
break;
}
}
}
@@ -405,7 +419,6 @@ class WebsocketHandler {
}
if (client['track-tx'] && txIds.indexOf(client['track-tx']) > -1) {
client['track-tx'] = null;
response['txConfirmed'] = true;
}