machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into nymkappa/bugfix/price

Browse Source
This commit is contained in:
wiz
2023-03-05 14:13:51 +09:00
committed by GitHub
parent adf9c12eb5 b674a340c8
commit 6bdfffce3c
17 changed files with 134 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/src/api/bitcoin/bitcoin.routes.ts
View File

@@ -6,7 +6,7 @@ import websocketHandler from '../websocket-handler';
import mempool from '../mempool';
import feeApi from '../fee-api';
import mempoolBlocks from '../mempool-blocks';
import bitcoinApi from './bitcoin-api-factory';
import bitcoinApi, { bitcoinCoreApi } from './bitcoin-api-factory';
import { Common } from '../common';
import backendInfo from '../backend-info';
import transactionUtils from '../transaction-utils';
@@ -469,7 +469,7 @@ class BitcoinRoutes {
returnBlocks.push(localBlock);
nextHash = localBlock.previousblockhash;
} else {
const block = await bitcoinApi.$getBlock(nextHash);
const block = await bitcoinCoreApi.$getBlock(nextHash);
returnBlocks.push(block);
nextHash = block.previousblockhash;
}

14
backend/src/api/blocks.ts
View File

@@ -1,5 +1,5 @@
import config from '../config';
import bitcoinApi from './bitcoin/bitcoin-api-factory';
import bitcoinApi, { bitcoinCoreApi } from './bitcoin/bitcoin-api-factory';
import logger from '../logger';
import memPool from './mempool';
import { BlockExtended, BlockExtension, BlockSummary, PoolTag, TransactionExtended, TransactionStripped, TransactionMinerInfo } from '../mempool.interfaces';
@@ -484,7 +484,7 @@ class Blocks {
loadingIndicators.setProgress('block-indexing', progress, false);
}
const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
@@ -532,13 +532,13 @@ class Blocks {
if (blockchainInfo.blocks === blockchainInfo.headers) {
const heightDiff = blockHeightTip % 2016;
const blockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff);
const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
this.lastDifficultyAdjustmentTime = block.timestamp;
this.currentDifficulty = block.difficulty;
if (blockHeightTip >= 2016) {
const previousPeriodBlockHash = await bitcoinApi.$getBlockHash(blockHeightTip - heightDiff - 2016);
const previousPeriodBlock: IEsploraApi.Block = await bitcoinApi.$getBlock(previousPeriodBlockHash);
const previousPeriodBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(previousPeriodBlockHash);
this.previousDifficultyRetarget = (block.difficulty - previousPeriodBlock.difficulty) / previousPeriodBlock.difficulty * 100;
logger.debug(`Initial difficulty adjustment data set.`);
}
@@ -662,7 +662,7 @@ class Blocks {
}
const blockHash = await bitcoinApi.$getBlockHash(height);
const block: IEsploraApi.Block = await bitcoinApi.$getBlock(blockHash);
const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(blockHash);
const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
const blockExtended = await this.$getBlockExtended(block, transactions);
@@ -685,11 +685,11 @@ class Blocks {
// Not Bitcoin network, return the block as it from the bitcoin backend
if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
return await bitcoinApi.$getBlock(hash);
return await bitcoinCoreApi.$getBlock(hash);
}
// Bitcoin network, add our custom data on top
const block: IEsploraApi.Block = await bitcoinApi.$getBlock(hash);
const block: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(hash);
return await this.$indexBlock(block.height);
}

13
backend/src/api/database-migration.ts
View File

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
import { RowDataPacket } from 'mysql2';
class DatabaseMigration {
private static currentVersion = 57;
private static currentVersion = 58;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@@ -505,6 +505,11 @@ class DatabaseMigration {
await this.$executeQuery(`ALTER TABLE nodes MODIFY updated_at datetime NULL`);
await this.updateToSchemaVersion(57);
}
if (databaseSchemaVersion < 58) {
// We only run some migration queries for this version
await this.updateToSchemaVersion(58);
}
}
/**
@@ -632,6 +637,11 @@ class DatabaseMigration {
queries.push(`INSERT INTO state(name, number, string) VALUES ('last_weekly_hashrates_indexing', 0, NULL)`);
}
if (version < 58) {
queries.push(`DELETE FROM state WHERE name = 'last_hashrates_indexing'`);
queries.push(`DELETE FROM state WHERE name = 'last_weekly_hashrates_indexing'`);
}
return queries;
}
@@ -1023,6 +1033,7 @@ class DatabaseMigration {
await this.$executeQuery(`TRUNCATE blocks`);
await this.$executeQuery(`TRUNCATE hashrates`);
await this.$executeQuery(`TRUNCATE difficulty_adjustments`);
await this.$executeQuery('DELETE FROM `pools`');
await this.$executeQuery('ALTER TABLE pools AUTO_INCREMENT = 1');
await this.$executeQuery(`UPDATE state SET string = NULL WHERE name = 'pools_json_sha'`);

31
backend/src/api/mempool-blocks.ts
View File

@@ -97,14 +97,14 @@ class MempoolBlocks {
blockSize += tx.size;
transactions.push(tx);
} else {
mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
mempoolBlocks.push(this.dataToMempoolBlocks(transactions, mempoolBlocks.length));
blockWeight = tx.weight;
blockSize = tx.size;
transactions = [tx];
}
});
if (transactions.length) {
mempoolBlocks.push(this.dataToMempoolBlocks(transactions, blockSize, blockWeight, mempoolBlocks.length));
mempoolBlocks.push(this.dataToMempoolBlocks(transactions, mempoolBlocks.length));
}
return mempoolBlocks;
@@ -281,7 +281,7 @@ class MempoolBlocks {
const mempoolBlocks = blocks.map((transactions, blockIndex) => {
return this.dataToMempoolBlocks(transactions.map(tx => {
return mempool[tx.txid] || null;
}).filter(tx => !!tx), undefined, undefined, blockIndex);
}).filter(tx => !!tx), blockIndex);
});
if (saveResults) {
@@ -293,18 +293,17 @@ class MempoolBlocks {
return mempoolBlocks;
}
private dataToMempoolBlocks(transactions: TransactionExtended[],
blockSize: number | undefined, blockWeight: number | undefined, blocksIndex: number): MempoolBlockWithTransactions {
let totalSize = blockSize || 0;
let totalWeight = blockWeight || 0;
if (blockSize === undefined && blockWeight === undefined) {
totalSize = 0;
totalWeight = 0;
transactions.forEach(tx => {
totalSize += tx.size;
totalWeight += tx.weight;
});
}
private dataToMempoolBlocks(transactions: TransactionExtended[], blocksIndex: number): MempoolBlockWithTransactions {
let totalSize = 0;
let totalWeight = 0;
const fitTransactions: TransactionExtended[] = [];
transactions.forEach(tx => {
totalSize += tx.size;
totalWeight += tx.weight;
if ((totalWeight + tx.weight) <= config.MEMPOOL.BLOCK_WEIGHT_UNITS * 1.2) {
fitTransactions.push(tx);
}
});
let rangeLength = 4;
if (blocksIndex === 0) {
rangeLength = 8;
@@ -322,7 +321,7 @@ class MempoolBlocks {
medianFee: Common.percentile(transactions.map((tx) => tx.effectiveFeePerVsize), config.MEMPOOL.RECOMMENDED_FEE_PERCENTILE),
feeRange: Common.getFeesInRange(transactions, rangeLength),
transactionIds: transactions.map((tx) => tx.txid),
transactions: transactions.map((tx) => Common.stripTransaction(tx)),
transactions: fitTransactions.map((tx) => Common.stripTransaction(tx)),
};
}
}

32
backend/src/api/mining/mining.ts
View File

@@ -11,14 +11,13 @@ import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjust
import config from '../../config';
import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
import PricesRepository from '../../repositories/PricesRepository';
import bitcoinApiFactory from '../bitcoin/bitcoin-api-factory';
import { bitcoinCoreApi } from '../bitcoin/bitcoin-api-factory';
import { IEsploraApi } from '../bitcoin/esplora-api.interface';
class Mining {
blocksPriceIndexingRunning = false;
constructor() {
}
private blocksPriceIndexingRunning = false;
public lastHashrateIndexingDate: number | null = null;
public lastWeeklyHashrateIndexingDate: number | null = null;
/**
* Get historical block predictions match rate
@@ -178,20 +177,21 @@ class Mining {
*/
public async $generatePoolHashrateHistory(): Promise<void> {
const now = new Date();
const lastestRunDate = await HashratesRepository.$getLatestRun('last_weekly_hashrates_indexing');
// Run only if:
// * lastestRunDate is set to 0 (node backend restart, reorg)
// * this.lastWeeklyHashrateIndexingDate is set to null (node backend restart, reorg)
// * we started a new week (around Monday midnight)
const runIndexing = lastestRunDate === 0 || now.getUTCDay() === 1 && lastestRunDate !== now.getUTCDate();
const runIndexing = this.lastWeeklyHashrateIndexingDate === null ||
now.getUTCDay() === 1 && this.lastWeeklyHashrateIndexingDate !== now.getUTCDate();
if (!runIndexing) {
logger.debug(`Pool hashrate history indexing is up to date, nothing to do`, logger.tags.mining);
return;
}
try {
const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
const genesisBlock: IEsploraApi.Block = await bitcoinApiFactory.$getBlock(await bitcoinClient.getBlockHash(0));
const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
const genesisTimestamp = genesisBlock.timestamp * 1000;
const indexedTimestamp = await HashratesRepository.$getWeeklyHashrateTimestamps();
@@ -266,7 +266,7 @@ class Mining {
++indexedThisRun;
++totalIndexed;
}
await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', new Date().getUTCDate());
this.lastWeeklyHashrateIndexingDate = new Date().getUTCDate();
if (newlyIndexed > 0) {
logger.notice(`Weekly mining pools hashrates indexing completed: indexed ${newlyIndexed}`, logger.tags.mining);
} else {
@@ -285,16 +285,16 @@ class Mining {
*/
public async $generateNetworkHashrateHistory(): Promise<void> {
// We only run this once a day around midnight
const latestRunDate = await HashratesRepository.$getLatestRun('last_hashrates_indexing');
const now = new Date().getUTCDate();
if (now === latestRunDate) {
const today = new Date().getUTCDate();
if (today === this.lastHashrateIndexingDate) {
logger.debug(`Network hashrate history indexing is up to date, nothing to do`, logger.tags.mining);
return;
}
const oldestConsecutiveBlockTimestamp = 1000 * (await BlocksRepository.$getOldestConsecutiveBlock()).timestamp;
try {
const genesisBlock: IEsploraApi.Block = await bitcoinApiFactory.$getBlock(await bitcoinClient.getBlockHash(0));
const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
const genesisTimestamp = genesisBlock.timestamp * 1000;
const indexedTimestamp = (await HashratesRepository.$getRawNetworkDailyHashrate(null)).map(hashrate => hashrate.timestamp);
const lastMidnight = this.getDateMidnight(new Date());
@@ -371,7 +371,7 @@ class Mining {
newlyIndexed += hashrates.length;
await HashratesRepository.$saveHashrates(hashrates);
await HashratesRepository.$setLatestRun('last_hashrates_indexing', new Date().getUTCDate());
this.lastHashrateIndexingDate = new Date().getUTCDate();
if (newlyIndexed > 0) {
logger.notice(`Daily network hashrate indexing completed: indexed ${newlyIndexed} days`, logger.tags.mining);
} else {
@@ -396,7 +396,7 @@ class Mining {
}
const blocks: any = await BlocksRepository.$getBlocksDifficulty();
const genesisBlock: IEsploraApi.Block = await bitcoinApiFactory.$getBlock(await bitcoinClient.getBlockHash(0));
const genesisBlock: IEsploraApi.Block = await bitcoinCoreApi.$getBlock(await bitcoinClient.getBlockHash(0));
let currentDifficulty = genesisBlock.difficulty;
let totalIndexed = 0;

3
backend/src/index.ts
View File

@@ -87,9 +87,6 @@ class Server {
await databaseMigration.$blocksReindexingTruncate();
}
await databaseMigration.$initializeOrMigrateDatabase();
if (Common.indexingEnabled()) {
await indexer.$resetHashratesIndexingState();
}
} catch (e) {
throw new Error(e instanceof Error ? e.message : 'Error');
}

12
backend/src/indexer.ts
View File

@@ -3,7 +3,6 @@ import blocks from './api/blocks';
import mempool from './api/mempool';
import mining from './api/mining/mining';
import logger from './logger';
import HashratesRepository from './repositories/HashratesRepository';
import bitcoinClient from './api/bitcoin/bitcoin-client';
import priceUpdater from './tasks/price-updater';
import PricesRepository from './repositories/PricesRepository';
@@ -131,7 +130,6 @@ class Indexer {
this.runSingleTask('blocksPrices');
await mining.$indexDifficultyAdjustments();
await this.$resetHashratesIndexingState(); // TODO - Remove this as it's not efficient
await mining.$generateNetworkHashrateHistory();
await mining.$generatePoolHashrateHistory();
await blocks.$generateBlocksSummariesDatabase();
@@ -150,16 +148,6 @@ class Indexer {
logger.debug(`Indexing completed. Next run planned at ${new Date(new Date().getTime() + runEvery).toUTCString()}`);
setTimeout(() => this.reindex(), runEvery);
}
async $resetHashratesIndexingState(): Promise<void> {
try {
await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0);
await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0);
} catch (e) {
logger.err(`Cannot reset hashrate indexing timestamps. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
}
export default new Indexer();

1
backend/src/repositories/DifficultyAdjustmentsRepository.ts
View File

@@ -1,5 +1,4 @@
import { Common } from '../api/common';
import config from '../config';
import DB from '../database';
import logger from '../logger';
import { IndexedDifficultyAdjustment } from '../mempool.interfaces';

23
backend/src/repositories/HashratesRepository.ts
View File

@@ -1,5 +1,6 @@
import { escape } from 'mysql2';
import { Common } from '../api/common';
import mining from '../api/mining/mining';
import DB from '../database';
import logger from '../logger';
import PoolsRepository from './PoolsRepository';
@@ -177,20 +178,6 @@ class HashratesRepository {
}
}
/**
* Set latest run timestamp
*/
public async $setLatestRun(key: string, val: number) {
const query = `UPDATE state SET number = ? WHERE name = ?`;
try {
await DB.query(query, [val, key]);
} catch (e) {
logger.err(`Cannot set last indexing run for ${key}. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Get latest run timestamp
*/
@@ -222,8 +209,8 @@ class HashratesRepository {
await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp = ?`, [row.timestamp]);
}
// Re-run the hashrate indexing to fill up missing data
await this.$setLatestRun('last_hashrates_indexing', 0);
await this.$setLatestRun('last_weekly_hashrates_indexing', 0);
mining.lastHashrateIndexingDate = null;
mining.lastWeeklyHashrateIndexingDate = null;
} catch (e) {
logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e));
}
@@ -238,8 +225,8 @@ class HashratesRepository {
try {
await DB.query(`DELETE FROM hashrates WHERE hashrate_timestamp >= FROM_UNIXTIME(?)`, [timestamp]);
// Re-run the hashrate indexing to fill up missing data
await this.$setLatestRun('last_hashrates_indexing', 0);
await this.$setLatestRun('last_weekly_hashrates_indexing', 0);
mining.lastHashrateIndexingDate = null;
mining.lastWeeklyHashrateIndexingDate = null;
} catch (e) {
logger.err('Cannot delete latest hashrates data points. Reason: ' + (e instanceof Error ? e.message : e));
}