machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[Indexing] Link blocks to their closest known price

Browse Source
This commit is contained in:
nymkappa 2022-07-09 16:53:29 +02:00
parent 80b3b91a82
commit 40634a0eb8
No known key found for this signature in database
GPG Key ID: E155910B16E8BD04
8 changed files with 338 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
backend/src/api/blocks.ts
View File

@ -18,13 +18,10 @@ import BlocksRepository from '../repositories/BlocksRepository';
import HashratesRepository from '../repositories/HashratesRepository'; import HashratesRepository from '../repositories/HashratesRepository';
import indexer from '../indexer'; import indexer from '../indexer';
import fiatConversion from './fiat-conversion'; import fiatConversion from './fiat-conversion';
import RatesRepository from '../repositories/RatesRepository';
import database from '../database';
import poolsParser from './pools-parser'; import poolsParser from './pools-parser';
import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository'; import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
import mining from './mining/mining'; import mining from './mining/mining';
import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository'; import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
import difficultyAdjustment from './difficulty-adjustment';
class Blocks { class Blocks {
private blocks: BlockExtended[] = []; private blocks: BlockExtended[] = [];

363
backend/src/api/database-migration.ts
View File

@ -4,7 +4,7 @@ import logger from '../logger';
import { Common } from './common'; import { Common } from './common';
class DatabaseMigration { class DatabaseMigration {
private static currentVersion = 29; private static currentVersion = 30;
private queryTimeout = 120000; private queryTimeout = 120000;
private statisticsAddedIndexed = false; private statisticsAddedIndexed = false;
private uniqueLogs: string[] = []; private uniqueLogs: string[] = [];
@ -12,8 +12,6 @@ class DatabaseMigration {
private blocksTruncatedMessage = `'blocks' table has been truncated. Re-indexing from scratch.`; private blocksTruncatedMessage = `'blocks' table has been truncated. Re-indexing from scratch.`;
private hashratesTruncatedMessage = `'hashrates' table has been truncated. Re-indexing from scratch.`; private hashratesTruncatedMessage = `'hashrates' table has been truncated. Re-indexing from scratch.`;
constructor() { }
/** /**
* Avoid printing multiple time the same message * Avoid printing multiple time the same message
*/ */
@ -104,195 +102,187 @@ class DatabaseMigration {
await this.$setStatisticsAddedIndexedFlag(databaseSchemaVersion); await this.$setStatisticsAddedIndexedFlag(databaseSchemaVersion);
const isBitcoin = ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK); const isBitcoin = ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK);
try {
await this.$executeQuery(this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs'));
await this.$executeQuery(this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
if (databaseSchemaVersion < 2 && this.statisticsAddedIndexed === false) {
await this.$executeQuery(`CREATE INDEX added ON statistics (added);`);
}
if (databaseSchemaVersion < 3) {
await this.$executeQuery(this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools'));
}
if (databaseSchemaVersion < 4) {
await this.$executeQuery('DROP table IF EXISTS blocks;');
await this.$executeQuery(this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
}
if (databaseSchemaVersion < 5 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
await this.$executeQuery('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 6 && isBitcoin === true) { await this.$executeQuery(this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs'));
this.uniqueLog(logger.notice, this.blocksTruncatedMessage); await this.$executeQuery(this.getCreateStatisticsQuery(), await this.$checkIfTableExists('statistics'));
await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index if (databaseSchemaVersion < 2 && this.statisticsAddedIndexed === false) {
// Cleanup original blocks fields type await this.$executeQuery(`CREATE INDEX added ON statistics (added);`);
await this.$executeQuery('ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"'); }
await this.$executeQuery('ALTER TABLE blocks MODIFY `tx_count` smallint unsigned NOT NULL DEFAULT "0"'); if (databaseSchemaVersion < 3) {
await this.$executeQuery('ALTER TABLE blocks MODIFY `size` integer unsigned NOT NULL DEFAULT "0"'); await this.$executeQuery(this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools'));
await this.$executeQuery('ALTER TABLE blocks MODIFY `weight` integer unsigned NOT NULL DEFAULT "0"'); }
await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` double NOT NULL DEFAULT "0"'); if (databaseSchemaVersion < 4) {
// We also fix the pools.id type so we need to drop/re-create the foreign key await this.$executeQuery('DROP table IF EXISTS blocks;');
await this.$executeQuery('ALTER TABLE blocks DROP FOREIGN KEY IF EXISTS `blocks_ibfk_1`'); await this.$executeQuery(this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
await this.$executeQuery('ALTER TABLE pools MODIFY `id` smallint unsigned AUTO_INCREMENT'); }
await this.$executeQuery('ALTER TABLE blocks MODIFY `pool_id` smallint unsigned NULL'); if (databaseSchemaVersion < 5 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE blocks ADD FOREIGN KEY (`pool_id`) REFERENCES `pools` (`id`)'); this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
// Add new block indexing fields await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
await this.$executeQuery('ALTER TABLE blocks ADD `version` integer unsigned NOT NULL DEFAULT "0"'); await this.$executeQuery('ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks ADD `bits` integer unsigned NOT NULL DEFAULT "0"'); }
await this.$executeQuery('ALTER TABLE blocks ADD `nonce` bigint unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks ADD `merkle_root` varchar(65) NOT NULL DEFAULT ""');
await this.$executeQuery('ALTER TABLE blocks ADD `previous_block_hash` varchar(65) NULL');
}
if (databaseSchemaVersion < 7 && isBitcoin === true) { if (databaseSchemaVersion < 6 && isBitcoin === true) {
await this.$executeQuery('DROP table IF EXISTS hashrates;'); this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery(this.getCreateDailyStatsTableQuery(), await this.$checkIfTableExists('hashrates')); await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
} // Cleanup original blocks fields type
await this.$executeQuery('ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `tx_count` smallint unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `size` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `weight` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` double NOT NULL DEFAULT "0"');
// We also fix the pools.id type so we need to drop/re-create the foreign key
await this.$executeQuery('ALTER TABLE blocks DROP FOREIGN KEY IF EXISTS `blocks_ibfk_1`');
await this.$executeQuery('ALTER TABLE pools MODIFY `id` smallint unsigned AUTO_INCREMENT');
await this.$executeQuery('ALTER TABLE blocks MODIFY `pool_id` smallint unsigned NULL');
await this.$executeQuery('ALTER TABLE blocks ADD FOREIGN KEY (`pool_id`) REFERENCES `pools` (`id`)');
// Add new block indexing fields
await this.$executeQuery('ALTER TABLE blocks ADD `version` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks ADD `bits` integer unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks ADD `nonce` bigint unsigned NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks ADD `merkle_root` varchar(65) NOT NULL DEFAULT ""');
await this.$executeQuery('ALTER TABLE blocks ADD `previous_block_hash` varchar(65) NULL');
}
if (databaseSchemaVersion < 8 && isBitcoin === true) { if (databaseSchemaVersion < 7 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.blocksTruncatedMessage); await this.$executeQuery('DROP table IF EXISTS hashrates;');
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index await this.$executeQuery(this.getCreateDailyStatsTableQuery(), await this.$checkIfTableExists('hashrates'));
await this.$executeQuery('ALTER TABLE `hashrates` DROP INDEX `PRIMARY`'); }
await this.$executeQuery('ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
await this.$executeQuery('ALTER TABLE `hashrates` ADD `share` float NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"');
}
if (databaseSchemaVersion < 9 && isBitcoin === true) { if (databaseSchemaVersion < 8 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.hashratesTruncatedMessage); this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery('ALTER TABLE `state` CHANGE `name` `name` varchar(100)'); await this.$executeQuery('ALTER TABLE `hashrates` DROP INDEX `PRIMARY`');
await this.$executeQuery('ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)'); await this.$executeQuery('ALTER TABLE `hashrates` ADD `id` int NOT NULL AUTO_INCREMENT PRIMARY KEY FIRST');
} await this.$executeQuery('ALTER TABLE `hashrates` ADD `share` float NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `hashrates` ADD `type` enum("daily", "weekly") DEFAULT "daily"');
}
if (databaseSchemaVersion < 10 && isBitcoin === true) { if (databaseSchemaVersion < 9 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)'); this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
} await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
await this.$executeQuery('ALTER TABLE `state` CHANGE `name` `name` varchar(100)');
await this.$executeQuery('ALTER TABLE `hashrates` ADD UNIQUE `hashrate_timestamp_pool_id` (`hashrate_timestamp`, `pool_id`)');
}
if (databaseSchemaVersion < 11 && isBitcoin === true) { if (databaseSchemaVersion < 10 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.blocksTruncatedMessage); await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `blockTimestamp` (`blockTimestamp`)');
await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index }
await this.$executeQuery(`ALTER TABLE blocks
ADD avg_fee INT UNSIGNED NULL,
ADD avg_fee_rate INT UNSIGNED NULL
`);
await this.$executeQuery('ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` INT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` INT UNSIGNED NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 12 && isBitcoin === true) { if (databaseSchemaVersion < 11 && isBitcoin === true) {
// No need to re-index because the new data type can contain larger values this.uniqueLog(logger.notice, this.blocksTruncatedMessage);
await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` BIGINT UNSIGNED NOT NULL DEFAULT "0"'); await this.$executeQuery('TRUNCATE blocks;'); // Need to re-index
} await this.$executeQuery(`ALTER TABLE blocks
ADD avg_fee INT UNSIGNED NULL,
ADD avg_fee_rate INT UNSIGNED NULL
`);
await this.$executeQuery('ALTER TABLE blocks MODIFY `reward` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` INT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` INT UNSIGNED NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 13 && isBitcoin === true) { if (databaseSchemaVersion < 12 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` DOUBLE UNSIGNED NOT NULL DEFAULT "0"'); // No need to re-index because the new data type can contain larger values
await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"'); await this.$executeQuery('ALTER TABLE blocks MODIFY `fees` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"'); }
await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee_rate` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 14 && isBitcoin === true) { if (databaseSchemaVersion < 13 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.hashratesTruncatedMessage); await this.$executeQuery('ALTER TABLE blocks MODIFY `difficulty` DOUBLE UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index await this.$executeQuery('ALTER TABLE blocks MODIFY `median_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`'); await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"'); await this.$executeQuery('ALTER TABLE blocks MODIFY `avg_fee_rate` BIGINT UNSIGNED NOT NULL DEFAULT "0"');
} }
if (databaseSchemaVersion < 16 && isBitcoin === true) { if (databaseSchemaVersion < 14 && isBitcoin === true) {
this.uniqueLog(logger.notice, this.hashratesTruncatedMessage); this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index because we changed timestamps await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index
} await this.$executeQuery('ALTER TABLE `hashrates` DROP FOREIGN KEY `hashrates_ibfk_1`');
await this.$executeQuery('ALTER TABLE `hashrates` MODIFY `pool_id` SMALLINT UNSIGNED NOT NULL DEFAULT "0"');
}
if (databaseSchemaVersion < 17 && isBitcoin === true) { if (databaseSchemaVersion < 16 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `pools` ADD `slug` CHAR(50) NULL'); this.uniqueLog(logger.notice, this.hashratesTruncatedMessage);
} await this.$executeQuery('TRUNCATE hashrates;'); // Need to re-index because we changed timestamps
}
if (databaseSchemaVersion < 18 && isBitcoin === true) { if (databaseSchemaVersion < 17 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `hash` (`hash`);'); await this.$executeQuery('ALTER TABLE `pools` ADD `slug` CHAR(50) NULL');
} }
if (databaseSchemaVersion < 19) { if (databaseSchemaVersion < 18 && isBitcoin === true) {
await this.$executeQuery(this.getCreateRatesTableQuery(), await this.$checkIfTableExists('rates')); await this.$executeQuery('ALTER TABLE `blocks` ADD INDEX `hash` (`hash`);');
} }
if (databaseSchemaVersion < 20 && isBitcoin === true) { if (databaseSchemaVersion < 19) {
await this.$executeQuery(this.getCreateBlocksSummariesTableQuery(), await this.$checkIfTableExists('blocks_summaries')); await this.$executeQuery(this.getCreateRatesTableQuery(), await this.$checkIfTableExists('rates'));
} }
if (databaseSchemaVersion < 21) { if (databaseSchemaVersion < 20 && isBitcoin === true) {
await this.$executeQuery('DROP TABLE IF EXISTS `rates`'); await this.$executeQuery(this.getCreateBlocksSummariesTableQuery(), await this.$checkIfTableExists('blocks_summaries'));
await this.$executeQuery(this.getCreatePricesTableQuery(), await this.$checkIfTableExists('prices')); }
}
if (databaseSchemaVersion < 22 && isBitcoin === true) { if (databaseSchemaVersion < 21) {
await this.$executeQuery('DROP TABLE IF EXISTS `difficulty_adjustments`'); await this.$executeQuery('DROP TABLE IF EXISTS `rates`');
await this.$executeQuery(this.getCreateDifficultyAdjustmentsTableQuery(), await this.$checkIfTableExists('difficulty_adjustments')); await this.$executeQuery(this.getCreatePricesTableQuery(), await this.$checkIfTableExists('prices'));
} }
if (databaseSchemaVersion < 23) { if (databaseSchemaVersion < 22 && isBitcoin === true) {
await this.$executeQuery('TRUNCATE `prices`'); await this.$executeQuery('DROP TABLE IF EXISTS `difficulty_adjustments`');
await this.$executeQuery('ALTER TABLE `prices` DROP `avg_prices`'); await this.$executeQuery(this.getCreateDifficultyAdjustmentsTableQuery(), await this.$checkIfTableExists('difficulty_adjustments'));
await this.$executeQuery('ALTER TABLE `prices` ADD `USD` float DEFAULT "0"'); }
await this.$executeQuery('ALTER TABLE `prices` ADD `EUR` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `GBP` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `CAD` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `CHF` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `AUD` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `JPY` float DEFAULT "0"');
}
if (databaseSchemaVersion < 24 && isBitcoin == true) { if (databaseSchemaVersion < 23) {
await this.$executeQuery('DROP TABLE IF EXISTS `blocks_audits`'); await this.$executeQuery('TRUNCATE `prices`');
await this.$executeQuery(this.getCreateBlocksAuditsTableQuery(), await this.$checkIfTableExists('blocks_audits')); await this.$executeQuery('ALTER TABLE `prices` DROP `avg_prices`');
} await this.$executeQuery('ALTER TABLE `prices` ADD `USD` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `EUR` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `GBP` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `CAD` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `CHF` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `AUD` float DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `prices` ADD `JPY` float DEFAULT "0"');
}
if (databaseSchemaVersion < 25 && isBitcoin === true) { if (databaseSchemaVersion < 24 && isBitcoin == true) {
await this.$executeQuery(`INSERT INTO state VALUES('last_node_stats', 0, '1970-01-01');`); await this.$executeQuery('DROP TABLE IF EXISTS `blocks_audits`');
await this.$executeQuery(this.getCreateLightningStatisticsQuery(), await this.$checkIfTableExists('lightning_stats')); await this.$executeQuery(this.getCreateBlocksAuditsTableQuery(), await this.$checkIfTableExists('blocks_audits'));
await this.$executeQuery(this.getCreateNodesQuery(), await this.$checkIfTableExists('nodes')); }
await this.$executeQuery(this.getCreateChannelsQuery(), await this.$checkIfTableExists('channels'));
await this.$executeQuery(this.getCreateNodesStatsQuery(), await this.$checkIfTableExists('node_stats'));
}
if (databaseSchemaVersion < 26 && isBitcoin === true) { if (databaseSchemaVersion < 25 && isBitcoin === true) {
this.uniqueLog(logger.notice, `'lightning_stats' table has been truncated. Will re-generate historical data from scratch.`); await this.$executeQuery(`INSERT INTO state VALUES('last_node_stats', 0, '1970-01-01');`);
await this.$executeQuery(`TRUNCATE lightning_stats`); await this.$executeQuery(this.getCreateLightningStatisticsQuery(), await this.$checkIfTableExists('lightning_stats'));
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD tor_nodes int(11) NOT NULL DEFAULT "0"'); await this.$executeQuery(this.getCreateNodesQuery(), await this.$checkIfTableExists('nodes'));
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_nodes int(11) NOT NULL DEFAULT "0"'); await this.$executeQuery(this.getCreateChannelsQuery(), await this.$checkIfTableExists('channels'));
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD unannounced_nodes int(11) NOT NULL DEFAULT "0"'); await this.$executeQuery(this.getCreateNodesStatsQuery(), await this.$checkIfTableExists('node_stats'));
} }
if (databaseSchemaVersion < 27 && isBitcoin === true) { if (databaseSchemaVersion < 26 && isBitcoin === true) {
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD avg_capacity bigint(20) unsigned NOT NULL DEFAULT "0"'); this.uniqueLog(logger.notice, `'lightning_stats' table has been truncated. Will re-generate historical data from scratch.`);
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD avg_fee_rate int(11) unsigned NOT NULL DEFAULT "0"'); await this.$executeQuery(`TRUNCATE lightning_stats`);
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD avg_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"'); await this.$executeQuery('ALTER TABLE `lightning_stats` ADD tor_nodes int(11) NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_capacity bigint(20) unsigned NOT NULL DEFAULT "0"'); await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_nodes int(11) NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_fee_rate int(11) unsigned NOT NULL DEFAULT "0"'); await this.$executeQuery('ALTER TABLE `lightning_stats` ADD unannounced_nodes int(11) NOT NULL DEFAULT "0"');
await this.$executeQuery('ALTER TABLE `lightning_stats` ADD med_base_fee_mtokens bigint(20) unsigned NOT NULL DEFAULT "0"'); }
}
if (databaseSchemaVersion < 28 && isBitcoin === true) { if (databaseSchemaVersion < 28 && isBitcoin === true) {
await this.$executeQuery(`TRUNCATE lightning_stats`); await this.$executeQuery(`TRUNCATE lightning_stats`);
await this.$executeQuery(`TRUNCATE node_stats`); await this.$executeQuery(`TRUNCATE node_stats`);
await this.$executeQuery(`ALTER TABLE lightning_stats MODIFY added DATE`); await this.$executeQuery(`ALTER TABLE lightning_stats MODIFY added DATE`);
} }
if (databaseSchemaVersion < 29 && isBitcoin === true) { if (databaseSchemaVersion < 29 && isBitcoin === true) {
await this.$executeQuery(this.getCreateGeoNamesTableQuery(), await this.$checkIfTableExists('geo_names')); await this.$executeQuery(this.getCreateGeoNamesTableQuery(), await this.$checkIfTableExists('geo_names'));
await this.$executeQuery('ALTER TABLE `nodes` ADD as_number int(11) unsigned NULL DEFAULT NULL'); await this.$executeQuery('ALTER TABLE `nodes` ADD as_number int(11) unsigned NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD city_id int(11) unsigned NULL DEFAULT NULL'); await this.$executeQuery('ALTER TABLE `nodes` ADD city_id int(11) unsigned NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD country_id int(11) unsigned NULL DEFAULT NULL'); await this.$executeQuery('ALTER TABLE `nodes` ADD country_id int(11) unsigned NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD accuracy_radius int(11) unsigned NULL DEFAULT NULL'); await this.$executeQuery('ALTER TABLE `nodes` ADD accuracy_radius int(11) unsigned NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD subdivision_id int(11) unsigned NULL DEFAULT NULL'); await this.$executeQuery('ALTER TABLE `nodes` ADD subdivision_id int(11) unsigned NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD longitude double NULL DEFAULT NULL'); await this.$executeQuery('ALTER TABLE `nodes` ADD longitude double NULL DEFAULT NULL');
await this.$executeQuery('ALTER TABLE `nodes` ADD latitude double NULL DEFAULT NULL'); await this.$executeQuery('ALTER TABLE `nodes` ADD latitude double NULL DEFAULT NULL');
} }
if (databaseSchemaVersion < 25 && isBitcoin == true) { // Link blocks to prices
} catch (e) { await this.$executeQuery('ALTER TABLE `prices` ADD `id` int NULL AUTO_INCREMENT UNIQUE');
throw e; await this.$executeQuery('DROP TABLE IF EXISTS `blocks_prices`');
await this.$executeQuery(this.getCreateBlocksPricesTableQuery(), await this.$checkIfTableExists('blocks_prices'));
} }
} }
@ -331,7 +321,7 @@ class DatabaseMigration {
/** /**
* Small query execution wrapper to log all executed queries * Small query execution wrapper to log all executed queries
*/ */
private async $executeQuery(query: string, silent: boolean = false): Promise<any> { private async $executeQuery(query: string, silent = false): Promise<any> {
if (!silent) { if (!silent) {
logger.debug('MIGRATIONS: Execute query:\n' + query); logger.debug('MIGRATIONS: Execute query:\n' + query);
} }
@ -360,21 +350,17 @@ class DatabaseMigration {
* Create the `state` table * Create the `state` table
*/ */
private async $createMigrationStateTable(): Promise<void> { private async $createMigrationStateTable(): Promise<void> {
try { const query = `CREATE TABLE IF NOT EXISTS state (
const query = `CREATE TABLE IF NOT EXISTS state ( name varchar(25) NOT NULL,
name varchar(25) NOT NULL, number int(11) NULL,
number int(11) NULL, string varchar(100) NULL,
string varchar(100) NULL, CONSTRAINT name_unique UNIQUE (name)
CONSTRAINT name_unique UNIQUE (name) ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; await this.$executeQuery(query);
await this.$executeQuery(query);
// Set initial values // Set initial values
await this.$executeQuery(`INSERT INTO state VALUES('schema_version', 0, NULL);`); await this.$executeQuery(`INSERT INTO state VALUES('schema_version', 0, NULL);`);
await this.$executeQuery(`INSERT INTO state VALUES('last_elements_block', 0, NULL);`); await this.$executeQuery(`INSERT INTO state VALUES('last_elements_block', 0, NULL);`);
} catch (e) {
throw e;
}
} }
/** /**
@ -714,6 +700,15 @@ class DatabaseMigration {
) ENGINE=InnoDB DEFAULT CHARSET=utf8;` ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`
} }
private getCreateBlocksPricesTableQuery(): string {
return `CREATE TABLE IF NOT EXISTS blocks_prices (
height int(10) unsigned NOT NULL,
price_id int(10) unsigned NOT NULL,
PRIMARY KEY (height),
INDEX (price_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
public async $truncateIndexedData(tables: string[]) { public async $truncateIndexedData(tables: string[]) {
const allowedTables = ['blocks', 'hashrates', 'prices']; const allowedTables = ['blocks', 'hashrates', 'prices'];

67
backend/src/api/mining/mining.ts
View File

@ -11,8 +11,11 @@ import indexer from '../../indexer';
import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository'; import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository';
import config from '../../config'; import config from '../../config';
import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository'; import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
import PricesRepository from '../repositories/PricesRepository';
class Mining { class Mining {
blocksPriceIndexingRunning = false;
constructor() { constructor() {
} }
@ -453,6 +456,70 @@ class Mining {
} }
} }
/**
* Create a link between blocks and the latest price at when they were mined
*/
public async $indexBlockPrices() {
if (this.blocksPriceIndexingRunning === true) {
return;
}
this.blocksPriceIndexingRunning = true;
try {
const prices: any[] = await PricesRepository.$getPricesTimesAndId();
const blocksWithoutPrices: any[] = await BlocksRepository.$getBlocksWithoutPrice();
let totalInserted = 0;
const blocksPrices: BlockPrice[] = [];
for (const block of blocksWithoutPrices) {
// Quick optimisation, out mtgox feed only goes back to 2010-07-19 02:00:00, so skip the first 68951 blocks
if (block.height < 68951) {
blocksPrices.push({
height: block.height,
priceId: prices[0].id,
});
continue;
}
for (const price of prices) {
if (block.timestamp < price.time) {
blocksPrices.push({
height: block.height,
priceId: price.id,
});
break;
};
}
if (blocksPrices.length >= 100000) {
totalInserted += blocksPrices.length;
if (blocksWithoutPrices.length > 200000) {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`);
} else {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`);
}
await BlocksRepository.$saveBlockPrices(blocksPrices);
blocksPrices.length = 0;
}
}
if (blocksPrices.length > 0) {
totalInserted += blocksPrices.length;
if (blocksWithoutPrices.length > 200000) {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price | Progress ${Math.round(totalInserted / blocksWithoutPrices.length * 100)}%`);
} else {
logger.debug(`Linking ${blocksPrices.length} newly indexed blocks to their closest price`);
}
await BlocksRepository.$saveBlockPrices(blocksPrices);
}
} catch (e) {
this.blocksPriceIndexingRunning = false;
throw e;
}
this.blocksPriceIndexingRunning = false;
}
private getDateMidnight(date: Date): Date { private getDateMidnight(date: Date): Date {
date.setUTCHours(0); date.setUTCHours(0);
date.setUTCMinutes(0); date.setUTCMinutes(0);

4
backend/src/index.ts
View File

@ -35,6 +35,8 @@ import miningRoutes from "./api/mining/mining-routes";
import bisqRoutes from "./api/bisq/bisq.routes"; import bisqRoutes from "./api/bisq/bisq.routes";
import liquidRoutes from "./api/liquid/liquid.routes"; import liquidRoutes from "./api/liquid/liquid.routes";
import bitcoinRoutes from "./api/bitcoin/bitcoin.routes"; import bitcoinRoutes from "./api/bitcoin/bitcoin.routes";
import BlocksAuditsRepository from './repositories/BlocksAuditsRepository';
import mining from "./api/mining";
class Server { class Server {
private wss: WebSocket.Server | undefined; private wss: WebSocket.Server | undefined;
@ -166,7 +168,7 @@ class Server {
await blocks.$updateBlocks(); await blocks.$updateBlocks();
await memPool.$updateMempool(); await memPool.$updateMempool();
indexer.$run(); indexer.$run();
priceUpdater.$run(); priceUpdater.$run().then(mining.$indexBlockPrices.bind(this));
setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS); setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
this.currentBackendRetryInterval = 5; this.currentBackendRetryInterval = 5;

2
backend/src/indexer.ts
View File

@ -48,7 +48,7 @@ class Indexer {
} }
await mining.$indexDifficultyAdjustments(); await mining.$indexDifficultyAdjustments();
await this.$resetHashratesIndexingState(); await this.$resetHashratesIndexingState(); // TODO - Remove this as it's not efficient
await mining.$generateNetworkHashrateHistory(); await mining.$generateNetworkHashrateHistory();
await mining.$generatePoolHashrateHistory(); await mining.$generatePoolHashrateHistory();
await blocks.$generateBlocksSummariesDatabase(); await blocks.$generateBlocksSummariesDatabase();

5
backend/src/mempool.interfaces.ts
View File

@ -121,6 +121,11 @@ export interface BlockSummary {
transactions: TransactionStripped[]; transactions: TransactionStripped[];
} }
export interface BlockPrice {
height: number;
priceId: number;
}
export interface TransactionMinerInfo { export interface TransactionMinerInfo {
vin: VinStrippedToScriptsig[]; vin: VinStrippedToScriptsig[];
vout: VoutStrippedToScriptPubkey[]; vout: VoutStrippedToScriptPubkey[];

93
backend/src/repositories/BlocksRepository.ts
View File

@ -1,4 +1,4 @@
import { BlockExtended } from '../mempool.interfaces'; import { BlockExtended, BlockPrice } from '../mempool.interfaces';
import DB from '../database'; import DB from '../database';
import logger from '../logger'; import logger from '../logger';
import { Common } from '../api/common'; import { Common } from '../api/common';
@ -275,9 +275,7 @@ class BlocksRepository {
previous_block_hash as previousblockhash, previous_block_hash as previousblockhash,
avg_fee, avg_fee,
avg_fee_rate, avg_fee_rate,
IFNULL(JSON_EXTRACT(rates.bisq_rates, '$.USD'), null) as usd
FROM blocks FROM blocks
LEFT JOIN rates on rates.height = blocks.height
WHERE pool_id = ?`; WHERE pool_id = ?`;
params.push(pool.id); params.push(pool.id);
@ -335,12 +333,10 @@ class BlocksRepository {
merkle_root, merkle_root,
previous_block_hash as previousblockhash, previous_block_hash as previousblockhash,
avg_fee, avg_fee,
avg_fee_rate, avg_fee_rate
IFNULL(JSON_EXTRACT(rates.bisq_rates, '$.USD'), null) as usd
FROM blocks FROM blocks
JOIN pools ON blocks.pool_id = pools.id JOIN pools ON blocks.pool_id = pools.id
LEFT JOIN rates on rates.height = blocks.height WHERE blocks.height = ${height}
WHERE blocks.height = ${height};
`); `);
if (rows.length <= 0) { if (rows.length <= 0) {
@ -365,10 +361,8 @@ class BlocksRepository {
pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug, pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.slug as pool_slug,
pools.addresses as pool_addresses, pools.regexes as pool_regexes, pools.addresses as pool_addresses, pools.regexes as pool_regexes,
previous_block_hash as previousblockhash, previous_block_hash as previousblockhash,
IFNULL(JSON_EXTRACT(rates.bisq_rates, '$.USD'), null) as usd
FROM blocks FROM blocks
JOIN pools ON blocks.pool_id = pools.id JOIN pools ON blocks.pool_id = pools.id
LEFT JOIN rates on rates.height = blocks.height
WHERE hash = '${hash}'; WHERE hash = '${hash}';
`; `;
const [rows]: any[] = await DB.query(query); const [rows]: any[] = await DB.query(query);
@ -393,7 +387,20 @@ class BlocksRepository {
const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty FROM blocks`); const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty FROM blocks`);
return rows; return rows;
} catch (e) { } catch (e) {
logger.err('Cannot generate difficulty history. Reason: ' + (e instanceof Error ? e.message : e)); logger.err('Cannot get blocks difficulty list from the db. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Return blocks height
*/
public async $getBlocksHeightsAndTimestamp(): Promise<object[]> {
try {
const [rows]: any[] = await DB.query(`SELECT height, blockTimestamp as timestamp FROM blocks`);
return rows;
} catch (e) {
logger.err('Cannot get blocks height and timestamp from the db. Reason: ' + (e instanceof Error ? e.message : e));
throw e; throw e;
} }
} }
@ -481,10 +488,9 @@ class BlocksRepository {
let query = `SELECT let query = `SELECT
CAST(AVG(blocks.height) as INT) as avgHeight, CAST(AVG(blocks.height) as INT) as avgHeight,
CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp, CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
CAST(AVG(fees) as INT) as avgFees, CAST(AVG(fees) as INT) as avgFees
IFNULL(JSON_EXTRACT(rates.bisq_rates, '$.USD'), null) as usd
FROM blocks FROM blocks
LEFT JOIN rates on rates.height = blocks.height`; `;
if (interval !== null) { if (interval !== null) {
query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`; query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
@ -508,10 +514,9 @@ class BlocksRepository {
let query = `SELECT let query = `SELECT
CAST(AVG(blocks.height) as INT) as avgHeight, CAST(AVG(blocks.height) as INT) as avgHeight,
CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp, CAST(AVG(UNIX_TIMESTAMP(blockTimestamp)) as INT) as timestamp,
CAST(AVG(reward) as INT) as avgRewards, CAST(AVG(reward) as INT) as avgRewards
IFNULL(JSON_EXTRACT(rates.bisq_rates, '$.USD'), null) as usd
FROM blocks FROM blocks
LEFT JOIN rates on rates.height = blocks.height`; `;
if (interval !== null) { if (interval !== null) {
query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`; query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
@ -638,6 +643,62 @@ class BlocksRepository {
throw e; throw e;
} }
} }
/**
* Get all blocks which have not be linked to a price yet
*/
public async $getBlocksWithoutPrice(): Promise<object[]> {
try {
const [rows]: any[] = await DB.query(`
SELECT UNIX_TIMESTAMP(blocks.blockTimestamp) as timestamp, blocks.height
FROM blocks
LEFT JOIN blocks_prices ON blocks.height = blocks_prices.height
WHERE blocks_prices.height IS NULL
ORDER BY blocks.height
`);
return rows;
} catch (e) {
logger.err('Cannot get blocks height and timestamp from the db. Reason: ' + (e instanceof Error ? e.message : e));
throw e;
}
}
/**
* Save block price
*/
public async $saveBlockPrice(blockPrice: BlockPrice): Promise<void> {
try {
await DB.query(`INSERT INTO blocks_prices(height, price_id) VALUE (?, ?)`, [blockPrice.height, blockPrice.priceId]);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`Cannot save block price for block ${blockPrice.height} because it has already been indexed, ignoring`);
} else {
logger.err(`Cannot save block price into db. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
}
/**
* Save block price by batch
*/
public async $saveBlockPrices(blockPrices: BlockPrice[]): Promise<void> {
try {
let query = `INSERT INTO blocks_prices(height, price_id) VALUES`;
for (const price of blockPrices) {
query += ` (${price.height}, ${price.priceId}),`
}
query = query.slice(0, -1);
await DB.query(query);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`Cannot save blocks prices for blocks [${blockPrices[0].height} to ${blockPrices[blockPrices.length - 1].height}] because it has already been indexed, ignoring`);
} else {
logger.err(`Cannot save blocks prices for blocks [${blockPrices[0].height} to ${blockPrices[blockPrices.length - 1].height}] into db. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
}
} }
export default new BlocksRepository(); export default new BlocksRepository();

7
backend/src/repositories/PricesRepository.ts
View File

@ -33,9 +33,14 @@ class PricesRepository {
} }
public async $getPricesTimes(): Promise<number[]> { public async $getPricesTimes(): Promise<number[]> {
const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1`); const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time from prices WHERE USD != -1 ORDER BY time`);
return times.map(time => time.time); return times.map(time => time.time);
} }
public async $getPricesTimesAndId(): Promise<number[]> {
const [times]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(time) as time, id, USD from prices ORDER BY time`);
return times;
}
} }
export default new PricesRepository(); export default new PricesRepository();