machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Index difficulty adjustments

Browse Source
This commit is contained in:
nymkappa 2022-06-25 12:14:32 +02:00
parent 570d8cfc74
commit acfdc8163b
No known key found for this signature in database
GPG Key ID: E155910B16E8BD04
14 changed files with 225 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
backend/src/api/blocks.ts
View File

@ -19,6 +19,9 @@ import HashratesRepository from '../repositories/HashratesRepository';
import indexer from '../indexer';
import poolsParser from './pools-parser';
import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
import mining from './mining';
import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
import difficultyAdjustment from './difficulty-adjustment';
class Blocks {
private blocks: BlockExtended[] = [];
@ -449,7 +452,10 @@ class Blocks {
const newBlock = await this.$indexBlock(lastBlock['height'] - i);
await this.$getStrippedBlockTransactions(newBlock.id, true, true);
}
logger.info(`Re-indexed 10 blocks and summaries`);
await mining.$indexDifficultyAdjustments();
await DifficultyAdjustmentsRepository.$deleteLastAdjustment();
logger.info(`Re-indexed 10 blocks and summaries. Also re-indexed the last difficulty adjustments. Will re-index latest hashrates in a few seconds.`);
indexer.reindex();
}
await blocksRepository.$saveBlockInDatabase(blockExtended);
@ -461,6 +467,15 @@ class Blocks {
}
if (block.height % 2016 === 0) {
if (Common.indexingEnabled()) {
await DifficultyAdjustmentsRepository.$saveAdjustments({
time: block.timestamp,
height: block.height,
difficulty: block.difficulty,
adjustment: Math.round((block.difficulty / this.currentDifficulty) * 1000000) / 1000000, // Remove float point noise
});
}
this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100;
this.lastDifficultyAdjustmentTime = block.timestamp;
this.currentDifficulty = block.difficulty;

20
backend/src/api/database-migration.ts
View File

@ -4,7 +4,7 @@ import logger from '../logger';
import { Common } from './common';
class DatabaseMigration {
private static currentVersion = 21;
private static currentVersion = 22;
private queryTimeout = 120000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@ -226,6 +226,11 @@ class DatabaseMigration {
await this.$executeQuery('DROP TABLE IF EXISTS `rates`');
await this.$executeQuery(this.getCreatePricesTableQuery(), await this.$checkIfTableExists('prices'));
}
if (databaseSchemaVersion < 22 && isBitcoin === true) {
await this.$executeQuery('DROP TABLE IF EXISTS `difficulty_adjustments`');
await this.$executeQuery(this.getCreateDifficultyAdjustmentsTableQuery(), await this.$checkIfTableExists('difficulty_adjustments'));
}
} catch (e) {
throw e;
}
@ -513,7 +518,7 @@ class DatabaseMigration {
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreateRatesTableQuery(): string {
private getCreateRatesTableQuery(): string { // This table has been replaced by the prices table
return `CREATE TABLE IF NOT EXISTS rates (
height int(10) unsigned NOT NULL,
bisq_rates JSON NOT NULL,
@ -539,6 +544,17 @@ class DatabaseMigration {
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
private getCreateDifficultyAdjustmentsTableQuery(): string {
return `CREATE TABLE IF NOT EXISTS difficulty_adjustments (
time timestamp NOT NULL,
height int(10) unsigned NOT NULL,
difficulty double unsigned NOT NULL,
adjustment float NOT NULL,
PRIMARY KEY (height),
INDEX (time)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
}
public async $truncateIndexedData(tables: string[]) {
const allowedTables = ['blocks', 'hashrates', 'prices'];

45
backend/src/api/mining.ts
View File

@ -1,4 +1,4 @@
import { PoolInfo, PoolStats, RewardStats } from '../mempool.interfaces';
import { IndexedDifficultyAdjustment, PoolInfo, PoolStats, RewardStats } from '../mempool.interfaces';
import BlocksRepository from '../repositories/BlocksRepository';
import PoolsRepository from '../repositories/PoolsRepository';
import HashratesRepository from '../repositories/HashratesRepository';
@ -7,6 +7,7 @@ import logger from '../logger';
import { Common } from './common';
import loadingIndicators from './loading-indicators';
import { escape } from 'mysql2';
import DifficultyAdjustmentsRepository from '../repositories/DifficultyAdjustmentsRepository';
class Mining {
constructor() {
@ -377,6 +378,48 @@ class Mining {
}
}
/**
* Index difficulty adjustments
*/
public async $indexDifficultyAdjustments(): Promise<void> {
const indexedHeightsArray = await DifficultyAdjustmentsRepository.$getAdjustmentsHeights();
const indexedHeights = {};
for (const height of indexedHeightsArray) {
indexedHeights[height] = true;
}
const blocks: any = await BlocksRepository.$getBlocksDifficulty();
let currentDifficulty = 0;
let totalIndexed = 0;
for (const block of blocks) {
if (block.difficulty !== currentDifficulty) {
if (block.height === 0 || indexedHeights[block.height] === true) { // Already indexed
currentDifficulty = block.difficulty;
continue;
}
let adjustment = block.difficulty / Math.max(1, currentDifficulty);
adjustment = Math.round(adjustment * 1000000) / 1000000; // Remove float point noise
await DifficultyAdjustmentsRepository.$saveAdjustments({
time: block.time,
height: block.height,
difficulty: block.difficulty,
adjustment: adjustment,
});
totalIndexed++;
currentDifficulty = block.difficulty;
}
}
if (totalIndexed > 0) {
logger.notice(`Indexed ${totalIndexed} difficulty adjustments`);
}
}
private getDateMidnight(date: Date): Date {
date.setUTCHours(0);
date.setUTCMinutes(0);

1
backend/src/index.ts
View File

@ -290,6 +290,7 @@ class Server {
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/rewards/:interval', routes.$getHistoricalBlockRewards)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/fee-rates/:interval', routes.$getHistoricalBlockFeeRates)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', routes.$getHistoricalBlockSizeAndWeight)
.get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments/:interval', routes.$getDifficultyAdjustments)
;
}

1
backend/src/indexer.ts
View File

@ -36,6 +36,7 @@ class Indexer {
return;
}
await mining.$indexDifficultyAdjustments();
await this.$resetHashratesIndexingState();
await mining.$generateNetworkHashrateHistory();
await mining.$generatePoolHashrateHistory();

7
backend/src/mempool.interfaces.ts
View File

@ -224,6 +224,13 @@ export interface IDifficultyAdjustment {
timeOffset: number;
}
export interface IndexedDifficultyAdjustment {
time: number; // UNIX timestamp
height: number; // Block height
difficulty: number;
adjustment: number;
}
export interface RewardStats {
totalReward: number;
totalFee: number;

65
backend/src/repositories/BlocksRepository.ts
View File

@ -7,6 +7,7 @@ import PoolsRepository from './PoolsRepository';
import HashratesRepository from './HashratesRepository';
import { escape } from 'mysql2';
import BlocksSummariesRepository from './BlocksSummariesRepository';
import DifficultyAdjustmentsRepository from './DifficultyAdjustmentsRepository';
class BlocksRepository {
/**
@ -381,48 +382,9 @@ class BlocksRepository {
/**
* Return blocks difficulty
*/
public async $getBlocksDifficulty(interval: string | null): Promise<object[]> {
interval = Common.getSqlInterval(interval);
// :D ... Yeah don't ask me about this one https://stackoverflow.com/a/40303162
// Basically, using temporary user defined fields, we are able to extract all
// difficulty adjustments from the blocks tables.
// This allow use to avoid indexing it in another table.
let query = `
SELECT
*
FROM
(
SELECT
UNIX_TIMESTAMP(blockTimestamp) as timestamp, difficulty, height,
IF(@prevStatus = YT.difficulty, @rn := @rn + 1,
IF(@prevStatus := YT.difficulty, @rn := 1, @rn := 1)
) AS rn
FROM blocks YT
CROSS JOIN
(
SELECT @prevStatus := -1, @rn := 1
) AS var
`;
if (interval) {
query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
}
query += `
ORDER BY YT.height
) AS t
WHERE t.rn = 1
ORDER BY t.height
`;
public async $getBlocksDifficulty(): Promise<object[]> {
try {
const [rows]: any[] = await DB.query(query);
for (const row of rows) {
delete row['rn'];
}
const [rows]: any[] = await DB.query(`SELECT UNIX_TIMESTAMP(blockTimestamp) as time, height, difficulty FROM blocks`);
return rows;
} catch (e) {
logger.err('Cannot generate difficulty history. Reason: ' + (e instanceof Error ? e.message : e));
@ -452,26 +414,6 @@ class BlocksRepository {
}
}
/*
* Check if the last 10 blocks chain is valid
*/
public async $validateRecentBlocks(): Promise<boolean> {
try {
const [lastBlocks]: any[] = await DB.query(`SELECT height, hash, previous_block_hash FROM blocks ORDER BY height DESC LIMIT 10`);
for (let i = 0; i < lastBlocks.length - 1; ++i) {
if (lastBlocks[i].previous_block_hash !== lastBlocks[i + 1].hash) {
logger.warn(`Chain divergence detected at block ${lastBlocks[i].height}, re-indexing most recent data`);
return false;
}
}
return true;
} catch (e) {
return true; // Don't do anything if there is a db error
}
}
/**
* Check if the chain of block hash is valid and delete data from the stale branch if needed
*/
@ -498,6 +440,7 @@ class BlocksRepository {
await this.$deleteBlocksFrom(blocks[idx - 1].height);
await BlocksSummariesRepository.$deleteBlocksFrom(blocks[idx - 1].height);
await HashratesRepository.$deleteHashratesFromTimestamp(blocks[idx - 1].timestamp - 604800);
await DifficultyAdjustmentsRepository.$deleteAdjustementsFromHeight(blocks[idx - 1].height);
return false;
}
++idx;

88
backend/src/repositories/DifficultyAdjustmentsRepository.ts Normal file
View File

@ -0,0 +1,88 @@
import { Common } from '../api/common';
import DB from '../database';
import logger from '../logger';
import { IndexedDifficultyAdjustment } from '../mempool.interfaces';
class DifficultyAdjustmentsRepository {
public async $saveAdjustments(adjustment: IndexedDifficultyAdjustment): Promise<void> {
if (adjustment.height === 1) {
return;
}
try {
const query = `INSERT INTO difficulty_adjustments(time, height, difficulty, adjustment) VALUE (FROM_UNIXTIME(?), ?, ?, ?)`;
const params: any[] = [
adjustment.time,
adjustment.height,
adjustment.difficulty,
adjustment.adjustment,
];
await DB.query(query, params);
} catch (e: any) {
if (e.errno === 1062) { // ER_DUP_ENTRY - This scenario is possible upon node backend restart
logger.debug(`Cannot save difficulty adjustment at block ${adjustment.height}, already indexed, ignoring`);
} else {
logger.err(`Cannot save difficulty adjustment at block ${adjustment.height}. Reason: ${e instanceof Error ? e.message : e}`);
throw e;
}
}
}
public async $getAdjustments(interval: string | null, descOrder: boolean = false): Promise<IndexedDifficultyAdjustment[]> {
interval = Common.getSqlInterval(interval);
let query = `SELECT UNIX_TIMESTAMP(time) as time, height, difficulty, adjustment
FROM difficulty_adjustments`;
if (interval) {
query += ` WHERE time BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
}
if (descOrder === true) {
query += ` ORDER BY time DESC`;
} else {
query += ` ORDER BY time`;
}
try {
const [rows] = await DB.query(query);
return rows as IndexedDifficultyAdjustment[];
} catch (e) {
logger.err(`Cannot get difficulty adjustments from the database. Reason: ` + (e instanceof Error ? e.message : e));
throw e;
}
}
public async $getAdjustmentsHeights(): Promise<number[]> {
try {
const [rows]: any[] = await DB.query(`SELECT height FROM difficulty_adjustments`);
return rows.map(block => block.height);
} catch (e: any) {
logger.err(`Cannot get difficulty adjustment block heights. Reason: ${e instanceof Error ? e.message : e}`);
throw e;
}
}
public async $deleteAdjustementsFromHeight(height: number): Promise<void> {
try {
logger.info(`Delete newer difficulty adjustments from height ${height} from the database`);
await DB.query(`DELETE FROM difficulty_adjustments WHERE height >= ?`, [height]);
} catch (e: any) {
logger.err(`Cannot delete difficulty adjustments from the database. Reason: ${e instanceof Error ? e.message : e}`);
throw e;
}
}
public async $deleteLastAdjustment(): Promise<void> {
try {
logger.info(`Delete last difficulty adjustment from the database`);
await DB.query(`DELETE FROM difficulty_adjustments ORDER BY time LIMIT 1`);
} catch (e: any) {
logger.err(`Cannot delete last difficulty adjustment from the database. Reason: ${e instanceof Error ? e.message : e}`);
throw e;
}
}
}
export default new DifficultyAdjustmentsRepository();

15
backend/src/routes.ts
View File

@ -26,6 +26,7 @@ import mining from './api/mining';
import BlocksRepository from './repositories/BlocksRepository';
import HashratesRepository from './repositories/HashratesRepository';
import difficultyAdjustment from './api/difficulty-adjustment';
import DifficultyAdjustmentsRepository from './repositories/DifficultyAdjustmentsRepository';
class Routes {
constructor() {}
@ -653,7 +654,7 @@ class Routes {
try {
const hashrates = await HashratesRepository.$getNetworkDailyHashrate(req.params.interval);
const difficulty = await BlocksRepository.$getBlocksDifficulty(req.params.interval);
const difficulty = await DifficultyAdjustmentsRepository.$getAdjustments(req.params.interval, false);
const blockCount = await BlocksRepository.$blockCount(null, null);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
@ -730,6 +731,18 @@ class Routes {
}
}
public async $getDifficultyAdjustments(req: Request, res: Response) {
try {
const difficulty = await DifficultyAdjustmentsRepository.$getAdjustments(req.params.interval, true);
res.header('Pragma', 'public');
res.header('Cache-control', 'public');
res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
res.json(difficulty.map(adj => [adj.time, adj.height, adj.difficulty, adj.adjustment]));
} catch (e) {
res.status(500).send(e instanceof Error ? e.message : e);
}
}
public async getBlock(req: Request, res: Response) {
try {
const block = await blocks.$getBlock(req.params.hash);

4
frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.component.html
View File

@ -9,7 +9,7 @@
</tr>
</thead>
<tbody *ngIf="(hashrateObservable$ | async) as data">
<tr *ngFor="let diffChange of data.difficulty">
<tr *ngFor="let diffChange of data">
<td class="d-none d-md-block"><a [routerLink]="['/block' | relativeUrl, diffChange.height]">{{ diffChange.height
}}</a></td>
<td class="text-left">
@ -17,7 +17,7 @@
</td>
<td class="text-right">{{ diffChange.difficultyShorten }}</td>
<td class="text-right" [style]="diffChange.change >= 0 ? 'color: #42B747' : 'color: #B74242'">
{{ diffChange.change >= 0 ? '+' : '' }}{{ diffChange.change | amountShortener }}%
{{ diffChange.change >= 0 ? '+' : '' }}{{ diffChange.change | amountShortener: 2 }}%
</td>
</tr>
</tbody>

23
frontend/src/app/components/difficulty-adjustments-table/difficulty-adjustments-table.components.ts
View File

@ -30,27 +30,24 @@ export class DifficultyAdjustmentsTable implements OnInit {
}
ngOnInit(): void {
this.hashrateObservable$ = this.apiService.getHistoricalHashrate$('1y')
this.hashrateObservable$ = this.apiService.getDifficultyAdjustments$('3m')
.pipe(
map((response) => {
const data = response.body;
const tableData = [];
for (let i = data.difficulty.length - 1; i > 0; --i) {
const selectedPowerOfTen: any = selectPowerOfTen(data.difficulty[i].difficulty);
const change = (data.difficulty[i].difficulty / data.difficulty[i - 1].difficulty - 1) * 100;
tableData.push(Object.assign(data.difficulty[i], {
change: Math.round(change * 100) / 100,
for (const adjustment of data) {
const selectedPowerOfTen: any = selectPowerOfTen(adjustment[2]);
tableData.push({
height: adjustment[1],
timestamp: adjustment[0],
change: (adjustment[3] - 1) * 100,
difficultyShorten: formatNumber(
data.difficulty[i].difficulty / selectedPowerOfTen.divider,
adjustment[2] / selectedPowerOfTen.divider,
this.locale, '1.2-2') + selectedPowerOfTen.unit
}));
});
}
this.isLoading = false;
return {
difficulty: tableData.slice(0, 6),
};
return tableData.slice(0, 6);
}),
);
}

3
frontend/src/app/components/hashrate-chart/hashrate-chart.component.ts
View File

@ -95,6 +95,7 @@ export class HashrateChartComponent implements OnInit {
.pipe(
tap((response) => {
const data = response.body;
// We generate duplicated data point so the tooltip works nicely
const diffFixed = [];
let diffIndex = 1;
@ -112,7 +113,7 @@ export class HashrateChartComponent implements OnInit {
}
while (hashIndex < data.hashrates.length && diffIndex < data.difficulty.length &&
data.hashrates[hashIndex].timestamp <= data.difficulty[diffIndex].timestamp
data.hashrates[hashIndex].timestamp <= data.difficulty[diffIndex].time
) {
diffFixed.push({
timestamp: data.hashrates[hashIndex].timestamp,

9
frontend/src/app/services/api.service.ts
View File

@ -140,7 +140,7 @@ export class ApiService {
this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/pools` +
(interval !== undefined ? `/${interval}` : ''), { observe: 'response' }
);
}
}
getPoolStats$(slug: string): Observable<PoolStat> {
return this.httpClient.get<PoolStat>(this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/pool/${slug}`);
@ -172,6 +172,13 @@ export class ApiService {
return this.httpClient.get<TransactionStripped[]>(this.apiBaseUrl + this.apiBasePath + '/api/v1/block/' + hash + '/summary');
}
getDifficultyAdjustments$(interval: string | undefined): Observable<any> {
return this.httpClient.get<any[]>(
this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/difficulty-adjustments` +
(interval !== undefined ? `/${interval}` : ''), { observe: 'response' }
);
}
getHistoricalHashrate$(interval: string | undefined): Observable<any> {
return this.httpClient.get<any[]>(
this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/hashrate` +

10
production/nginx-cache-warmer
View File

@ -67,6 +67,16 @@ do for url in / \
'/api/v1/mining/blocks/fee-rates/2y' \
'/api/v1/mining/blocks/fee-rates/3y' \
'/api/v1/mining/blocks/fee-rates/all' \
'/api/v1/mining/difficulty-adjustments/24h' \
'/api/v1/mining/difficulty-adjustments/3d' \
'/api/v1/mining/difficulty-adjustments/1w' \
'/api/v1/mining/difficulty-adjustments/1m' \
'/api/v1/mining/difficulty-adjustments/3m' \
'/api/v1/mining/difficulty-adjustments/6m' \
'/api/v1/mining/difficulty-adjustments/1y' \
'/api/v1/mining/difficulty-adjustments/2y' \
'/api/v1/mining/difficulty-adjustments/3y' \
'/api/v1/mining/difficulty-adjustments/all' \
do
curl -s "https://${hostname}${url}" >/dev/null