machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #4887 from mempool/mononaut/local-acceleration-data

Browse Source 
Local acceleration data
This commit is contained in:
softsimon 2024-04-08 20:43:46 +09:00 committed by GitHub
commit 03867ada49
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 103 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
backend/src/api/acceleration/acceleration.routes.ts
View File

@ -1,12 +1,14 @@
import { Application, Request, Response } from "express";
import config from "../../config";
import axios from "axios";
import logger from "../../logger";
import { Application, Request, Response } from 'express';
import config from '../../config';
import axios from 'axios';
import logger from '../../logger';
import mempool from '../mempool';
import AccelerationRepository from '../../repositories/AccelerationRepository';
class AccelerationRoutes {
private tag = 'Accelerator';
public initRoutes(app: Application) {
public initRoutes(app: Application): void {
app
.get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations', this.$getAcceleratorAccelerations.bind(this))
.get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/history', this.$getAcceleratorAccelerationsHistory.bind(this))
@ -15,35 +17,27 @@ class AccelerationRoutes {
;
}
private async $getAcceleratorAccelerations(req: Request, res: Response) {
const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
try {
const response = await axios.get(url, { responseType: 'stream', timeout: 10000 });
for (const key in response.headers) {
res.setHeader(key, response.headers[key]);
}
response.data.pipe(res);
} catch (e) {
logger.err(`Unable to get current accelerations from ${url} in $getAcceleratorAccelerations(), ${e}`, this.tag);
res.status(500).end();
}
private async $getAcceleratorAccelerations(req: Request, res: Response): Promise<void> {
const accelerations = mempool.getAccelerations();
res.status(200).send(Object.values(accelerations));
}
private async $getAcceleratorAccelerationsHistory(req: Request, res: Response) {
const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
try {
const response = await axios.get(url, { responseType: 'stream', timeout: 10000 });
for (const key in response.headers) {
res.setHeader(key, response.headers[key]);
}
response.data.pipe(res);
} catch (e) {
logger.err(`Unable to get acceleration history from ${url} in $getAcceleratorAccelerationsHistory(), ${e}`, this.tag);
res.status(500).end();
}
private async $getAcceleratorAccelerationsHistory(req: Request, res: Response): Promise<void> {
const history = await AccelerationRepository.$getAccelerationInfo(null, req.query.blockHeight ? parseInt(req.query.blockHeight as string, 10) : null);
res.status(200).send(history.map(accel => ({
txid: accel.txid,
added: accel.added,
status: 'completed',
effectiveFee: accel.effective_fee,
effectiveVsize: accel.effective_vsize,
boostRate: accel.boost_rate,
boostCost: accel.boost_cost,
blockHeight: accel.height,
pools: [accel.pool],
})));
}
private async $getAcceleratorAccelerationsHistoryAggregated(req: Request, res: Response) {
private async $getAcceleratorAccelerationsHistoryAggregated(req: Request, res: Response): Promise<void> {
const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
try {
const response = await axios.get(url, { responseType: 'stream', timeout: 10000 });
@ -57,7 +51,7 @@ class AccelerationRoutes {
}
}
private async $getAcceleratorAccelerationsStats(req: Request, res: Response) {
private async $getAcceleratorAccelerationsStats(req: Request, res: Response): Promise<void> {
const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
try {
const response = await axios.get(url, { responseType: 'stream', timeout: 10000 });

2
backend/src/api/blocks.ts
View File

@ -29,6 +29,7 @@ import websocketHandler from './websocket-handler';
import redisCache from './redis-cache';
import rbfCache from './rbf-cache';
import { calcBitsDifference } from './difficulty-adjustment';
import AccelerationRepository from '../repositories/AccelerationRepository';
class Blocks {
private blocks: BlockExtended[] = [];
@ -872,6 +873,7 @@ class Blocks {
await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10);
await HashratesRepository.$deleteLastEntries();
await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10);
await AccelerationRepository.$deleteAccelerationsFrom(lastBlock.height - 10);
this.blocks = this.blocks.slice(0, -10);
this.updateTimerProgress(timer, `rolled back chain divergence from ${this.currentBlockHeight}`);
for (let i = 10; i >= 0; --i) {

7
backend/src/api/database-migration.ts
View File

@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
import { RowDataPacket } from 'mysql2';
class DatabaseMigration {
private static currentVersion = 76;
private static currentVersion = 77;
private queryTimeout = 3600_000;
private statisticsAddedIndexed = false;
private uniqueLogs: string[] = [];
@ -664,6 +664,11 @@ class DatabaseMigration {
await this.$executeQuery('ALTER TABLE `blocks_audits` ADD prioritized_txs JSON DEFAULT "[]"');
await this.updateToSchemaVersion(76);
}
if (databaseSchemaVersion < 77 && config.MEMPOOL.NETWORK === 'mainnet') {
await this.$executeQuery('ALTER TABLE `accelerations` ADD requested datetime DEFAULT NULL');
await this.updateToSchemaVersion(77);
}
}
/**

3
backend/src/api/services/acceleration.ts
View File

@ -5,6 +5,9 @@ import axios from 'axios';
export interface Acceleration {
txid: string,
added: number,
effectiveVsize: number,
effectiveFee: number,
feeDelta: number,
pools: number[],
};

47
backend/src/repositories/AccelerationRepository.ts
View File

@ -6,7 +6,7 @@ import { IEsploraApi } from '../api/bitcoin/esplora-api.interface';
import { Common } from '../api/common';
import config from '../config';
import blocks from '../api/blocks';
import accelerationApi, { Acceleration } from '../api/services/acceleration';
import accelerationApi, { Acceleration, AccelerationHistory } from '../api/services/acceleration';
import accelerationCosts from '../api/acceleration/acceleration';
import bitcoinApi from '../api/bitcoin/bitcoin-api-factory';
import transactionUtils from '../api/transaction-utils';
@ -15,6 +15,7 @@ import { BlockExtended, MempoolTransactionExtended } from '../mempool.interfaces
export interface PublicAcceleration {
txid: string,
height: number,
added: number,
pool: {
id: number,
slug: string,
@ -29,15 +30,20 @@ export interface PublicAcceleration {
class AccelerationRepository {
private bidBoostV2Activated = 831580;
public async $saveAcceleration(acceleration: AccelerationInfo, block: IEsploraApi.Block, pool_id: number): Promise<void> {
public async $saveAcceleration(acceleration: AccelerationInfo, block: IEsploraApi.Block, pool_id: number, accelerationData: Acceleration[]): Promise<void> {
const accelerationMap: { [txid: string]: Acceleration } = {};
for (const acc of accelerationData) {
accelerationMap[acc.txid] = acc;
}
try {
await DB.query(`
INSERT INTO accelerations(txid, added, height, pool, effective_vsize, effective_fee, boost_rate, boost_cost)
VALUE (?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?)
INSERT INTO accelerations(txid, requested, added, height, pool, effective_vsize, effective_fee, boost_rate, boost_cost)
VALUE (?, FROM_UNIXTIME(?), FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?)
ON DUPLICATE KEY UPDATE
height = ?
`, [
acceleration.txSummary.txid,
accelerationMap[acceleration.txSummary.txid].added,
block.timestamp,
block.height,
pool_id,
@ -64,7 +70,7 @@ class AccelerationRepository {
}
let query = `
SELECT * FROM accelerations
SELECT *, UNIX_TIMESTAMP(requested) as requested_timestamp, UNIX_TIMESTAMP(added) as block_timestamp FROM accelerations
JOIN pools on pools.unique_id = accelerations.pool
`;
let params: any[] = [];
@ -99,6 +105,7 @@ class AccelerationRepository {
return rows.map(row => ({
txid: row.txid,
height: row.height,
added: row.requested_timestamp || row.block_timestamp,
pool: {
id: row.id,
slug: row.slug,
@ -202,7 +209,7 @@ class AccelerationRepository {
const tx = blockTxs[acc.txid];
const accelerationInfo = accelerationCosts.getAccelerationInfo(tx, boostRate, transactions);
accelerationInfo.cost = Math.max(0, Math.min(acc.feeDelta, accelerationInfo.cost));
this.$saveAcceleration(accelerationInfo, block, block.extras.pool.id);
this.$saveAcceleration(accelerationInfo, block, block.extras.pool.id, successfulAccelerations);
}
}
const lastSyncedHeight = await this.$getLastSyncedHeight();
@ -230,7 +237,7 @@ class AccelerationRepository {
logger.debug(`Fetching accelerations between block ${lastSyncedHeight} and ${currentHeight}`);
// Fetch accelerations from mempool.space since the last synced block;
const accelerationsByBlock = {};
const accelerationsByBlock: {[height: number]: AccelerationHistory[]} = {};
const blockHashes = {};
let done = false;
let page = 1;
@ -297,12 +304,16 @@ class AccelerationRepository {
const feeStats = Common.calcEffectiveFeeStatistics(template);
boostRate = feeStats.medianFee;
}
const accelerationSummaries = accelerations.map(acc => ({
...acc,
pools: acc.pools.map(pool => pool.pool_unique_id),
}))
for (const acc of accelerations) {
if (blockTxs[acc.txid]) {
const tx = blockTxs[acc.txid];
const accelerationInfo = accelerationCosts.getAccelerationInfo(tx, boostRate, transactions);
accelerationInfo.cost = Math.max(0, Math.min(acc.feeDelta, accelerationInfo.cost));
await this.$saveAcceleration(accelerationInfo, block, block.extras.pool.id);
await this.$saveAcceleration(accelerationInfo, block, block.extras.pool.id, accelerationSummaries);
}
}
await this.$setLastSyncedHeight(height);
@ -317,6 +328,26 @@ class AccelerationRepository {
logger.debug(`Indexing accelerations completed`);
}
/**
* Delete accelerations from the database above blockHeight
*/
public async $deleteAccelerationsFrom(blockHeight: number): Promise<void> {
logger.info(`Delete newer accelerations from height ${blockHeight} from the database`);
try {
const currentSyncedHeight = await this.$getLastSyncedHeight();
if (currentSyncedHeight >= blockHeight) {
await DB.query(`
UPDATE state
SET number = ?
WHERE name = 'last_acceleration_block'
`, [blockHeight - 1]);
}
await DB.query(`DELETE FROM accelerations where height >= ${blockHeight}`);
} catch (e) {
logger.err('Cannot delete indexed accelerations. Reason: ' + (e instanceof Error ? e.message : e));
}
}
}
export default new AccelerationRepository();

6
frontend/src/app/components/acceleration/accelerations-list/accelerations-list.component.html
View File

@ -39,10 +39,10 @@
</td>
</ng-container>
<ng-container *ngIf="!pending">
<td *ngIf="acceleration.feePaid" class="fee text-right">
{{ (acceleration.boost) | number }} <span class="symbol" i18n="shared.sat|sat">sat</span>
<td *ngIf="acceleration.boost != null" class="fee text-right">
{{ acceleration.boost | number }} <span class="symbol" i18n="shared.sat|sat">sat</span>
</td>
<td *ngIf="!acceleration.feePaid" class="fee text-right">
<td *ngIf="acceleration.boost == null" class="fee text-right">
~
</td>
<td class="block text-right">

2
frontend/src/app/components/acceleration/accelerations-list/accelerations-list.component.ts
View File

@ -58,7 +58,7 @@ export class AccelerationsListComponent implements OnInit {
}
}
for (const acc of accelerations) {
acc.boost = acc.feePaid - acc.baseFee - acc.vsizeFee;
acc.boost = acc.boostCost != null ? acc.boostCost : (acc.feePaid - acc.baseFee - acc.vsizeFee);
}
if (this.widget) {
return of(accelerations.slice(0, 6));

12
frontend/src/app/components/acceleration/accelerator-dashboard/accelerator-dashboard.component.ts
View File

@ -116,15 +116,15 @@ export class AcceleratorDashboardComponent implements OnInit {
switchMap(([accelerations, blocks]) => {
const blockMap = {};
for (const block of blocks) {
blockMap[block.id] = block;
blockMap[block.height] = block;
}
const accelerationsByBlock: { [ hash: string ]: Acceleration[] } = {};
const accelerationsByBlock: { [ height: number ]: Acceleration[] } = {};
for (const acceleration of accelerations) {
if (['completed_provisional', 'failed_provisional', 'completed'].includes(acceleration.status) && acceleration.pools.includes(blockMap[acceleration.blockHash]?.extras.pool.id)) {
if (!accelerationsByBlock[acceleration.blockHash]) {
accelerationsByBlock[acceleration.blockHash] = [];
if (['completed_provisional', 'failed_provisional', 'completed'].includes(acceleration.status) && acceleration.pools.includes(blockMap[acceleration.blockHeight]?.extras.pool.id)) {
if (!accelerationsByBlock[acceleration.blockHeight]) {
accelerationsByBlock[acceleration.blockHeight] = [];
}
accelerationsByBlock[acceleration.blockHash].push(acceleration);
accelerationsByBlock[acceleration.blockHeight].push(acceleration);
}
}
return of(blocks.slice(0, 6).map(block => {

2
frontend/src/app/components/block/block-preview.component.ts
View File

@ -136,7 +136,7 @@ export class BlockPreviewComponent implements OnInit, OnDestroy {
return of(transactions);
})
),
this.stateService.env.ACCELERATOR === true && block.height > 819500 ? this.servicesApiService.getAccelerationHistory$({ blockHash: block.id }) : of([])
this.stateService.env.ACCELERATOR === true && block.height > 819500 ? this.servicesApiService.getAccelerationHistory$({ blockHeight: block.height }) : of([])
]);
}
),

2
frontend/src/app/components/block/block.component.ts
View File

@ -345,7 +345,7 @@ export class BlockComponent implements OnInit, OnDestroy {
return of(null);
})
),
this.stateService.env.ACCELERATOR === true && block.height > 819500 ? this.servicesApiService.getAccelerationHistory$({ blockHash: block.id }) : of([])
this.stateService.env.ACCELERATOR === true && block.height > 819500 ? this.servicesApiService.getAccelerationHistory$({ blockHeight: block.height }) : of([])
]);
})
)

15
frontend/src/app/components/transaction/transaction.component.ts
View File

@ -98,7 +98,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
fetchCpfp$ = new Subject<string>();
fetchRbfHistory$ = new Subject<string>();
fetchCachedTx$ = new Subject<string>();
fetchAcceleration$ = new Subject<string>();
fetchAcceleration$ = new Subject<number>();
fetchMiningInfo$ = new Subject<{ hash: string, height: number, txid: string }>();
isCached: boolean = false;
now = Date.now();
@ -288,8 +288,8 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
tap(() => {
this.accelerationInfo = null;
}),
switchMap((blockHash: string) => {
return this.servicesApiService.getAccelerationHistory$({ blockHash });
switchMap((blockHeight: number) => {
return this.servicesApiService.getAccelerationHistory$({ blockHeight });
}),
catchError(() => {
return of(null);
@ -297,7 +297,10 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
).subscribe((accelerationHistory) => {
for (const acceleration of accelerationHistory) {
if (acceleration.txid === this.txId && (acceleration.status === 'completed' || acceleration.status === 'completed_provisional')) {
acceleration.acceleratedFeeRate = Math.max(acceleration.effectiveFee, acceleration.effectiveFee + acceleration.feePaid - acceleration.baseFee - acceleration.vsizeFee) / acceleration.effectiveVsize;
const boostCost = acceleration.boostCost || (acceleration.feePaid - acceleration.baseFee - acceleration.vsizeFee);
acceleration.acceleratedFeeRate = Math.max(acceleration.effectiveFee, acceleration.effectiveFee + boostCost) / acceleration.effectiveVsize;
acceleration.boost = boostCost;
this.accelerationInfo = acceleration;
}
}
@ -482,7 +485,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
this.getTransactionTime();
}
} else {
this.fetchAcceleration$.next(tx.status.block_hash);
this.fetchAcceleration$.next(tx.status.block_height);
this.fetchMiningInfo$.next({ hash: tx.status.block_hash, height: tx.status.block_height, txid: tx.txid });
this.transactionTime = 0;
}
@ -544,7 +547,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
} else {
this.audioService.playSound('magic');
}
this.fetchAcceleration$.next(block.id);
this.fetchAcceleration$.next(block.height);
this.fetchMiningInfo$.next({ hash: block.id, height: block.height, txid: this.tx.txid });
}
});

3
frontend/src/app/interfaces/node-api.interface.ts
View File

@ -396,6 +396,9 @@ export interface Acceleration {
acceleratedFeeRate?: number;
boost?: number;
boostCost?: number;
boostRate?: number;
}
export interface AccelerationHistoryParams {