Merge pull request #4887 from mempool/mononaut/local-acceleration-data

Local acceleration data

backend/src/api/acceleration/acceleration.routes.ts

@@ -1,12 +1,14 @@
-import { Application, Request, Response } from "express";
+import { Application, Request, Response } from 'express';
-import config from "../../config";
+import config from '../../config';
-import axios from "axios";
+import axios from 'axios';
-import logger from "../../logger";
+import logger from '../../logger';
+import mempool from '../mempool';
+import AccelerationRepository from '../../repositories/AccelerationRepository';
 
 class AccelerationRoutes {
   private tag = 'Accelerator';
 
-  public initRoutes(app: Application) {
+  public initRoutes(app: Application): void {
     app
       .get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations', this.$getAcceleratorAccelerations.bind(this))
       .get(config.MEMPOOL.API_URL_PREFIX + 'services/accelerator/accelerations/history', this.$getAcceleratorAccelerationsHistory.bind(this))
@@ -15,41 +17,33 @@ class AccelerationRoutes {
     ;
   }
 
-  private async $getAcceleratorAccelerations(req: Request, res: Response) {
+  private async $getAcceleratorAccelerations(req: Request, res: Response): Promise<void> {
-    const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
+    const accelerations = mempool.getAccelerations();
-    try {
+    res.status(200).send(Object.values(accelerations));
-      const response = await axios.get(url, { responseType: 'stream', timeout: 10000 });
-      for (const key in response.headers) {
-        res.setHeader(key, response.headers[key]); 
-      }      
-      response.data.pipe(res);
-    } catch (e) {
-      logger.err(`Unable to get current accelerations from ${url} in $getAcceleratorAccelerations(), ${e}`, this.tag);
-      res.status(500).end();
-    }
   }
 
-  private async $getAcceleratorAccelerationsHistory(req: Request, res: Response) {
+  private async $getAcceleratorAccelerationsHistory(req: Request, res: Response): Promise<void> {
-    const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
+    const history = await AccelerationRepository.$getAccelerationInfo(null, req.query.blockHeight ? parseInt(req.query.blockHeight as string, 10) : null);
-    try {
+    res.status(200).send(history.map(accel => ({
-      const response = await axios.get(url, { responseType: 'stream', timeout: 10000 });
+      txid: accel.txid,
-      for (const key in response.headers) {
+      added: accel.added,
-        res.setHeader(key, response.headers[key]); 
+      status: 'completed',
-      }      
+      effectiveFee: accel.effective_fee,
-      response.data.pipe(res);
+      effectiveVsize: accel.effective_vsize,
-    } catch (e) {
+      boostRate: accel.boost_rate,
-      logger.err(`Unable to get acceleration history from ${url} in $getAcceleratorAccelerationsHistory(), ${e}`, this.tag);
+      boostCost: accel.boost_cost,
-      res.status(500).end();
+      blockHeight: accel.height,
-    }
+      pools: [accel.pool],
+    })));
   }
 
-  private async $getAcceleratorAccelerationsHistoryAggregated(req: Request, res: Response) {
+  private async $getAcceleratorAccelerationsHistoryAggregated(req: Request, res: Response): Promise<void> {
     const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
     try {
       const response = await axios.get(url, { responseType: 'stream', timeout: 10000 });
       for (const key in response.headers) {
-        res.setHeader(key, response.headers[key]); 
+        res.setHeader(key, response.headers[key]);
-      }      
+      }
       response.data.pipe(res);
     } catch (e) {
       logger.err(`Unable to get aggregated acceleration history from ${url} in $getAcceleratorAccelerationsHistoryAggregated(), ${e}`, this.tag);
@@ -57,13 +51,13 @@ class AccelerationRoutes {
     }
   }
 
-  private async $getAcceleratorAccelerationsStats(req: Request, res: Response) {
+  private async $getAcceleratorAccelerationsStats(req: Request, res: Response): Promise<void> {
     const url = `${config.MEMPOOL_SERVICES.API}/${req.originalUrl.replace('/api/v1/services/', '')}`;
     try {
       const response = await axios.get(url, { responseType: 'stream', timeout: 10000 });
       for (const key in response.headers) {
-        res.setHeader(key, response.headers[key]); 
+        res.setHeader(key, response.headers[key]);
-      }      
+      }
       response.data.pipe(res);
     } catch (e) {
       logger.err(`Unable to get acceleration stats from ${url} in $getAcceleratorAccelerationsStats(), ${e}`, this.tag);

backend/src/api/blocks.ts

@@ -29,6 +29,7 @@ import websocketHandler from './websocket-handler';
 import redisCache from './redis-cache';
 import rbfCache from './rbf-cache';
 import { calcBitsDifference } from './difficulty-adjustment';
+import AccelerationRepository from '../repositories/AccelerationRepository';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -872,6 +873,7 @@ class Blocks {
             await BlocksRepository.$deleteBlocksFrom(lastBlock.height - 10);
             await HashratesRepository.$deleteLastEntries();
             await cpfpRepository.$deleteClustersFrom(lastBlock.height - 10);
+            await AccelerationRepository.$deleteAccelerationsFrom(lastBlock.height - 10);
             this.blocks = this.blocks.slice(0, -10);
             this.updateTimerProgress(timer, `rolled back chain divergence from ${this.currentBlockHeight}`);
             for (let i = 10; i >= 0; --i) {

backend/src/api/database-migration.ts

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 76;
+  private static currentVersion = 77;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -664,6 +664,11 @@ class DatabaseMigration {
       await this.$executeQuery('ALTER TABLE `blocks_audits` ADD prioritized_txs JSON DEFAULT "[]"');
       await this.updateToSchemaVersion(76);
     }
+
+    if (databaseSchemaVersion < 77 && config.MEMPOOL.NETWORK === 'mainnet') {
+      await this.$executeQuery('ALTER TABLE `accelerations` ADD requested datetime DEFAULT NULL');
+      await this.updateToSchemaVersion(77);
+    }
   }
 
   /**

backend/src/api/services/acceleration.ts

@@ -5,6 +5,9 @@ import axios from 'axios';
 
 export interface Acceleration {
   txid: string,
+  added: number,
+  effectiveVsize: number,
+  effectiveFee: number,
   feeDelta: number,
   pools: number[],
 };

backend/src/repositories/AccelerationRepository.ts

@@ -6,7 +6,7 @@ import { IEsploraApi } from '../api/bitcoin/esplora-api.interface';
 import { Common } from '../api/common';
 import config from '../config';
 import blocks from '../api/blocks';
-import accelerationApi, { Acceleration } from '../api/services/acceleration';
+import accelerationApi, { Acceleration, AccelerationHistory } from '../api/services/acceleration';
 import accelerationCosts from '../api/acceleration/acceleration';
 import bitcoinApi from '../api/bitcoin/bitcoin-api-factory';
 import transactionUtils from '../api/transaction-utils';
@@ -15,6 +15,7 @@ import { BlockExtended, MempoolTransactionExtended } from '../mempool.interfaces
 export interface PublicAcceleration {
   txid: string,
   height: number,
+  added: number,
   pool: {
     id: number,
     slug: string,
@@ -29,15 +30,20 @@ export interface PublicAcceleration {
 class AccelerationRepository {
   private bidBoostV2Activated = 831580;
 
-  public async $saveAcceleration(acceleration: AccelerationInfo, block: IEsploraApi.Block, pool_id: number): Promise<void> {
+  public async $saveAcceleration(acceleration: AccelerationInfo, block: IEsploraApi.Block, pool_id: number, accelerationData: Acceleration[]): Promise<void> {
+    const accelerationMap: { [txid: string]: Acceleration } = {};
+    for (const acc of accelerationData) {
+      accelerationMap[acc.txid] = acc;
+    }
     try {
       await DB.query(`
-        INSERT INTO accelerations(txid, added, height, pool, effective_vsize, effective_fee, boost_rate, boost_cost)
+        INSERT INTO accelerations(txid, requested, added, height, pool, effective_vsize, effective_fee, boost_rate, boost_cost)
-        VALUE (?, FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?)
+        VALUE (?, FROM_UNIXTIME(?), FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?)
         ON DUPLICATE KEY UPDATE
           height = ?
       `, [
         acceleration.txSummary.txid,
+        accelerationMap[acceleration.txSummary.txid].added,
         block.timestamp,
         block.height,
         pool_id,
@@ -64,7 +70,7 @@ class AccelerationRepository {
     }
 
     let query = `
-      SELECT * FROM accelerations
+      SELECT *, UNIX_TIMESTAMP(requested) as requested_timestamp, UNIX_TIMESTAMP(added) as block_timestamp FROM accelerations
       JOIN pools on pools.unique_id = accelerations.pool
     `;
     let params: any[] = [];
@@ -99,6 +105,7 @@ class AccelerationRepository {
         return rows.map(row => ({
           txid: row.txid,
           height: row.height,
+          added: row.requested_timestamp || row.block_timestamp,
           pool: {
             id: row.id,
             slug: row.slug,
@@ -202,7 +209,7 @@ class AccelerationRepository {
         const tx = blockTxs[acc.txid];
         const accelerationInfo = accelerationCosts.getAccelerationInfo(tx, boostRate, transactions);
         accelerationInfo.cost = Math.max(0, Math.min(acc.feeDelta, accelerationInfo.cost));
-        this.$saveAcceleration(accelerationInfo, block, block.extras.pool.id);
+        this.$saveAcceleration(accelerationInfo, block, block.extras.pool.id, successfulAccelerations);
       }
     }
     const lastSyncedHeight = await this.$getLastSyncedHeight();
@@ -230,7 +237,7 @@ class AccelerationRepository {
     logger.debug(`Fetching accelerations between block ${lastSyncedHeight} and ${currentHeight}`);
 
     // Fetch accelerations from mempool.space since the last synced block;
-    const accelerationsByBlock = {};
+    const accelerationsByBlock: {[height: number]: AccelerationHistory[]} = {};
     const blockHashes = {};
     let done = false;
     let page = 1;
@@ -297,12 +304,16 @@ class AccelerationRepository {
           const feeStats = Common.calcEffectiveFeeStatistics(template);
           boostRate = feeStats.medianFee;
         }
+        const accelerationSummaries = accelerations.map(acc => ({
+          ...acc,
+          pools: acc.pools.map(pool => pool.pool_unique_id),
+        }))
         for (const acc of accelerations) {
           if (blockTxs[acc.txid]) {
             const tx = blockTxs[acc.txid];
             const accelerationInfo = accelerationCosts.getAccelerationInfo(tx, boostRate, transactions);
             accelerationInfo.cost = Math.max(0, Math.min(acc.feeDelta, accelerationInfo.cost));
-            await this.$saveAcceleration(accelerationInfo, block, block.extras.pool.id);
+            await this.$saveAcceleration(accelerationInfo, block, block.extras.pool.id, accelerationSummaries);
           }
         }
         await this.$setLastSyncedHeight(height);
@@ -317,6 +328,26 @@ class AccelerationRepository {
 
     logger.debug(`Indexing accelerations completed`);
   }
+
+  /**
+   * Delete accelerations from the database above blockHeight
+   */
+  public async $deleteAccelerationsFrom(blockHeight: number): Promise<void> {
+    logger.info(`Delete newer accelerations from height ${blockHeight} from the database`);
+    try {
+      const currentSyncedHeight = await this.$getLastSyncedHeight();
+      if (currentSyncedHeight >= blockHeight) {
+        await DB.query(`
+          UPDATE state
+          SET number = ?
+          WHERE name = 'last_acceleration_block'
+        `, [blockHeight - 1]);
+      }
+      await DB.query(`DELETE FROM accelerations where height >= ${blockHeight}`);
+    } catch (e) {
+      logger.err('Cannot delete indexed accelerations. Reason: ' + (e instanceof Error ? e.message : e));
+    }
+  }
 }
 
 export default new AccelerationRepository();

frontend/src/app/components/acceleration/accelerations-list/accelerations-list.component.html

@@ -39,10 +39,10 @@
             </td>
           </ng-container>
           <ng-container *ngIf="!pending">
-            <td *ngIf="acceleration.feePaid" class="fee text-right">
+            <td *ngIf="acceleration.boost != null" class="fee text-right">
-              {{ (acceleration.boost) | number }} <span class="symbol" i18n="shared.sat|sat">sat</span>
+              {{ acceleration.boost | number }} <span class="symbol" i18n="shared.sat|sat">sat</span>
             </td>
-            <td *ngIf="!acceleration.feePaid" class="fee text-right">
+            <td *ngIf="acceleration.boost == null" class="fee text-right">
               ~
             </td>
             <td class="block text-right">

frontend/src/app/components/acceleration/accelerations-list/accelerations-list.component.ts

@@ -58,7 +58,7 @@ export class AccelerationsListComponent implements OnInit {
               }
             }
             for (const acc of accelerations) {
-              acc.boost = acc.feePaid - acc.baseFee - acc.vsizeFee;
+              acc.boost = acc.boostCost != null ? acc.boostCost : (acc.feePaid - acc.baseFee - acc.vsizeFee);
             }
             if (this.widget) {
               return of(accelerations.slice(0, 6));

frontend/src/app/components/acceleration/accelerator-dashboard/accelerator-dashboard.component.ts

@@ -116,15 +116,15 @@ export class AcceleratorDashboardComponent implements OnInit {
       switchMap(([accelerations, blocks]) => {
         const blockMap = {};
         for (const block of blocks) {
-          blockMap[block.id] = block;
+          blockMap[block.height] = block;
         }
-        const accelerationsByBlock: { [ hash: string ]: Acceleration[] } = {};
+        const accelerationsByBlock: { [ height: number ]: Acceleration[] } = {};
         for (const acceleration of accelerations) {
-          if (['completed_provisional', 'failed_provisional', 'completed'].includes(acceleration.status) && acceleration.pools.includes(blockMap[acceleration.blockHash]?.extras.pool.id)) {
+          if (['completed_provisional', 'failed_provisional', 'completed'].includes(acceleration.status) && acceleration.pools.includes(blockMap[acceleration.blockHeight]?.extras.pool.id)) {
-            if (!accelerationsByBlock[acceleration.blockHash]) {
+            if (!accelerationsByBlock[acceleration.blockHeight]) {
-              accelerationsByBlock[acceleration.blockHash] = [];
+              accelerationsByBlock[acceleration.blockHeight] = [];
             }
-            accelerationsByBlock[acceleration.blockHash].push(acceleration);
+            accelerationsByBlock[acceleration.blockHeight].push(acceleration);
           }
         }
         return of(blocks.slice(0, 6).map(block => {

frontend/src/app/components/block/block-preview.component.ts

@@ -136,7 +136,7 @@ export class BlockPreviewComponent implements OnInit, OnDestroy {
                   return of(transactions);
                 })
               ),
-            this.stateService.env.ACCELERATOR === true && block.height > 819500 ? this.servicesApiService.getAccelerationHistory$({ blockHash: block.id }) : of([])
+            this.stateService.env.ACCELERATOR === true && block.height > 819500 ? this.servicesApiService.getAccelerationHistory$({ blockHeight: block.height }) : of([])
           ]);
         }
       ),

frontend/src/app/components/block/block.component.ts

@@ -345,7 +345,7 @@ export class BlockComponent implements OnInit, OnDestroy {
                 return of(null);
               })
             ),
-          this.stateService.env.ACCELERATOR === true && block.height > 819500 ? this.servicesApiService.getAccelerationHistory$({ blockHash: block.id }) : of([])
+          this.stateService.env.ACCELERATOR === true && block.height > 819500 ? this.servicesApiService.getAccelerationHistory$({ blockHeight: block.height }) : of([])
         ]);
       })
     )

frontend/src/app/components/transaction/transaction.component.ts

@@ -98,7 +98,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
   fetchCpfp$ = new Subject<string>();
   fetchRbfHistory$ = new Subject<string>();
   fetchCachedTx$ = new Subject<string>();
-  fetchAcceleration$ = new Subject<string>();
+  fetchAcceleration$ = new Subject<number>();
   fetchMiningInfo$ = new Subject<{ hash: string, height: number, txid: string }>();
   isCached: boolean = false;
   now = Date.now();
@@ -288,8 +288,8 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
       tap(() => {
         this.accelerationInfo = null;
       }),
-      switchMap((blockHash: string) => {
+      switchMap((blockHeight: number) => {
-        return this.servicesApiService.getAccelerationHistory$({ blockHash });
+        return this.servicesApiService.getAccelerationHistory$({ blockHeight });
       }),
       catchError(() => {
         return of(null);
@@ -297,7 +297,10 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
     ).subscribe((accelerationHistory) => {
       for (const acceleration of accelerationHistory) {
         if (acceleration.txid === this.txId && (acceleration.status === 'completed' || acceleration.status === 'completed_provisional')) {
-          acceleration.acceleratedFeeRate = Math.max(acceleration.effectiveFee, acceleration.effectiveFee + acceleration.feePaid - acceleration.baseFee - acceleration.vsizeFee) / acceleration.effectiveVsize;
+          const boostCost = acceleration.boostCost || (acceleration.feePaid - acceleration.baseFee - acceleration.vsizeFee);
+          acceleration.acceleratedFeeRate = Math.max(acceleration.effectiveFee, acceleration.effectiveFee + boostCost) / acceleration.effectiveVsize;
+          acceleration.boost = boostCost;
+
           this.accelerationInfo = acceleration;
         }
       }
@@ -482,7 +485,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
               this.getTransactionTime();
             }
           } else {
-            this.fetchAcceleration$.next(tx.status.block_hash);
+            this.fetchAcceleration$.next(tx.status.block_height);
             this.fetchMiningInfo$.next({ hash: tx.status.block_hash, height: tx.status.block_height, txid: tx.txid });
             this.transactionTime = 0;
           }
@@ -544,7 +547,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
         } else {
           this.audioService.playSound('magic');
         }
-        this.fetchAcceleration$.next(block.id);
+        this.fetchAcceleration$.next(block.height);
         this.fetchMiningInfo$.next({ hash: block.id, height: block.height, txid: this.tx.txid });
       }
     });

frontend/src/app/interfaces/node-api.interface.ts

@@ -396,6 +396,9 @@ export interface Acceleration {
 
   acceleratedFeeRate?: number;
   boost?: number;
+
+  boostCost?: number;
+  boostRate?: number;
 }
 
 export interface AccelerationHistoryParams {