add FIRST_SEEN_INDEXING_DAYS config, auto-deletion

backend/mempool-config.sample.json

@@ -27,7 +27,8 @@
     "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
     "ADVANCED_GBT_AUDIT": false,
     "ADVANCED_GBT_MEMPOOL": false,
-    "TRANSACTION_INDEXING": false
+    "TRANSACTION_INDEXING": false,
+    "FIRST_SEEN_INDEXING_DAYS": 0
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",

backend/src/__fixtures__/mempool-config.template.json

@@ -28,7 +28,8 @@
     "POOLS_JSON_URL": "__POOLS_JSON_URL__",
     "ADVANCED_GBT_AUDIT": "__ADVANCED_GBT_AUDIT__",
     "ADVANCED_GBT_MEMPOOL": "__ADVANCED_GBT_MEMPOOL__",
-    "TRANSACTION_INDEXING": "__TRANSACTION_INDEXING__"
+    "TRANSACTION_INDEXING": "__TRANSACTION_INDEXING__",
+    "FIRST_SEEN_INDEXING_DAYS": "__FIRST_SEEN_INDEXING_DAYS__"
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",

backend/src/__tests__/config.test.ts

@@ -41,6 +41,7 @@ describe('Mempool Backend Config', () => {
         ADVANCED_GBT_AUDIT: false,
         ADVANCED_GBT_MEMPOOL: false,
         TRANSACTION_INDEXING: false,
+        FIRST_SEEN_INDEXING_DAYS: 0,
       });
 
       expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -25,6 +25,7 @@ class BitcoinRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'transaction-times', this.getTransactionTimes)
       .get(config.MEMPOOL.API_URL_PREFIX + 'outspends', this.$getBatchedOutspends)
       .get(config.MEMPOOL.API_URL_PREFIX + 'cpfp/:txId', this.$getCpfpInfo)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'extras/:txId', this.$getTransactionExtras)
       .get(config.MEMPOOL.API_URL_PREFIX + 'difficulty-adjustment', this.getDifficultyChange)
       .get(config.MEMPOOL.API_URL_PREFIX + 'fees/recommended', this.getRecommendedFees)
       .get(config.MEMPOOL.API_URL_PREFIX + 'fees/mempool-blocks', this.getMempoolBlocks)
@@ -221,6 +222,42 @@ class BitcoinRoutes {
     res.status(404).send(`Transaction has no CPFP info available.`);
   }
 
+  private async $getTransactionExtras(req: Request, res: Response): Promise<void> {
+    if (!/^[a-fA-F0-9]{64}$/.test(req.params.txId)) {
+      res.status(501).send(`Invalid transaction ID.`);
+      return;
+    }
+
+    const tx = mempool.getMempool()[req.params.txId];
+    if (tx) {
+      if (tx?.cpfpChecked) {
+        res.json({
+          ancestors: tx.ancestors,
+          bestDescendant: tx.bestDescendant || null,
+          descendants: tx.descendants || null,
+          effectiveFeePerVsize: tx.effectiveFeePerVsize || null,
+          firstSeen: tx.firstSeen,
+        });
+        return;
+      }
+
+      const cpfpInfo = Common.setRelativesAndGetCpfpInfo(tx, mempool.getMempool());
+
+      res.json({
+        ...cpfpInfo,
+        firstSeen: tx.firstSeen,
+      });
+      return;
+    } else {
+      const extras = await transactionRepository.$getTransactionExtras(req.params.txId);
+      if (extras) {
+        res.json(extras);
+        return;
+      }
+    }
+    res.status(404).send(`Transaction has no extra info available.`);
+  }
+
   private getBackendInfo(req: Request, res: Response) {
     res.json(backendInfo.getBackendInfo());
   }

backend/src/api/common.ts

@@ -194,6 +194,13 @@ export class Common {
     );
   }
 
+  static firstSeenIndexingEnabled(): boolean {
+    return (
+      Common.indexingEnabled() &&
+      config.MEMPOOL.FIRST_SEEN_INDEXING_DAYS !== 0
+    );
+  }
+
   static setDateMidnight(date: Date): void {
     date.setUTCHours(0);
     date.setUTCMinutes(0);

backend/src/api/database-migration.ts

@@ -4,7 +4,7 @@ import logger from '../logger';
 import { Common } from './common';
 
 class DatabaseMigration {
-  private static currentVersion = 49;
+  private static currentVersion = 50;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -442,6 +442,11 @@ class DatabaseMigration {
       await this.$executeQuery('TRUNCATE TABLE `blocks_audits`');
       await this.updateToSchemaVersion(49);
     }
+
+    if (databaseSchemaVersion < 50 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `transactions` ADD first_seen datetime DEFAULT NULL, ADD INDEX (first_seen)');
+      await this.updateToSchemaVersion(50);
+    }
   }
 
   /**

backend/src/api/mempool.ts

@@ -9,6 +9,7 @@ import loadingIndicators from './loading-indicators';
 import bitcoinClient from './bitcoin/bitcoin-client';
 import bitcoinSecondClient from './bitcoin/bitcoin-second-client';
 import rbfCache from './rbf-cache';
+import transactionRepository from '../repositories/TransactionRepository';
 
 class Mempool {
   private static WEBSOCKET_REFRESH_RATE_MS = 10000;
@@ -217,6 +218,14 @@ class Mempool {
     }
   }
 
+  public async $saveTxFirstSeenTimes(transactions: TransactionExtended[], mempool: { [txid: string]: TransactionExtended }) {
+    for (const tx of transactions) {
+      if (mempool[tx.txid]) {
+         await transactionRepository.$saveTxFirstSeen(tx.txid, tx.firstSeen || Date.now());
+      }
+    }
+  }
+
   private updateTxPerSecond() {
     const nowMinusTimeSpan = new Date().getTime() - (1000 * config.STATISTICS.TX_PER_SECOND_SAMPLE_PERIOD);
     this.txPerSecondArray = this.txPerSecondArray.filter((unixTime) => unixTime > nowMinusTimeSpan);

backend/src/api/websocket-handler.ts

@@ -19,6 +19,7 @@ import feeApi from './fee-api';
 import BlocksAuditsRepository from '../repositories/BlocksAuditsRepository';
 import BlocksSummariesRepository from '../repositories/BlocksSummariesRepository';
 import Audit from './audit';
+import mempool from './mempool';
 
 class WebsocketHandler {
   private wss: WebSocket.Server | undefined;
@@ -462,6 +463,10 @@ class WebsocketHandler {
       }
     }
 
+    if (Common.firstSeenIndexingEnabled()) {
+      await mempool.$saveTxFirstSeenTimes(transactions, _memPool);
+    }
+
     const removed: string[] = [];
     // Update mempool to remove transactions included in the new block
     for (const txId of txIds) {

backend/src/config.ts

@@ -32,6 +32,7 @@ interface IConfig {
     ADVANCED_GBT_AUDIT: boolean;
     ADVANCED_GBT_MEMPOOL: boolean;
     TRANSACTION_INDEXING: boolean;
+    FIRST_SEEN_INDEXING_DAYS: number;
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -153,6 +154,7 @@ const defaults: IConfig = {
     'ADVANCED_GBT_AUDIT': false,
     'ADVANCED_GBT_MEMPOOL': false,
     'TRANSACTION_INDEXING': false,
+    'FIRST_SEEN_INDEXING_DAYS': 0,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',

backend/src/indexer.ts

@@ -7,6 +7,7 @@ import HashratesRepository from './repositories/HashratesRepository';
 import bitcoinClient from './api/bitcoin/bitcoin-client';
 import priceUpdater from './tasks/price-updater';
 import PricesRepository from './repositories/PricesRepository';
+import TransactionRepository from './repositories/TransactionRepository';
 
 class Indexer {
   runIndexer = true;
@@ -78,6 +79,7 @@ class Indexer {
       await mining.$generatePoolHashrateHistory();
       await blocks.$generateBlocksSummariesDatabase();
       await blocks.$generateCPFPDatabase();
+      await TransactionRepository.$clearOldFirstSeen();
     } catch (e) {
       this.indexerRunning = false;
       logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e));

backend/src/mempool.interfaces.ts

@@ -136,6 +136,10 @@ export interface CpfpInfo {
   effectiveFeePerVsize?: number;
 }
 
+export interface TransactionExtras extends CpfpInfo {
+  firstSeen?: number;
+}
+
 export interface TransactionStripped {
   txid: string;
   fee: number;

backend/src/repositories/TransactionRepository.ts

@@ -1,14 +1,16 @@
+import config from '../config';
 import DB from '../database';
 import logger from '../logger';
-import { Ancestor, CpfpInfo } from '../mempool.interfaces';
+import { Ancestor, CpfpInfo, TransactionExtras } from '../mempool.interfaces';
 
-interface CpfpSummary {
+interface TxInfo {
   txid: string;
   cluster: string;
   root: string;
   txs: Ancestor[];
   height: number;
   fee_rate: number;
+  firstSeen: number;
 }
 
 class TransactionRepository {
@@ -33,6 +35,46 @@ class TransactionRepository {
     }
   }
 
+  public async $saveTxFirstSeen(txid: string, seenAt: number) {
+    try {
+      await DB.query(
+        `
+          INSERT INTO transactions
+          (
+            txid,
+            first_seen
+          )
+          VALUE (?, FROM_UNIXTIME(?))
+          ON DUPLICATE KEY UPDATE
+            first_seen = FROM_UNIXTIME(?)
+        ;`,
+        [txid, seenAt, seenAt]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot save transaction first seen time into db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  public async $getTransactionExtras(txid: string): Promise<TransactionExtras | void> {
+    try {
+      let query = `
+        SELECT *, UNIX_TIMESTAMP(first_seen) as firstSeen
+        FROM transactions
+        LEFT JOIN cpfp_clusters AS cluster ON cluster.root = transactions.cluster
+        WHERE transactions.txid = ?
+      `;
+      const [rows]: any = await DB.query(query, [txid]);
+      if (rows.length) {
+        rows[0].txs = JSON.parse(rows[0].txs) as Ancestor[];
+        return this.convertCpfp(rows[0]);
+      }
+    } catch (e) {
+      logger.err('Cannot get transaction cpfp info from db. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   public async $getCpfpInfo(txid: string): Promise<CpfpInfo | void> {
     try {
       let query = `
@@ -54,12 +96,34 @@ class TransactionRepository {
     }
   }
 
-  private convertCpfp(cpfp: CpfpSummary): CpfpInfo {
+  public async $clearOldFirstSeen() {
+    if (config.MEMPOOL.FIRST_SEEN_INDEXING_DAYS > 0) {
+      const cutoff = Math.floor(Date.now() / 1000) - (config.MEMPOOL.FIRST_SEEN_INDEXING_DAYS * 86400);
+      await this.$clearFirstSeenBefore(cutoff);
+    }
+  }
+
+  private async $clearFirstSeenBefore(cutoff: number) {
+    try {
+      const result = await DB.query(
+        `
+          DELETE FROM transactions
+          WHERE cluster is null AND first_seen < FROM_UNIXTIME(?)
+        ;`,
+        [cutoff]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot clear old tx first seen times from db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
+  private convertCpfp(info: TxInfo): TransactionExtras {
     const descendants: Ancestor[] = [];
     const ancestors: Ancestor[] = [];
     let matched = false;
-    for (const tx of cpfp.txs) {
-      if (tx.txid === cpfp.txid) {
+    for (const tx of (info.txs || [])) {
+      if (tx.txid === info.txid) {
         matched = true;
       } else if (!matched) {
         descendants.push(tx);
@@ -68,9 +132,10 @@ class TransactionRepository {
       }
     }
     return {
-      descendants,
-      ancestors,
-      effectiveFeePerVsize: cpfp.fee_rate
+      descendants: descendants?.length ? descendants : undefined,
+      ancestors: ancestors,
+      effectiveFeePerVsize: info.fee_rate,
+      firstSeen: info.firstSeen || undefined,
     };
   }
 }

frontend/src/app/components/time-span/time-span.component.ts

@@ -53,9 +53,6 @@ export class TimeSpanComponent implements OnInit, OnChanges, OnDestroy {
 
   calculate() {
     const seconds = Math.floor(this.time);
-    if (seconds < 60) {
-      return $localize`:@@date-base.just-now:Just now`;
-    }
     let counter: number;
     for (const i in this.intervals) {
       if (this.intervals.hasOwnProperty(i)) {

frontend/src/app/components/transaction/transaction.component.html

@@ -54,6 +54,18 @@
                     </div>
                   </td>
                 </tr>
+                <ng-template [ngIf]="transactionTime !== 0">
+                  <tr *ngIf="transactionTime === -1; else firstSeenTmpl">
+                    <td><span class="skeleton-loader"></span></td>
+                    <td><span class="skeleton-loader"></span></td>
+                  </tr>
+                  <ng-template #firstSeenTmpl>
+                    <tr>
+                      <td i18n="transaction.first-seen|Transaction first seen">First seen</td>
+                      <td><i><app-time-since [time]="transactionTime" [fastRender]="true"></app-time-since></i></td>
+                    </tr>
+                  </ng-template>
+                </ng-template>
                 <tr *ngIf="latestBlock && tx.status.block_height <= latestBlock.height - 8">
                   <td class="td-width" i18n="transaction.included-in-block|Transaction included in block">Included in block</td>
                   <td>
@@ -63,10 +75,10 @@
                 <ng-template [ngIf]="transactionTime > 0">
                   <tr>
                     <td i18n="transaction.confirmed|Transaction Confirmed state">Confirmed</td>
-                    <td><app-time-span [time]="tx.status.block_time - transactionTime" [fastRender]="true"></app-time-span></td>
+                    <td><app-time-span [time]="tx.status.block_time - transactionTime"></app-time-span></td>
                   </tr>
                 </ng-template>
-                <tr *ngIf="network !== 'liquid' && network !== 'liquidtestnet'">
+                <tr *ngIf="network !== 'liquid' && network !== 'liquidtestnet' && (cpfpInfo && (cpfpInfo?.bestDescendant || cpfpInfo?.descendants?.length || cpfpInfo?.ancestors?.length) || !(transactionTime > 0))">
                   <td class="td-width" i18n="transaction.features|Transaction features">Features</td>
                   <td>
                     <app-tx-features [tx]="tx"></app-tx-features>
@@ -497,6 +509,12 @@
           <button type="button" class="btn btn-outline-info btn-sm btn-small-height float-right" (click)="showCpfpDetails = !showCpfpDetails">CPFP <fa-icon [icon]="['fas', 'info-circle']" [fixedWidth]="true"></fa-icon></button>
         </td>
       </tr>
+      <tr *ngIf="tx?.status?.confirmed && (!cpfpInfo || (!cpfpInfo?.bestDescendant && !cpfpInfo?.descendants?.length && !cpfpInfo?.ancestors?.length)) && transactionTime > 0 && network !== 'liquid' && network !== 'liquidtestnet'">
+        <td class="td-width" i18n="transaction.features|Transaction Features">Features</td>
+        <td>
+          <app-tx-features [tx]="tx"></app-tx-features>
+        </td>
+      </tr>
     </tbody>
   </table>
 </ng-template>

frontend/src/app/components/transaction/transaction.component.ts

@@ -110,7 +110,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
       .pipe(
         switchMap((txId) =>
           this.apiService
-            .getCpfpinfo$(txId)
+            .getTransactionExtras$(txId)
             .pipe(retryWhen((errors) => errors.pipe(
               mergeMap((error) => {
                 if (!this.tx?.status || this.tx.status.confirmed) {
@@ -156,6 +156,9 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
             txFeePerVSize: this.tx.effectiveFeePerVsize,
           });
         }
+        if (cpfpInfo.firstSeen) {
+          this.transactionTime = cpfpInfo.firstSeen;
+        }
         this.cpfpInfo = cpfpInfo;
       });
 

frontend/src/app/interfaces/node-api.interface.ts

@@ -27,6 +27,10 @@ export interface CpfpInfo {
   effectiveFeePerVsize?: number;
 }
 
+export interface TransactionExtras extends CpfpInfo {
+  firstSeen?: number;
+}
+
 export interface DifficultyAdjustment {
   progressPercent: number;
   difficultyChange: number;

frontend/src/app/services/api.service.ts

@@ -1,7 +1,7 @@
 import { Injectable } from '@angular/core';
 import { HttpClient, HttpParams } from '@angular/common/http';
 import { CpfpInfo, OptimizedMempoolStats, AddressInformation, LiquidPegs, ITranslators,
-  PoolStat, BlockExtended, TransactionStripped, RewardStats, AuditScore } from '../interfaces/node-api.interface';
+  PoolStat, BlockExtended, TransactionStripped, RewardStats, AuditScore, TransactionExtras } from '../interfaces/node-api.interface';
 import { Observable } from 'rxjs';
 import { StateService } from './state.service';
 import { WebsocketResponse } from '../interfaces/websocket.interface';
@@ -115,6 +115,10 @@ export class ApiService {
     return this.httpClient.get<CpfpInfo>(this.apiBaseUrl + this.apiBasePath + '/api/v1/cpfp/' + txid);
   }
 
+  getTransactionExtras$(txid: string): Observable<TransactionExtras> {
+    return this.httpClient.get<TransactionExtras>(this.apiBaseUrl + this.apiBasePath + '/api/v1/extras/' + txid);
+  }
+
   validateAddress$(address: string): Observable<AddressInformation> {
     return this.httpClient.get<AddressInformation>(this.apiBaseUrl + this.apiBasePath + '/api/v1/validate-address/' + address);
   }