add FIRST_SEEN_INDEXING_DAYS config, auto-deletion

backend/mempool-config.sample.json

@@ -27,7 +27,8 @@
     "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master",
     "ADVANCED_GBT_AUDIT": false,
     "ADVANCED_GBT_MEMPOOL": false,
-    "TRANSACTION_INDEXING": false
+    "TRANSACTION_INDEXING": false,
+    "FIRST_SEEN_INDEXING_DAYS": 0
   },
   "CORE_RPC": {
     "HOST": "127.0.0.1",

backend/src/__fixtures__/mempool-config.template.json

@@ -28,7 +28,8 @@
     "POOLS_JSON_URL": "__POOLS_JSON_URL__",
     "ADVANCED_GBT_AUDIT": "__ADVANCED_GBT_AUDIT__",
     "ADVANCED_GBT_MEMPOOL": "__ADVANCED_GBT_MEMPOOL__",
-    "TRANSACTION_INDEXING": "__TRANSACTION_INDEXING__"
+    "TRANSACTION_INDEXING": "__TRANSACTION_INDEXING__",
+    "FIRST_SEEN_INDEXING_DAYS": "__FIRST_SEEN_INDEXING_DAYS__"
   },
   "CORE_RPC": {
     "HOST": "__CORE_RPC_HOST__",

backend/src/__tests__/config.test.ts

@@ -41,6 +41,7 @@ describe('Mempool Backend Config', () => {
         ADVANCED_GBT_AUDIT: false,
         ADVANCED_GBT_MEMPOOL: false,
         TRANSACTION_INDEXING: false,
+        FIRST_SEEN_INDEXING_DAYS: 0,
       });
 
       expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });

backend/src/api/common.ts

@@ -194,6 +194,13 @@ export class Common {
     );
   }
 
+  static firstSeenIndexingEnabled(): boolean {
+    return (
+      Common.indexingEnabled() &&
+      config.MEMPOOL.FIRST_SEEN_INDEXING_DAYS !== 0
+    );
+  }
+
   static setDateMidnight(date: Date): void {
     date.setUTCHours(0);
     date.setUTCMinutes(0);

backend/src/api/database-migration.ts

@@ -444,8 +444,8 @@ class DatabaseMigration {
     }
 
     if (databaseSchemaVersion < 50 && isBitcoin === true) {
-      await this.$executeQuery('ALTER TABLE `transactions` ADD first_seen datetime DEFAULT NULL');
-      await this.updateToSchemaVersion(49);
+      await this.$executeQuery('ALTER TABLE `transactions` ADD first_seen datetime DEFAULT NULL, ADD INDEX (first_seen)');
+      await this.updateToSchemaVersion(50);
     }
   }
 

backend/src/api/websocket-handler.ts

@@ -463,7 +463,7 @@ class WebsocketHandler {
       }
     }
 
-    if (config.MEMPOOL.TRANSACTION_INDEXING) {
+    if (Common.firstSeenIndexingEnabled()) {
       await mempool.$saveTxFirstSeenTimes(transactions, _memPool);
     }
 

backend/src/config.ts

@@ -32,6 +32,7 @@ interface IConfig {
     ADVANCED_GBT_AUDIT: boolean;
     ADVANCED_GBT_MEMPOOL: boolean;
     TRANSACTION_INDEXING: boolean;
+    FIRST_SEEN_INDEXING_DAYS: number;
   };
   ESPLORA: {
     REST_API_URL: string;
@@ -153,6 +154,7 @@ const defaults: IConfig = {
     'ADVANCED_GBT_AUDIT': false,
     'ADVANCED_GBT_MEMPOOL': false,
     'TRANSACTION_INDEXING': false,
+    'FIRST_SEEN_INDEXING_DAYS': 0,
   },
   'ESPLORA': {
     'REST_API_URL': 'http://127.0.0.1:3000',

backend/src/indexer.ts

@@ -7,6 +7,7 @@ import HashratesRepository from './repositories/HashratesRepository';
 import bitcoinClient from './api/bitcoin/bitcoin-client';
 import priceUpdater from './tasks/price-updater';
 import PricesRepository from './repositories/PricesRepository';
+import TransactionRepository from './repositories/TransactionRepository';
 
 class Indexer {
   runIndexer = true;
@@ -78,6 +79,7 @@ class Indexer {
       await mining.$generatePoolHashrateHistory();
       await blocks.$generateBlocksSummariesDatabase();
       await blocks.$generateCPFPDatabase();
+      await TransactionRepository.$clearOldFirstSeen();
     } catch (e) {
       this.indexerRunning = false;
       logger.err(`Indexer failed, trying again in 10 seconds. Reason: ` + (e instanceof Error ? e.message : e));

backend/src/repositories/TransactionRepository.ts

@@ -1,6 +1,7 @@
+import config from '../config';
 import DB from '../database';
 import logger from '../logger';
-import { Ancestor, CpfpInfo, TransactionExtended, TransactionExtras } from '../mempool.interfaces';
+import { Ancestor, CpfpInfo, TransactionExtras } from '../mempool.interfaces';
 
 interface TxInfo {
   txid: string;
@@ -95,6 +96,28 @@ class TransactionRepository {
     }
   }
 
+  public async $clearOldFirstSeen() {
+    if (config.MEMPOOL.FIRST_SEEN_INDEXING_DAYS > 0) {
+      const cutoff = Math.floor(Date.now() / 1000) - (config.MEMPOOL.FIRST_SEEN_INDEXING_DAYS * 86400);
+      await this.$clearFirstSeenBefore(cutoff);
+    }
+  }
+
+  private async $clearFirstSeenBefore(cutoff: number) {
+    try {
+      const result = await DB.query(
+        `
+          DELETE FROM transactions
+          WHERE cluster is null AND first_seen < FROM_UNIXTIME(?)
+        ;`,
+        [cutoff]
+      );
+    } catch (e: any) {
+      logger.err(`Cannot clear old tx first seen times from db. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   private convertCpfp(info: TxInfo): TransactionExtras {
     const descendants: Ancestor[] = [];
     const ancestors: Ancestor[] = [];