Analytic form for ETA algorithm

Consider incoming transactions flow into ETA calculation
statistics: class incoming transactions by fee rate
2024-09-09 16:03:40 +02:00 · 2024-09-07 23:07:16 +02:00 · 2024-09-07 17:49:21 +02:00
8 changed files with 550 additions and 63 deletions
--- a/backend/src/api/database-migration.ts
+++ b/backend/src/api/database-migration.ts
@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';

 class DatabaseMigration {
-  private static currentVersion = 82;
+  private static currentVersion = 83;
  private queryTimeout = 3600_000;
  private statisticsAddedIndexed = false;
  private uniqueLogs: string[] = [];
@ -705,6 +705,11 @@ class DatabaseMigration {
      await this.$fixBadV1AuditBlocks();
      await this.updateToSchemaVersion(82);
    }
+
+    if (databaseSchemaVersion < 83) {
+      await this.$addPerSecondVsizeToStatistics();
+      await this.updateToSchemaVersion(83);
+    }
  }

  /**
@ -1341,6 +1346,47 @@ class DatabaseMigration {
      }
    }
  }
+
+  private async $addPerSecondVsizeToStatistics(): Promise<void> {
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_1` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_2` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_3` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_4` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_5` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_6` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_8` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_10` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_12` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_15` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_20` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_30` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_40` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_50` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_60` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_70` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_80` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_90` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_100` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_125` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_150` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_175` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_200` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_250` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_300` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_350` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_400` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_500` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_600` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_700` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_800` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_900` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_1000` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_1200` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_1400` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_1600` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_1800` int(11) NOT NULL DEFAULT 0');
+    await this.$executeQuery('ALTER TABLE `statistics` ADD `vsize_ps_2000` int(11) NOT NULL DEFAULT 0');
+  }
 }

 export default new DatabaseMigration();
--- a/backend/src/api/mempool.ts
+++ b/backend/src/api/mempool.ts
@ -12,6 +12,7 @@ import rbfCache from './rbf-cache';
 import { Acceleration } from './services/acceleration';
 import redisCache from './redis-cache';
 import blocks from './blocks';
+import { logFees } from './statistics/statistics';

 class Mempool {
  private inSync: boolean = false;
@ -34,6 +35,7 @@ class Mempool {

  private vBytesPerSecondArray: VbytesPerSecond[] = [];
  private vBytesPerSecond: number = 0;
+  private vBytesPerSecondByFeeRate: { [feePerWU: number]: number } = {};
  private mempoolProtection = 0;
  private latestTransactions: any[] = [];

@ -193,6 +195,10 @@ class Mempool {
    return this.vBytesPerSecond;
  }

+  public getVBytesPerSecondByFeeRate(): { [feePerWU: number]: number } {
+    return this.vBytesPerSecondByFeeRate;
+  }
+
  public getFirstSeenForTransactions(txIds: string[]): number[] {
    const txTimes: number[] = [];
    txIds.forEach((txId: string) => {
@ -270,6 +276,7 @@ class Mempool {
            this.vBytesPerSecondArray.push({
              unixTime: new Date().getTime(),
              vSize: transaction.vsize,
+              effectiveFeePerVsize: transaction.effectiveFeePerVsize
            });
          }
          hasChange = true;
@ -588,6 +595,21 @@ class Mempool {
      this.vBytesPerSecond = Math.round(
        this.vBytesPerSecondArray.map((data) => data.vSize).reduce((a, b) => a + b) / config.STATISTICS.TX_PER_SECOND_SAMPLE_PERIOD
      );
+
+      if (!Common.isLiquid()) {
+        this.vBytesPerSecondByFeeRate = {};
+        for (const tx of this.vBytesPerSecondArray) {
+          for (let i = 0; i < logFees.length; i++) {
+            if (tx.effectiveFeePerVsize < logFees[i + 1] || i === logFees.length - 1) {
+              this.vBytesPerSecondByFeeRate[logFees[i]] = (this.vBytesPerSecondByFeeRate[logFees[i]] || 0) + tx.vSize;
+              break;
+            }
+          }
+        }
+        for (const feeRate of Object.keys(this.vBytesPerSecondByFeeRate)) {
+          this.vBytesPerSecondByFeeRate[feeRate] = Math.round(this.vBytesPerSecondByFeeRate[feeRate] / config.STATISTICS.TX_PER_SECOND_SAMPLE_PERIOD);
+        }
+      }
    }
  }

--- a/backend/src/api/statistics/statistics-api.ts
+++ b/backend/src/api/statistics/statistics-api.ts
@ -53,7 +53,45 @@ class StatisticsApi {
              vsize_1400,
              vsize_1600,
              vsize_1800,
-              vsize_2000
+              vsize_2000,
+              vsize_ps_1,
+              vsize_ps_2,
+              vsize_ps_3,
+              vsize_ps_4,
+              vsize_ps_5,
+              vsize_ps_6,
+              vsize_ps_8,
+              vsize_ps_10,
+              vsize_ps_12,
+              vsize_ps_15,
+              vsize_ps_20,
+              vsize_ps_30,
+              vsize_ps_40,
+              vsize_ps_50,
+              vsize_ps_60,
+              vsize_ps_70,
+              vsize_ps_80,
+              vsize_ps_90,
+              vsize_ps_100,
+              vsize_ps_125,
+              vsize_ps_150,
+              vsize_ps_175,
+              vsize_ps_200,
+              vsize_ps_250,
+              vsize_ps_300,
+              vsize_ps_350,
+              vsize_ps_400,
+              vsize_ps_500,
+              vsize_ps_600,
+              vsize_ps_700,
+              vsize_ps_800,
+              vsize_ps_900,
+              vsize_ps_1000,
+              vsize_ps_1200,
+              vsize_ps_1400,
+              vsize_ps_1600,
+              vsize_ps_1800,
+              vsize_ps_2000
            )
            VALUES (NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
               0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)`;
@ -67,56 +105,94 @@ class StatisticsApi {
  public async $create(statistics: Statistic, convertToDatetime = false): Promise<number | undefined> {
    try {
      const query = `INSERT INTO statistics(
-              added,
-              unconfirmed_transactions,
-              tx_per_second,
-              vbytes_per_second,
-              mempool_byte_weight,
-              fee_data,
-              total_fee,
-              min_fee,
-              vsize_1,
-              vsize_2,
-              vsize_3,
-              vsize_4,
-              vsize_5,
-              vsize_6,
-              vsize_8,
-              vsize_10,
-              vsize_12,
-              vsize_15,
-              vsize_20,
-              vsize_30,
-              vsize_40,
-              vsize_50,
-              vsize_60,
-              vsize_70,
-              vsize_80,
-              vsize_90,
-              vsize_100,
-              vsize_125,
-              vsize_150,
-              vsize_175,
-              vsize_200,
-              vsize_250,
-              vsize_300,
-              vsize_350,
-              vsize_400,
-              vsize_500,
-              vsize_600,
-              vsize_700,
-              vsize_800,
-              vsize_900,
-              vsize_1000,
-              vsize_1200,
-              vsize_1400,
-              vsize_1600,
-              vsize_1800,
-              vsize_2000
-            )
-            VALUES (${convertToDatetime ? `FROM_UNIXTIME(${statistics.added})` : statistics.added}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
-               ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
-
+        added,
+        unconfirmed_transactions,
+        tx_per_second,
+        vbytes_per_second,
+        mempool_byte_weight,
+        fee_data,
+        total_fee,
+        min_fee,
+        vsize_1,
+        vsize_2,
+        vsize_3,
+        vsize_4,
+        vsize_5,
+        vsize_6,
+        vsize_8,
+        vsize_10,
+        vsize_12,
+        vsize_15,
+        vsize_20,
+        vsize_30,
+        vsize_40,
+        vsize_50,
+        vsize_60,
+        vsize_70,
+        vsize_80,
+        vsize_90,
+        vsize_100,
+        vsize_125,
+        vsize_150,
+        vsize_175,
+        vsize_200,
+        vsize_250,
+        vsize_300,
+        vsize_350,
+        vsize_400,
+        vsize_500,
+        vsize_600,
+        vsize_700,
+        vsize_800,
+        vsize_900,
+        vsize_1000,
+        vsize_1200,
+        vsize_1400,
+        vsize_1600,
+        vsize_1800,
+        vsize_2000,
+        vsize_ps_1,
+        vsize_ps_2,
+        vsize_ps_3,
+        vsize_ps_4,
+        vsize_ps_5,
+        vsize_ps_6,
+        vsize_ps_8,
+        vsize_ps_10,
+        vsize_ps_12,
+        vsize_ps_15,
+        vsize_ps_20,
+        vsize_ps_30,
+        vsize_ps_40,
+        vsize_ps_50,
+        vsize_ps_60,
+        vsize_ps_70,
+        vsize_ps_80,
+        vsize_ps_90,
+        vsize_ps_100,
+        vsize_ps_125,
+        vsize_ps_150,
+        vsize_ps_175,
+        vsize_ps_200,
+        vsize_ps_250,
+        vsize_ps_300,
+        vsize_ps_350,
+        vsize_ps_400,
+        vsize_ps_500,
+        vsize_ps_600,
+        vsize_ps_700,
+        vsize_ps_800,
+        vsize_ps_900,
+        vsize_ps_1000,
+        vsize_ps_1200,
+        vsize_ps_1400,
+        vsize_ps_1600,
+        vsize_ps_1800,
+        vsize_ps_2000
+      )
+      VALUES (${convertToDatetime ? `FROM_UNIXTIME(${statistics.added})` : statistics.added}, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 
+          ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
+      
      const params: (string | number)[] = [
        statistics.unconfirmed_transactions,
        statistics.tx_per_second,
@ -163,6 +239,44 @@ class StatisticsApi {
        statistics.vsize_1600,
        statistics.vsize_1800,
        statistics.vsize_2000,
+        statistics.vsize_ps_1,
+        statistics.vsize_ps_2,
+        statistics.vsize_ps_3,
+        statistics.vsize_ps_4,
+        statistics.vsize_ps_5,
+        statistics.vsize_ps_6,
+        statistics.vsize_ps_8,
+        statistics.vsize_ps_10,
+        statistics.vsize_ps_12,
+        statistics.vsize_ps_15,
+        statistics.vsize_ps_20,
+        statistics.vsize_ps_30,
+        statistics.vsize_ps_40,
+        statistics.vsize_ps_50,
+        statistics.vsize_ps_60,
+        statistics.vsize_ps_70,
+        statistics.vsize_ps_80,
+        statistics.vsize_ps_90,
+        statistics.vsize_ps_100,
+        statistics.vsize_ps_125,
+        statistics.vsize_ps_150,
+        statistics.vsize_ps_175,
+        statistics.vsize_ps_200,
+        statistics.vsize_ps_250,
+        statistics.vsize_ps_300,
+        statistics.vsize_ps_350,
+        statistics.vsize_ps_400,
+        statistics.vsize_ps_500,
+        statistics.vsize_ps_600,
+        statistics.vsize_ps_700,
+        statistics.vsize_ps_800,
+        statistics.vsize_ps_900,
+        statistics.vsize_ps_1000,
+        statistics.vsize_ps_1200,
+        statistics.vsize_ps_1400,
+        statistics.vsize_ps_1600,
+        statistics.vsize_ps_1800,
+        statistics.vsize_ps_2000,
      ];
      const [result]: any = await DB.query(query, params);
      return result.insertId;
@ -214,7 +328,45 @@ class StatisticsApi {
      CAST(avg(vsize_1400) as DOUBLE) as vsize_1400,
      CAST(avg(vsize_1600) as DOUBLE) as vsize_1600,
      CAST(avg(vsize_1800) as DOUBLE) as vsize_1800,
-      CAST(avg(vsize_2000) as DOUBLE) as vsize_2000 \
+      CAST(avg(vsize_2000) as DOUBLE) as vsize_2000,
+      CAST(avg(vsize_ps_1) as DOUBLE) as vsize_ps_1,
+      CAST(avg(vsize_ps_2) as DOUBLE) as vsize_ps_2,
+      CAST(avg(vsize_ps_3) as DOUBLE) as vsize_ps_3,
+      CAST(avg(vsize_ps_4) as DOUBLE) as vsize_ps_4,
+      CAST(avg(vsize_ps_5) as DOUBLE) as vsize_ps_5,
+      CAST(avg(vsize_ps_6) as DOUBLE) as vsize_ps_6,
+      CAST(avg(vsize_ps_8) as DOUBLE) as vsize_ps_8,
+      CAST(avg(vsize_ps_10) as DOUBLE) as vsize_ps_10,
+      CAST(avg(vsize_ps_12) as DOUBLE) as vsize_ps_12,
+      CAST(avg(vsize_ps_15) as DOUBLE) as vsize_ps_15,
+      CAST(avg(vsize_ps_20) as DOUBLE) as vsize_ps_20,
+      CAST(avg(vsize_ps_30) as DOUBLE) as vsize_ps_30,
+      CAST(avg(vsize_ps_40) as DOUBLE) as vsize_ps_40,
+      CAST(avg(vsize_ps_50) as DOUBLE) as vsize_ps_50,
+      CAST(avg(vsize_ps_60) as DOUBLE) as vsize_ps_60,
+      CAST(avg(vsize_ps_70) as DOUBLE) as vsize_ps_70,
+      CAST(avg(vsize_ps_80) as DOUBLE) as vsize_ps_80,
+      CAST(avg(vsize_ps_90) as DOUBLE) as vsize_ps_90,
+      CAST(avg(vsize_ps_100) as DOUBLE) as vsize_ps_100,
+      CAST(avg(vsize_ps_125) as DOUBLE) as vsize_ps_125,
+      CAST(avg(vsize_ps_150) as DOUBLE) as vsize_ps_150,
+      CAST(avg(vsize_ps_175) as DOUBLE) as vsize_ps_175,
+      CAST(avg(vsize_ps_200) as DOUBLE) as vsize_ps_200,
+      CAST(avg(vsize_ps_250) as DOUBLE) as vsize_ps_250,
+      CAST(avg(vsize_ps_300) as DOUBLE) as vsize_ps_300,
+      CAST(avg(vsize_ps_350) as DOUBLE) as vsize_ps_350,
+      CAST(avg(vsize_ps_400) as DOUBLE) as vsize_ps_400,
+      CAST(avg(vsize_ps_500) as DOUBLE) as vsize_ps_500,
+      CAST(avg(vsize_ps_600) as DOUBLE) as vsize_ps_600,
+      CAST(avg(vsize_ps_700) as DOUBLE) as vsize_ps_700,
+      CAST(avg(vsize_ps_800) as DOUBLE) as vsize_ps_800,
+      CAST(avg(vsize_ps_900) as DOUBLE) as vsize_ps_900,
+      CAST(avg(vsize_ps_1000) as DOUBLE) as vsize_ps_1000,
+      CAST(avg(vsize_ps_1200) as DOUBLE) as vsize_ps_1200,
+      CAST(avg(vsize_ps_1400) as DOUBLE) as vsize_ps_1400,
+      CAST(avg(vsize_ps_1600) as DOUBLE) as vsize_ps_1600,
+      CAST(avg(vsize_ps_1800) as DOUBLE) as vsize_ps_1800,
+      CAST(avg(vsize_ps_2000) as DOUBLE) as vsize_ps_2000 \
      FROM statistics \
      ${interval === 'all' ? '' : `WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`} \
      GROUP BY UNIX_TIMESTAMP(added) DIV ${div} \
@ -264,7 +416,45 @@ class StatisticsApi {
      vsize_1400,
      vsize_1600,
      vsize_1800,
-      vsize_2000 \
+      vsize_2000,
+      vsize_ps_1,
+      vsize_ps_2,
+      vsize_ps_3,
+      vsize_ps_4,
+      vsize_ps_5,
+      vsize_ps_6,
+      vsize_ps_8,
+      vsize_ps_10,
+      vsize_ps_12,
+      vsize_ps_15,
+      vsize_ps_20,
+      vsize_ps_30,
+      vsize_ps_40,
+      vsize_ps_50,
+      vsize_ps_60,
+      vsize_ps_70,
+      vsize_ps_80,
+      vsize_ps_90,
+      vsize_ps_100,
+      vsize_ps_125,
+      vsize_ps_150,
+      vsize_ps_175,
+      vsize_ps_200,
+      vsize_ps_250,
+      vsize_ps_300,
+      vsize_ps_350,
+      vsize_ps_400,
+      vsize_ps_500,
+      vsize_ps_600,
+      vsize_ps_700,
+      vsize_ps_800,
+      vsize_ps_900,
+      vsize_ps_1000,
+      vsize_ps_1200,
+      vsize_ps_1400,
+      vsize_ps_1600,
+      vsize_ps_1800,
+      vsize_ps_2000 \
      FROM statistics \
      ${interval === 'all' ? '' : `WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`} \
      GROUP BY UNIX_TIMESTAMP(added) DIV ${div} \
@ -452,7 +642,47 @@ class StatisticsApi {
          s.vsize_1600,
          s.vsize_1800,
          s.vsize_2000,
-        ]
+        ],
+        vsizes_ps: [
+          s.vsize_ps_1,
+          s.vsize_ps_2,
+          s.vsize_ps_3,
+          s.vsize_ps_4,
+          s.vsize_ps_5,
+          s.vsize_ps_6,
+          s.vsize_ps_8,
+          s.vsize_ps_10,
+          s.vsize_ps_12,
+          s.vsize_ps_15,
+          s.vsize_ps_20,
+          s.vsize_ps_30,
+          s.vsize_ps_40,
+          s.vsize_ps_50,
+          s.vsize_ps_60,
+          s.vsize_ps_70,
+          s.vsize_ps_80,
+          s.vsize_ps_90,
+          s.vsize_ps_100,
+          s.vsize_ps_125,
+          s.vsize_ps_150,
+          s.vsize_ps_175,
+          s.vsize_ps_200,
+          s.vsize_ps_250,
+          s.vsize_ps_300,
+          s.vsize_ps_350,
+          s.vsize_ps_400,
+          s.vsize_ps_500,
+          s.vsize_ps_600,
+          s.vsize_ps_700,
+          s.vsize_ps_800,
+          s.vsize_ps_900,
+          s.vsize_ps_1000,
+          s.vsize_ps_1200,
+          s.vsize_ps_1400,
+          s.vsize_ps_1600,
+          s.vsize_ps_1800,
+          s.vsize_ps_2000,
+        ],
      };
    });
  }
@ -506,6 +736,44 @@ class StatisticsApi {
        vsize_1600: s.vsizes[35],
        vsize_1800: s.vsizes[36],
        vsize_2000: s.vsizes[37],
+        vsize_ps_1: s.vsizes_ps?.[0] || 0,
+        vsize_ps_2: s.vsizes_ps?.[1] || 0,
+        vsize_ps_3: s.vsizes_ps?.[2] || 0,
+        vsize_ps_4: s.vsizes_ps?.[3] || 0,
+        vsize_ps_5: s.vsizes_ps?.[4] || 0,
+        vsize_ps_6: s.vsizes_ps?.[5] || 0,
+        vsize_ps_8: s.vsizes_ps?.[6] || 0,
+        vsize_ps_10: s.vsizes_ps?.[7] || 0,
+        vsize_ps_12: s.vsizes_ps?.[8] || 0,
+        vsize_ps_15: s.vsizes_ps?.[9] || 0,
+        vsize_ps_20: s.vsizes_ps?.[10] || 0,
+        vsize_ps_30: s.vsizes_ps?.[11] || 0,
+        vsize_ps_40: s.vsizes_ps?.[12] || 0,
+        vsize_ps_50: s.vsizes_ps?.[13] || 0,
+        vsize_ps_60: s.vsizes_ps?.[14] || 0,
+        vsize_ps_70: s.vsizes_ps?.[15] || 0,
+        vsize_ps_80: s.vsizes_ps?.[16] || 0,
+        vsize_ps_90: s.vsizes_ps?.[17] || 0,
+        vsize_ps_100: s.vsizes_ps?.[18] || 0,
+        vsize_ps_125: s.vsizes_ps?.[19] || 0,
+        vsize_ps_150: s.vsizes_ps?.[20] || 0,
+        vsize_ps_175: s.vsizes_ps?.[21] || 0,
+        vsize_ps_200: s.vsizes_ps?.[22] || 0,
+        vsize_ps_250: s.vsizes_ps?.[23] || 0,
+        vsize_ps_300: s.vsizes_ps?.[24] || 0,
+        vsize_ps_350: s.vsizes_ps?.[25] || 0,
+        vsize_ps_400: s.vsizes_ps?.[26] || 0,
+        vsize_ps_500: s.vsizes_ps?.[27] || 0,
+        vsize_ps_600: s.vsizes_ps?.[28] || 0,
+        vsize_ps_700: s.vsizes_ps?.[29] || 0,
+        vsize_ps_800: s.vsizes_ps?.[30] || 0,
+        vsize_ps_900: s.vsizes_ps?.[31] || 0,
+        vsize_ps_1000: s.vsizes_ps?.[32] || 0,
+        vsize_ps_1200: s.vsizes_ps?.[33] || 0,
+        vsize_ps_1400: s.vsizes_ps?.[34] || 0,
+        vsize_ps_1600: s.vsizes_ps?.[35] || 0,
+        vsize_ps_1800: s.vsizes_ps?.[36] || 0,
+        vsize_ps_2000: s.vsizes_ps?.[37] || 0,
      }
    });
  }
--- a/backend/src/api/statistics/statistics.ts
+++ b/backend/src/api/statistics/statistics.ts
@ -4,6 +4,9 @@ import { TransactionExtended, OptimizedStatistic } from '../../mempool.interface
 import { Common } from '../common';
 import statisticsApi from './statistics-api';

+export const logFees = [1, 2, 3, 4, 5, 6, 8, 10, 12, 15, 20, 30, 40, 50, 60, 70, 80, 90, 100, 125, 150, 175, 200,
+  250, 300, 350, 400, 500, 600, 700, 800, 900, 1000, 1200, 1400, 1600, 1800, 2000];
+
 class Statistics {
  protected intervalTimer: NodeJS.Timer | undefined;
  protected lastRun: number = 0;
@ -42,6 +45,7 @@ class Statistics {
    const currentMempool = memPool.getMempool();
    const txPerSecond = memPool.getTxPerSecond();
    const vBytesPerSecond = memPool.getVBytesPerSecond();
+    const vBytesPerSecondByFeeRate = memPool.getVBytesPerSecondByFeeRate();

    logger.debug('Running statistics');

@ -73,9 +77,6 @@ class Statistics {
    const totalWeight = memPoolArray.map((tx) => tx.vsize).reduce((acc, curr) => acc + curr) * 4;
    const totalFee = memPoolArray.map((tx) => tx.fee).reduce((acc, curr) => acc + curr);

-    const logFees = [1, 2, 3, 4, 5, 6, 8, 10, 12, 15, 20, 30, 40, 50, 60, 70, 80, 90, 100, 125, 150, 175, 200,
-      250, 300, 350, 400, 500, 600, 700, 800, 900, 1000, 1200, 1400, 1600, 1800, 2000];
-
    const weightVsizeFees: { [feePerWU: number]: number } = {};
    const lastItem = logFees.length - 1;

@ -147,6 +148,44 @@ class Statistics {
        vsize_1600: weightVsizeFees['1600'] || 0,
        vsize_1800: weightVsizeFees['1800'] || 0,
        vsize_2000: weightVsizeFees['2000'] || 0,
+        vsize_ps_1: vBytesPerSecondByFeeRate['1'] || 0,
+        vsize_ps_2: vBytesPerSecondByFeeRate['2'] || 0,
+        vsize_ps_3: vBytesPerSecondByFeeRate['3'] || 0,
+        vsize_ps_4: vBytesPerSecondByFeeRate['4'] || 0,
+        vsize_ps_5: vBytesPerSecondByFeeRate['5'] || 0,
+        vsize_ps_6: vBytesPerSecondByFeeRate['6'] || 0,
+        vsize_ps_8: vBytesPerSecondByFeeRate['8'] || 0,
+        vsize_ps_10: vBytesPerSecondByFeeRate['10'] || 0,
+        vsize_ps_12: vBytesPerSecondByFeeRate['12'] || 0,
+        vsize_ps_15: vBytesPerSecondByFeeRate['15'] || 0,
+        vsize_ps_20: vBytesPerSecondByFeeRate['20'] || 0,
+        vsize_ps_30: vBytesPerSecondByFeeRate['30'] || 0,
+        vsize_ps_40: vBytesPerSecondByFeeRate['40'] || 0,
+        vsize_ps_50: vBytesPerSecondByFeeRate['50'] || 0,
+        vsize_ps_60: vBytesPerSecondByFeeRate['60'] || 0,
+        vsize_ps_70: vBytesPerSecondByFeeRate['70'] || 0,
+        vsize_ps_80: vBytesPerSecondByFeeRate['80'] || 0,
+        vsize_ps_90: vBytesPerSecondByFeeRate['90'] || 0,
+        vsize_ps_100: vBytesPerSecondByFeeRate['100'] || 0,
+        vsize_ps_125: vBytesPerSecondByFeeRate['125'] || 0,
+        vsize_ps_150: vBytesPerSecondByFeeRate['150'] || 0,
+        vsize_ps_175: vBytesPerSecondByFeeRate['175'] || 0,
+        vsize_ps_200: vBytesPerSecondByFeeRate['200'] || 0,
+        vsize_ps_250: vBytesPerSecondByFeeRate['250'] || 0,
+        vsize_ps_300: vBytesPerSecondByFeeRate['300'] || 0,
+        vsize_ps_350: vBytesPerSecondByFeeRate['350'] || 0,
+        vsize_ps_400: vBytesPerSecondByFeeRate['400'] || 0,
+        vsize_ps_500: vBytesPerSecondByFeeRate['500'] || 0,
+        vsize_ps_600: vBytesPerSecondByFeeRate['600'] || 0,
+        vsize_ps_700: vBytesPerSecondByFeeRate['700'] || 0,
+        vsize_ps_800: vBytesPerSecondByFeeRate['800'] || 0,
+        vsize_ps_900: vBytesPerSecondByFeeRate['900'] || 0,
+        vsize_ps_1000: vBytesPerSecondByFeeRate['1000'] || 0,
+        vsize_ps_1200: vBytesPerSecondByFeeRate['1200'] || 0,
+        vsize_ps_1400: vBytesPerSecondByFeeRate['1400'] || 0,
+        vsize_ps_1600: vBytesPerSecondByFeeRate['1600'] || 0,
+        vsize_ps_1800: vBytesPerSecondByFeeRate['1800'] || 0,
+        vsize_ps_2000: vBytesPerSecondByFeeRate['2000'] || 0,
      });

      if (this.newStatisticsEntryCallback && insertId) {
--- a/backend/src/mempool.interfaces.ts
+++ b/backend/src/mempool.interfaces.ts
@ -439,6 +439,46 @@ export interface Statistic {
  vsize_1600: number;
  vsize_1800: number;
  vsize_2000: number;
+
+  vsize_ps_1: number;
+  vsize_ps_2: number;
+  vsize_ps_3: number;
+  vsize_ps_4: number;
+  vsize_ps_5: number;
+  vsize_ps_6: number;
+  vsize_ps_8: number;
+  vsize_ps_10: number;
+  vsize_ps_12: number;
+  vsize_ps_15: number;
+  vsize_ps_20: number;
+  vsize_ps_30: number;
+  vsize_ps_40: number;
+  vsize_ps_50: number;
+  vsize_ps_60: number;
+  vsize_ps_70: number;
+  vsize_ps_80: number;
+  vsize_ps_90: number;
+  vsize_ps_100: number;
+  vsize_ps_125: number;
+  vsize_ps_150: number;
+  vsize_ps_175: number;
+  vsize_ps_200: number;
+  vsize_ps_250: number;
+  vsize_ps_300: number;
+  vsize_ps_350: number;
+  vsize_ps_400: number;
+  vsize_ps_500: number;
+  vsize_ps_600: number;
+  vsize_ps_700: number;
+  vsize_ps_800: number;
+  vsize_ps_900: number;
+  vsize_ps_1000: number;
+  vsize_ps_1200: number;
+  vsize_ps_1400: number;
+  vsize_ps_1600: number;
+  vsize_ps_1800: number;
+  vsize_ps_2000: number;
+  
 }

 export interface OptimizedStatistic {
@ -449,6 +489,7 @@ export interface OptimizedStatistic {
  mempool_byte_weight: number;
  min_fee: number;
  vsizes: number[];
+  vsizes_ps: number[];
 }

 export interface TxTrackingInfo {
@ -481,6 +522,7 @@ export interface WebsocketResponse {
 export interface VbytesPerSecond {
  unixTime: number;
  vSize: number;
+  effectiveFeePerVsize: number;
 }

 export interface RequiredSpec { [name: string]: RequiredParams; }
--- a/frontend/src/app/components/transaction/transaction.component.ts
+++ b/frontend/src/app/components/transaction/transaction.component.ts
@ -27,7 +27,7 @@ import { StorageService } from '../../services/storage.service';
 import { seoDescriptionNetwork } from '../../shared/common.utils';
 import { getTransactionFlags, getUnacceleratedFeeRate } from '../../shared/transaction.utils';
 import { Filter, TransactionFlags, toFilters } from '../../shared/filters.utils';
-import { BlockExtended, CpfpInfo, RbfTree, MempoolPosition, DifficultyAdjustment, Acceleration, AccelerationPosition } from '../../interfaces/node-api.interface';
+import { BlockExtended, CpfpInfo, RbfTree, MempoolPosition, DifficultyAdjustment, Acceleration, AccelerationPosition, OptimizedMempoolStats } from '../../interfaces/node-api.interface';
 import { LiquidUnblinding } from './liquid-ublinding';
 import { RelativeUrlPipe } from '../../shared/pipes/relative-url/relative-url.pipe';
 import { PriceService } from '../../services/price.service';
@ -139,6 +139,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
  firstLoad = true;
  waitingForAccelerationInfo: boolean = false;
  isLoadingFirstSeen = false;
+  mempoolStats: OptimizedMempoolStats[] = null;

  featuresEnabled: boolean;
  segwitEnabled: boolean;
@ -196,7 +197,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
      });
    }

-    this.websocketService.want(['blocks', 'mempool-blocks']);
+    this.websocketService.want(['blocks', 'mempool-blocks', 'live-2h-chart']);
    this.stateService.networkChanged$.subscribe(
      (network) => {
        this.network = network;
@ -769,8 +770,20 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
      this.stateService.difficultyAdjustment$.pipe(startWith(null)),
      this.isAccelerated$,
      this.txChanged$,
+      this.apiService.list2HStatistics$(),
+      this.stateService.live2Chart$.pipe(startWith(null)),
    ]).pipe(
-      map(([position, mempoolBlocks, da, isAccelerated]) => {
+      map(([position, mempoolBlocks, da, isAccelerated, _, mempoolStats, mempoolStat]) => {
+
+        if (this.mempoolStats === null) {
+          this.mempoolStats = mempoolStats;
+        }
+
+        if (this.mempoolStats.length && mempoolStat && this.mempoolStats[0].added !== mempoolStat.added) {
+          this.mempoolStats.pop();
+          this.mempoolStats.unshift(mempoolStat);
+        }
+
        return this.etaService.calculateETA(
          this.network,
          this.tx,
@ -780,6 +793,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
          this.miningStats,
          isAccelerated,
          this.accelerationPositions,
+          this.mempoolStats,
        );
      })
    );
@ -977,6 +991,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
    this.isAcceleration = false;
    this.isAccelerated$.next(this.isAcceleration);
    this.eligibleForAcceleration = false;
+    this.mempoolStats = null;
    this.leaveTransaction();
  }

--- a/frontend/src/app/interfaces/node-api.interface.ts
+++ b/frontend/src/app/interfaces/node-api.interface.ts
@ -7,6 +7,7 @@ export interface OptimizedMempoolStats {
  total_fee: number;
  mempool_byte_weight: number;
  vsizes: number[];
+  vsizes_ps: number[];
 }

 interface Ancestor {
--- a/frontend/src/app/services/eta.service.ts
+++ b/frontend/src/app/services/eta.service.ts
@ -1,5 +1,5 @@
 import { Injectable } from '@angular/core';
-import { AccelerationPosition, CpfpInfo, DifficultyAdjustment, MempoolPosition, SinglePoolStats } from '../interfaces/node-api.interface';
+import { AccelerationPosition, CpfpInfo, DifficultyAdjustment, MempoolPosition, OptimizedMempoolStats, SinglePoolStats } from '../interfaces/node-api.interface';
 import { StateService } from './state.service';
 import { MempoolBlock } from '../interfaces/websocket.interface';
 import { Transaction } from '../interfaces/electrs.interface';
@ -7,6 +7,7 @@ import { MiningService, MiningStats } from './mining.service';
 import { getUnacceleratedFeeRate } from '../shared/transaction.utils';
 import { AccelerationEstimate } from '../components/accelerate-checkout/accelerate-checkout.component';
 import { Observable, combineLatest, map, of, share, shareReplay, tap } from 'rxjs';
+import { feeLevels } from '../app.constants';

 export interface ETA {
  now: number, // time at which calculation performed
@ -113,6 +114,7 @@ export class EtaService {
    miningStats: MiningStats,
    isAccelerated: boolean,
    accelerationPositions: AccelerationPosition[],
+    mempoolStats: OptimizedMempoolStats[] = [],
  ): ETA | null {
    // return this.calculateETA(tx, this.accelerationPositions, position, mempoolBlocks, da, isAccelerated)
    if (!tx || !mempoolBlocks) {
@ -143,7 +145,17 @@ export class EtaService {

    if (!isAccelerated) {
      const blocks = mempoolPosition.block + 1;
-      const wait = da.adjustedTimeAvg * (mempoolPosition.block + 1);
+
+      const vsizeAhead = mempoolPosition.vsize + this.stateService.blockVSize * mempoolPosition.block;
+      const incomingVsizePerBlock = this.estimateVsizePerSecond(tx, mempoolStats) * da.adjustedTimeAvg / 1000;
+      const vsizeConsumedPerBlock = Math.max(
+        this.stateService.blockVSize - incomingVsizePerBlock,
+        0.05 * this.stateService.blockVSize // So that we don't return infinite ETA
+      )
+
+      const blocksUntilMined = Math.ceil(vsizeAhead / vsizeConsumedPerBlock);
+      const wait = blocksUntilMined * da.adjustedTimeAvg;
+
      return {
        now,
        time: wait + now + da.timeOffset,
@ -279,4 +291,46 @@ export class EtaService {
    return tx.fee / (tx.weight / 4);

  }
+
+  estimateVsizePerSecond(tx: Transaction, mempoolStats: OptimizedMempoolStats[], timeWindow: number = 15 * 60 * 1000): number {
+    const nowMinusTimeSpan = (new Date().getTime() - timeWindow) / 1000;
+    const vsizeAboveTransaction = mempoolStats
+      // Remove datapoints older than now - timeWindow
+      .filter(stat => stat.added > nowMinusTimeSpan)
+      // Remove datapoints less than 45 seconds apart from the previous one
+      .filter((el, i, arr) => {
+        if (i === 0) {
+          return true;
+        }
+        return arr[i - 1].added - el.added > 45;
+      })
+      // For each datapoint, compute the total vsize of transactions with higher fee rate
+      .map(stat => {
+        let vsizeAbove = 0;
+        for (let i = feeLevels.length - 1; i >= 0; i--) {
+          if (feeLevels[i] > tx.effectiveFeePerVsize) {
+            vsizeAbove += stat.vsizes_ps[i];
+          } else {
+            break;
+          }
+        }
+        return vsizeAbove;
+      });
+    
+    // vsizeAboveTransaction is a temporal series of past vsize values above the transaction's fee rate
+    // From this array we need to estimate the future vsize per second
+    // Naive first approach: take the median of the series
+    if (!vsizeAboveTransaction.length) {
+      return 0;
+    }
+
+    const sorted = Array.from(vsizeAboveTransaction).sort((a, b) => a - b);
+    const middle = Math.floor(sorted.length / 2);
+
+    if (sorted.length % 2 === 0) {
+        return (sorted[middle - 1] + sorted[middle]) / 2;
+    }
+
+    return sorted[middle];
+  }
 }
Author	SHA1	Message	Date
natsoni	428cc9eacd	Analytic form for ETA algorithm	2024-09-09 16:03:40 +02:00
natsoni	638acffbad	Consider incoming transactions flow into ETA calculation	2024-09-07 23:07:16 +02:00
natsoni	ae9125d316	statistics: class incoming transactions by fee rate	2024-09-07 17:49:21 +02:00