Fetch missing block audit scores

backend/src/api/audit.ts

@@ -1,5 +1,10 @@
 import config from '../config';
-import { BlockExtended, TransactionExtended, MempoolBlockWithTransactions } from '../mempool.interfaces';
+import bitcoinApi from './bitcoin/bitcoin-api-factory';
+import { Common } from './common';
+import { TransactionExtended, MempoolBlockWithTransactions, AuditScore } from '../mempool.interfaces';
+import blocksRepository from '../repositories/BlocksRepository';
+import blocksAuditsRepository from '../repositories/BlocksAuditsRepository';
+import blocks from '../api/blocks';
 
 const PROPAGATION_MARGIN = 180; // in seconds, time since a transaction is first seen after which it is assumed to have propagated to all miners
 
@@ -81,7 +86,7 @@ class Audit {
         }
         overflowWeight += tx.weight;
       }
-      totalWeight += tx.weight
+      totalWeight += tx.weight;
     }
 
     // transactions missing from near the end of our template are probably not being censored
@@ -97,7 +102,7 @@ class Audit {
         }
         if (mempool[txid].effectiveFeePerVsize > maxOverflowRate) {
           maxOverflowRate = mempool[txid].effectiveFeePerVsize;
-          rateThreshold = (Math.ceil(maxOverflowRate * 100) / 100) + 0.005
+          rateThreshold = (Math.ceil(maxOverflowRate * 100) / 100) + 0.005;
         }
       } else if (mempool[txid].effectiveFeePerVsize <= rateThreshold) { // tolerance of 0.01 sat/vb + rounding
         if (isCensored[txid]) {
@@ -117,6 +122,45 @@ class Audit {
       score
     };
   }
+
+  public async $getBlockAuditScores(fromHeight?: number, limit: number = 15): Promise<AuditScore[]> {
+    let currentHeight = fromHeight !== undefined ? fromHeight : await blocksRepository.$mostRecentBlockHeight();
+    const returnScores: AuditScore[] = [];
+
+    if (currentHeight < 0) {
+      return returnScores;
+    }
+
+    for (let i = 0; i < limit && currentHeight >= 0; i++) {
+      const block = blocks.getBlocks().find((b) => b.height === currentHeight);
+      if (block?.extras?.matchRate != null) {
+        returnScores.push({
+          hash: block.id,
+          matchRate: block.extras.matchRate
+        });
+      } else {
+        let currentHash;
+        if (!currentHash && Common.indexingEnabled()) {
+          const dbBlock = await blocksRepository.$getBlockByHeight(currentHeight);
+          if (dbBlock && dbBlock['id']) {
+            currentHash = dbBlock['id'];
+          }
+        }
+        if (!currentHash) {
+          currentHash = await bitcoinApi.$getBlockHash(currentHeight);
+        }
+        if (currentHash) {
+          const auditScore = await blocksAuditsRepository.$getBlockAuditScore(currentHash);
+          returnScores.push({
+            hash: currentHash,
+            matchRate: auditScore?.matchRate
+          });
+        }
+      }
+      currentHeight--;
+    }
+    return returnScores;
+  }
 }
 
 export default new Audit();

backend/src/api/blocks.ts

@@ -195,9 +195,9 @@ class Blocks {
         };
       }
 
-      const auditSummary = await BlocksAuditsRepository.$getShortBlockAudit(block.id);
+      const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
-      if (auditSummary) {
+      if (auditScore != null) {
-        blockExtended.extras.matchRate = auditSummary.matchRate;
+        blockExtended.extras.matchRate = auditScore.matchRate;
       }
     }
 

backend/src/api/mining/mining-routes.ts

@@ -1,6 +1,7 @@
 import { Application, Request, Response } from 'express';
 import config from "../../config";
 import logger from '../../logger';
+import audits from '../audit';
 import BlocksAuditsRepository from '../../repositories/BlocksAuditsRepository';
 import BlocksRepository from '../../repositories/BlocksRepository';
 import DifficultyAdjustmentsRepository from '../../repositories/DifficultyAdjustmentsRepository';
@@ -26,6 +27,9 @@ class MiningRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/sizes-weights/:interval', this.$getHistoricalBlockSizeAndWeight)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty-adjustments/:interval', this.$getDifficultyAdjustments)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/predictions/:interval', this.$getHistoricalBlockPrediction)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/scores', this.$getBlockAuditScores)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/scores/:height', this.$getBlockAuditScores)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/score/:hash', this.$getBlockAuditScore)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/audit/:hash', this.$getBlockAudit)
       .get(config.MEMPOOL.API_URL_PREFIX + 'mining/blocks/timestamp/:timestamp', this.$getHeightFromTimestamp)
     ;
@@ -276,6 +280,29 @@ class MiningRoutes {
       res.status(500).send(e instanceof Error ? e.message : e);
     }
   }
+
+  private async $getBlockAuditScores(req: Request, res: Response) {
+    try {
+      const height = req.params.height === undefined ? undefined : parseInt(req.params.height, 10);
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(await audits.$getBlockAuditScores(height, 15));
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  public async $getBlockAuditScore(req: Request, res: Response) {
+    try {
+      const audit = await BlocksAuditsRepository.$getBlockAuditScore(req.params.hash);
+
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 3600 * 24).toUTCString());
+      res.json(audit || 'null');
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
 }
 
 export default new MiningRoutes();

backend/src/mempool.interfaces.ts

@@ -32,6 +32,11 @@ export interface BlockAudit {
   matchRate: number,
 }
 
+export interface AuditScore {
+  hash: string,
+  matchRate?: number,
+}
+
 export interface MempoolBlock {
   blockSize: number;
   blockVSize: number;

backend/src/repositories/BlocksAuditsRepository.ts

@@ -1,6 +1,6 @@
 import DB from '../database';
 import logger from '../logger';
-import { BlockAudit } from '../mempool.interfaces';
+import { BlockAudit, AuditScore } from '../mempool.interfaces';
 
 class BlocksAuditRepositories {
   public async $saveAudit(audit: BlockAudit): Promise<void> {
@@ -72,10 +72,10 @@ class BlocksAuditRepositories {
     }
   }
 
-  public async $getShortBlockAudit(hash: string): Promise<any> {
+  public async $getBlockAuditScore(hash: string): Promise<AuditScore> {
     try {
       const [rows]: any[] = await DB.query(
-        `SELECT hash as id, match_rate as matchRate
+        `SELECT hash, match_rate as matchRate
         FROM blocks_audits
         WHERE blocks_audits.hash = "${hash}"
       `);

frontend/src/app/components/block/block.component.html

@@ -114,7 +114,7 @@
                   <td i18n="block.health">Block health</td>
                   <td>
                     <a *ngIf="block.extras?.matchRate != null" [routerLink]="['/block-audit/' | relativeUrl, blockHash]">{{ block.extras.matchRate }}%</a>
-                    <span *ngIf="block.extras?.matchRate == null" i18n="unknown">Unknown</span>
+                    <span *ngIf="block.extras?.matchRate === null" i18n="unknown">Unknown</span>
                   </td>
                 </tr>
               </ng-template>

frontend/src/app/components/block/block.component.ts

@@ -4,7 +4,7 @@ import { ActivatedRoute, ParamMap, Router } from '@angular/router';
 import { ElectrsApiService } from '../../services/electrs-api.service';
 import { switchMap, tap, throttleTime, catchError, map, shareReplay, startWith, pairwise } from 'rxjs/operators';
 import { Transaction, Vout } from '../../interfaces/electrs.interface';
-import { Observable, of, Subscription, asyncScheduler, EMPTY } from 'rxjs';
+import { Observable, of, Subscription, asyncScheduler, EMPTY, Subject } from 'rxjs';
 import { StateService } from '../../services/state.service';
 import { SeoService } from '../../services/seo.service';
 import { WebsocketService } from '../../services/websocket.service';
@@ -60,6 +60,8 @@ export class BlockComponent implements OnInit, OnDestroy {
   nextBlockTxListSubscription: Subscription = undefined;
   timeLtrSubscription: Subscription;
   timeLtr: boolean;
+  fetchAuditScore$ = new Subject<string>();
+  fetchAuditScoreSubscription: Subscription;
 
   @ViewChild('blockGraph') blockGraph: BlockOverviewGraphComponent;
 
@@ -105,12 +107,30 @@ export class BlockComponent implements OnInit, OnDestroy {
 
         if (block.id === this.blockHash) {
           this.block = block;
+          if (this.block.id && this.block?.extras?.matchRate == null) {
+            this.fetchAuditScore$.next(this.block.id);
+          }
           if (block?.extras?.reward != undefined) {
             this.fees = block.extras.reward / 100000000 - this.blockSubsidy;
           }
         }
       });
 
+    if (this.indexingAvailable) {
+      this.fetchAuditScoreSubscription = this.fetchAuditScore$
+        .pipe(
+          switchMap((hash) => this.apiService.getBlockAuditScore$(hash)),
+          catchError(() => EMPTY),
+        )
+        .subscribe((score) => {
+          if (score && score.hash === this.block.id) {
+            this.block.extras.matchRate = score.matchRate || null;
+          } else {
+            this.block.extras.matchRate = null;
+          }
+        });
+    }
+
     const block$ = this.route.paramMap.pipe(
       switchMap((params: ParamMap) => {
         const blockHash: string = params.get('id') || '';
@@ -209,6 +229,9 @@ export class BlockComponent implements OnInit, OnDestroy {
           this.fees = block.extras.reward / 100000000 - this.blockSubsidy;
         }
         this.stateService.markBlock$.next({ blockHeight: this.blockHeight });
+        if (this.block.id && this.block?.extras?.matchRate == null) {
+          this.fetchAuditScore$.next(this.block.id);
+        }
         this.isLoadingTransactions = true;
         this.transactions = null;
         this.transactionsError = null;
@@ -311,6 +334,7 @@ export class BlockComponent implements OnInit, OnDestroy {
     this.networkChangedSubscription.unsubscribe();
     this.queryParamsSubscription.unsubscribe();
     this.timeLtrSubscription.unsubscribe();
+    this.fetchAuditScoreSubscription?.unsubscribe();
     this.unsubscribeNextBlockSubscriptions();
   }
 

frontend/src/app/components/blocks-list/blocks-list.component.html

@@ -46,22 +46,17 @@
             <app-time-since [time]="block.timestamp" [fastRender]="true"></app-time-since>
           </td>
           <td *ngIf="indexingAvailable" class="health text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
-            <a *ngIf="block.extras?.matchRate != null" class="clear-link" [routerLink]="['/block-audit/' | relativeUrl, block.id]">
+            <a class="clear-link" [routerLink]="auditScores[block.id] != null ? ['/block-audit/' | relativeUrl, block.id] : null">
               <div class="progress progress-health">
                 <div class="progress-bar progress-bar-health" role="progressbar"
-                  [ngStyle]="{'width': (100 - (block.extras?.matchRate || 0)) + '%' }"></div>
+                  [ngStyle]="{'width': (100 - (auditScores[block.id] || 0)) + '%' }"></div>
                 <div class="progress-text">
-                  <span>{{ block.extras.matchRate }}%</span>
+                  <span *ngIf="auditScores[block.id] != null;">{{ auditScores[block.id] }}%</span>
+                  <span *ngIf="auditScores[block.id] === undefined" class="skeleton-loader"></span>
+                  <span *ngIf="auditScores[block.id] === null">~</span>
                 </div>
               </div>
             </a>
-            <div *ngIf="block.extras?.matchRate == null" class="progress progress-health">
-              <div class="progress-bar progress-bar-health" role="progressbar"
-                [ngStyle]="{'width': '100%' }"></div>
-              <div class="progress-text">
-                <span>~</span>
-              </div>
-            </div>
           </td>
           <td *ngIf="indexingAvailable" class="reward text-right" [ngClass]="{'widget': widget, 'legacy': !indexingAvailable}">
             <app-amount [satoshis]="block.extras.reward" [noFiat]="true" digitsInfo="1.2-2"></app-amount>

frontend/src/app/components/blocks-list/blocks-list.component.scss

@@ -196,6 +196,10 @@ tr, td, th {
   @media (max-width: 950px) {
     display: none;
   }
+
+  .progress-text .skeleton-loader {
+    top: -8.5px;
+  }
 }
 .health.widget {
   width: 25%;

frontend/src/app/components/blocks-list/blocks-list.component.ts

@@ -1,6 +1,6 @@
-import { Component, OnInit, ChangeDetectionStrategy, Input } from '@angular/core';
+import { Component, OnInit, OnDestroy, ChangeDetectionStrategy, Input } from '@angular/core';
-import { BehaviorSubject, combineLatest, concat, Observable, timer } from 'rxjs';
+import { BehaviorSubject, combineLatest, concat, Observable, timer, EMPTY, Subscription, of } from 'rxjs';
-import { delayWhen, map, retryWhen, scan, skip, switchMap, tap } from 'rxjs/operators';
+import { catchError, delayWhen, map, retryWhen, scan, skip, switchMap, tap } from 'rxjs/operators';
 import { BlockExtended } from '../../interfaces/node-api.interface';
 import { ApiService } from '../../services/api.service';
 import { StateService } from '../../services/state.service';
@@ -12,10 +12,14 @@ import { WebsocketService } from '../../services/websocket.service';
   styleUrls: ['./blocks-list.component.scss'],
   changeDetection: ChangeDetectionStrategy.OnPush,
 })
-export class BlocksList implements OnInit {
+export class BlocksList implements OnInit, OnDestroy {
   @Input() widget: boolean = false;
 
   blocks$: Observable<BlockExtended[]> = undefined;
+  auditScores: { [hash: string]: number | void } = {};
+
+  auditScoreSubscription: Subscription;
+  latestScoreSubscription: Subscription;
 
   indexingAvailable = false;
   isLoading = true;
@@ -105,6 +109,53 @@ export class BlocksList implements OnInit {
           return acc;
         }, [])
       );
+
+    if (this.indexingAvailable) {
+      this.auditScoreSubscription = this.fromHeightSubject.pipe(
+        switchMap((fromBlockHeight) => {
+          return this.apiService.getBlockAuditScores$(this.page === 1 ? undefined : fromBlockHeight)
+            .pipe(
+              catchError(() => {
+                return EMPTY;
+              })
+            );
+        })
+      ).subscribe((scores) => {
+        Object.values(scores).forEach(score => {
+          this.auditScores[score.hash] = score?.matchRate != null ? score.matchRate : null;
+        });
+      });
+
+      this.latestScoreSubscription = this.stateService.blocks$.pipe(
+        switchMap((block) => {
+          if (block[0]?.extras?.matchRate != null) {
+            return of({
+              hash: block[0].id,
+              matchRate: block[0]?.extras?.matchRate,
+            });
+          }
+          else if (block[0]?.id && this.auditScores[block[0].id] === undefined) {
+            return this.apiService.getBlockAuditScore$(block[0].id)
+              .pipe(
+                catchError(() => {
+                  return EMPTY;
+                })
+              );
+          } else {
+            return EMPTY;
+          }
+        }),
+      ).subscribe((score) => {
+        if (score && score.hash) {
+          this.auditScores[score.hash] = score?.matchRate != null ? score.matchRate : null;
+        }
+      });
+    }
+  }
+
+  ngOnDestroy(): void {
+    this.auditScoreSubscription?.unsubscribe();
+    this.latestScoreSubscription?.unsubscribe();
   }
 
   pageChange(page: number) {

frontend/src/app/interfaces/node-api.interface.ts

@@ -152,6 +152,11 @@ export interface RewardStats {
   totalTx: number;
 }
 
+export interface AuditScore {
+  hash: string;
+  matchRate?: number;
+}
+
 export interface ITopNodesPerChannels {
   publicKey: string,
   alias: string,

frontend/src/app/services/api.service.ts

@@ -1,7 +1,7 @@
 import { Injectable } from '@angular/core';
 import { HttpClient, HttpParams } from '@angular/common/http';
 import { CpfpInfo, OptimizedMempoolStats, AddressInformation, LiquidPegs, ITranslators,
-  PoolStat, BlockExtended, TransactionStripped, RewardStats } from '../interfaces/node-api.interface';
+  PoolStat, BlockExtended, TransactionStripped, RewardStats, AuditScore } from '../interfaces/node-api.interface';
 import { Observable } from 'rxjs';
 import { StateService } from './state.service';
 import { WebsocketResponse } from '../interfaces/websocket.interface';
@@ -234,6 +234,19 @@ export class ApiService {
     );
   }
 
+  getBlockAuditScores$(from: number): Observable<AuditScore[]> {
+    return this.httpClient.get<AuditScore[]>(
+      this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/blocks/audit/scores` +
+      (from !== undefined ? `/${from}` : ``)
+    );
+  }
+
+  getBlockAuditScore$(hash: string) : Observable<any> {
+    return this.httpClient.get<any>(
+      this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/blocks/audit/score/` + hash
+    );
+  }
+
   getRewardStats$(blockCount: number = 144): Observable<RewardStats> {
     return this.httpClient.get<RewardStats>(this.apiBaseUrl + this.apiBasePath + `/api/v1/mining/reward-stats/${blockCount}`);
   }