Merge branch 'master' into feature/pool-stats-page

backend/src/api/bitcoin/bitcoin-api.interface.ts

@@ -4,6 +4,7 @@ export namespace IBitcoinApi {
     size: number;                    //  (numeric) Current tx count
     bytes: number;                   //  (numeric) Sum of all virtual transaction sizes as defined in BIP 141.
     usage: number;                   //  (numeric) Total memory usage for the mempool
+    total_fee: number;               //  (numeric) Total fees of transactions in the mempool
     maxmempool: number;              //  (numeric) Maximum memory usage for the mempool
     mempoolminfee: number;           //  (numeric) Minimum fee rate in BTC/kB for tx to be accepted.
     minrelaytxfee: number;           //  (numeric) Current minimum relay fee for transactions

backend/src/api/blocks.ts

@@ -61,7 +61,7 @@ class Blocks {
         // optimize here by directly fetching txs in the "outdated" mempool
         transactions.push(mempool[txIds[i]]);
         transactionsFound++;
-      } else if (config.MEMPOOL.BACKEND === 'esplora' || memPool.isInSync() || i === 0) {
+      } else if (config.MEMPOOL.BACKEND === 'esplora' || !memPool.hasPriority() || i === 0) {
         // Otherwise we fetch the tx data through backend services (esplora, electrum, core rpc...)
         if (!quiet && (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length)) { // Avoid log spam
           logger.debug(`Indexing tx ${i + 1} of ${txIds.length} in block #${blockHeight}`);
@@ -116,10 +116,7 @@ class Blocks {
     blockExtended.extras.feeRange = transactionsTmp.length > 0 ?
       Common.getFeesInRange(transactionsTmp, 8) : [0, 0];
 
-    const indexingAvailable =
+    if (Common.indexingEnabled()) {
-      ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) &&
-      config.DATABASE.ENABLED === true;
-    if (indexingAvailable) {
       let pool: PoolTag;
       if (blockExtended.extras?.coinbaseTx !== undefined) {
         pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
@@ -173,11 +170,9 @@ class Blocks {
    * Index all blocks metadata for the mining dashboard
    */
   public async $generateBlockDatabase() {
-    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false || // Bitcoin only
+    if (this.blockIndexingStarted === true ||
-      config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === 0 || // Indexing of older blocks must be enabled
+      !Common.indexingEnabled() ||
-      !memPool.isInSync() || // We sync the mempool first
+      memPool.hasPriority()
-      this.blockIndexingStarted === true || // Indexing must not already be in progress
-      config.DATABASE.ENABLED === false
     ) {
       return;
     }
@@ -293,10 +288,7 @@ class Blocks {
       const transactions = await this.$getTransactionsExtended(blockHash, block.height, false);
       const blockExtended: BlockExtended = await this.$getBlockExtended(block, transactions);
 
-      const indexingAvailable =
+      if (Common.indexingEnabled()) {
-        ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) &&
-        config.DATABASE.ENABLED === true;
-      if (indexingAvailable) {
         await blocksRepository.$saveBlockInDatabase(blockExtended);
       }
 
@@ -314,7 +306,7 @@ class Blocks {
       if (this.newBlockCallbacks.length) {
         this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
       }
-      if (memPool.isInSync()) {
+      if (!memPool.hasPriority()) {
         diskCache.$saveCacheToDisk();
       }
     }
@@ -340,10 +332,6 @@ class Blocks {
   }
 
   public async $getBlocksExtras(fromHeight: number): Promise<BlockExtended[]> {
-    const indexingAvailable =
-      ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) &&
-      config.DATABASE.ENABLED === true;
-
     try {
       loadingIndicators.setProgress('blocks', 0);
 
@@ -366,7 +354,7 @@ class Blocks {
       let nextHash = startFromHash;
       for (let i = 0; i < 10 && currentHeight >= 0; i++) {
         let block = this.getBlocks().find((b) => b.height === currentHeight);
-        if (!block && indexingAvailable) {
+        if (!block && Common.indexingEnabled()) {
           block = this.prepareBlock(await this.$indexBlock(currentHeight));
         } else if (!block) {
           block = this.prepareBlock(await bitcoinApi.$getBlock(nextHash));

backend/src/api/common.ts

@@ -169,4 +169,12 @@ export class Common {
       default: return null;
     }
   }
+  
+  static indexingEnabled(): boolean {
+    return (
+      ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) &&
+      config.DATABASE.ENABLED === true &&
+      config.MEMPOOL.INDEXING_BLOCKS_AMOUNT != 0
+    );
+  }
 }

backend/src/api/mempool.ts

@@ -13,8 +13,9 @@ class Mempool {
   private static WEBSOCKET_REFRESH_RATE_MS = 10000;
   private static LAZY_DELETE_AFTER_SECONDS = 30;
   private inSync: boolean = false;
+  private mempoolCacheDelta: number = -1;
   private mempoolCache: { [txId: string]: TransactionExtended } = {};
-  private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0,
+  private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
                                                     maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 };
   private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
     deletedTransactions: TransactionExtended[]) => void) | undefined;
@@ -32,6 +33,17 @@ class Mempool {
     setInterval(this.deleteExpiredTransactions.bind(this), 20000);
   }
 
+  /**
+   * Return true if we should leave resources available for mempool tx caching
+   */
+  public hasPriority(): boolean {
+    if (this.inSync) {
+      return false;
+    } else {
+      return this.mempoolCacheDelta == -1 || this.mempoolCacheDelta > 25;
+    }
+  }
+
   public isInSync(): boolean {
     return this.inSync;
   }
@@ -100,6 +112,8 @@ class Mempool {
     const diff = transactions.length - currentMempoolSize;
     const newTransactions: TransactionExtended[] = [];
 
+    this.mempoolCacheDelta = Math.abs(diff);
+
     if (!this.inSync) {
       loadingIndicators.setProgress('mempool', Object.keys(this.mempoolCache).length / transactions.length * 100);
     }
@@ -168,13 +182,14 @@ class Mempool {
     const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx));
     this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6);
 
-    const syncedThreshold = 0.99; // If we synced 99% of the mempool tx count, consider we're synced
+    if (!this.inSync && transactions.length === Object.keys(this.mempoolCache).length) {
-    if (!this.inSync && Object.keys(this.mempoolCache).length >= transactions.length * syncedThreshold) {
       this.inSync = true;
       logger.notice('The mempool is now in sync!');
       loadingIndicators.setProgress('mempool', 100);
     }
 
+    this.mempoolCacheDelta = Math.abs(transactions.length - Object.keys(this.mempoolCache).length);
+
     if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
       this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
     }

backend/src/routes.ts

@@ -729,7 +729,13 @@ class Routes {
   }
 
   public async getMempool(req: Request, res: Response) {
-    res.status(501).send('Not implemented');
+    const info = mempool.getMempoolInfo();
+    res.json({
+      count: info.size,
+      vsize: info.bytes,
+      total_fee: info.total_fee * 1e8,
+      fee_histogram: []
+    });
   }
 
   public async getMempoolTxIds(req: Request, res: Response) {

frontend/cypress/integration/mainnet/mainnet.spec.ts

@@ -274,113 +274,19 @@ describe('Mainnet', () => {
             });
           });
         });
-      });
-    });
 
-
+        it('loads genesis block and click on the arrow left', () => {
-    it('loads skeleton when changes between networks', () => {
+          cy.viewport('macbook-16');
-      cy.visit('/');
+          cy.visit('/block/0');
-      cy.waitForSkeletonGone();
+          cy.waitForSkeletonGone();
-
+          cy.waitForPageIdle();
-      cy.changeNetwork("testnet");
+          cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
-      cy.changeNetwork("signet");
+          cy.get('[ngbtooltip="Previous Block"] > .ng-fa-icon > .svg-inline--fa').should('not.exist');
-      cy.changeNetwork("mainnet");
+          cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').click().then(() => {
-    });
+            cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
-
+            cy.get('[ngbtooltip="Previous Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
-    it.skip('loads the dashboard with the skeleton blocks', () => {
+          });
-      cy.mockMempoolSocket();
+        });
-      cy.visit("/");
-      cy.get(':nth-child(1) > #bitcoin-block-0').should('be.visible');
-      cy.get(':nth-child(2) > #bitcoin-block-0').should('be.visible');
-      cy.get(':nth-child(3) > #bitcoin-block-0').should('be.visible');
-      cy.get('#mempool-block-0').should('be.visible');
-      cy.get('#mempool-block-1').should('be.visible');
-      cy.get('#mempool-block-2').should('be.visible');
-
-      emitMempoolInfo({
-        'params': {
-          command: 'init'
-        }
-      });
-
-      cy.get(':nth-child(1) > #bitcoin-block-0').should('not.exist');
-      cy.get(':nth-child(2) > #bitcoin-block-0').should('not.exist');
-      cy.get(':nth-child(3) > #bitcoin-block-0').should('not.exist');
-    });
-
-    it('loads the pools screen', () => {
-      cy.visit('/');
-      cy.waitForSkeletonGone();
-      cy.get('#btn-pools').click().then(() => {
-        cy.waitForPageIdle();
-      });
-    });
-
-    it('loads the graphs screen', () => {
-      cy.visit('/');
-      cy.waitForSkeletonGone();
-      cy.get('#btn-graphs').click().then(() => {
-        cy.wait(1000);
-      });
-    });
-
-    describe('graphs page', () => {
-      it('check buttons - mobile', () => {
-        cy.viewport('iphone-6');
-        cy.visit('/graphs');
-        cy.waitForSkeletonGone();
-        cy.get('.small-buttons > :nth-child(2)').should('be.visible');
-        cy.get('#dropdownFees').should('be.visible');
-        cy.get('.btn-group').should('be.visible');
-      });
-      it('check buttons - tablet', () => {
-        cy.viewport('ipad-2');
-        cy.visit('/graphs');
-        cy.waitForSkeletonGone();
-        cy.get('.small-buttons > :nth-child(2)').should('be.visible');
-        cy.get('#dropdownFees').should('be.visible');
-        cy.get('.btn-group').should('be.visible');
-      });
-      it('check buttons - desktop', () => {
-        cy.viewport('macbook-16');
-        cy.visit('/graphs');
-        cy.waitForSkeletonGone();
-        cy.get('.small-buttons > :nth-child(2)').should('be.visible');
-        cy.get('#dropdownFees').should('be.visible');
-        cy.get('.btn-group').should('be.visible');
-      });
-    });
-
-    it('loads the tv screen - desktop', () => {
-      cy.viewport('macbook-16');
-      cy.visit('/');
-      cy.waitForSkeletonGone();
-      cy.get('#btn-tv').click().then(() => {
-        cy.viewport('macbook-16');
-        cy.get('.chart-holder');
-        cy.get('.blockchain-wrapper').should('be.visible');
-        cy.get('#mempool-block-0').should('be.visible');
-      });
-    });
-
-    it('loads the tv screen - mobile', () => {
-      cy.viewport('iphone-6');
-      cy.visit('/tv');
-      cy.waitForSkeletonGone();
-      cy.get('.chart-holder');
-      cy.get('.blockchain-wrapper').should('not.visible');
-    });
-
-    it('loads genesis block and click on the arrow left', () => {
-      cy.viewport('macbook-16');
-      cy.visit('/block/0');
-      cy.waitForSkeletonGone();
-      cy.waitForPageIdle();
-      cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
-      cy.get('[ngbtooltip="Previous Block"] > .ng-fa-icon > .svg-inline--fa').should('not.exist');
-      cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').click().then(() => {
-        cy.get('[ngbtooltip="Next Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
-        cy.get('[ngbtooltip="Previous Block"] > .ng-fa-icon > .svg-inline--fa').should('be.visible');
       });
     });
 

frontend/mempool-frontend-config.sample.json

@@ -15,5 +15,6 @@
   "BASE_MODULE": "mempool",
   "MEMPOOL_WEBSITE_URL": "https://mempool.space",
   "LIQUID_WEBSITE_URL": "https://liquid.network",
-  "BISQ_WEBSITE_URL": "https://bisq.markets"
+  "BISQ_WEBSITE_URL": "https://bisq.markets",
+  "MINING_DASHBOARD": true
 }

frontend/src/app/components/master-page/master-page.component.html

@@ -31,9 +31,12 @@
       <li class="nav-item" routerLinkActive="active" [routerLinkActiveOptions]="{exact: true}" id="btn-home">
         <a class="nav-link" [routerLink]="['/' | relativeUrl]" (click)="collapse()"><fa-icon [icon]="['fas', 'tachometer-alt']" [fixedWidth]="true" i18n-title="master-page.dashboard" title="Dashboard"></fa-icon></a>
       </li>
-      <li class="nav-item" routerLinkActive="active" id="btn-pools">
+      <li class="nav-item" routerLinkActive="active" id="btn-pools" *ngIf="stateService.env.MINING_DASHBOARD">
         <a class="nav-link" [routerLink]="['/mining/pools' | relativeUrl]" (click)="collapse()"><fa-icon [icon]="['fas', 'hammer']" [fixedWidth]="true" i18n-title="master-page.mining-pools" title="Mining Pools"></fa-icon></a>
       </li>
+      <li class="nav-item" routerLinkActive="active" id="btn-blocks" *ngIf="!stateService.env.MINING_DASHBOARD">
+        <a class="nav-link" [routerLink]="['/blocks' | relativeUrl]" (click)="collapse()"><fa-icon [icon]="['fas', 'cubes']" [fixedWidth]="true" i18n-title="master-page.blocks" title="Blocks"></fa-icon></a>
+      </li>
       <li class="nav-item" routerLinkActive="active" id="btn-graphs">
         <a class="nav-link" [routerLink]="['/graphs' | relativeUrl]" (click)="collapse()"><fa-icon [icon]="['fas', 'chart-area']" [fixedWidth]="true" i18n-title="master-page.graphs" title="Graphs"></fa-icon></a>
       </li>

frontend/src/app/components/master-page/master-page.component.ts

@@ -18,7 +18,7 @@ export class MasterPageComponent implements OnInit {
   urlLanguage: string;
 
   constructor(
-    private stateService: StateService,
+    public stateService: StateService,
     private languageService: LanguageService,
   ) { }
 

frontend/src/app/services/state.service.ts

@@ -36,6 +36,7 @@ export interface Env {
   MEMPOOL_WEBSITE_URL: string;
   LIQUID_WEBSITE_URL: string;
   BISQ_WEBSITE_URL: string;
+  MINING_DASHBOARD: boolean;
 }
 
 const defaultEnv: Env = {
@@ -59,6 +60,7 @@ const defaultEnv: Env = {
   'MEMPOOL_WEBSITE_URL': 'https://mempool.space',
   'LIQUID_WEBSITE_URL': 'https://liquid.network',
   'BISQ_WEBSITE_URL': 'https://bisq.markets',
+  'MINING_DASHBOARD': true
 };
 
 @Injectable({