Merge branch 'master' into regtest-1

backend/src/api/bitcoin/bitcoin-api.interface.ts

@@ -4,6 +4,7 @@ export namespace IBitcoinApi {
     size: number;                    //  (numeric) Current tx count
     bytes: number;                   //  (numeric) Sum of all virtual transaction sizes as defined in BIP 141.
     usage: number;                   //  (numeric) Total memory usage for the mempool
+    total_fee: number;               //  (numeric) Total fees of transactions in the mempool
     maxmempool: number;              //  (numeric) Maximum memory usage for the mempool
     mempoolminfee: number;           //  (numeric) Minimum fee rate in BTC/kB for tx to be accepted.
     minrelaytxfee: number;           //  (numeric) Current minimum relay fee for transactions

backend/src/api/blocks.ts

@@ -10,6 +10,7 @@ import bitcoinClient from './bitcoin/bitcoin-client';
 import { IEsploraApi } from './bitcoin/esplora-api.interface';
 import poolsRepository from '../repositories/PoolsRepository';
 import blocksRepository from '../repositories/BlocksRepository';
+import loadingIndicators from './loading-indicators';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -41,7 +42,12 @@ class Blocks {
    * @param onlyCoinbase - Set to true if you only need the coinbase transaction
    * @returns Promise<TransactionExtended[]>
    */
-  private async $getTransactionsExtended(blockHash: string, blockHeight: number, onlyCoinbase: boolean): Promise<TransactionExtended[]> {
+  private async $getTransactionsExtended(
+    blockHash: string,
+    blockHeight: number,
+    onlyCoinbase: boolean,
+    quiet: boolean = false,
+  ): Promise<TransactionExtended[]> {
     const transactions: TransactionExtended[] = [];
     const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
 
@@ -55,9 +61,9 @@ class Blocks {
         // optimize here by directly fetching txs in the "outdated" mempool
         transactions.push(mempool[txIds[i]]);
         transactionsFound++;
-      } else if (config.MEMPOOL.BACKEND === 'esplora' || memPool.isInSync() || i === 0) {
+      } else if (config.MEMPOOL.BACKEND === 'esplora' || !memPool.hasPriority() || i === 0) {
         // Otherwise we fetch the tx data through backend services (esplora, electrum, core rpc...)
-        if (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length) { // Avoid log spam
+        if (!quiet && (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length)) { // Avoid log spam
           logger.debug(`Indexing tx ${i + 1} of ${txIds.length} in block #${blockHeight}`);
         }
         try {
@@ -83,7 +89,9 @@ class Blocks {
       }
     });
 
-    logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${transactionsFetched} fetched through backend service.`);
+    if (!quiet) {
+      logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${transactionsFetched} fetched through backend service.`);
+    }
 
     return transactions;
   }
@@ -94,13 +102,10 @@ class Blocks {
    * @param transactions
    * @returns BlockExtended
    */
-  private getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): BlockExtended {
-    const blockExtended: BlockExtended = Object.assign({}, block);
-
-    blockExtended.extras = {
-      reward: transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0),
-      coinbaseTx: transactionUtils.stripCoinbaseTransaction(transactions[0]),
-    };
+   private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
+    const blockExtended: BlockExtended = Object.assign({extras: {}}, block);
+    blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
+    blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
 
     const transactionsTmp = [...transactions];
     transactionsTmp.shift();
@@ -111,6 +116,19 @@ class Blocks {
     blockExtended.extras.feeRange = transactionsTmp.length > 0 ?
       Common.getFeesInRange(transactionsTmp, 8) : [0, 0];
 
+    if (Common.indexingEnabled()) {
+      let pool: PoolTag;
+      if (blockExtended.extras?.coinbaseTx !== undefined) {
+        pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
+      } else {
+        pool = await poolsRepository.$getUnknownPool();
+      }
+      blockExtended.extras.pool = {
+        id: pool.id,
+        name: pool.name
+      };
+    }
+
     return blockExtended;
   }
 
@@ -152,20 +170,20 @@ class Blocks {
    * Index all blocks metadata for the mining dashboard
    */
   public async $generateBlockDatabase() {
-    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false || // Bitcoin only
-      config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === 0 || // Indexing must be enabled
-      !memPool.isInSync() || // We sync the mempool first
-      this.blockIndexingStarted === true // Indexing must not already be in progress
+    if (this.blockIndexingStarted === true ||
+      !Common.indexingEnabled() ||
+      memPool.hasPriority()
     ) {
       return;
     }
 
     const blockchainInfo = await bitcoinClient.getBlockchainInfo();
-    if (blockchainInfo.blocks !== blockchainInfo.headers) {
+    if (blockchainInfo.blocks !== blockchainInfo.headers) { // Wait for node to sync
       return;
     }
 
     this.blockIndexingStarted = true;
+    const startedAt = new Date().getTime() / 1000;
 
     try {
       let currentBlockHeight = blockchainInfo.blocks;
@@ -180,6 +198,8 @@ class Blocks {
       logger.info(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`);
 
       const chunkSize = 10000;
+      let totaIndexed = await blocksRepository.$blockCount(null, null);
+      let indexedThisRun = 0;
       while (currentBlockHeight >= lastBlockToIndex) {
         const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1);
 
@@ -198,21 +218,19 @@ class Blocks {
             break;
           }
           try {
-            logger.debug(`Indexing block #${blockHeight}`);
+            ++indexedThisRun;
+            if (++totaIndexed % 100 === 0 || blockHeight === lastBlockToIndex) {
+              const elapsedSeconds = Math.max(1, Math.round((new Date().getTime() / 1000) - startedAt));
+              const blockPerSeconds = Math.max(1, Math.round(indexedThisRun / elapsedSeconds));
+              const progress = Math.round(totaIndexed / indexingBlockAmount * 100);
+              const timeLeft = Math.round((indexingBlockAmount - totaIndexed) / blockPerSeconds);
+              logger.debug(`Indexing block #${blockHeight} | ~${blockPerSeconds} blocks/sec | total: ${totaIndexed}/${indexingBlockAmount} (${progress}%) | elapsed: ${elapsedSeconds} seconds | left: ~${timeLeft} seconds`);
+            }
             const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
             const block = await bitcoinApi.$getBlock(blockHash);
-            const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
-            const blockExtended = this.getBlockExtended(block, transactions);
-
-            let miner: PoolTag;
-            if (blockExtended?.extras?.coinbaseTx) {
-              miner = await this.$findBlockMiner(blockExtended.extras.coinbaseTx);
-            } else {
-              miner = await poolsRepository.$getUnknownPool();
-            }
-
-            const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true);
-            await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner);
+            const transactions = await this.$getTransactionsExtended(blockHash, block.height, true, true);
+            const blockExtended = await this.$getBlockExtended(block, transactions);
+            await blocksRepository.$saveBlockInDatabase(blockExtended);
           } catch (e) {
             logger.err(`Something went wrong while indexing blocks.` + e);
           }
@@ -273,17 +291,10 @@ class Blocks {
       const block = await bitcoinApi.$getBlock(blockHash);
       const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
       const transactions = await this.$getTransactionsExtended(blockHash, block.height, false);
-      const blockExtended: BlockExtended = this.getBlockExtended(block, transactions);
-      const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true);
+      const blockExtended: BlockExtended = await this.$getBlockExtended(block, transactions);
 
-      if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === true) {
-        let miner: PoolTag;
-        if (blockExtended?.extras?.coinbaseTx) {
-          miner = await this.$findBlockMiner(blockExtended.extras.coinbaseTx);
-        } else {
-          miner = await poolsRepository.$getUnknownPool();
-        }
-        await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner);
+      if (Common.indexingEnabled()) {
+        await blocksRepository.$saveBlockInDatabase(blockExtended);
       }
 
       if (block.height % 2016 === 0) {
@@ -300,12 +311,98 @@ class Blocks {
       if (this.newBlockCallbacks.length) {
         this.newBlockCallbacks.forEach((cb) => cb(blockExtended, txIds, transactions));
       }
-      if (memPool.isInSync()) {
+      if (!memPool.hasPriority()) {
         diskCache.$saveCacheToDisk();
       }
     }
   }
 
+  /**
+   * Index a block if it's missing from the database. Returns the block after indexing
+   */
+   public async $indexBlock(height: number): Promise<BlockExtended> {
+    const dbBlock = await blocksRepository.$getBlockByHeight(height);
+    if (dbBlock != null) {
+      return this.prepareBlock(dbBlock);
+    }
+
+    const blockHash = await bitcoinApi.$getBlockHash(height);
+    const block = await bitcoinApi.$getBlock(blockHash);
+    const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
+    const blockExtended = await this.$getBlockExtended(block, transactions);
+
+    await blocksRepository.$saveBlockInDatabase(blockExtended);
+
+    return blockExtended;
+  }
+
+  public async $getBlocksExtras(fromHeight: number): Promise<BlockExtended[]> {
+    try {
+      loadingIndicators.setProgress('blocks', 0);
+
+      let currentHeight = fromHeight ? fromHeight : this.getCurrentBlockHeight();
+      const returnBlocks: BlockExtended[] = [];
+
+      if (currentHeight < 0) {
+        return returnBlocks;
+      }
+
+      // Check if block height exist in local cache to skip the hash lookup
+      const blockByHeight = this.getBlocks().find((b) => b.height === currentHeight);
+      let startFromHash: string | null = null;
+      if (blockByHeight) {
+        startFromHash = blockByHeight.id;
+      } else {
+        startFromHash = await bitcoinApi.$getBlockHash(currentHeight);
+      }
+
+      let nextHash = startFromHash;
+      for (let i = 0; i < 10 && currentHeight >= 0; i++) {
+        let block = this.getBlocks().find((b) => b.height === currentHeight);
+        if (!block && Common.indexingEnabled()) {
+          block = this.prepareBlock(await this.$indexBlock(currentHeight));
+        } else if (!block) {
+          block = this.prepareBlock(await bitcoinApi.$getBlock(nextHash));
+        }
+        returnBlocks.push(block);
+        nextHash = block.previousblockhash;
+        loadingIndicators.setProgress('blocks', i / 10 * 100);
+        currentHeight--;
+      }
+
+      return returnBlocks;
+    } catch (e) {
+      loadingIndicators.setProgress('blocks', 100);
+      throw e;
+    }
+  }
+
+  private prepareBlock(block: any): BlockExtended {
+    return <BlockExtended>{
+      id: block.id ?? block.hash, // hash for indexed block
+      timestamp: block?.timestamp ?? block?.blockTimestamp, // blockTimestamp for indexed block
+      height: block?.height,
+      version: block?.version,
+      bits: block?.bits,
+      nonce: block?.nonce,
+      difficulty: block?.difficulty,
+      merkle_root: block?.merkle_root,
+      tx_count: block?.tx_count,
+      size: block?.size,
+      weight: block?.weight,
+      previousblockhash: block?.previousblockhash,
+      extras: {
+        medianFee: block?.medianFee,
+        feeRange: block?.feeRange ?? [], // TODO
+        reward: block?.reward,
+        pool: block?.extras?.pool ?? (block?.pool_id ? {
+          id: block?.pool_id,
+          name: block?.pool_name,
+        } : undefined),
+      }
+    };
+  }
+
   public getLastDifficultyAdjustmentTime(): number {
     return this.lastDifficultyAdjustmentTime;
   }

backend/src/api/common.ts

@@ -154,4 +154,27 @@ export class Common {
     });
     return parents;
   }
+
+  static getSqlInterval(interval: string | null): string | null {
+    switch (interval) {
+      case '24h': return '1 DAY';
+      case '3d': return '3 DAY';
+      case '1w': return '1 WEEK';
+      case '1m': return '1 MONTH';
+      case '3m': return '3 MONTH';
+      case '6m': return '6 MONTH';
+      case '1y': return '1 YEAR';
+      case '2y': return '2 YEAR';
+      case '3y': return '3 YEAR';
+      default: return null;
+    }
+  }
+  
+  static indexingEnabled(): boolean {
+    return (
+      ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) &&
+      config.DATABASE.ENABLED === true &&
+      config.MEMPOOL.INDEXING_BLOCKS_AMOUNT != 0
+    );
+  }
 }

backend/src/api/database-migration.ts

@@ -6,7 +6,7 @@ import logger from '../logger';
 const sleep = (ms: number) => new Promise(res => setTimeout(res, ms));
 
 class DatabaseMigration {
-  private static currentVersion = 4;
+  private static currentVersion = 6;
   private queryTimeout = 120000;
   private statisticsAddedIndexed = false;
 
@@ -76,6 +76,7 @@ class DatabaseMigration {
   private async $createMissingTablesAndIndexes(databaseSchemaVersion: number) {
     await this.$setStatisticsAddedIndexedFlag(databaseSchemaVersion);
 
+    const isBitcoin = ['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK);
     const connection = await DB.pool.getConnection();
     try {
       await this.$executeQuery(connection, this.getCreateElementsTableQuery(), await this.$checkIfTableExists('elements_pegs'));
@@ -90,6 +91,31 @@ class DatabaseMigration {
         await this.$executeQuery(connection, 'DROP table IF EXISTS blocks;');
         await this.$executeQuery(connection, this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
       }
+      if (databaseSchemaVersion < 5 && isBitcoin === true) {
+        await this.$executeQuery(connection, 'TRUNCATE blocks;'); // Need to re-index
+        await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `reward` double unsigned NOT NULL DEFAULT "0"');
+      }
+
+      if (databaseSchemaVersion < 6 && isBitcoin === true) {
+        await this.$executeQuery(connection, 'TRUNCATE blocks;');  // Need to re-index
+        // Cleanup original blocks fields type
+        await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `height` integer unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `tx_count` smallint unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `size` integer unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `weight` integer unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `difficulty` double NOT NULL DEFAULT "0"');
+        // We also fix the pools.id type so we need to drop/re-create the foreign key
+        await this.$executeQuery(connection, 'ALTER TABLE blocks DROP FOREIGN KEY IF EXISTS `blocks_ibfk_1`');
+        await this.$executeQuery(connection, 'ALTER TABLE pools MODIFY `id` smallint unsigned AUTO_INCREMENT');
+        await this.$executeQuery(connection, 'ALTER TABLE blocks MODIFY `pool_id` smallint unsigned NULL');
+        await this.$executeQuery(connection, 'ALTER TABLE blocks ADD FOREIGN KEY (`pool_id`) REFERENCES `pools` (`id`)');
+        // Add new block indexing fields
+        await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `version` integer unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `bits` integer unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `nonce` bigint unsigned NOT NULL DEFAULT "0"');
+        await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `merkle_root` varchar(65) NOT NULL DEFAULT ""');
+        await this.$executeQuery(connection, 'ALTER TABLE blocks ADD `previous_block_hash` varchar(65) NULL');
+      }
       connection.release();
     } catch (e) {
       connection.release();

backend/src/api/mempool.ts

@@ -13,8 +13,9 @@ class Mempool {
   private static WEBSOCKET_REFRESH_RATE_MS = 10000;
   private static LAZY_DELETE_AFTER_SECONDS = 30;
   private inSync: boolean = false;
+  private mempoolCacheDelta: number = -1;
   private mempoolCache: { [txId: string]: TransactionExtended } = {};
-  private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0,
+  private mempoolInfo: IBitcoinApi.MempoolInfo = { loaded: false, size: 0, bytes: 0, usage: 0, total_fee: 0,
                                                     maxmempool: 300000000, mempoolminfee: 0.00001000, minrelaytxfee: 0.00001000 };
   private mempoolChangedCallback: ((newMempool: {[txId: string]: TransactionExtended; }, newTransactions: TransactionExtended[],
     deletedTransactions: TransactionExtended[]) => void) | undefined;
@@ -32,6 +33,17 @@ class Mempool {
     setInterval(this.deleteExpiredTransactions.bind(this), 20000);
   }
 
+  /**
+   * Return true if we should leave resources available for mempool tx caching
+   */
+  public hasPriority(): boolean {
+    if (this.inSync) {
+      return false;
+    } else {
+      return this.mempoolCacheDelta == -1 || this.mempoolCacheDelta > 25;
+    }
+  }
+
   public isInSync(): boolean {
     return this.inSync;
   }
@@ -100,6 +112,8 @@ class Mempool {
     const diff = transactions.length - currentMempoolSize;
     const newTransactions: TransactionExtended[] = [];
 
+    this.mempoolCacheDelta = Math.abs(diff);
+
     if (!this.inSync) {
       loadingIndicators.setProgress('mempool', Object.keys(this.mempoolCache).length / transactions.length * 100);
     }
@@ -168,13 +182,14 @@ class Mempool {
     const newTransactionsStripped = newTransactions.map((tx) => Common.stripTransaction(tx));
     this.latestTransactions = newTransactionsStripped.concat(this.latestTransactions).slice(0, 6);
 
-    const syncedThreshold = 0.99; // If we synced 99% of the mempool tx count, consider we're synced
-    if (!this.inSync && Object.keys(this.mempoolCache).length >= transactions.length * syncedThreshold) {
+    if (!this.inSync && transactions.length === Object.keys(this.mempoolCache).length) {
       this.inSync = true;
       logger.notice('The mempool is now in sync!');
       loadingIndicators.setProgress('mempool', 100);
     }
 
+    this.mempoolCacheDelta = Math.abs(transactions.length - Object.keys(this.mempoolCache).length);
+
     if (this.mempoolChangedCallback && (hasChange || deletedTransactions.length)) {
       this.mempoolChangedCallback(this.mempoolCache, newTransactions, deletedTransactions);
     }

backend/src/api/mining.ts

@@ -11,24 +11,10 @@ class Mining {
    * Generate high level overview of the pool ranks and general stats
    */
   public async $getPoolsStats(interval: string | null) : Promise<object> {
-    let sqlInterval: string | null = null;
-    switch (interval) {
-      case '24h': sqlInterval = '1 DAY'; break;
-      case '3d': sqlInterval = '3 DAY'; break;
-      case '1w': sqlInterval = '1 WEEK'; break;
-      case '1m': sqlInterval = '1 MONTH'; break;
-      case '3m': sqlInterval = '3 MONTH'; break;
-      case '6m': sqlInterval = '6 MONTH'; break;
-      case '1y': sqlInterval = '1 YEAR'; break;
-      case '2y': sqlInterval = '2 YEAR'; break;
-      case '3y': sqlInterval = '3 YEAR'; break;
-      default: sqlInterval = null; break;
-    }
-
     const poolsStatistics = {};
 
-    const poolsInfo: PoolInfo[] = await PoolsRepository.$getPoolsInfo(sqlInterval);
-    const emptyBlocks: EmptyBlocks[] = await BlocksRepository.$countEmptyBlocks(sqlInterval);
+    const poolsInfo: PoolInfo[] = await PoolsRepository.$getPoolsInfo(interval);
+    const emptyBlocks: EmptyBlocks[] = await BlocksRepository.$getEmptyBlocks(null, interval);
 
     const poolsStats: PoolStats[] = [];
     let rank = 1;
@@ -55,7 +41,7 @@ class Mining {
     const oldestBlock = new Date(await BlocksRepository.$oldestBlockTimestamp());
     poolsStatistics['oldestIndexedBlockTimestamp'] = oldestBlock.getTime();
 
-    const blockCount: number = await BlocksRepository.$blockCount(sqlInterval);
+    const blockCount: number = await BlocksRepository.$blockCount(null, interval);
     poolsStatistics['blockCount'] = blockCount;
 
     const blockHeightTip = await bitcoinClient.getBlockCount();
@@ -64,6 +50,38 @@ class Mining {
 
     return poolsStatistics;
   }
+
+  /**
+   * Get all mining pool stats for a pool
+   */
+  public async $getPoolStat(interval: string | null, poolId: number): Promise<object> {
+    const pool = await PoolsRepository.$getPool(poolId);
+    if (!pool) {
+      throw new Error(`This mining pool does not exist`);
+    }
+
+    const blockCount: number = await BlocksRepository.$blockCount(poolId, interval);
+    const emptyBlocks: EmptyBlocks[] = await BlocksRepository.$getEmptyBlocks(poolId, interval);
+
+    return {
+      pool: pool,
+      blockCount: blockCount,
+      emptyBlocks: emptyBlocks,
+    };
+  }
+
+  /**
+   * Return the historical difficulty adjustments and oldest indexed block timestamp
+   */
+  public async $getHistoricalDifficulty(interval: string | null): Promise<object> {
+    const difficultyAdjustments = await BlocksRepository.$getBlocksDifficulty(interval);
+    const oldestBlock = new Date(await BlocksRepository.$oldestBlockTimestamp());
+
+    return {
+      adjustments: difficultyAdjustments,
+      oldestIndexedBlockTimestamp: oldestBlock.getTime(),
+    }
+  }
 }
 
 export default new Mining();

backend/src/index.ts

@@ -256,6 +256,11 @@ class Server {
         .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1y', routes.$getStatisticsByTime.bind(routes, '1y'))
         .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2y', routes.$getStatisticsByTime.bind(routes, '2y'))
         .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3y', routes.$getStatisticsByTime.bind(routes, '3y'))
+        ;
+    }
+
+    if (Common.indexingEnabled()) {
+      this.app
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/24h', routes.$getPools.bind(routes, '24h'))
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/3d', routes.$getPools.bind(routes, '3d'))
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/1w', routes.$getPools.bind(routes, '1w'))
@@ -266,7 +271,12 @@ class Server {
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/2y', routes.$getPools.bind(routes, '2y'))
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/3y', routes.$getPools.bind(routes, '3y'))
         .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/all', routes.$getPools.bind(routes, 'all'))
-        ;
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:poolId/blocks', routes.$getPoolBlocks)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:poolId/blocks/:height', routes.$getPoolBlocks)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:poolId', routes.$getPool)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pool/:poolId/:interval', routes.$getPool)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty', routes.$getHistoricalDifficulty)
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/difficulty/:interval', routes.$getHistoricalDifficulty);
     }
 
     if (config.BISQ.ENABLED) {
@@ -290,6 +300,10 @@ class Server {
         ;
     }
 
+    this.app
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-extras', routes.getBlocksExtras)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'blocks-extras/:height', routes.getBlocksExtras);
+
     if (config.MEMPOOL.BACKEND !== 'esplora') {
       this.app
         .get(config.MEMPOOL.API_URL_PREFIX + 'mempool', routes.getMempool)

backend/src/mempool.interfaces.ts

@@ -1,23 +1,23 @@
 import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
 
 export interface PoolTag {
-  id: number | null, // mysql row id
-  name: string,
-  link: string,
-  regexes: string, // JSON array
-  addresses: string, // JSON array
+  id: number; // mysql row id
+  name: string;
+  link: string;
+  regexes: string; // JSON array
+  addresses: string; // JSON array
 }
 
 export interface PoolInfo {
-  poolId: number, // mysql row id
-  name: string,
-  link: string,
-  blockCount: number,
+  poolId: number; // mysql row id
+  name: string;
+  link: string;
+  blockCount: number;
 }
 
 export interface PoolStats extends PoolInfo {
-  rank: number,
-  emptyBlocks: number,
+  rank: number;
+  emptyBlocks: number;
 }
 
 export interface MempoolBlock {
@@ -83,10 +83,14 @@ export interface BlockExtension {
   reward?: number;
   coinbaseTx?: TransactionMinerInfo;
   matchRate?: number;
+  pool?: {
+    id: number;
+    name: string;
+  }
 }
 
 export interface BlockExtended extends IEsploraApi.Block {
-  extras?: BlockExtension;
+  extras: BlockExtension;
 }
 
 export interface TransactionMinerInfo {

backend/src/repositories/BlocksRepository.ts

@@ -1,6 +1,7 @@
 import { BlockExtended, PoolTag } from '../mempool.interfaces';
 import { DB } from '../database';
 import logger from '../logger';
+import { Common } from '../api/common';
 
 export interface EmptyBlocks {
   emptyBlocks: number;
@@ -11,40 +12,46 @@ class BlocksRepository {
   /**
    * Save indexed block data in the database
    */
-  public async $saveBlockInDatabase(
-    block: BlockExtended,
-    blockHash: string,
-    coinbaseHex: string | undefined,
-    poolTag: PoolTag
-  ) {
+  public async $saveBlockInDatabase(block: BlockExtended) {
     const connection = await DB.pool.getConnection();
 
     try {
       const query = `INSERT INTO blocks(
         height,  hash,     blockTimestamp, size,
         weight,  tx_count, coinbase_raw,   difficulty,
-        pool_id, fees,     fee_span,       median_fee
+        pool_id, fees,     fee_span,       median_fee,
+        reward,  version,  bits,           nonce,
+        merkle_root,       previous_block_hash
       ) VALUE (
         ?, ?, FROM_UNIXTIME(?), ?,
         ?, ?, ?, ?,
-        ?, ?, ?, ?
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?,    ?
       )`;
 
       const params: any[] = [
         block.height,
-        blockHash,
+        block.id,
         block.timestamp,
         block.size,
         block.weight,
         block.tx_count,
-        coinbaseHex ? coinbaseHex : '',
+        '',
         block.difficulty,
-        poolTag.id,
+        block.extras.pool?.id, // Should always be set to something
         0,
         '[]',
-        block.extras ? block.extras.medianFee : 0,
+        block.extras.medianFee ?? 0,
+        block.extras.reward ?? 0,
+        block.version,
+        block.bits,
+        block.nonce,
+        block.merkle_root,
+        block.previousblockhash
       ];
 
+      // logger.debug(query);
       await connection.query(query, params);
     } catch (e: any) {
       if (e.errno === 1062) { // ER_DUP_ENTRY
@@ -66,35 +73,45 @@ class BlocksRepository {
     }
 
     const connection = await DB.pool.getConnection();
-    const [rows] : any[] = await connection.query(`
+    const [rows]: any[] = await connection.query(`
       SELECT height
       FROM blocks
-      WHERE height <= ${startHeight} AND height >= ${endHeight}
+      WHERE height <= ? AND height >= ?
       ORDER BY height DESC;
-    `);
+    `, [startHeight, endHeight]);
     connection.release();
 
     const indexedBlockHeights: number[] = [];
     rows.forEach((row: any) => { indexedBlockHeights.push(row.height); });
     const seekedBlocks: number[] = Array.from(Array(startHeight - endHeight + 1).keys(), n => n + endHeight).reverse();
-    const missingBlocksHeights =  seekedBlocks.filter(x => indexedBlockHeights.indexOf(x) === -1);
+    const missingBlocksHeights = seekedBlocks.filter(x => indexedBlockHeights.indexOf(x) === -1);
 
     return missingBlocksHeights;
   }
 
   /**
-   * Count empty blocks for all pools
+   * Get empty blocks for one or all pools
    */
-  public async $countEmptyBlocks(interval: string | null): Promise<EmptyBlocks[]> {
-    const query = `
-      SELECT pool_id as poolId
-      FROM blocks
-      WHERE tx_count = 1` +
-      (interval != null ? ` AND blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()` : ``)
-    ;
+  public async $getEmptyBlocks(poolId: number | null, interval: string | null = null): Promise<EmptyBlocks[]> {
+    interval = Common.getSqlInterval(interval);
 
+    const params: any[] = [];
+    let query = `SELECT height, hash, tx_count, size, pool_id, weight, UNIX_TIMESTAMP(blockTimestamp) as timestamp
+      FROM blocks
+      WHERE tx_count = 1`;
+
+    if (poolId) {
+      query += ` AND pool_id = ?`;
+      params.push(poolId);
+    }
+
+    if (interval) {
+      query += ` AND blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+    }
+
+    // logger.debug(query);
     const connection = await DB.pool.getConnection();
-    const [rows] = await connection.query(query);
+    const [rows] = await connection.query(query, params);
     connection.release();
 
     return <EmptyBlocks[]>rows;
@@ -103,15 +120,30 @@ class BlocksRepository {
   /**
    * Get blocks count for a period
    */
-   public async $blockCount(interval: string | null): Promise<number> {
-    const query = `
-      SELECT count(height) as blockCount
-      FROM blocks` +
-      (interval != null ? ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()` : ``)
-    ;
+  public async $blockCount(poolId: number | null, interval: string | null): Promise<number> {
+    interval = Common.getSqlInterval(interval);
 
+    const params: any[] = [];
+    let query = `SELECT count(height) as blockCount
+      FROM blocks`;
+
+    if (poolId) {
+      query += ` WHERE pool_id = ?`;
+      params.push(poolId);
+    }
+
+    if (interval) {
+      if (poolId) {
+        query += ` AND`;
+      } else {
+        query += ` WHERE`;
+      }
+      query += ` blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+    }
+
+    // logger.debug(query);
     const connection = await DB.pool.getConnection();
-    const [rows] = await connection.query(query);
+    const [rows] = await connection.query(query, params);
     connection.release();
 
     return <number>rows[0].blockCount;
@@ -121,13 +153,15 @@ class BlocksRepository {
    * Get the oldest indexed block
    */
   public async $oldestBlockTimestamp(): Promise<number> {
-    const connection = await DB.pool.getConnection();
-    const [rows]: any[] = await connection.query(`
-      SELECT blockTimestamp
+    const query = `SELECT blockTimestamp
       FROM blocks
       ORDER BY height
-      LIMIT 1;
-    `);
+      LIMIT 1;`;
+
+
+    // logger.debug(query);
+    const connection = await DB.pool.getConnection();
+    const [rows]: any[] = await connection.query(query);
     connection.release();
 
     if (rows.length <= 0) {
@@ -136,6 +170,83 @@ class BlocksRepository {
 
     return <number>rows[0].blockTimestamp;
   }
+
+  /**
+   * Get blocks mined by a specific mining pool
+   */
+  public async $getBlocksByPool(
+    poolId: number,
+    startHeight: number | null = null
+  ): Promise<object[]> {
+    const params: any[] = [];
+    let query = `SELECT height, hash as id, tx_count, size, weight, pool_id, UNIX_TIMESTAMP(blockTimestamp) as timestamp, reward
+      FROM blocks
+      WHERE pool_id = ?`;
+    params.push(poolId);
+
+    if (startHeight) {
+      query += ` AND height < ?`;
+      params.push(startHeight);
+    }
+
+    query += ` ORDER BY height DESC
+      LIMIT 10`;
+
+    // logger.debug(query);
+    const connection = await DB.pool.getConnection();
+    const [rows] = await connection.query(query, params);
+    connection.release();
+
+    for (const block of <object[]>rows) {
+      delete block['blockTimestamp'];
+    }
+
+    return <object[]>rows;
+  }
+
+  /**
+   * Get one block by height
+   */
+   public async $getBlockByHeight(height: number): Promise<object | null> {
+    const connection = await DB.pool.getConnection();
+    const [rows]: any[] = await connection.query(`
+      SELECT *, UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp, pools.id as pool_id, pools.name as pool_name, pools.link as pool_link, pools.addresses as pool_addresses, pools.regexes as pool_regexes
+      FROM blocks
+      JOIN pools ON blocks.pool_id = pools.id
+      WHERE height = ${height};
+    `);
+    connection.release();
+
+    if (rows.length <= 0) {
+      return null;
+    }
+
+    return rows[0];
+  }
+
+  /**
+   * Return blocks difficulty
+   */
+   public async $getBlocksDifficulty(interval: string | null): Promise<object[]> {
+    interval = Common.getSqlInterval(interval);
+
+    const connection = await DB.pool.getConnection();
+
+    let query = `SELECT MIN(UNIX_TIMESTAMP(blockTimestamp)) as timestamp, difficulty, height
+      FROM blocks`;
+
+    if (interval) {
+      query += ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+    }
+
+    query += ` GROUP BY difficulty
+      ORDER BY blockTimestamp DESC`;
+
+    const [rows]: any[] = await connection.query(query);
+    connection.release();
+
+    return rows;
+  }
 }
 
-export default new BlocksRepository();
+export default new BlocksRepository();

backend/src/repositories/PoolsRepository.ts

@@ -1,4 +1,6 @@
+import { Common } from '../api/common';
 import { DB } from '../database';
+import logger from '../logger';
 import { PoolInfo, PoolTag } from '../mempool.interfaces';
 
 class PoolsRepository {
@@ -7,7 +9,7 @@ class PoolsRepository {
    */
   public async $getPools(): Promise<PoolTag[]> {
     const connection = await DB.pool.getConnection();
-    const [rows] = await connection.query('SELECT * FROM pools;');
+    const [rows] = await connection.query('SELECT id, name, addresses, regexes FROM pools;');
     connection.release();
     return <PoolTag[]>rows;
   }
@@ -17,7 +19,7 @@ class PoolsRepository {
    */
   public async $getUnknownPool(): Promise<PoolTag> {
     const connection = await DB.pool.getConnection();
-    const [rows] = await connection.query('SELECT * FROM pools where name = "Unknown"');
+    const [rows] = await connection.query('SELECT id, name FROM pools where name = "Unknown"');
     connection.release();
     return <PoolTag>rows[0];
   }
@@ -25,22 +27,47 @@ class PoolsRepository {
   /**
    * Get basic pool info and block count
    */
-  public async $getPoolsInfo(interval: string | null): Promise<PoolInfo[]> {
-    const query = `
-      SELECT COUNT(height) as blockCount, pool_id as poolId, pools.name as name, pools.link as link
-      FROM blocks
-      JOIN pools on pools.id = pool_id` +
-      (interval != null ? ` WHERE blocks.blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()` : ``) +
-      ` GROUP BY pool_id
-      ORDER BY COUNT(height) DESC
-    `;
+  public async $getPoolsInfo(interval: string | null = null): Promise<PoolInfo[]> {
+    interval = Common.getSqlInterval(interval);
 
+    let query = `SELECT COUNT(height) as blockCount, pool_id as poolId, pools.name as name, pools.link as link
+      FROM blocks
+      JOIN pools on pools.id = pool_id`;
+
+    if (interval) {
+      query += ` WHERE blocks.blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
+    }
+
+    query += ` GROUP BY pool_id
+      ORDER BY COUNT(height) DESC`;
+
+    // logger.debug(query);
     const connection = await DB.pool.getConnection();
     const [rows] = await connection.query(query);
     connection.release();
 
     return <PoolInfo[]>rows;
   }
+
+  /**
+   * Get mining pool statistics for one pool
+   */
+   public async $getPool(poolId: any): Promise<object> {
+    const query = `
+      SELECT *
+      FROM pools
+      WHERE pools.id = ?`;
+
+    // logger.debug(query);
+    const connection = await DB.pool.getConnection();
+    const [rows] = await connection.query(query, [poolId]);
+    connection.release();
+
+    rows[0].regexes = JSON.parse(rows[0].regexes);
+    rows[0].addresses = JSON.parse(rows[0].addresses);
+
+    return rows[0];
+  }
 }
 
 export default new PoolsRepository();

backend/src/routes.ts

@@ -22,6 +22,9 @@ import elementsParser from './api/liquid/elements-parser';
 import icons from './api/liquid/icons';
 import miningStats from './api/mining';
 import axios from 'axios';
+import PoolsRepository from './repositories/PoolsRepository';
+import mining from './api/mining';
+import BlocksRepository from './repositories/BlocksRepository';
 
 class Routes {
   constructor() {}
@@ -533,9 +536,9 @@ class Routes {
     }
   }
 
-  public async $getPools(interval: string, req: Request, res: Response) {
+  public async $getPool(req: Request, res: Response) {
     try {
-      let stats = await miningStats.$getPoolsStats(interval);
+      const stats = await mining.$getPoolStat(req.params.interval ?? null, parseInt(req.params.poolId, 10));
       res.header('Pragma', 'public');
       res.header('Cache-control', 'public');
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
@@ -545,6 +548,45 @@ class Routes {
     }
   }
 
+  public async $getPoolBlocks(req: Request, res: Response) {
+    try {
+      const poolBlocks = await BlocksRepository.$getBlocksByPool(
+        parseInt(req.params.poolId, 10),
+        parseInt(req.params.height, 10) ?? null,
+      );
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(poolBlocks);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  public async $getPools(interval: string, req: Request, res: Response) {
+    try {
+      const stats = await miningStats.$getPoolsStats(interval);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(stats);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  public async $getHistoricalDifficulty(req: Request, res: Response) {
+    try {
+      const stats = await mining.$getHistoricalDifficulty(req.params.interval ?? null);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 300).toUTCString());
+      res.json(stats);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   public async getBlock(req: Request, res: Response) {
     try {
       const result = await bitcoinApi.$getBlock(req.params.hash);
@@ -564,6 +606,14 @@ class Routes {
     }
   }
 
+  public async getBlocksExtras(req: Request, res: Response) {
+    try {
+      res.json(await blocks.$getBlocksExtras(parseInt(req.params.height, 10)))
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+  
   public async getBlocks(req: Request, res: Response) {
     try {
       loadingIndicators.setProgress('blocks', 0);
@@ -691,7 +741,13 @@ class Routes {
   }
 
   public async getMempool(req: Request, res: Response) {
-    res.status(501).send('Not implemented');
+    const info = mempool.getMempoolInfo();
+    res.json({
+      count: info.size,
+      vsize: info.bytes,
+      total_fee: info.total_fee * 1e8,
+      fee_histogram: []
+    });
   }
 
   public async getMempoolTxIds(req: Request, res: Response) {