Merge branch 'master' into log-priority

fix conflicts in backend/src/config.ts and backend/mempool-config.sample.json

backend/src/api/bitcoin/bitcoin-api.ts

@@ -107,14 +107,25 @@ class BitcoinApi implements AbstractBitcoinApi {
     const outSpends: IEsploraApi.Outspend[] = [];
     const tx = await this.$getRawTransaction(txId, true, false);
     for (let i = 0; i < tx.vout.length; i++) {
-      const txOut = await this.bitcoindClient.getTxOut(txId, i);
-      outSpends.push({
-        spent: txOut === null,
-      });
+      if (tx.status && tx.status.block_height == 0) {
+        outSpends.push({
+          spent: false
+        });
+      } else {
+        const txOut = await this.bitcoindClient.getTxOut(txId, i);
+        outSpends.push({
+          spent: txOut === null,
+        });
+      }
     }
     return outSpends;
   }
 
+  $getEstimatedHashrate(blockHeight: number): Promise<number> {
+    // 120 is the default block span in Core
+    return this.bitcoindClient.getNetworkHashPs(120, blockHeight);
+  }
+
   protected async $convertTransaction(transaction: IBitcoinApi.Transaction, addPrevout: boolean): Promise<IEsploraApi.Transaction> {
     let esploraTransaction: IEsploraApi.Transaction = {
       txid: transaction.txid,

backend/src/api/blocks.ts

@@ -2,11 +2,14 @@ import config from '../config';
 import bitcoinApi from './bitcoin/bitcoin-api-factory';
 import logger from '../logger';
 import memPool from './mempool';
-import { BlockExtended, TransactionExtended } from '../mempool.interfaces';
+import { BlockExtended, PoolTag, TransactionExtended, TransactionMinerInfo } from '../mempool.interfaces';
 import { Common } from './common';
 import diskCache from './disk-cache';
 import transactionUtils from './transaction-utils';
 import bitcoinClient from './bitcoin/bitcoin-client';
+import { IEsploraApi } from './bitcoin/esplora-api.interface';
+import poolsRepository from '../repositories/PoolsRepository';
+import blocksRepository from '../repositories/BlocksRepository';
 
 class Blocks {
   private blocks: BlockExtended[] = [];
@@ -15,6 +18,7 @@ class Blocks {
   private lastDifficultyAdjustmentTime = 0;
   private previousDifficultyRetarget = 0;
   private newBlockCallbacks: ((block: BlockExtended, txIds: string[], transactions: TransactionExtended[]) => void)[] = [];
+  private blockIndexingStarted = false;
 
   constructor() { }
 
@@ -30,6 +34,186 @@ class Blocks {
     this.newBlockCallbacks.push(fn);
   }
 
+  /**
+   * Return the list of transaction for a block
+   * @param blockHash
+   * @param blockHeight
+   * @param onlyCoinbase - Set to true if you only need the coinbase transaction
+   * @returns Promise<TransactionExtended[]>
+   */
+  private async $getTransactionsExtended(blockHash: string, blockHeight: number, onlyCoinbase: boolean): Promise<TransactionExtended[]> {
+    const transactions: TransactionExtended[] = [];
+    const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
+
+    const mempool = memPool.getMempool();
+    let transactionsFound = 0;
+    let transactionsFetched = 0;
+
+    for (let i = 0; i < txIds.length; i++) {
+      if (mempool[txIds[i]]) {
+        // We update blocks before the mempool (index.ts), therefore we can
+        // optimize here by directly fetching txs in the "outdated" mempool
+        transactions.push(mempool[txIds[i]]);
+        transactionsFound++;
+      } else if (config.MEMPOOL.BACKEND === 'esplora' || memPool.isInSync() || i === 0) {
+        // Otherwise we fetch the tx data through backend services (esplora, electrum, core rpc...)
+        if (i % (Math.round((txIds.length) / 10)) === 0 || i + 1 === txIds.length) { // Avoid log spam
+          logger.debug(`Indexing tx ${i + 1} of ${txIds.length} in block #${blockHeight}`);
+        }
+        try {
+          const tx = await transactionUtils.$getTransactionExtended(txIds[i]);
+          transactions.push(tx);
+          transactionsFetched++;
+        } catch (e) {
+          logger.debug('Error fetching block tx: ' + (e instanceof Error ? e.message : e));
+          if (i === 0) {
+            throw new Error('Failed to fetch Coinbase transaction: ' + txIds[i]);
+          }
+        }
+      }
+
+      if (onlyCoinbase === true) {
+        break; // Fetch the first transaction and exit
+      }
+    }
+
+    transactions.forEach((tx) => {
+      if (!tx.cpfpChecked) {
+        Common.setRelativesAndGetCpfpInfo(tx, mempool); // Child Pay For Parent
+      }
+    });
+
+    logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${transactionsFetched} fetched through backend service.`);
+
+    return transactions;
+  }
+
+  /**
+   * Return a block with additional data (reward, coinbase, fees...)
+   * @param block
+   * @param transactions
+   * @returns BlockExtended
+   */
+  private getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): BlockExtended {
+    const blockExtended: BlockExtended = Object.assign({}, block);
+    blockExtended.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
+    blockExtended.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
+
+    const transactionsTmp = [...transactions];
+    transactionsTmp.shift();
+    transactionsTmp.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize);
+    blockExtended.medianFee = transactionsTmp.length > 0 ? Common.median(transactionsTmp.map((tx) => tx.effectiveFeePerVsize)) : 0;
+    blockExtended.feeRange = transactionsTmp.length > 0 ? Common.getFeesInRange(transactionsTmp, 8) : [0, 0];
+
+    return blockExtended;
+  }
+
+  /**
+   * Try to find which miner found the block
+   * @param txMinerInfo
+   * @returns
+   */
+  private async $findBlockMiner(txMinerInfo: TransactionMinerInfo | undefined): Promise<PoolTag> {
+    if (txMinerInfo === undefined || txMinerInfo.vout.length < 1) {
+      return await poolsRepository.$getUnknownPool();
+    }
+
+    const asciiScriptSig = transactionUtils.hex2ascii(txMinerInfo.vin[0].scriptsig);
+    const address = txMinerInfo.vout[0].scriptpubkey_address;
+
+    const pools: PoolTag[] = await poolsRepository.$getPools();
+    for (let i = 0; i < pools.length; ++i) {
+      if (address !== undefined) {
+        const addresses: string[] = JSON.parse(pools[i].addresses);
+        if (addresses.indexOf(address) !== -1) {
+          return pools[i];
+        }
+      }
+
+      const regexes: string[] = JSON.parse(pools[i].regexes);
+      for (let y = 0; y < regexes.length; ++y) {
+        const match = asciiScriptSig.match(regexes[y]);
+        if (match !== null) {
+          return pools[i];
+        }
+      }
+    }
+
+    return await poolsRepository.$getUnknownPool();
+  }
+
+  /**
+   * Index all blocks metadata for the mining dashboard
+   */
+  public async $generateBlockDatabase() {
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false || // Bitcoin only
+      config.MEMPOOL.INDEXING_BLOCKS_AMOUNT === 0 || // Indexing must be enabled
+      !memPool.isInSync() || // We sync the mempool first
+      this.blockIndexingStarted === true // Indexing must not already be in progress
+    ) {
+      return;
+    }
+
+    const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+    if (blockchainInfo.blocks !== blockchainInfo.headers) {
+      return;
+    }
+
+    this.blockIndexingStarted = true;
+
+    try {
+      let currentBlockHeight = blockchainInfo.blocks;
+
+      let indexingBlockAmount = config.MEMPOOL.INDEXING_BLOCKS_AMOUNT;
+      if (indexingBlockAmount <= -1) {
+        indexingBlockAmount = currentBlockHeight + 1;
+      }
+
+      const lastBlockToIndex = Math.max(0, currentBlockHeight - indexingBlockAmount + 1);
+
+      logger.info(`Indexing blocks from #${currentBlockHeight} to #${lastBlockToIndex}`);
+
+      const chunkSize = 10000;
+      while (currentBlockHeight >= lastBlockToIndex) {
+        const endBlock = Math.max(0, lastBlockToIndex, currentBlockHeight - chunkSize + 1);
+
+        const missingBlockHeights: number[] = await blocksRepository.$getMissingBlocksBetweenHeights(
+          currentBlockHeight, endBlock);
+        if (missingBlockHeights.length <= 0) {
+          logger.debug(`No missing blocks between #${currentBlockHeight} to #${endBlock}`);
+          currentBlockHeight -= chunkSize;
+          continue;
+        }
+
+        logger.debug(`Indexing ${missingBlockHeights.length} blocks from #${currentBlockHeight} to #${endBlock}`);
+
+        for (const blockHeight of missingBlockHeights) {
+          if (blockHeight < lastBlockToIndex) {
+            break;
+          }
+          try {
+            logger.debug(`Indexing block #${blockHeight}`);
+            const blockHash = await bitcoinApi.$getBlockHash(blockHeight);
+            const block = await bitcoinApi.$getBlock(blockHash);
+            const transactions = await this.$getTransactionsExtended(blockHash, block.height, true);
+            const blockExtended = this.getBlockExtended(block, transactions);
+            const miner = await this.$findBlockMiner(blockExtended.coinbaseTx);
+            const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true);
+            await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner);
+          } catch (e) {
+            logger.err(`Something went wrong while indexing blocks.` + e);
+          }
+        }
+
+        currentBlockHeight -= chunkSize;
+      }
+      logger.info('Block indexing completed');
+    } catch (e) {
+      logger.err('An error occured in $generateBlockDatabase(). Skipping block indexing. ' + e);
+      console.log(e);
+    }
+  }
+
   public async $updateBlocks() {
     const blockHeightTip = await bitcoinApi.$getBlockHeightTip();
 
@@ -70,49 +254,18 @@ class Blocks {
         logger.debug(`New block found (#${this.currentBlockHeight})!`);
       }
 
-      const transactions: TransactionExtended[] = [];
-
       const blockHash = await bitcoinApi.$getBlockHash(this.currentBlockHeight);
       const block = await bitcoinApi.$getBlock(blockHash);
       const txIds: string[] = await bitcoinApi.$getTxIdsForBlock(blockHash);
+      const transactions = await this.$getTransactionsExtended(blockHash, block.height, false);
+      const blockExtended: BlockExtended = this.getBlockExtended(block, transactions);
+      const coinbase: IEsploraApi.Transaction = await bitcoinApi.$getRawTransaction(transactions[0].txid, true);
 
-      const mempool = memPool.getMempool();
-      let transactionsFound = 0;
-
-      for (let i = 0; i < txIds.length; i++) {
-        if (mempool[txIds[i]]) {
-          transactions.push(mempool[txIds[i]]);
-          transactionsFound++;
-        } else if (config.MEMPOOL.BACKEND === 'esplora' || memPool.isInSync() || i === 0) {
-          logger.debug(`Fetching block tx ${i} of ${txIds.length}`);
-          try {
-            const tx = await transactionUtils.$getTransactionExtended(txIds[i]);
-            transactions.push(tx);
-          } catch (e) {
-            logger.debug('Error fetching block tx: ' + (e instanceof Error ? e.message : e));
-            if (i === 0) {
-              throw new Error('Failed to fetch Coinbase transaction: ' + txIds[i]);
-            }
-          }
-        }
+      if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === true) {
+        const miner = await this.$findBlockMiner(blockExtended.coinbaseTx);
+        await blocksRepository.$saveBlockInDatabase(blockExtended, blockHash, coinbase.hex, miner);
       }
 
-      transactions.forEach((tx) => {
-        if (!tx.cpfpChecked) {
-          Common.setRelativesAndGetCpfpInfo(tx, mempool);
-        }
-      });
-
-      logger.debug(`${transactionsFound} of ${txIds.length} found in mempool. ${txIds.length - transactionsFound} not found.`);
-
-      const blockExtended: BlockExtended = Object.assign({}, block);
-      blockExtended.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
-      blockExtended.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
-      transactions.shift();
-      transactions.sort((a, b) => b.effectiveFeePerVsize - a.effectiveFeePerVsize);
-      blockExtended.medianFee = transactions.length > 0 ? Common.median(transactions.map((tx) => tx.effectiveFeePerVsize)) : 0;
-      blockExtended.feeRange = transactions.length > 0 ? Common.getFeesInRange(transactions, 8) : [0, 0];
-
       if (block.height % 2016 === 0) {
         this.previousDifficultyRetarget = (block.difficulty - this.currentDifficulty) / this.currentDifficulty * 100;
         this.lastDifficultyAdjustmentTime = block.timestamp;
@@ -130,6 +283,8 @@ class Blocks {
       if (memPool.isInSync()) {
         diskCache.$saveCacheToDisk();
       }
+
+      return;
     }
   }
 

backend/src/api/database-migration.ts

@@ -3,10 +3,10 @@ import config from '../config';
 import { DB } from '../database';
 import logger from '../logger';
 
-const sleep = (ms: number) => new Promise( res => setTimeout(res, ms));
+const sleep = (ms: number) => new Promise(res => setTimeout(res, ms));
 
 class DatabaseMigration {
-  private static currentVersion = 2;
+  private static currentVersion = 4;
   private queryTimeout = 120000;
   private statisticsAddedIndexed = false;
 
@@ -83,6 +83,13 @@ class DatabaseMigration {
       if (databaseSchemaVersion < 2 && this.statisticsAddedIndexed === false) {
         await this.$executeQuery(connection, `CREATE INDEX added ON statistics (added);`);
       }
+      if (databaseSchemaVersion < 3) {
+        await this.$executeQuery(connection, this.getCreatePoolsTableQuery(), await this.$checkIfTableExists('pools'));
+      }
+      if (databaseSchemaVersion < 4) {
+        await this.$executeQuery(connection, 'DROP table IF EXISTS blocks;');
+        await this.$executeQuery(connection, this.getCreateBlocksTableQuery(), await this.$checkIfTableExists('blocks'));
+      }
       connection.release();
     } catch (e) {
       connection.release();
@@ -197,7 +204,6 @@ class DatabaseMigration {
     const connection = await DB.pool.getConnection();
     try {
       await this.$executeQuery(connection, 'START TRANSACTION;');
-      await this.$executeQuery(connection, 'SET autocommit = 0;');
       for (const query of transactionQueries) {
         await this.$executeQuery(connection, query);
       }
@@ -335,6 +341,37 @@ class DatabaseMigration {
       final_tx int(11) NOT NULL
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
+
+  private getCreatePoolsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS pools (
+      id int(11) NOT NULL AUTO_INCREMENT,
+      name varchar(50) NOT NULL,
+      link varchar(255) NOT NULL,
+      addresses text NOT NULL,
+      regexes text NOT NULL,
+      PRIMARY KEY (id)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;`;
+  }
+
+  private getCreateBlocksTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS blocks (
+      height int(11) unsigned NOT NULL,
+      hash varchar(65) NOT NULL,
+      blockTimestamp timestamp NOT NULL,
+      size int(11) unsigned NOT NULL,
+      weight int(11) unsigned NOT NULL,
+      tx_count int(11) unsigned NOT NULL,
+      coinbase_raw text,
+      difficulty bigint(20) unsigned NOT NULL,
+      pool_id int(11) DEFAULT -1,
+      fees double unsigned NOT NULL,
+      fee_span json NOT NULL,
+      median_fee double unsigned NOT NULL,
+      PRIMARY KEY (height),
+      INDEX (pool_id),
+      FOREIGN KEY (pool_id) REFERENCES pools (id)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
 }
 
 export default new DatabaseMigration();

backend/src/api/mining.ts

@@ -0,0 +1,69 @@
+import { PoolInfo, PoolStats } from '../mempool.interfaces';
+import BlocksRepository, { EmptyBlocks } from '../repositories/BlocksRepository';
+import PoolsRepository from '../repositories/PoolsRepository';
+import bitcoinClient from './bitcoin/bitcoin-client';
+
+class Mining {
+  constructor() {
+  }
+
+  /**
+   * Generate high level overview of the pool ranks and general stats
+   */
+  public async $getPoolsStats(interval: string | null) : Promise<object> {
+    let sqlInterval: string | null = null;
+    switch (interval) {
+      case '24h': sqlInterval = '1 DAY'; break;
+      case '3d': sqlInterval = '3 DAY'; break;
+      case '1w': sqlInterval = '1 WEEK'; break;
+      case '1m': sqlInterval = '1 MONTH'; break;
+      case '3m': sqlInterval = '3 MONTH'; break;
+      case '6m': sqlInterval = '6 MONTH'; break;
+      case '1y': sqlInterval = '1 YEAR'; break;
+      case '2y': sqlInterval = '2 YEAR'; break;
+      case '3y': sqlInterval = '3 YEAR'; break;
+      default: sqlInterval = null; break;
+    }
+
+    const poolsStatistics = {};
+
+    const poolsInfo: PoolInfo[] = await PoolsRepository.$getPoolsInfo(sqlInterval);
+    const emptyBlocks: EmptyBlocks[] = await BlocksRepository.$countEmptyBlocks(sqlInterval);
+
+    const poolsStats: PoolStats[] = [];
+    let rank = 1;
+
+    poolsInfo.forEach((poolInfo: PoolInfo) => {
+      const poolStat: PoolStats = {
+        poolId: poolInfo.poolId, // mysql row id
+        name: poolInfo.name,
+        link: poolInfo.link,
+        blockCount: poolInfo.blockCount,
+        rank: rank++,
+        emptyBlocks: 0, 
+      }
+      for (let i = 0; i < emptyBlocks.length; ++i) {
+        if (emptyBlocks[i].poolId === poolInfo.poolId) {
+          poolStat.emptyBlocks++;
+        }
+      }
+      poolsStats.push(poolStat);
+    });
+
+    poolsStatistics['pools'] = poolsStats;
+
+    const oldestBlock = new Date(await BlocksRepository.$oldestBlockTimestamp());
+    poolsStatistics['oldestIndexedBlockTimestamp'] = oldestBlock.getTime();
+
+    const blockCount: number = await BlocksRepository.$blockCount(sqlInterval);
+    poolsStatistics['blockCount'] = blockCount;
+
+    const blockHeightTip = await bitcoinClient.getBlockCount();
+    const lastBlockHashrate = await bitcoinClient.getNetworkHashPs(120, blockHeightTip);
+    poolsStatistics['lastEstimatedHashrate'] = lastBlockHashrate;
+
+    return poolsStatistics;
+  }
+}
+
+export default new Mining();

backend/src/api/pools-parser.ts

@@ -0,0 +1,173 @@
+import { readFileSync } from 'fs';
+import { DB } from '../database';
+import logger from '../logger';
+import config from '../config';
+
+interface Pool {
+  name: string;
+  link: string;
+  regexes: string[];
+  addresses: string[];
+}
+
+class PoolsParser {
+  /**
+   * Parse the pools.json file, consolidate the data and dump it into the database
+   */
+  public async migratePoolsJson() {
+    if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) {
+      return;
+    }
+
+    logger.debug('Importing pools.json to the database, open ./pools.json');
+
+    let poolsJson: object = {};
+    try {
+      const fileContent: string = readFileSync('./pools.json', 'utf8');
+      poolsJson = JSON.parse(fileContent);
+    } catch (e) {
+      logger.err('Unable to open ./pools.json, does the file exist?');
+      await this.insertUnknownPool();
+      return;
+    }
+
+    // First we save every entries without paying attention to pool duplication
+    const poolsDuplicated: Pool[] = [];
+
+    logger.debug('Parse coinbase_tags');
+    const coinbaseTags = Object.entries(poolsJson['coinbase_tags']);
+    for (let i = 0; i < coinbaseTags.length; ++i) {
+      poolsDuplicated.push({
+        'name': (<Pool>coinbaseTags[i][1]).name,
+        'link': (<Pool>coinbaseTags[i][1]).link,
+        'regexes': [coinbaseTags[i][0]],
+        'addresses': [],
+      });
+    }
+    logger.debug('Parse payout_addresses');
+    const addressesTags = Object.entries(poolsJson['payout_addresses']);
+    for (let i = 0; i < addressesTags.length; ++i) {
+      poolsDuplicated.push({
+        'name': (<Pool>addressesTags[i][1]).name,
+        'link': (<Pool>addressesTags[i][1]).link,
+        'regexes': [],
+        'addresses': [addressesTags[i][0]],
+      });
+    }
+
+    // Then, we find unique mining pool names
+    logger.debug('Identify unique mining pools');
+    const poolNames: string[] = [];
+    for (let i = 0; i < poolsDuplicated.length; ++i) {
+      if (poolNames.indexOf(poolsDuplicated[i].name) === -1) {
+        poolNames.push(poolsDuplicated[i].name);
+      }
+    }
+    logger.debug(`Found ${poolNames.length} unique mining pools`);
+
+    // Get existing pools from the db
+    const connection = await DB.pool.getConnection();
+    let existingPools;
+    try {
+      [existingPools] = await connection.query<any>({ sql: 'SELECT * FROM pools;', timeout: 120000 });
+    } catch (e) {
+      logger.err('Unable to get existing pools from the database, skipping pools.json import');
+      connection.release();
+      return;
+    }
+
+    // Finally, we generate the final consolidated pools data
+    const finalPoolDataAdd: Pool[] = [];
+    const finalPoolDataUpdate: Pool[] = [];
+    for (let i = 0; i < poolNames.length; ++i) {
+      let allAddresses: string[] = [];
+      let allRegexes: string[] = [];
+      const match = poolsDuplicated.filter((pool: Pool) => pool.name === poolNames[i]);
+
+      for (let y = 0; y < match.length; ++y) {
+        allAddresses = allAddresses.concat(match[y].addresses);
+        allRegexes = allRegexes.concat(match[y].regexes);
+      }
+
+      const finalPoolName = poolNames[i].replace(`'`, `''`); // To support single quote in names when doing db queries
+
+      if (existingPools.find((pool) => pool.name === poolNames[i]) !== undefined) {
+        logger.debug(`Update '${finalPoolName}' mining pool`);
+        finalPoolDataUpdate.push({
+          'name': finalPoolName,
+          'link': match[0].link,
+          'regexes': allRegexes,
+          'addresses': allAddresses,
+        });
+      } else {
+        logger.debug(`Add '${finalPoolName}' mining pool`);
+        finalPoolDataAdd.push({
+          'name': finalPoolName,
+          'link': match[0].link,
+          'regexes': allRegexes,
+          'addresses': allAddresses,
+        });
+      }
+    }
+
+    logger.debug(`Update pools table now`);
+
+    // Add new mining pools into the database
+    let queryAdd: string = 'INSERT INTO pools(name, link, regexes, addresses) VALUES ';
+    for (let i = 0; i < finalPoolDataAdd.length; ++i) {
+      queryAdd += `('${finalPoolDataAdd[i].name}', '${finalPoolDataAdd[i].link}',
+      '${JSON.stringify(finalPoolDataAdd[i].regexes)}', '${JSON.stringify(finalPoolDataAdd[i].addresses)}'),`;
+    }
+    queryAdd = queryAdd.slice(0, -1) + ';';
+
+    // Add new mining pools into the database
+    const updateQueries: string[] = [];
+    for (let i = 0; i < finalPoolDataUpdate.length; ++i) {
+      updateQueries.push(`
+        UPDATE pools
+        SET name='${finalPoolDataUpdate[i].name}', link='${finalPoolDataUpdate[i].link}',
+        regexes='${JSON.stringify(finalPoolDataUpdate[i].regexes)}', addresses='${JSON.stringify(finalPoolDataUpdate[i].addresses)}'
+        WHERE name='${finalPoolDataUpdate[i].name}'
+      ;`);
+    }
+
+    try {
+      if (finalPoolDataAdd.length > 0) {
+        await connection.query<any>({ sql: queryAdd, timeout: 120000 });
+      }
+      for (const query of updateQueries) {
+        await connection.query<any>({ sql: query, timeout: 120000 });
+      }
+      await this.insertUnknownPool();
+      connection.release();
+      logger.info('Mining pools.json import completed');
+    } catch (e) {
+      connection.release();
+      logger.err(`Unable to import pools in the database!`);
+      throw e;
+    }
+  }
+
+  /**
+   * Manually add the 'unknown pool'
+   */
+  private async insertUnknownPool() {
+    const connection = await DB.pool.getConnection();
+    try {
+      const [rows]: any[] = await connection.query({ sql: 'SELECT name from pools where name="Unknown"', timeout: 120000 });
+      if (rows.length === 0) {
+        logger.debug('Manually inserting "Unknown" mining pool into the databse');
+        await connection.query({
+          sql: `INSERT INTO pools(name, link, regexes, addresses)
+          VALUES("Unknown", "https://learnmeabitcoin.com/technical/coinbase-transaction", "[]", "[]");
+        `});
+      }
+    } catch (e) {
+      logger.err('Unable to insert "Unknown" mining pool');
+    }
+
+    connection.release();
+  }
+}
+
+export default new PoolsParser();

backend/src/api/statistics.ts

@@ -53,12 +53,16 @@ class Statistics {
     memPoolArray = memPoolArray.filter((tx) => tx.effectiveFeePerVsize);
 
     if (!memPoolArray.length) {
-      const insertIdZeroed = await this.$createZeroedStatistic();
-      if (this.newStatisticsEntryCallback && insertIdZeroed) {
-        const newStats = await this.$get(insertIdZeroed);
-        if (newStats) {
-          this.newStatisticsEntryCallback(newStats);
+      try {
+        const insertIdZeroed = await this.$createZeroedStatistic();
+        if (this.newStatisticsEntryCallback && insertIdZeroed) {
+          const newStats = await this.$get(insertIdZeroed);
+          if (newStats) {
+            this.newStatisticsEntryCallback(newStats);
+          }
         }
+      } catch (e) {
+        logger.err('Unable to insert zeroed statistics. ' + e);
       }
       return;
     }
@@ -90,59 +94,63 @@ class Statistics {
       }
     });
 
-    const insertId = await this.$create({
-      added: 'NOW()',
-      unconfirmed_transactions: memPoolArray.length,
-      tx_per_second: txPerSecond,
-      vbytes_per_second: Math.round(vBytesPerSecond),
-      mempool_byte_weight: totalWeight,
-      total_fee: totalFee,
-      fee_data: '',
-      vsize_1: weightVsizeFees['1'] || 0,
-      vsize_2: weightVsizeFees['2'] || 0,
-      vsize_3: weightVsizeFees['3'] || 0,
-      vsize_4: weightVsizeFees['4'] || 0,
-      vsize_5: weightVsizeFees['5'] || 0,
-      vsize_6: weightVsizeFees['6'] || 0,
-      vsize_8: weightVsizeFees['8'] || 0,
-      vsize_10: weightVsizeFees['10'] || 0,
-      vsize_12: weightVsizeFees['12'] || 0,
-      vsize_15: weightVsizeFees['15'] || 0,
-      vsize_20: weightVsizeFees['20'] || 0,
-      vsize_30: weightVsizeFees['30'] || 0,
-      vsize_40: weightVsizeFees['40'] || 0,
-      vsize_50: weightVsizeFees['50'] || 0,
-      vsize_60: weightVsizeFees['60'] || 0,
-      vsize_70: weightVsizeFees['70'] || 0,
-      vsize_80: weightVsizeFees['80'] || 0,
-      vsize_90: weightVsizeFees['90'] || 0,
-      vsize_100: weightVsizeFees['100'] || 0,
-      vsize_125: weightVsizeFees['125'] || 0,
-      vsize_150: weightVsizeFees['150'] || 0,
-      vsize_175: weightVsizeFees['175'] || 0,
-      vsize_200: weightVsizeFees['200'] || 0,
-      vsize_250: weightVsizeFees['250'] || 0,
-      vsize_300: weightVsizeFees['300'] || 0,
-      vsize_350: weightVsizeFees['350'] || 0,
-      vsize_400: weightVsizeFees['400'] || 0,
-      vsize_500: weightVsizeFees['500'] || 0,
-      vsize_600: weightVsizeFees['600'] || 0,
-      vsize_700: weightVsizeFees['700'] || 0,
-      vsize_800: weightVsizeFees['800'] || 0,
-      vsize_900: weightVsizeFees['900'] || 0,
-      vsize_1000: weightVsizeFees['1000'] || 0,
-      vsize_1200: weightVsizeFees['1200'] || 0,
-      vsize_1400: weightVsizeFees['1400'] || 0,
-      vsize_1600: weightVsizeFees['1600'] || 0,
-      vsize_1800: weightVsizeFees['1800'] || 0,
-      vsize_2000: weightVsizeFees['2000'] || 0,
-    });
+    try {
+      const insertId = await this.$create({
+        added: 'NOW()',
+        unconfirmed_transactions: memPoolArray.length,
+        tx_per_second: txPerSecond,
+        vbytes_per_second: Math.round(vBytesPerSecond),
+        mempool_byte_weight: totalWeight,
+        total_fee: totalFee,
+        fee_data: '',
+        vsize_1: weightVsizeFees['1'] || 0,
+        vsize_2: weightVsizeFees['2'] || 0,
+        vsize_3: weightVsizeFees['3'] || 0,
+        vsize_4: weightVsizeFees['4'] || 0,
+        vsize_5: weightVsizeFees['5'] || 0,
+        vsize_6: weightVsizeFees['6'] || 0,
+        vsize_8: weightVsizeFees['8'] || 0,
+        vsize_10: weightVsizeFees['10'] || 0,
+        vsize_12: weightVsizeFees['12'] || 0,
+        vsize_15: weightVsizeFees['15'] || 0,
+        vsize_20: weightVsizeFees['20'] || 0,
+        vsize_30: weightVsizeFees['30'] || 0,
+        vsize_40: weightVsizeFees['40'] || 0,
+        vsize_50: weightVsizeFees['50'] || 0,
+        vsize_60: weightVsizeFees['60'] || 0,
+        vsize_70: weightVsizeFees['70'] || 0,
+        vsize_80: weightVsizeFees['80'] || 0,
+        vsize_90: weightVsizeFees['90'] || 0,
+        vsize_100: weightVsizeFees['100'] || 0,
+        vsize_125: weightVsizeFees['125'] || 0,
+        vsize_150: weightVsizeFees['150'] || 0,
+        vsize_175: weightVsizeFees['175'] || 0,
+        vsize_200: weightVsizeFees['200'] || 0,
+        vsize_250: weightVsizeFees['250'] || 0,
+        vsize_300: weightVsizeFees['300'] || 0,
+        vsize_350: weightVsizeFees['350'] || 0,
+        vsize_400: weightVsizeFees['400'] || 0,
+        vsize_500: weightVsizeFees['500'] || 0,
+        vsize_600: weightVsizeFees['600'] || 0,
+        vsize_700: weightVsizeFees['700'] || 0,
+        vsize_800: weightVsizeFees['800'] || 0,
+        vsize_900: weightVsizeFees['900'] || 0,
+        vsize_1000: weightVsizeFees['1000'] || 0,
+        vsize_1200: weightVsizeFees['1200'] || 0,
+        vsize_1400: weightVsizeFees['1400'] || 0,
+        vsize_1600: weightVsizeFees['1600'] || 0,
+        vsize_1800: weightVsizeFees['1800'] || 0,
+        vsize_2000: weightVsizeFees['2000'] || 0,
+      });
 
-    if (this.newStatisticsEntryCallback && insertId) {
-      const newStats = await this.$get(insertId);
-      if (newStats) {
-        this.newStatisticsEntryCallback(newStats);
+      if (this.newStatisticsEntryCallback && insertId) {
+        const newStats = await this.$get(insertId);
+        if (newStats) {
+          this.newStatisticsEntryCallback(newStats);
+        }
       }
+    } catch (e) {
+      logger.err('Unable to insert statistics. ' + e);
     }
   }
 

backend/src/api/transaction-utils.ts

@@ -44,6 +44,14 @@ class TransactionUtils {
     }
     return transactionExtended;
   }
+
+  public hex2ascii(hex: string) {
+    let str = '';
+    for (let i = 0; i < hex.length; i += 2) {
+      str += String.fromCharCode(parseInt(hex.substr(i, 2), 16));
+    }
+    return str;
+  }
 }
 
 export default new TransactionUtils();

backend/src/config.ts

@@ -14,6 +14,7 @@ interface IConfig {
     BLOCK_WEIGHT_UNITS: number;
     INITIAL_BLOCKS_AMOUNT: number;
     MEMPOOL_BLOCKS_AMOUNT: number;
+    INDEXING_BLOCKS_AMOUNT: number;
     PRICE_FEED_UPDATE_INTERVAL: number;
     USE_SECOND_NODE_FOR_MINFEE: boolean;
     EXTERNAL_ASSETS: string[];
@@ -78,9 +79,12 @@ const defaults: IConfig = {
     'BLOCK_WEIGHT_UNITS': 4000000,
     'INITIAL_BLOCKS_AMOUNT': 8,
     'MEMPOOL_BLOCKS_AMOUNT': 8,
+    'INDEXING_BLOCKS_AMOUNT': 1100, // 0 = disable indexing, -1 = index all blocks
     'PRICE_FEED_UPDATE_INTERVAL': 3600,
     'USE_SECOND_NODE_FOR_MINFEE': false,
-    'EXTERNAL_ASSETS': [],
+    'EXTERNAL_ASSETS': [
+      'https://mempool.space/resources/pools.json'
+    ],
     'STDOUT_LOG_MIN_PRIORITY': 'debug',
   },
   'ESPLORA': {

backend/src/index.ts

@@ -22,6 +22,7 @@ import loadingIndicators from './api/loading-indicators';
 import mempool from './api/mempool';
 import elementsParser from './api/liquid/elements-parser';
 import databaseMigration from './api/database-migration';
+import poolsParser from './api/pools-parser';
 import syncAssets from './sync-assets';
 import icons from './api/liquid/icons';
 import { Common } from './api/common';
@@ -88,6 +89,7 @@ class Server {
       await checkDbConnection();
       try {
         await databaseMigration.$initializeOrMigrateDatabase();
+        await poolsParser.migratePoolsJson();
       } catch (e) {
         throw new Error(e instanceof Error ? e.message : 'Error');
       }
@@ -136,6 +138,8 @@ class Server {
       }
       await blocks.$updateBlocks();
       await memPool.$updateMempool();
+      blocks.$generateBlockDatabase();
+
       setTimeout(this.runMainUpdateLoop.bind(this), config.MEMPOOL.POLL_RATE_MS);
       this.currentBackendRetryInterval = 5;
     } catch (e) {
@@ -252,6 +256,16 @@ class Server {
         .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/1y', routes.$getStatisticsByTime.bind(routes, '1y'))
         .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/2y', routes.$getStatisticsByTime.bind(routes, '2y'))
         .get(config.MEMPOOL.API_URL_PREFIX + 'statistics/3y', routes.$getStatisticsByTime.bind(routes, '3y'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/24h', routes.$getPools.bind(routes, '24h'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/3d', routes.$getPools.bind(routes, '3d'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/1w', routes.$getPools.bind(routes, '1w'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/1m', routes.$getPools.bind(routes, '1m'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/3m', routes.$getPools.bind(routes, '3m'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/6m', routes.$getPools.bind(routes, '6m'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/1y', routes.$getPools.bind(routes, '1y'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/2y', routes.$getPools.bind(routes, '2y'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/3y', routes.$getPools.bind(routes, '3y'))
+        .get(config.MEMPOOL.API_URL_PREFIX + 'mining/pools/all', routes.$getPools.bind(routes, 'all'))
         ;
     }
 

backend/src/mempool.interfaces.ts

@@ -1,5 +1,25 @@
 import { IEsploraApi } from './api/bitcoin/esplora-api.interface';
 
+export interface PoolTag {
+  id: number | null, // mysql row id
+  name: string,
+  link: string,
+  regexes: string, // JSON array
+  addresses: string, // JSON array
+}
+
+export interface PoolInfo {
+  poolId: number, // mysql row id
+  name: string,
+  link: string,
+  blockCount: number,
+}
+
+export interface PoolStats extends PoolInfo {
+  rank: number,
+  emptyBlocks: number,
+}
+
 export interface MempoolBlock {
   blockSize: number;
   blockVSize: number;

backend/src/repositories/BlocksRepository.ts

@@ -0,0 +1,128 @@
+import { BlockExtended, PoolTag } from '../mempool.interfaces';
+import { DB } from '../database';
+import logger from '../logger';
+
+export interface EmptyBlocks {
+  emptyBlocks: number;
+  poolId: number;
+}
+
+class BlocksRepository {
+  /**
+   * Save indexed block data in the database
+   */
+  public async $saveBlockInDatabase(
+    block: BlockExtended,
+    blockHash: string,
+    coinbaseHex: string | undefined,
+    poolTag: PoolTag
+  ) {
+    const connection = await DB.pool.getConnection();
+
+    try {
+      const query = `INSERT INTO blocks(
+        height,  hash,     blockTimestamp, size,
+        weight,  tx_count, coinbase_raw,   difficulty,
+        pool_id, fees,     fee_span,       median_fee
+      ) VALUE (
+        ?, ?, FROM_UNIXTIME(?), ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?
+      )`;
+
+      const params: any[] = [
+        block.height, blockHash, block.timestamp, block.size,
+        block.weight, block.tx_count, coinbaseHex ? coinbaseHex : '', block.difficulty,
+        poolTag.id, 0, '[]', block.medianFee,
+      ];
+
+      await connection.query(query, params);
+    } catch (e) {
+      logger.err('$saveBlockInDatabase() error' + (e instanceof Error ? e.message : e));
+    }
+
+    connection.release();
+  }
+
+  /**
+   * Get all block height that have not been indexed between [startHeight, endHeight]
+   */
+  public async $getMissingBlocksBetweenHeights(startHeight: number, endHeight: number): Promise<number[]> {
+    if (startHeight < endHeight) {
+      return [];
+    }
+
+    const connection = await DB.pool.getConnection();
+    const [rows] : any[] = await connection.query(`
+      SELECT height
+      FROM blocks
+      WHERE height <= ${startHeight} AND height >= ${endHeight}
+      ORDER BY height DESC;
+    `);
+    connection.release();
+
+    const indexedBlockHeights: number[] = [];
+    rows.forEach((row: any) => { indexedBlockHeights.push(row.height); });
+    const seekedBlocks: number[] = Array.from(Array(startHeight - endHeight + 1).keys(), n => n + endHeight).reverse();
+    const missingBlocksHeights =  seekedBlocks.filter(x => indexedBlockHeights.indexOf(x) === -1);
+
+    return missingBlocksHeights;
+  }
+
+  /**
+   * Count empty blocks for all pools
+   */
+  public async $countEmptyBlocks(interval: string | null): Promise<EmptyBlocks[]> {
+    const query = `
+      SELECT pool_id as poolId
+      FROM blocks
+      WHERE tx_count = 1` +
+      (interval != null ? ` AND blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()` : ``)
+    ;
+
+    const connection = await DB.pool.getConnection();
+    const [rows] = await connection.query(query);
+    connection.release();
+
+    return <EmptyBlocks[]>rows;
+  }
+
+  /**
+   * Get blocks count for a period
+   */
+   public async $blockCount(interval: string | null): Promise<number> {
+    const query = `
+      SELECT count(height) as blockCount
+      FROM blocks` +
+      (interval != null ? ` WHERE blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()` : ``)
+    ;
+
+    const connection = await DB.pool.getConnection();
+    const [rows] = await connection.query(query);
+    connection.release();
+
+    return <number>rows[0].blockCount;
+  }
+
+  /**
+   * Get the oldest indexed block
+   */
+  public async $oldestBlockTimestamp(): Promise<number> {
+    const connection = await DB.pool.getConnection();
+    const [rows]: any[] = await connection.query(`
+      SELECT blockTimestamp
+      FROM blocks
+      ORDER BY height
+      LIMIT 1;
+    `);
+    connection.release();
+
+    if (rows.length <= 0) {
+      return -1;
+    }
+
+    return <number>rows[0].blockTimestamp;
+  }
+}
+
+export default new BlocksRepository();

backend/src/repositories/PoolsRepository.ts

@@ -0,0 +1,46 @@
+import { DB } from '../database';
+import { PoolInfo, PoolTag } from '../mempool.interfaces';
+
+class PoolsRepository {
+  /**
+   * Get all pools tagging info
+   */
+  public async $getPools(): Promise<PoolTag[]> {
+    const connection = await DB.pool.getConnection();
+    const [rows] = await connection.query('SELECT * FROM pools;');
+    connection.release();
+    return <PoolTag[]>rows;
+  }
+
+  /**
+   * Get unknown pool tagging info
+   */
+  public async $getUnknownPool(): Promise<PoolTag> {
+    const connection = await DB.pool.getConnection();
+    const [rows] = await connection.query('SELECT * FROM pools where name = "Unknown"');
+    connection.release();
+    return <PoolTag>rows[0];
+  }
+
+  /**
+   * Get basic pool info and block count
+   */
+  public async $getPoolsInfo(interval: string | null): Promise<PoolInfo[]> {
+    const query = `
+      SELECT COUNT(height) as blockCount, pool_id as poolId, pools.name as name, pools.link as link
+      FROM blocks
+      JOIN pools on pools.id = pool_id` +
+      (interval != null ? ` WHERE blocks.blockTimestamp BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()` : ``) +
+      ` GROUP BY pool_id
+      ORDER BY COUNT(height) DESC
+    `;
+
+    const connection = await DB.pool.getConnection();
+    const [rows] = await connection.query(query);
+    connection.release();
+
+    return <PoolInfo[]>rows;
+  }
+}
+
+export default new PoolsRepository();

backend/src/routes.ts

@@ -20,6 +20,7 @@ import { Common } from './api/common';
 import bitcoinClient from './api/bitcoin/bitcoin-client';
 import elementsParser from './api/liquid/elements-parser';
 import icons from './api/liquid/icons';
+import miningStats from './api/mining';
 
 class Routes {
   constructor() {}
@@ -531,6 +532,18 @@ class Routes {
     }
   }
 
+  public async $getPools(interval: string, req: Request, res: Response) {
+    try {
+      let stats = await miningStats.$getPoolsStats(interval);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
+      res.json(stats);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   public async getBlock(req: Request, res: Response) {
     try {
       const result = await bitcoinApi.$getBlock(req.params.hash);