Implemented coinstatsindex indexing

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -407,7 +407,10 @@ class BitcoinRoutes {
   private async getBlocksByBulk(req: Request, res: Response) {
     try {
       if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK) === false) { // Liquid, Bisq - Not implemented
-        return res.status(404).send(`Not implemented`);
+        return res.status(404).send(`This API is only available for Bitcoin networks`);
+      }
+      if (!Common.indexingEnabled()) {
+        return res.status(404).send(`Indexing is required for this API`);
       }
 
       const from = parseInt(req.params.from, 10);
@@ -423,7 +426,7 @@ class BitcoinRoutes {
       }
 
       res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
-      res.json(await blocks.$getBlocksByBulk(from, to));
+      res.json(await blocks.$getBlocksBetweenHeight(from, to));
 
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);

backend/src/api/bitcoin/esplora-api.interface.ts

@@ -88,6 +88,7 @@ export namespace IEsploraApi {
     size: number;
     weight: number;
     previousblockhash: string;
+    medianTime?: number;
   }
 
   export interface Address {

backend/src/api/blocks.ts

@@ -165,33 +165,75 @@ class Blocks {
    * @returns BlockExtended
    */
   private async $getBlockExtended(block: IEsploraApi.Block, transactions: TransactionExtended[]): Promise<BlockExtended> {
-    const blockExtended: BlockExtended = Object.assign({ extras: {} }, block);
+    const blk: BlockExtended = Object.assign({ extras: {} }, block);
-    blockExtended.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
+    blk.extras.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
-    blockExtended.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
+    blk.extras.coinbaseTx = transactionUtils.stripCoinbaseTransaction(transactions[0]);
-    blockExtended.extras.coinbaseRaw = blockExtended.extras.coinbaseTx.vin[0].scriptsig;
+    blk.extras.coinbaseRaw = blk.extras.coinbaseTx.vin[0].scriptsig;
-    blockExtended.extras.usd = priceUpdater.latestPrices.USD;
+    blk.extras.usd = priceUpdater.latestPrices.USD;
 
     if (block.height === 0) {
-      blockExtended.extras.medianFee = 0; // 50th percentiles
+      blk.extras.medianFee = 0; // 50th percentiles
-      blockExtended.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
+      blk.extras.feeRange = [0, 0, 0, 0, 0, 0, 0];
-      blockExtended.extras.totalFees = 0;
+      blk.extras.totalFees = 0;
-      blockExtended.extras.avgFee = 0;
+      blk.extras.avgFee = 0;
-      blockExtended.extras.avgFeeRate = 0;
+      blk.extras.avgFeeRate = 0;
+      blk.extras.utxoSetChange = 0;
+      blk.extras.avgTxSize = 0;
+      blk.extras.totalInputs = 0;
+      blk.extras.totalOutputs = 1;
+      blk.extras.totalOutputAmt = 0;
+      blk.extras.segwitTotalTxs = 0;
+      blk.extras.segwitTotalSize = 0;
+      blk.extras.segwitTotalWeight = 0;
     } else {
-      const stats = await bitcoinClient.getBlockStats(block.id, [
+      const stats = await bitcoinClient.getBlockStats(block.id);
-        'feerate_percentiles', 'minfeerate', 'maxfeerate', 'totalfee', 'avgfee', 'avgfeerate'
+      blk.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
-      ]);
+      blk.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
-      blockExtended.extras.medianFee = stats.feerate_percentiles[2]; // 50th percentiles
+      blk.extras.totalFees = stats.totalfee;
-      blockExtended.extras.feeRange = [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat();
+      blk.extras.avgFee = stats.avgfee;
-      blockExtended.extras.totalFees = stats.totalfee;
+      blk.extras.avgFeeRate = stats.avgfeerate;
-      blockExtended.extras.avgFee = stats.avgfee;
+      blk.extras.utxoSetChange = stats.utxo_increase;
-      blockExtended.extras.avgFeeRate = stats.avgfeerate;
+      blk.extras.avgTxSize = Math.round(stats.total_size / stats.txs * 100) * 0.01;
+      blk.extras.totalInputs = stats.ins;
+      blk.extras.totalOutputs = stats.outs;
+      blk.extras.totalOutputAmt = stats.total_out;
+      blk.extras.segwitTotalTxs = stats.swtxs;
+      blk.extras.segwitTotalSize = stats.swtotal_size;
+      blk.extras.segwitTotalWeight = stats.swtotal_weight;
+    }
+
+    blk.extras.feePercentiles = [], // TODO
+    blk.extras.medianFeeAmt = 0; // TODO
+    blk.extras.medianTimestamp = block.medianTime; // TODO
+    blk.extras.blockTime = 0; // TODO
+    blk.extras.orphaned = false; // TODO
+  
+    blk.extras.virtualSize = block.weight / 4.0;
+    if (blk.extras.coinbaseTx.vout.length > 0) {
+      blk.extras.coinbaseAddress = blk.extras.coinbaseTx.vout[0].scriptpubkey_address ?? null;
+      blk.extras.coinbaseSignature = blk.extras.coinbaseTx.vout[0].scriptpubkey_asm ?? null;
+    } else {
+      blk.extras.coinbaseAddress = null;
+      blk.extras.coinbaseSignature = null;
+    }
+
+    const header = await bitcoinClient.getBlockHeader(block.id, false);
+    blk.extras.header = header;
+
+    const coinStatsIndex = indexer.isCoreIndexReady('coinstatsindex');
+    if (coinStatsIndex !== null && coinStatsIndex.best_block_height >= block.height) {
+      const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
+      blk.extras.utxoSetSize = txoutset.txouts,
+      blk.extras.totalInputAmt = Math.round(txoutset.block_info.prevout_spent * 100000000);
+    } else {
+      blk.extras.utxoSetSize = null;
+      blk.extras.totalInputAmt = null;
     }
 
     if (['mainnet', 'testnet', 'signet'].includes(config.MEMPOOL.NETWORK)) {
       let pool: PoolTag;
-      if (blockExtended.extras?.coinbaseTx !== undefined) {
+      if (blk.extras?.coinbaseTx !== undefined) {
-        pool = await this.$findBlockMiner(blockExtended.extras?.coinbaseTx);
+        pool = await this.$findBlockMiner(blk.extras?.coinbaseTx);
       } else {
         if (config.DATABASE.ENABLED === true) {
           pool = await poolsRepository.$getUnknownPool();
@@ -201,10 +243,10 @@ class Blocks {
       }
 
       if (!pool) { // We should never have this situation in practise
-        logger.warn(`Cannot assign pool to block ${blockExtended.height} and 'unknown' pool does not exist. ` +
+        logger.warn(`Cannot assign pool to block ${blk.height} and 'unknown' pool does not exist. ` +
           `Check your "pools" table entries`);
       } else {
-        blockExtended.extras.pool = {
+        blk.extras.pool = {
           id: pool.id,
           name: pool.name,
           slug: pool.slug,
@@ -214,12 +256,12 @@ class Blocks {
       if (config.MEMPOOL.AUDIT) {
         const auditScore = await BlocksAuditsRepository.$getBlockAuditScore(block.id);
         if (auditScore != null) {
-          blockExtended.extras.matchRate = auditScore.matchRate;
+          blk.extras.matchRate = auditScore.matchRate;
         }
       }
     }
 
-    return blockExtended;
+    return blk;
   }
 
   /**
@@ -727,60 +769,28 @@ class Blocks {
     return returnBlocks;
   }
 
-  public async $getBlocksByBulk(start: number, end: number) {
+  /**
-    start = Math.max(1, start);
+   * Used for bulk block data query
+   * 
+   * @param fromHeight 
+   * @param toHeight 
+   */
+  public async $getBlocksBetweenHeight(fromHeight: number, toHeight: number): Promise<any> {
+    if (!Common.indexingEnabled()) {
+      return [];
+    }
 
     const blocks: any[] = [];
-    for (let i = end; i >= start; --i) {
-      const blockHash = await bitcoinApi.$getBlockHash(i);
-      const coreBlock = await bitcoinClient.getBlock(blockHash);
-      const electrsBlock = await bitcoinApi.$getBlock(blockHash);
-      const txs = await this.$getTransactionsExtended(blockHash, i, true);
-      const stats = await bitcoinClient.getBlockStats(blockHash);
-      const header = await bitcoinClient.getBlockHeader(blockHash, false);
-      const txoutset = await bitcoinClient.getTxoutSetinfo('none', i);
 
-      const formatted = {
+    while (fromHeight <= toHeight) {
-        blockhash: coreBlock.id,
+      let block = await blocksRepository.$getBlockByHeight(fromHeight);
-        blockheight: coreBlock.height,
+      if (!block) {
-        prev_blockhash: coreBlock.previousblockhash,
+        block = await this.$indexBlock(fromHeight);
-        timestamp: coreBlock.timestamp,
-        median_timestamp: coreBlock.mediantime,
-        // @ts-ignore
-        blocktime: coreBlock.time,
-        orphaned: null,
-        header: header,
-        version: coreBlock.version,
-        difficulty: coreBlock.difficulty,
-        merkle_root: coreBlock.merkle_root,
-        bits: coreBlock.bits,
-        nonce: coreBlock.nonce,
-        coinbase_scriptsig: txs[0].vin[0].scriptsig,
-        coinbase_address: txs[0].vout[0].scriptpubkey_address,
-        coinbase_signature: txs[0].vout[0].scriptpubkey_asm,
-        size: coreBlock.size,
-        virtual_size: coreBlock.weight / 4.0,
-        weight: coreBlock.weight,
-        utxoset_size: txoutset.txouts,
-        utxoset_change: stats.utxo_increase,
-        total_txs: coreBlock.tx_count,
-        avg_tx_size: Math.round(stats.total_size / stats.txs * 100) * 0.01,
-        total_inputs: stats.ins,
-        total_outputs: stats.outs,
-        total_input_amt: Math.round(txoutset.block_info.prevout_spent * 100000000),
-        total_output_amt: stats.total_out,
-        block_subsidy: txs[0].vout.reduce((acc, curr) => acc + curr.value, 0),
-        total_fee: stats.totalfee,
-        avg_feerate: stats.avgfeerate,
-        feerate_percentiles: [stats.minfeerate, stats.feerate_percentiles, stats.maxfeerate].flat(),
-        avg_fee: stats.avgfee,
-        fee_percentiles: null,
-        segwit_total_txs: stats.swtxs,
-        segwit_total_size: stats.swtotal_size,
-        segwit_total_weight: stats.swtotal_weight,
-      };
-      blocks.push(formatted);
       }
+      blocks.push(block);
+      fromHeight++;
+    }
+
     return blocks;
   }
 

backend/src/api/database-migration.ts

@@ -7,7 +7,7 @@ import cpfpRepository from '../repositories/CpfpRepository';
 import { RowDataPacket } from 'mysql2';
 
 class DatabaseMigration {
-  private static currentVersion = 54;
+  private static currentVersion = 55;
   private queryTimeout = 3600_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -483,6 +483,11 @@ class DatabaseMigration {
       }
       await this.updateToSchemaVersion(54);
     }
+
+    if (databaseSchemaVersion < 55) {
+      await this.$executeQuery(this.getAdditionalBlocksDataQuery());
+      await this.updateToSchemaVersion(55);
+    }
   }
 
   /**
@@ -756,6 +761,28 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getAdditionalBlocksDataQuery(): string {
+    return `ALTER TABLE blocks
+      ADD median_timestamp timestamp NOT NULL,
+      ADD block_time int unsigned NOT NULL,
+      ADD coinbase_address varchar(100) NULL,
+      ADD coinbase_signature varchar(500) NULL,
+      ADD avg_tx_size double unsigned NOT NULL,
+      ADD total_inputs int unsigned NOT NULL,
+      ADD total_outputs int unsigned NOT NULL,
+      ADD total_output_amt bigint unsigned NOT NULL,
+      ADD fee_percentiles longtext NULL,
+      ADD median_fee_amt int unsigned NOT NULL,
+      ADD segwit_total_txs int unsigned NOT NULL,
+      ADD segwit_total_size int unsigned NOT NULL,
+      ADD segwit_total_weight int unsigned NOT NULL,
+      ADD header varchar(160) NOT NULL,
+      ADD utxoset_change int NOT NULL,
+      ADD utxoset_size int unsigned NULL,
+      ADD total_input_amt bigint unsigned NULL
+    `;
+  }
+
   private getCreateDailyStatsTableQuery(): string {
     return `CREATE TABLE IF NOT EXISTS hashrates (
       hashrate_timestamp timestamp NOT NULL,

backend/src/api/mining/mining.ts

@@ -172,7 +172,7 @@ class Mining {
   }
 
   /**
-   * [INDEXING] Generate weekly mining pool hashrate history
+   * Generate weekly mining pool hashrate history
    */
   public async $generatePoolHashrateHistory(): Promise<void> {
     const now = new Date();
@@ -279,7 +279,7 @@ class Mining {
   }
 
   /**
-   * [INDEXING] Generate daily hashrate data
+   * Generate daily hashrate data
    */
   public async $generateNetworkHashrateHistory(): Promise<void> {
     // We only run this once a day around midnight
@@ -459,7 +459,7 @@ class Mining {
   /**
    * Create a link between blocks and the latest price at when they were mined
    */
-  public async $indexBlockPrices() {
+  public async $indexBlockPrices(): Promise<void> {
     if (this.blocksPriceIndexingRunning === true) {
       return;
     }
@@ -520,6 +520,41 @@ class Mining {
     this.blocksPriceIndexingRunning = false;
   }
 
+  /**
+   * Index core coinstatsindex
+   */
+  public async $indexCoinStatsIndex(): Promise<void> {
+    let timer = new Date().getTime() / 1000;
+    let totalIndexed = 0;
+
+    const blockchainInfo = await bitcoinClient.getBlockchainInfo();
+    let currentBlockHeight = blockchainInfo.blocks;
+
+    while (currentBlockHeight > 0) {
+      const indexedBlocks = await BlocksRepository.$getBlocksMissingCoinStatsIndex(
+        currentBlockHeight, currentBlockHeight - 10000);
+        
+      for (const block of indexedBlocks) {
+        const txoutset = await bitcoinClient.getTxoutSetinfo('none', block.height);
+        await BlocksRepository.$updateCoinStatsIndexData(block.hash, txoutset.txouts,
+          Math.round(txoutset.block_info.prevout_spent * 100000000));        
+        ++totalIndexed;
+
+        const elapsedSeconds = Math.max(1, new Date().getTime() / 1000 - timer);
+        if (elapsedSeconds > 5) {
+          logger.info(`Indexing coinstatsindex data for block #${block.height}. Indexed ${totalIndexed} blocks.`, logger.tags.mining);
+          timer = new Date().getTime() / 1000;
+        }
+      }
+
+      currentBlockHeight -= 10000;
+    }
+
+    if (totalIndexed) {
+      logger.info(`Indexing missing coinstatsindex data completed`, logger.tags.mining);
+    }
+  }
+
   private getDateMidnight(date: Date): Date {
     date.setUTCHours(0);
     date.setUTCMinutes(0);

backend/src/api/transaction-utils.ts

@@ -14,6 +14,7 @@ class TransactionUtils {
       vout: tx.vout
         .map((vout) => ({
           scriptpubkey_address: vout.scriptpubkey_address,
+          scriptpubkey_asm: vout.scriptpubkey_asm,
           value: vout.value
         }))
         .filter((vout) => vout.value)

backend/src/index.ts

@@ -36,6 +36,7 @@ import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
 import fundingTxFetcher from './tasks/lightning/sync-tasks/funding-tx-fetcher';
 import forensicsService from './tasks/lightning/forensics.service';
 import priceUpdater from './tasks/price-updater';
+import mining from './api/mining/mining';
 import { AxiosError } from 'axios';
 
 class Server {

backend/src/indexer.ts

@@ -8,18 +8,67 @@ import bitcoinClient from './api/bitcoin/bitcoin-client';
 import priceUpdater from './tasks/price-updater';
 import PricesRepository from './repositories/PricesRepository';
 
+export interface CoreIndex {
+  name: string;
+  synced: boolean;
+  best_block_height: number;
+}
+
 class Indexer {
   runIndexer = true;
   indexerRunning = false;
   tasksRunning: string[] = [];
+  coreIndexes: CoreIndex[] = [];
 
-  public reindex() {
+  /**
+   * Check which core index is available for indexing
+   */
+  public async checkAvailableCoreIndexes(): Promise<void> {
+    const updatedCoreIndexes: CoreIndex[] = [];
+
+    const indexes: any = await bitcoinClient.getIndexInfo();
+    for (const indexName in indexes) {
+      const newState = {
+        name: indexName,
+        synced: indexes[indexName].synced,
+        best_block_height: indexes[indexName].best_block_height,
+      };
+      logger.info(`Core index '${indexName}' is ${indexes[indexName].synced ? 'synced' : 'not synced'}. Best block height is ${indexes[indexName].best_block_height}`);      
+      updatedCoreIndexes.push(newState);
+
+      if (indexName === 'coinstatsindex' && newState.synced === true) {
+        const previousState = this.isCoreIndexReady('coinstatsindex');
+        // if (!previousState || previousState.synced === false) {
+          this.runSingleTask('coinStatsIndex');
+        // }
+      }
+    }
+
+    this.coreIndexes = updatedCoreIndexes;
+  }
+
+  /**
+   * Return the best block height if a core index is available, or 0 if not
+   * 
+   * @param name 
+   * @returns 
+   */
+  public isCoreIndexReady(name: string): CoreIndex | null {
+    for (const index of this.coreIndexes) {
+      if (index.name === name && index.synced === true) {
+        return index;
+      }
+    }
+    return null;
+  }
+
+  public reindex(): void {
     if (Common.indexingEnabled()) {
       this.runIndexer = true;
     }
   }
 
-  public async runSingleTask(task: 'blocksPrices') {
+  public async runSingleTask(task: 'blocksPrices' | 'coinStatsIndex'): Promise<void> {
     if (!Common.indexingEnabled()) {
       return;
     }
@@ -28,20 +77,27 @@ class Indexer {
       this.tasksRunning.push(task);
       const lastestPriceId = await PricesRepository.$getLatestPriceId();
       if (priceUpdater.historyInserted === false || lastestPriceId === null) {
-        logger.debug(`Blocks prices indexer is waiting for the price updater to complete`)
+        logger.debug(`Blocks prices indexer is waiting for the price updater to complete`);
         setTimeout(() => {
-          this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
+          this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
           this.runSingleTask('blocksPrices');
         }, 10000);
       } else {
-        logger.debug(`Blocks prices indexer will run now`)
+        logger.debug(`Blocks prices indexer will run now`);
         await mining.$indexBlockPrices();
-        this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask != task)
+        this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
-      }
       }
     }
 
-  public async $run() {
+    if (task === 'coinStatsIndex' && !this.tasksRunning.includes(task)) {
+      this.tasksRunning.push(task);
+      logger.debug(`Indexing coinStatsIndex now`);
+      await mining.$indexCoinStatsIndex();
+      this.tasksRunning = this.tasksRunning.filter(runningTask => runningTask !== task);
+    }
+  }
+
+  public async $run(): Promise<void> {
     if (!Common.indexingEnabled() || this.runIndexer === false ||
       this.indexerRunning === true || mempool.hasPriority()
     ) {
@@ -57,7 +113,9 @@ class Indexer {
     this.runIndexer = false;
     this.indexerRunning = true;
 
-    logger.debug(`Running mining indexer`);
+    logger.info(`Running mining indexer`);
+
+    await this.checkAvailableCoreIndexes();
 
     try {
       await priceUpdater.$run();
@@ -93,7 +151,7 @@ class Indexer {
     setTimeout(() => this.reindex(), runEvery);
   }
 
-  async $resetHashratesIndexingState() {
+  async $resetHashratesIndexingState(): Promise<void> {
     try {
       await HashratesRepository.$setLatestRun('last_hashrates_indexing', 0);
       await HashratesRepository.$setLatestRun('last_weekly_hashrates_indexing', 0);

backend/src/mempool.interfaces.ts

@@ -64,6 +64,7 @@ interface VinStrippedToScriptsig {
 
 interface VoutStrippedToScriptPubkey {
   scriptpubkey_address: string | undefined;
+  scriptpubkey_asm: string | undefined;
   value: number;
 }
 
@@ -160,6 +161,26 @@ export interface BlockExtension {
   avgFeeRate?: number;
   coinbaseRaw?: string;
   usd?: number | null;
+  medianTimestamp?: number;
+  blockTime?: number;
+  orphaned?: boolean;
+  coinbaseAddress?: string | null;
+  coinbaseSignature?: string | null;
+  virtualSize?: number;
+  avgTxSize?: number;
+  totalInputs?: number;
+  totalOutputs?: number;
+  totalOutputAmt?: number;
+  medianFeeAmt?: number;
+  feePercentiles?: number[],
+  segwitTotalTxs?: number;
+  segwitTotalSize?: number;
+  segwitTotalWeight?: number;
+  header?: string;
+  utxoSetChange?: number;
+  // Requires coinstatsindex, will be set to NULL otherwise
+  utxoSetSize?: number | null;
+  totalInputAmt?: number | null;
 }
 
 export interface BlockExtended extends IEsploraApi.Block {

backend/src/repositories/BlocksRepository.ts

@@ -22,13 +22,23 @@ class BlocksRepository {
         weight,             tx_count,            coinbase_raw,      difficulty,
         pool_id,            fees,                fee_span,          median_fee,
         reward,             version,             bits,              nonce,
-        merkle_root,      previous_block_hash, avg_fee,        avg_fee_rate
+        merkle_root,        previous_block_hash, avg_fee,           avg_fee_rate,
+        median_timestamp,   block_time,          header,            coinbase_address,
+        coinbase_signature, utxoset_size,        utxoset_change,    avg_tx_size,
+        total_inputs,       total_outputs,       total_input_amt,   total_output_amt,
+        fee_percentiles,    segwit_total_txs,    segwit_total_size, segwit_total_weight,
+        median_fee_amt
       ) VALUE (
         ?, ?, FROM_UNIXTIME(?), ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
         ?, ?, ?, ?,
-        ?, ?, ?, ?
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?, ?, ?, ?,
+        ?
       )`;
 
       const params: any[] = [
@@ -52,6 +62,23 @@ class BlocksRepository {
         block.previousblockhash,
         block.extras.avgFee,
         block.extras.avgFeeRate,
+        block.extras.medianTimestamp,
+        block.extras.blockTime,
+        block.extras.header,
+        block.extras.coinbaseAddress,
+        block.extras.coinbaseSignature,
+        block.extras.utxoSetSize,
+        block.extras.utxoSetChange,
+        block.extras.avgTxSize,
+        block.extras.totalInputs,
+        block.extras.totalOutputs,
+        block.extras.totalInputAmt,
+        block.extras.totalOutputAmt,
+        JSON.stringify(block.extras.feePercentiles),
+        block.extras.segwitTotalTxs,
+        block.extras.segwitTotalSize,
+        block.extras.segwitTotalWeight,
+        block.extras.medianFeeAmt,
       ];
 
       await DB.query(query, params);
@@ -65,6 +92,33 @@ class BlocksRepository {
     }
   }
 
+  /**
+   * Save newly indexed data from core coinstatsindex
+   * 
+   * @param utxoSetSize 
+   * @param totalInputAmt 
+   */
+  public async $updateCoinStatsIndexData(blockHash: string, utxoSetSize: number,
+    totalInputAmt: number
+  ) : Promise<void> {
+    try {
+      const query = `
+        UPDATE blocks
+        SET utxoset_size = ?, total_input_amt = ?
+        WHERE hash = ?
+      `;
+      const params: any[] = [
+        utxoSetSize,
+        totalInputAmt,
+        blockHash
+      ];
+      await DB.query(query, params);
+    } catch (e: any) {
+      logger.err('Cannot update indexed block coinstatsindex. Reason: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   /**
    * Get all block height that have not been indexed between [startHeight, endHeight]
    */
@@ -310,32 +364,16 @@ class BlocksRepository {
   public async $getBlockByHeight(height: number): Promise<object | null> {
     try {
       const [rows]: any[] = await DB.query(`SELECT
-        blocks.height,
+        blocks.*,
-        hash,
         hash as id,
         UNIX_TIMESTAMP(blocks.blockTimestamp) as blockTimestamp,
-        size,
-        weight,
-        tx_count,
-        coinbase_raw,
-        difficulty,
         pools.id as pool_id,
         pools.name as pool_name,
         pools.link as pool_link,
         pools.slug as pool_slug,
         pools.addresses as pool_addresses,
         pools.regexes as pool_regexes,
-        fees,
+        previous_block_hash as previousblockhash
-        fee_span,
-        median_fee,
-        reward,
-        version,
-        bits,
-        nonce,
-        merkle_root,
-        previous_block_hash as previousblockhash,
-        avg_fee,
-        avg_fee_rate
         FROM blocks
         JOIN pools ON blocks.pool_id = pools.id
         WHERE blocks.height = ${height}
@@ -694,7 +732,6 @@ class BlocksRepository {
       logger.err('Cannot fetch CPFP unindexed blocks. Reason: ' + (e instanceof Error ? e.message : e));
       throw e;
     }
-    return [];
   }
 
   /**
@@ -741,7 +778,7 @@ class BlocksRepository {
     try {
       let query = `INSERT INTO blocks_prices(height, price_id) VALUES`;
       for (const price of blockPrices) {
-        query += ` (${price.height}, ${price.priceId}),`
+        query += ` (${price.height}, ${price.priceId}),`;
       }
       query = query.slice(0, -1);
       await DB.query(query);
@@ -754,6 +791,24 @@ class BlocksRepository {
       }
     }
   }
+
+  /**
+   * Get all indexed blocsk with missing coinstatsindex data
+   */
+  public async $getBlocksMissingCoinStatsIndex(maxHeight: number, minHeight: number): Promise<any> {
+    try {
+      const [blocks] = await DB.query(`
+        SELECT height, hash
+        FROM blocks
+        WHERE height >= ${minHeight} AND height <= ${maxHeight} AND
+          (utxoset_size IS NULL OR total_input_amt IS NULL)
+      `);
+      return blocks;
+    } catch (e) {
+      logger.err(`Cannot get blocks with missing coinstatsindex. Reason: ` + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
 }
 
 export default new BlocksRepository();

backend/src/rpc-api/commands.ts

@@ -89,5 +89,6 @@ module.exports = {
   walletLock: 'walletlock',
   walletPassphrase: 'walletpassphrase',
   walletPassphraseChange: 'walletpassphrasechange',
-  getTxoutSetinfo: 'gettxoutsetinfo'
+  getTxoutSetinfo: 'gettxoutsetinfo',
-}
+  getIndexInfo: 'getindexinfo',
+};