Merge branch 'master' into nymkappa/halving-widget

backend/mempool-config.sample.json

@@ -16,6 +16,7 @@
     "MEMPOOL_BLOCKS_AMOUNT": 8,
     "INDEXING_BLOCKS_AMOUNT": 11000,
     "BLOCKS_SUMMARIES_INDEXING": false,
+    "GOGGLES_INDEXING": false,
     "USE_SECOND_NODE_FOR_MINFEE": false,
     "EXTERNAL_ASSETS": [],
     "EXTERNAL_MAX_RETRY": 1,

backend/src/__fixtures__/mempool-config.template.json

@@ -4,6 +4,7 @@
     "NETWORK": "__MEMPOOL_NETWORK__",
     "BACKEND": "__MEMPOOL_BACKEND__",
     "BLOCKS_SUMMARIES_INDEXING": true,
+    "GOGGLES_INDEXING": false,
     "HTTP_PORT": 1,
     "SPAWN_CLUSTER_PROCS": 2,
     "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",

backend/src/__tests__/config.test.ts

@@ -17,6 +17,7 @@ describe('Mempool Backend Config', () => {
         NETWORK: 'mainnet',
         BACKEND: 'none',
         BLOCKS_SUMMARIES_INDEXING: false,
+        GOGGLES_INDEXING: false,
         HTTP_PORT: 8999,
         SPAWN_CLUSTER_PROCS: 0,
         API_URL_PREFIX: '/api/v1/',

backend/src/api/blocks.ts

@@ -566,7 +566,7 @@ class Blocks {
    */
   public async $classifyBlocks(): Promise<void> {
     // classification requires an esplora backend
-    if (!Common.blocksSummariesIndexingEnabled() || config.MEMPOOL.BACKEND !== 'esplora') {
+    if (!Common.gogglesIndexingEnabled() || config.MEMPOOL.BACKEND !== 'esplora') {
       return;
     }
 
@@ -617,6 +617,7 @@ class Blocks {
           // classify
           const { transactions: classifiedTxs } = this.summarizeBlockTransactions(blockHash, cpfpSummary.transactions);
           await BlocksSummariesRepository.$saveTransactions(height, blockHash, classifiedTxs, 1);
+          await Common.sleep$(250);
         }
         if (unclassifiedTemplates[height]) {
           // classify template
@@ -656,6 +657,7 @@ class Blocks {
             });
           }
           await BlocksSummariesRepository.$saveTemplate({ height, template: { id: blockHash, transactions: classifiedTemplate }, version: 1 });
+          await Common.sleep$(250);
         }
       } catch (e) {
         logger.warn(`Failed to classify template or block summary at ${height}`, logger.tags.goggles);

backend/src/api/common.ts

@@ -508,6 +508,13 @@ export class Common {
     );
   }
 
+  static gogglesIndexingEnabled(): boolean {
+    return (
+      Common.blocksSummariesIndexingEnabled() &&
+      config.MEMPOOL.GOGGLES_INDEXING === true
+    );
+  }
+
   static cpfpIndexingEnabled(): boolean {
     return (
       Common.indexingEnabled() &&

backend/src/config.ts

@@ -20,6 +20,7 @@ interface IConfig {
     MEMPOOL_BLOCKS_AMOUNT: number;
     INDEXING_BLOCKS_AMOUNT: number;
     BLOCKS_SUMMARIES_INDEXING: boolean;
+    GOGGLES_INDEXING: boolean;
     USE_SECOND_NODE_FOR_MINFEE: boolean;
     EXTERNAL_ASSETS: string[];
     EXTERNAL_MAX_RETRY: number;
@@ -175,6 +176,7 @@ const defaults: IConfig = {
     'MEMPOOL_BLOCKS_AMOUNT': 8,
     'INDEXING_BLOCKS_AMOUNT': 11000, // 0 = disable indexing, -1 = index all blocks
     'BLOCKS_SUMMARIES_INDEXING': false,
+    'GOGGLES_INDEXING': false,
     'USE_SECOND_NODE_FOR_MINFEE': false,
     'EXTERNAL_ASSETS': [],
     'EXTERNAL_MAX_RETRY': 1,

docker/backend/mempool-config.json

@@ -22,6 +22,7 @@
     "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
     "INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__,
     "BLOCKS_SUMMARIES_INDEXING": __MEMPOOL_BLOCKS_SUMMARIES_INDEXING__,
+    "GOGGLES_INDEXING": __MEMPOOL_GOGGLES_INDEXING__,
     "AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__,
     "AUDIT": __MEMPOOL_AUDIT__,
     "ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__,

docker/backend/start.sh

@@ -17,6 +17,7 @@ __MEMPOOL_INITIAL_BLOCKS_AMOUNT__=${MEMPOOL_INITIAL_BLOCKS_AMOUNT:=8}
 __MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__=${MEMPOOL_MEMPOOL_BLOCKS_AMOUNT:=8}
 __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=11000}
 __MEMPOOL_BLOCKS_SUMMARIES_INDEXING__=${MEMPOOL_BLOCKS_SUMMARIES_INDEXING:=false}
+__MEMPOOL_GOGGLES_INDEXING__=${MEMPOOL_GOGGLES_INDEXING:=false}
 __MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__=${MEMPOOL_USE_SECOND_NODE_FOR_MINFEE:=false}
 __MEMPOOL_EXTERNAL_ASSETS__=${MEMPOOL_EXTERNAL_ASSETS:=[]}
 __MEMPOOL_EXTERNAL_MAX_RETRY__=${MEMPOOL_EXTERNAL_MAX_RETRY:=1}
@@ -170,6 +171,7 @@ sed -i "s!__MEMPOOL_INITIAL_BLOCKS_AMOUNT__!${__MEMPOOL_INITIAL_BLOCKS_AMOUNT__}
 sed -i "s!__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__!${__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_INDEXING_BLOCKS_AMOUNT__!${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__!${__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__}!g" mempool-config.json
+sed -i "s!__MEMPOOL_GOGGLES_INDEXING__!${__MEMPOOL_GOGGLES_INDEXING__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__!${__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_EXTERNAL_ASSETS__!${__MEMPOOL_EXTERNAL_ASSETS__}!g" mempool-config.json
 sed -i "s!__MEMPOOL_EXTERNAL_MAX_RETRY__!${__MEMPOOL_EXTERNAL_MAX_RETRY__}!g" mempool-config.json

production/mempool-config.mainnet.json

@@ -11,6 +11,7 @@
     "POLL_RATE_MS": 1000,
     "INDEXING_BLOCKS_AMOUNT": -1,
     "BLOCKS_SUMMARIES_INDEXING": true,
+    "GOGGLES_INDEXING": true,
     "AUDIT": true,
     "CPFP_INDEXING": true,
     "ADVANCED_GBT_AUDIT": true,