From 7f903b033112d05e5b04d63d9bfcac661c733bf5 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 21 Dec 2022 09:39:22 -0600 Subject: [PATCH 1/4] Add missing vars to docker config & readme --- .../__fixtures__/mempool-config.template.json | 6 +++--- docker/README.md | 18 +++++++++++++++++- docker/backend/mempool-config.json | 5 ++++- docker/backend/start.sh | 5 +++++ 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 2e9221c7a..7971b65ca 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -26,9 +26,9 @@ "INDEXING_BLOCKS_AMOUNT": 14, "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__", "POOLS_JSON_URL": "__POOLS_JSON_URL__", - "ADVANCED_GBT_AUDIT": "__ADVANCED_GBT_AUDIT__", - "ADVANCED_GBT_MEMPOOL": "__ADVANCED_GBT_MEMPOOL__", - "TRANSACTION_INDEXING": "__TRANSACTION_INDEXING__" + "ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__", + "ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__", + "TRANSACTION_INDEXING": "__MEMPOOL_TRANSACTION_INDEXING__" }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/docker/README.md b/docker/README.md index 4061f420c..b891748d0 100644 --- a/docker/README.md +++ b/docker/README.md @@ -100,12 +100,18 @@ Below we list all settings from `mempool-config.json` and the corresponding over "BLOCK_WEIGHT_UNITS": 4000000, "INITIAL_BLOCKS_AMOUNT": 8, "MEMPOOL_BLOCKS_AMOUNT": 8, + "BLOCKS_SUMMARIES_INDEXING": false, "PRICE_FEED_UPDATE_INTERVAL": 600, "USE_SECOND_NODE_FOR_MINFEE": false, "EXTERNAL_ASSETS": ["https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json"], "STDOUT_LOG_MIN_PRIORITY": "info", + "INDEXING_BLOCKS_AMOUNT": false, + "AUTOMATIC_BLOCK_REINDEXING": false, "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json", - "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master" + "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master", + "ADVANCED_GBT_AUDIT": false, + "ADVANCED_GBT_MEMPOOL": false, + "TRANSACTION_INDEXING": false, }, ``` @@ -125,15 +131,25 @@ Corresponding `docker-compose.yml` overrides: MEMPOOL_BLOCK_WEIGHT_UNITS: "" MEMPOOL_INITIAL_BLOCKS_AMOUNT: "" MEMPOOL_MEMPOOL_BLOCKS_AMOUNT: "" + MEMPOOL_BLOCKS_SUMMARIES_INDEXING: "" MEMPOOL_PRICE_FEED_UPDATE_INTERVAL: "" MEMPOOL_USE_SECOND_NODE_FOR_MINFEE: "" MEMPOOL_EXTERNAL_ASSETS: "" MEMPOOL_STDOUT_LOG_MIN_PRIORITY: "" + MEMPOOL_INDEXING_BLOCKS_AMOUNT: "" + MEMPOOL_AUTOMATIC_BLOCK_REINDEXING: "" MEMPOOL_POOLS_JSON_URL: "" MEMPOOL_POOLS_JSON_TREE_URL: "" + MEMPOOL_ADVANCED_GBT_AUDIT: "" + MEMPOOL_ADVANCED_GBT_MEMPOOL: "" + MEMPOOL_TRANSACTION_INDEXING: "" ... ``` +`ADVANCED_GBT_AUDIT` AND `ADVANCED_GBT_MEMPOOL` enable a more accurate (but slower) block prediction algorithm for the block audit feature and the projected mempool-blocks respectively. + +`TRANSACTION_INDEXING` enables indexing CPFP (Child Pays For Parent) information for the last `INDEXING_BLOCKS_AMOUNT` blocks. +
`mempool-config.json`: diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 378bba8db..8f5eae65e 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -22,7 +22,10 @@ "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__", "INDEXING_BLOCKS_AMOUNT": __MEMPOOL_INDEXING_BLOCKS_AMOUNT__, "BLOCKS_SUMMARIES_INDEXING": __MEMPOOL_BLOCKS_SUMMARIES_INDEXING__, - "AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__ + "AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__, + "ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__, + "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__, + "TRANSACTION_INDEXING": __MEMPOOL_TRANSACTION_INDEXING__ }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/docker/backend/start.sh b/docker/backend/start.sh index c8e2a1502..370b329ba 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -27,6 +27,9 @@ __MEMPOOL_INDEXING_BLOCKS_AMOUNT__=${MEMPOOL_INDEXING_BLOCKS_AMOUNT:=false} __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__=${MEMPOOL_AUTOMATIC_BLOCK_REINDEXING:=false} __MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json} __MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master} +__MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false} +__MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false} +__MEMPOOL_TRANSACTION_INDEXING__=${MEMPOOL_TRANSACTION_INDEXING:=false} # CORE_RPC __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1} @@ -136,6 +139,8 @@ sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT_ sed -i "s/__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__/${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}/g" mempool-config.json sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-config.json sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json +sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json +sed -i "s!__MEMPOOL_TRANSACTION_INDEXING__!${__MEMPOOL_TRANSACTION_INDEXING__}!g" mempool-config.json sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json From 2e45dab4b7c9b248716b48849388d466e6b227ad Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 28 Dec 2022 05:28:37 -0600 Subject: [PATCH 2/4] Rename TRANSACTION_INDEXING to CPFP_INDEXING and add to mainnet prod config --- backend/mempool-config.sample.json | 2 +- backend/src/__fixtures__/mempool-config.template.json | 2 +- backend/src/__tests__/config.test.ts | 2 +- backend/src/api/blocks.ts | 4 ++-- backend/src/api/common.ts | 2 +- backend/src/config.ts | 4 ++-- production/mempool-config.mainnet.json | 1 + 7 files changed, 9 insertions(+), 8 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index dbbc8412d..1f64214ce 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -27,7 +27,7 @@ "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master", "ADVANCED_GBT_AUDIT": false, "ADVANCED_GBT_MEMPOOL": false, - "TRANSACTION_INDEXING": false + "CPFP_INDEXING": false }, "CORE_RPC": { "HOST": "127.0.0.1", diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 7971b65ca..e699c9458 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -28,7 +28,7 @@ "POOLS_JSON_URL": "__POOLS_JSON_URL__", "ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__", "ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__", - "TRANSACTION_INDEXING": "__MEMPOOL_TRANSACTION_INDEXING__" + "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__" }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 58cf3a214..4158d3df1 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -40,7 +40,7 @@ describe('Mempool Backend Config', () => { POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json', ADVANCED_GBT_AUDIT: false, ADVANCED_GBT_MEMPOOL: false, - TRANSACTION_INDEXING: false, + CPFP_INDEXING: false, }); expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true }); diff --git a/backend/src/api/blocks.ts b/backend/src/api/blocks.ts index 8292fe241..1b551f0b5 100644 --- a/backend/src/api/blocks.ts +++ b/backend/src/api/blocks.ts @@ -530,7 +530,7 @@ class Blocks { for (let i = 10; i >= 0; --i) { const newBlock = await this.$indexBlock(lastBlock['height'] - i); await this.$getStrippedBlockTransactions(newBlock.id, true, true); - if (config.MEMPOOL.TRANSACTION_INDEXING) { + if (config.MEMPOOL.CPFP_INDEXING) { await this.$indexCPFP(newBlock.id, lastBlock['height'] - i); } } @@ -558,7 +558,7 @@ class Blocks { if (Common.blocksSummariesIndexingEnabled() === true) { await this.$getStrippedBlockTransactions(blockExtended.id, true); } - if (config.MEMPOOL.TRANSACTION_INDEXING) { + if (config.MEMPOOL.CPFP_INDEXING) { this.$indexCPFP(blockExtended.id, this.currentBlockHeight); } } diff --git a/backend/src/api/common.ts b/backend/src/api/common.ts index 621f021ba..f0c5c6b88 100644 --- a/backend/src/api/common.ts +++ b/backend/src/api/common.ts @@ -190,7 +190,7 @@ export class Common { static cpfpIndexingEnabled(): boolean { return ( Common.indexingEnabled() && - config.MEMPOOL.TRANSACTION_INDEXING === true + config.MEMPOOL.CPFP_INDEXING === true ); } diff --git a/backend/src/config.ts b/backend/src/config.ts index e97deb5e5..fb06c84fb 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -31,7 +31,7 @@ interface IConfig { POOLS_JSON_TREE_URL: string, ADVANCED_GBT_AUDIT: boolean; ADVANCED_GBT_MEMPOOL: boolean; - TRANSACTION_INDEXING: boolean; + CPFP_INDEXING: boolean; }; ESPLORA: { REST_API_URL: string; @@ -152,7 +152,7 @@ const defaults: IConfig = { 'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master', 'ADVANCED_GBT_AUDIT': false, 'ADVANCED_GBT_MEMPOOL': false, - 'TRANSACTION_INDEXING': false, + 'CPFP_INDEXING': false, }, 'ESPLORA': { 'REST_API_URL': 'http://127.0.0.1:3000', diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index ab2fa69c1..fc22e6b03 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -10,6 +10,7 @@ "POLL_RATE_MS": 1000, "INDEXING_BLOCKS_AMOUNT": -1, "BLOCKS_SUMMARIES_INDEXING": true, + "CPFP_INDEXING": true, "ADVANCED_GBT_AUDIT": true, "ADVANCED_GBT_MEMPOOL": false, "USE_SECOND_NODE_FOR_MINFEE": true From f5922b7b71e3b81552e1461bdcfd03e514e2082a Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 9 Jan 2023 10:34:55 -0600 Subject: [PATCH 3/4] Swap TRANSACTION_INDEXING for CPFP_INDEXING in docker config --- docker/README.md | 6 +++--- docker/backend/mempool-config.json | 2 +- docker/backend/start.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/README.md b/docker/README.md index b891748d0..0c53cee97 100644 --- a/docker/README.md +++ b/docker/README.md @@ -111,7 +111,7 @@ Below we list all settings from `mempool-config.json` and the corresponding over "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master", "ADVANCED_GBT_AUDIT": false, "ADVANCED_GBT_MEMPOOL": false, - "TRANSACTION_INDEXING": false, + "CPFP_INDEXING": false, }, ``` @@ -142,13 +142,13 @@ Corresponding `docker-compose.yml` overrides: MEMPOOL_POOLS_JSON_TREE_URL: "" MEMPOOL_ADVANCED_GBT_AUDIT: "" MEMPOOL_ADVANCED_GBT_MEMPOOL: "" - MEMPOOL_TRANSACTION_INDEXING: "" + MEMPOOL_CPFP_INDEXING: "" ... ``` `ADVANCED_GBT_AUDIT` AND `ADVANCED_GBT_MEMPOOL` enable a more accurate (but slower) block prediction algorithm for the block audit feature and the projected mempool-blocks respectively. -`TRANSACTION_INDEXING` enables indexing CPFP (Child Pays For Parent) information for the last `INDEXING_BLOCKS_AMOUNT` blocks. +`CPFP_INDEXING` enables indexing CPFP (Child Pays For Parent) information for the last `INDEXING_BLOCKS_AMOUNT` blocks.
diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index 8f5eae65e..2e3826f1d 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -25,7 +25,7 @@ "AUTOMATIC_BLOCK_REINDEXING": __MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__, "ADVANCED_GBT_AUDIT": __MEMPOOL_ADVANCED_GBT_AUDIT__, "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__, - "TRANSACTION_INDEXING": __MEMPOOL_TRANSACTION_INDEXING__ + "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__ }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 370b329ba..b5b6e863d 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -29,7 +29,7 @@ __MEMPOOL_POOLS_JSON_URL__=${MEMPOOL_POOLS_JSON_URL:=https://raw.githubuserconte __MEMPOOL_POOLS_JSON_TREE_URL__=${MEMPOOL_POOLS_JSON_TREE_URL:=https://api.github.com/repos/mempool/mining-pools/git/trees/master} __MEMPOOL_ADVANCED_GBT_AUDIT__=${MEMPOOL_ADVANCED_GBT_AUDIT:=false} __MEMPOOL_ADVANCED_GBT_MEMPOOL__=${MEMPOOL_ADVANCED_GBT_MEMPOOL:=false} -__MEMPOOL_TRANSACTION_INDEXING__=${MEMPOOL_TRANSACTION_INDEXING:=false} +__MEMPOOL_CPFP_INDEXING__=${MEMPOOL_CPFP_INDEXING:=false} # CORE_RPC __CORE_RPC_HOST__=${CORE_RPC_HOST:=127.0.0.1} @@ -140,7 +140,7 @@ sed -i "s/__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__/${__MEMPOOL_AUTOMATIC_BLOCK_REI sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-config.json sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json sed -i "s!__MEMPOOL_ADVANCED_GBT_MEMPOOL__!${__MEMPOOL_ADVANCED_GBT_MEMPOOL__}!g" mempool-config.json -sed -i "s!__MEMPOOL_TRANSACTION_INDEXING__!${__MEMPOOL_TRANSACTION_INDEXING__}!g" mempool-config.json +sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-config.json sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json From 649d14011e1b9c81edf5c76d66ffb23591523d7e Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 9 Jan 2023 10:35:25 -0600 Subject: [PATCH 4/4] Remove cpfp_indexing=true from mainnet prod config --- production/mempool-config.mainnet.json | 1 - 1 file changed, 1 deletion(-) diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index fc22e6b03..ab2fa69c1 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -10,7 +10,6 @@ "POLL_RATE_MS": 1000, "INDEXING_BLOCKS_AMOUNT": -1, "BLOCKS_SUMMARIES_INDEXING": true, - "CPFP_INDEXING": true, "ADVANCED_GBT_AUDIT": true, "ADVANCED_GBT_MEMPOOL": false, "USE_SECOND_NODE_FOR_MINFEE": true