From 178da3df26b514ee0523aa6a2d93c561f73532f9 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sun, 14 May 2023 13:13:37 -0700 Subject: [PATCH 01/10] Use the same delimiter for all substitutions --- docker/backend/start.sh | 100 ++++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/docker/backend/start.sh b/docker/backend/start.sh index a54f16ec6..54f0825d5 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -46,7 +46,7 @@ __ELECTRUM_TLS_ENABLED__=${ELECTRUM_TLS_ENABLED:=false} # ESPLORA __ESPLORA_REST_API_URL__=${ESPLORA_REST_API_URL:=http://127.0.0.1:3000} -__ESPLORA_UNIX_SOCKET_PATH__=${ESPLORA_UNIX_SOCKET_PATH:=null} +__ESPLORA_UNIX_SOCKET_PATH__=${ESPLORA_UNIX_SOCKET_PATH:="null"} __ESPLORA_RETRY_UNIX_SOCKET_AFTER__=${ESPLORA_RETRY_UNIX_SOCKET_AFTER:=30000} # SECOND_CORE_RPC @@ -127,28 +127,28 @@ __MAXMIND_GEOIP2_ISP__=${MAXMIND_GEOIP2_ISP:=""} mkdir -p "${__MEMPOOL_CACHE_DIR__}" -sed -i "s/__MEMPOOL_NETWORK__/${__MEMPOOL_NETWORK__}/g" mempool-config.json -sed -i "s/__MEMPOOL_BACKEND__/${__MEMPOOL_BACKEND__}/g" mempool-config.json -sed -i "s/__MEMPOOL_ENABLED__/${__MEMPOOL_ENABLED__}/g" mempool-config.json -sed -i "s/__MEMPOOL_HTTP_PORT__/${__MEMPOOL_HTTP_PORT__}/g" mempool-config.json -sed -i "s/__MEMPOOL_SPAWN_CLUSTER_PROCS__/${__MEMPOOL_SPAWN_CLUSTER_PROCS__}/g" mempool-config.json +sed -i "s!__MEMPOOL_NETWORK__!${__MEMPOOL_NETWORK__}!g" mempool-config.json +sed -i "s!__MEMPOOL_BACKEND__!${__MEMPOOL_BACKEND__}!g" mempool-config.json +sed -i "s!__MEMPOOL_ENABLED__!${__MEMPOOL_ENABLED__}!g" mempool-config.json +sed -i "s!__MEMPOOL_HTTP_PORT__!${__MEMPOOL_HTTP_PORT__}!g" mempool-config.json +sed -i "s!__MEMPOOL_SPAWN_CLUSTER_PROCS__!${__MEMPOOL_SPAWN_CLUSTER_PROCS__}!g" mempool-config.json sed -i "s!__MEMPOOL_API_URL_PREFIX__!${__MEMPOOL_API_URL_PREFIX__}!g" mempool-config.json -sed -i "s/__MEMPOOL_POLL_RATE_MS__/${__MEMPOOL_POLL_RATE_MS__}/g" mempool-config.json +sed -i "s!__MEMPOOL_POLL_RATE_MS__!${__MEMPOOL_POLL_RATE_MS__}!g" mempool-config.json sed -i "s!__MEMPOOL_CACHE_DIR__!${__MEMPOOL_CACHE_DIR__}!g" mempool-config.json -sed -i "s/__MEMPOOL_CLEAR_PROTECTION_MINUTES__/${__MEMPOOL_CLEAR_PROTECTION_MINUTES__}/g" mempool-config.json -sed -i "s/__MEMPOOL_RECOMMENDED_FEE_PERCENTILE__/${__MEMPOOL_RECOMMENDED_FEE_PERCENTILE__}/g" mempool-config.json -sed -i "s/__MEMPOOL_BLOCK_WEIGHT_UNITS__/${__MEMPOOL_BLOCK_WEIGHT_UNITS__}/g" mempool-config.json -sed -i "s/__MEMPOOL_INITIAL_BLOCKS_AMOUNT__/${__MEMPOOL_INITIAL_BLOCKS_AMOUNT__}/g" mempool-config.json -sed -i "s/__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__/${__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__}/g" mempool-config.json -sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json -sed -i "s/__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__/${__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__}/g" mempool-config.json -sed -i "s/__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__/${__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__}/g" mempool-config.json +sed -i "s!__MEMPOOL_CLEAR_PROTECTION_MINUTES__!${__MEMPOOL_CLEAR_PROTECTION_MINUTES__}!g" mempool-config.json +sed -i "s!__MEMPOOL_RECOMMENDED_FEE_PERCENTILE__!${__MEMPOOL_RECOMMENDED_FEE_PERCENTILE__}!g" mempool-config.json +sed -i "s!__MEMPOOL_BLOCK_WEIGHT_UNITS__!${__MEMPOOL_BLOCK_WEIGHT_UNITS__}!g" mempool-config.json +sed -i "s!__MEMPOOL_INITIAL_BLOCKS_AMOUNT__!${__MEMPOOL_INITIAL_BLOCKS_AMOUNT__}!g" mempool-config.json +sed -i "s!__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__!${__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__}!g" mempool-config.json +sed -i "s!__MEMPOOL_INDEXING_BLOCKS_AMOUNT__!${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}!g" mempool-config.json +sed -i "s!__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__!${__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__}!g" mempool-config.json +sed -i "s!__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__!${__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__}!g" mempool-config.json sed -i "s!__MEMPOOL_EXTERNAL_ASSETS__!${__MEMPOOL_EXTERNAL_ASSETS__}!g" mempool-config.json sed -i "s!__MEMPOOL_EXTERNAL_MAX_RETRY__!${__MEMPOOL_EXTERNAL_MAX_RETRY__}!g" mempool-config.json sed -i "s!__MEMPOOL_EXTERNAL_RETRY_INTERVAL__!${__MEMPOOL_EXTERNAL_RETRY_INTERVAL__}!g" mempool-config.json sed -i "s!__MEMPOOL_USER_AGENT__!${__MEMPOOL_USER_AGENT__}!g" mempool-config.json -sed -i "s/__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__/${__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__}/g" mempool-config.json -sed -i "s/__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__/${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}/g" mempool-config.json +sed -i "s!__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__!${__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__}!g" mempool-config.json +sed -i "s!__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__!${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}!g" mempool-config.json sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-config.json sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json @@ -158,53 +158,53 @@ sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-conf sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json -sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json -sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json -sed -i "s/__CORE_RPC_USERNAME__/${__CORE_RPC_USERNAME__}/g" mempool-config.json -sed -i "s/__CORE_RPC_PASSWORD__/${__CORE_RPC_PASSWORD__}/g" mempool-config.json -sed -i "s/__CORE_RPC_TIMEOUT__/${__CORE_RPC_TIMEOUT__}/g" mempool-config.json +sed -i "s!__CORE_RPC_HOST__!${__CORE_RPC_HOST__}!g" mempool-config.json +sed -i "s!__CORE_RPC_PORT__!${__CORE_RPC_PORT__}!g" mempool-config.json +sed -i "s!__CORE_RPC_USERNAME__!${__CORE_RPC_USERNAME__}!g" mempool-config.json +sed -i "s!__CORE_RPC_PASSWORD__!${__CORE_RPC_PASSWORD__}!g" mempool-config.json +sed -i "s!__CORE_RPC_TIMEOUT__!${__CORE_RPC_TIMEOUT__}!g" mempool-config.json -sed -i "s/__ELECTRUM_HOST__/${__ELECTRUM_HOST__}/g" mempool-config.json -sed -i "s/__ELECTRUM_PORT__/${__ELECTRUM_PORT__}/g" mempool-config.json -sed -i "s/__ELECTRUM_TLS_ENABLED__/${__ELECTRUM_TLS_ENABLED__}/g" mempool-config.json +sed -i "s!__ELECTRUM_HOST__!${__ELECTRUM_HOST__}!g" mempool-config.json +sed -i "s!__ELECTRUM_PORT__!${__ELECTRUM_PORT__}!g" mempool-config.json +sed -i "s!__ELECTRUM_TLS_ENABLED__!${__ELECTRUM_TLS_ENABLED__}!g" mempool-config.json sed -i "s!__ESPLORA_REST_API_URL__!${__ESPLORA_REST_API_URL__}!g" mempool-config.json sed -i "s!__ESPLORA_UNIX_SOCKET_PATH__!${__ESPLORA_UNIX_SOCKET_PATH__}!g" mempool-config.json sed -i "s!__ESPLORA_RETRY_UNIX_SOCKET_AFTER__!${__ESPLORA_RETRY_UNIX_SOCKET_AFTER__}!g" mempool-config.json -sed -i "s/__SECOND_CORE_RPC_HOST__/${__SECOND_CORE_RPC_HOST__}/g" mempool-config.json -sed -i "s/__SECOND_CORE_RPC_PORT__/${__SECOND_CORE_RPC_PORT__}/g" mempool-config.json -sed -i "s/__SECOND_CORE_RPC_USERNAME__/${__SECOND_CORE_RPC_USERNAME__}/g" mempool-config.json -sed -i "s/__SECOND_CORE_RPC_PASSWORD__/${__SECOND_CORE_RPC_PASSWORD__}/g" mempool-config.json -sed -i "s/__SECOND_CORE_RPC_TIMEOUT__/${__SECOND_CORE_RPC_TIMEOUT__}/g" mempool-config.json +sed -i "s!__SECOND_CORE_RPC_HOST__!${__SECOND_CORE_RPC_HOST__}!g" mempool-config.json +sed -i "s!__SECOND_CORE_RPC_PORT__!${__SECOND_CORE_RPC_PORT__}!g" mempool-config.json +sed -i "s!__SECOND_CORE_RPC_USERNAME__!${__SECOND_CORE_RPC_USERNAME__}!g" mempool-config.json +sed -i "s!__SECOND_CORE_RPC_PASSWORD__!${__SECOND_CORE_RPC_PASSWORD__}!g" mempool-config.json +sed -i "s!__SECOND_CORE_RPC_TIMEOUT__!${__SECOND_CORE_RPC_TIMEOUT__}!g" mempool-config.json -sed -i "s/__DATABASE_ENABLED__/${__DATABASE_ENABLED__}/g" mempool-config.json -sed -i "s/__DATABASE_HOST__/${__DATABASE_HOST__}/g" mempool-config.json +sed -i "s!__DATABASE_ENABLED__!${__DATABASE_ENABLED__}!g" mempool-config.json +sed -i "s!__DATABASE_HOST__!${__DATABASE_HOST__}!g" mempool-config.json sed -i "s!__DATABASE_SOCKET__!${__DATABASE_SOCKET__}!g" mempool-config.json -sed -i "s/__DATABASE_PORT__/${__DATABASE_PORT__}/g" mempool-config.json -sed -i "s/__DATABASE_DATABASE__/${__DATABASE_DATABASE__}/g" mempool-config.json -sed -i "s/__DATABASE_USERNAME__/${__DATABASE_USERNAME__}/g" mempool-config.json -sed -i "s/__DATABASE_PASSWORD__/${__DATABASE_PASSWORD__}/g" mempool-config.json +sed -i "s!__DATABASE_PORT__!${__DATABASE_PORT__}!g" mempool-config.json +sed -i "s!__DATABASE_DATABASE__!${__DATABASE_DATABASE__}!g" mempool-config.json +sed -i "s!__DATABASE_USERNAME__!${__DATABASE_USERNAME__}!g" mempool-config.json +sed -i "s!__DATABASE_PASSWORD__!${__DATABASE_PASSWORD__}!g" mempool-config.json -sed -i "s/__SYSLOG_ENABLED__/${__SYSLOG_ENABLED__}/g" mempool-config.json -sed -i "s/__SYSLOG_HOST__/${__SYSLOG_HOST__}/g" mempool-config.json -sed -i "s/__SYSLOG_PORT__/${__SYSLOG_PORT__}/g" mempool-config.json -sed -i "s/__SYSLOG_MIN_PRIORITY__/${__SYSLOG_MIN_PRIORITY__}/g" mempool-config.json -sed -i "s/__SYSLOG_FACILITY__/${__SYSLOG_FACILITY__}/g" mempool-config.json +sed -i "s!__SYSLOG_ENABLED__!${__SYSLOG_ENABLED__}!g" mempool-config.json +sed -i "s!__SYSLOG_HOST__!${__SYSLOG_HOST__}!g" mempool-config.json +sed -i "s!__SYSLOG_PORT__!${__SYSLOG_PORT__}!g" mempool-config.json +sed -i "s!__SYSLOG_MIN_PRIORITY__!${__SYSLOG_MIN_PRIORITY__}!g" mempool-config.json +sed -i "s!__SYSLOG_FACILITY__!${__SYSLOG_FACILITY__}!g" mempool-config.json -sed -i "s/__STATISTICS_ENABLED__/${__STATISTICS_ENABLED__}/g" mempool-config.json -sed -i "s/__STATISTICS_TX_PER_SECOND_SAMPLE_PERIOD__/${__STATISTICS_TX_PER_SECOND_SAMPLE_PERIOD__}/g" mempool-config.json +sed -i "s!__STATISTICS_ENABLED__!${__STATISTICS_ENABLED__}!g" mempool-config.json +sed -i "s!__STATISTICS_TX_PER_SECOND_SAMPLE_PERIOD__!${__STATISTICS_TX_PER_SECOND_SAMPLE_PERIOD__}!g" mempool-config.json -sed -i "s/__BISQ_ENABLED__/${__BISQ_ENABLED__}/g" mempool-config.json +sed -i "s!__BISQ_ENABLED__!${__BISQ_ENABLED__}!g" mempool-config.json sed -i "s!__BISQ_DATA_PATH__!${__BISQ_DATA_PATH__}!g" mempool-config.json -sed -i "s/__SOCKS5PROXY_ENABLED__/${__SOCKS5PROXY_ENABLED__}/g" mempool-config.json -sed -i "s/__SOCKS5PROXY_USE_ONION__/${__SOCKS5PROXY_USE_ONION__}/g" mempool-config.json -sed -i "s/__SOCKS5PROXY_HOST__/${__SOCKS5PROXY_HOST__}/g" mempool-config.json -sed -i "s/__SOCKS5PROXY_PORT__/${__SOCKS5PROXY_PORT__}/g" mempool-config.json -sed -i "s/__SOCKS5PROXY_USERNAME__/${__SOCKS5PROXY_USERNAME__}/g" mempool-config.json -sed -i "s/__SOCKS5PROXY_PASSWORD__/${__SOCKS5PROXY_PASSWORD__}/g" mempool-config.json +sed -i "s!__SOCKS5PROXY_ENABLED__!${__SOCKS5PROXY_ENABLED__}!g" mempool-config.json +sed -i "s!__SOCKS5PROXY_USE_ONION__!${__SOCKS5PROXY_USE_ONION__}!g" mempool-config.json +sed -i "s!__SOCKS5PROXY_HOST__!${__SOCKS5PROXY_HOST__}!g" mempool-config.json +sed -i "s!__SOCKS5PROXY_PORT__!${__SOCKS5PROXY_PORT__}!g" mempool-config.json +sed -i "s!__SOCKS5PROXY_USERNAME__!${__SOCKS5PROXY_USERNAME__}!g" mempool-config.json +sed -i "s!__SOCKS5PROXY_PASSWORD__!${__SOCKS5PROXY_PASSWORD__}!g" mempool-config.json sed -i "s!__PRICE_DATA_SERVER_TOR_URL__!${__PRICE_DATA_SERVER_TOR_URL__}!g" mempool-config.json sed -i "s!__PRICE_DATA_SERVER_CLEARNET_URL__!${__PRICE_DATA_SERVER_CLEARNET_URL__}!g" mempool-config.json From 3f8fefcb0e310642db4a13d5f1939feb200ea24b Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sun, 14 May 2023 13:14:44 -0700 Subject: [PATCH 02/10] Add a test to check if all the vars are on the Docker script --- backend/src/__tests__/config.test.ts | 48 ++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index aa287308b..4971403af 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -152,4 +152,52 @@ describe('Mempool Backend Config', () => { expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER); }); }); + + test('should ensure the docker start.sh script has default values', () => { + jest.isolateModules(() => { + const startSh = fs.readFileSync(`${__dirname}/../../../docker/backend/start.sh`, 'utf-8'); + const fixture = JSON.parse(fs.readFileSync(`${__dirname}/../__fixtures__/mempool-config.template.json`, 'utf8')); + + function parseJson(jsonObj, root?) { + for (const [key, value] of Object.entries(jsonObj)) { + // We have a few cases where we can't follow the pattern + if (root === 'MEMPOOL' && key === 'HTTP_PORT') { + console.log('skipping check for MEMPOOL_HTTP_PORT'); + return; + } + switch (typeof value) { + case 'object': { + if (Array.isArray(value)) { + return; + } else { + parseJson(value, key); + } + break; + } + default: { + //The flattened string, i.e, __MEMPOOL_ENABLED__ + const replaceStr = `${root ? '__' + root + '_' : '__'}${key}__`; + + //The string used as the environment variable, i.e, MEMPOOL_ENABLED + const envVarStr = `${root ? root : ''}_${key}`; + + //The string used as the default value, to be checked as a regex, i.e, __MEMPOOL_ENABLED__=${MEMPOOL_ENABLED:=(.*)} + const defaultEntry = replaceStr + '=' + '\\${' + envVarStr + ':=(.*)' + '}'; + + console.log(`looking for ${defaultEntry} in the start.sh script`); + const re = new RegExp(defaultEntry); + expect(startSh).toMatch(re); + + //The string that actually replaces the values in the config file + const sedStr = 'sed -i "s!' + replaceStr + '!${' + replaceStr + '}!g" mempool-config.json'; + console.log(`looking for ${sedStr} in the start.sh script`); + expect(startSh).toContain(sedStr); + break; + } + } + } + } + parseJson(fixture); + }); + }); }); From 804485a526adbb09d5fd3b4bef284ef1be8dd690 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sun, 14 May 2023 13:20:18 -0700 Subject: [PATCH 03/10] Add missing DATABASE_TIMEOUT config --- docker/backend/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 54f0825d5..f1ddba3fd 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -181,11 +181,11 @@ sed -i "s!__SECOND_CORE_RPC_TIMEOUT__!${__SECOND_CORE_RPC_TIMEOUT__}!g" mempool- sed -i "s!__DATABASE_ENABLED__!${__DATABASE_ENABLED__}!g" mempool-config.json sed -i "s!__DATABASE_HOST__!${__DATABASE_HOST__}!g" mempool-config.json sed -i "s!__DATABASE_SOCKET__!${__DATABASE_SOCKET__}!g" mempool-config.json - sed -i "s!__DATABASE_PORT__!${__DATABASE_PORT__}!g" mempool-config.json sed -i "s!__DATABASE_DATABASE__!${__DATABASE_DATABASE__}!g" mempool-config.json sed -i "s!__DATABASE_USERNAME__!${__DATABASE_USERNAME__}!g" mempool-config.json sed -i "s!__DATABASE_PASSWORD__!${__DATABASE_PASSWORD__}!g" mempool-config.json +sed -i "s!__DATABASE_TIMEOUT__!${__DATABASE_TIMEOUT__}!g" mempool-config.json sed -i "s!__SYSLOG_ENABLED__!${__SYSLOG_ENABLED__}!g" mempool-config.json sed -i "s!__SYSLOG_HOST__!${__SYSLOG_HOST__}!g" mempool-config.json From de434bb398827a306af6bb65f7e75bd2f88d5b5c Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sun, 14 May 2023 13:26:35 -0700 Subject: [PATCH 04/10] Add missing Lightning config --- docker/backend/start.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/backend/start.sh b/docker/backend/start.sh index f1ddba3fd..cb1108a05 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -108,6 +108,8 @@ __LIGHTNING_TOPOLOGY_FOLDER__=${LIGHTNING_TOPOLOGY_FOLDER:=""} __LIGHTNING_STATS_REFRESH_INTERVAL__=${LIGHTNING_STATS_REFRESH_INTERVAL:=600} __LIGHTNING_GRAPH_REFRESH_INTERVAL__=${LIGHTNING_GRAPH_REFRESH_INTERVAL:=600} __LIGHTNING_LOGGER_UPDATE_INTERVAL__=${LIGHTNING_LOGGER_UPDATE_INTERVAL:=30} +__LIGHTNING_FORENSICS_INTERVAL__=${LIGHTNING_FORENSICS_INTERVAL:=43200} +__LIGHTNING_FORENSICS_RATE_LIMIT__=${LIGHTNING_FORENSICS_RATE_LIMIT:=20} # LND __LND_TLS_CERT_PATH__=${LND_TLS_CERT_PATH:=""} @@ -223,6 +225,8 @@ sed -i "s!__LIGHTNING_TOPOLOGY_FOLDER__!${__LIGHTNING_TOPOLOGY_FOLDER__}!g" memp sed -i "s!__LIGHTNING_STATS_REFRESH_INTERVAL__!${__LIGHTNING_STATS_REFRESH_INTERVAL__}!g" mempool-config.json sed -i "s!__LIGHTNING_GRAPH_REFRESH_INTERVAL__!${__LIGHTNING_GRAPH_REFRESH_INTERVAL__}!g" mempool-config.json sed -i "s!__LIGHTNING_LOGGER_UPDATE_INTERVAL__!${__LIGHTNING_LOGGER_UPDATE_INTERVAL__}!g" mempool-config.json +sed -i "s!__LIGHTNING_FORENSICS_INTERVAL__!${__LIGHTNING_FORENSICS_INTERVAL__}!g" mempool-config.json +sed -i "s!__LIGHTNING_FORENSICS_RATE_LIMIT__!${__LIGHTNING_FORENSICS_RATE_LIMIT__}!g" mempool-config.json # LND sed -i "s!__LND_TLS_CERT_PATH__!${__LND_TLS_CERT_PATH__}!g" mempool-config.json From 184ca3c66220e1f82240df5f4bc81af398b926ac Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sun, 14 May 2023 14:17:53 -0700 Subject: [PATCH 05/10] Add a test to check that the docker json is in sync with the template --- backend/src/__tests__/config.test.ts | 42 ++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 4971403af..278d83f50 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -200,4 +200,46 @@ describe('Mempool Backend Config', () => { parseJson(fixture); }); }); + + test('should ensure that the mempool-config.json Docker template has all the keys', () => { + jest.isolateModules(() => { + const fixture = JSON.parse(fs.readFileSync(`${__dirname}/../__fixtures__/mempool-config.template.json`, 'utf8')); + const dockerJson = fs.readFileSync(`${__dirname}/../../../docker/backend/mempool-config.json`, 'utf-8'); + + function parseJson(jsonObj, root?) { + for (const [key, value] of Object.entries(jsonObj)) { + switch (typeof value) { + case 'object': { + if (Array.isArray(value)) { + // numbers, arrays and booleans won't be enclosed by quotes + const replaceStr = `${root ? '__' + root + '_' : '__'}${key}__`; + expect(dockerJson).toContain(`"${key}": ${replaceStr}`); + break; + } else { + //Check for top level config keys + expect(dockerJson).toContain(`"${key}"`); + parseJson(value, key); + break; + } + } + case 'string': { + // strings should be enclosed by quotes + const replaceStr = `${root ? '__' + root + '_' : '__'}${key}__`; + expect(dockerJson).toContain(`"${key}": "${replaceStr}"`); + break; + } + default: { + // numbers, arrays and booleans won't be enclosed by quotes + const replaceStr = `${root ? '__' + root + '_' : '__'}${key}__`; + expect(dockerJson).toContain(`"${key}": ${replaceStr}`); + break; + } + } + }; + } + parseJson(fixture); + }); + }); + + }); From 6333f3aa4702d7aae8c58ce47f1c1c4a8df7a6b4 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sun, 14 May 2023 14:25:23 -0700 Subject: [PATCH 06/10] Update the backend config fixture --- .../__fixtures__/mempool-config.template.json | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index eb082d89f..919784464 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -16,7 +16,7 @@ "INITIAL_BLOCKS_AMOUNT": 7, "MEMPOOL_BLOCKS_AMOUNT": 8, "USE_SECOND_NODE_FOR_MINFEE": 10, - "EXTERNAL_ASSETS": 11, + "EXTERNAL_ASSETS": [], "EXTERNAL_MAX_RETRY": 12, "EXTERNAL_RETRY_INTERVAL": 13, "USER_AGENT": "__MEMPOOL_USER_AGENT__", @@ -24,19 +24,19 @@ "INDEXING_BLOCKS_AMOUNT": 14, "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__", "POOLS_JSON_URL": "__POOLS_JSON_URL__", - "AUDIT": "__MEMPOOL_AUDIT__", - "ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__", - "ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__", - "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__", - "MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__", - "DISK_CACHE_BLOCK_INTERVAL": "__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__" + "AUDIT": true, + "ADVANCED_GBT_AUDIT": true, + "ADVANCED_GBT_MEMPOOL": true, + "CPFP_INDEXING": true, + "MAX_BLOCKS_BULK_QUERY": 999, + "DISK_CACHE_BLOCK_INTERVAL": 999 }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", "PORT": 15, "USERNAME": "__CORE_RPC_USERNAME__", "PASSWORD": "__CORE_RPC_PASSWORD__", - "TIMEOUT": "__CORE_RPC_TIMEOUT__" + "TIMEOUT": 1000 }, "ELECTRUM": { "HOST": "__ELECTRUM_HOST__", @@ -46,14 +46,14 @@ "ESPLORA": { "REST_API_URL": "__ESPLORA_REST_API_URL__", "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__", - "RETRY_UNIX_SOCKET_AFTER": "__ESPLORA_RETRY_UNIX_SOCKET_AFTER__" + "RETRY_UNIX_SOCKET_AFTER": 888 }, "SECOND_CORE_RPC": { "HOST": "__SECOND_CORE_RPC_HOST__", "PORT": 17, "USERNAME": "__SECOND_CORE_RPC_USERNAME__", "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__", - "TIMEOUT": "__SECOND_CORE_RPC_TIMEOUT__" + "TIMEOUT": 2000 }, "DATABASE": { "ENABLED": false, @@ -63,7 +63,7 @@ "DATABASE": "__DATABASE_DATABASE__", "USERNAME": "__DATABASE_USERNAME__", "PASSWORD": "__DATABASE_PASSWORD__", - "TIMEOUT": "__DATABASE_TIMEOUT__" + "TIMEOUT": 3000 }, "SYSLOG": { "ENABLED": false, @@ -101,14 +101,14 @@ "BISQ_ONION": "__EXTERNAL_DATA_SERVER_BISQ_ONION__" }, "LIGHTNING": { - "ENABLED": "__LIGHTNING_ENABLED__", + "ENABLED": true, "BACKEND": "__LIGHTNING_BACKEND__", "TOPOLOGY_FOLDER": "__LIGHTNING_TOPOLOGY_FOLDER__", "STATS_REFRESH_INTERVAL": 600, "GRAPH_REFRESH_INTERVAL": 600, "LOGGER_UPDATE_INTERVAL": 30, "FORENSICS_INTERVAL": 43200, - "FORENSICS_RATE_LIMIT": "__FORENSICS_RATE_LIMIT__" + "FORENSICS_RATE_LIMIT": 1234 }, "LND": { "TLS_CERT_PATH": "", @@ -119,4 +119,4 @@ "CLIGHTNING": { "SOCKET": "__CLIGHTNING_SOCKET__" } -} +} \ No newline at end of file From b00e0b6a735ecda0f2d87f58f7369355f3aed96e Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sun, 14 May 2023 14:26:38 -0700 Subject: [PATCH 07/10] Fix Docker config file to use the correct types --- docker/backend/mempool-config.json | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index fd8abaf02..06ed51f41 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -27,7 +27,9 @@ "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__, "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__, "MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__, - "DISK_CACHE_BLOCK_INTERVAL": __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__ + "DISK_CACHE_BLOCK_INTERVAL": __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__, + "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", + "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__" }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", @@ -61,7 +63,7 @@ "DATABASE": "__DATABASE_DATABASE__", "USERNAME": "__DATABASE_USERNAME__", "PASSWORD": "__DATABASE_PASSWORD__", - "TIMEOUT": "__DATABASE_TIMEOUT__" + "TIMEOUT": __DATABASE_TIMEOUT__ }, "SYSLOG": { "ENABLED": __SYSLOG_ENABLED__, @@ -84,13 +86,15 @@ "STATS_REFRESH_INTERVAL": __LIGHTNING_STATS_REFRESH_INTERVAL__, "GRAPH_REFRESH_INTERVAL": __LIGHTNING_GRAPH_REFRESH_INTERVAL__, "LOGGER_UPDATE_INTERVAL": __LIGHTNING_LOGGER_UPDATE_INTERVAL__, - "TOPOLOGY_FOLDER": "__LIGHTNING_TOPOLOGY_FOLDER__" + "TOPOLOGY_FOLDER": "__LIGHTNING_TOPOLOGY_FOLDER__", + "FORENSICS_INTERVAL": __LIGHTNING_FORENSICS_INTERVAL__, + "FORENSICS_RATE_LIMIT": __LIGHTNING_FORENSICS_RATE_LIMIT__ }, "LND": { "TLS_CERT_PATH": "__LND_TLS_CERT_PATH__", "MACAROON_PATH": "__LND_MACAROON_PATH__", "REST_API_URL": "__LND_REST_API_URL__", - "TIMEOUT": "__LND_TIMEOUT__" + "TIMEOUT": __LND_TIMEOUT__ }, "CLIGHTNING": { "SOCKET": "__CLIGHTNING_SOCKET__" @@ -121,4 +125,4 @@ "GEOLITE2_ASN": "__MAXMIND_GEOLITE2_ASN__", "GEOIP2_ISP": "__MAXMIND_GEOIP2_ISP__" } -} +} \ No newline at end of file From 264c02eca1f38b97a3b3b86b639cc207aacff8dc Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Thu, 25 May 2023 01:29:55 -0400 Subject: [PATCH 08/10] Add bull bitcoin as enterprise sponsor --- frontend/src/app/components/about/about.component.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index e67facfe1..aa604f678 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -119,6 +119,10 @@ Gemini + + + Bull Bitcoin + From 5711c939d75b18524b37b8c7ed1ef18b7ef07d6e Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Thu, 25 May 2023 15:58:48 -0400 Subject: [PATCH 09/10] Make svg pretty --- frontend/src/app/components/about/about.component.html | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index aa604f678..8655ab800 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -120,7 +120,15 @@ Gemini - + Bull Bitcoin From 9fa1eebc02d2aac516bee0141abde45e0071c814 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Fri, 26 May 2023 17:25:52 -0400 Subject: [PATCH 10/10] Switch tm to r for registered trademarks --- frontend/src/app/components/about/about.component.html | 4 ++-- .../trademark-policy/trademark-policy.component.html | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index 9a778be44..c59f700a8 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -1,7 +1,7 @@
- ™ + ®
v{{ packetJsonVersion }} [{{ frontendGitCommitHash }}] @@ -396,7 +396,7 @@ Trademark Notice

- The Mempool Open Source Project™, mempool.space™, the mempool logo™, the mempool.space logos™, the mempool square logo™, and the mempool blocks logo™ are either registered trademarks or trademarks of Mempool Space K.K in Japan, the United States, and/or other countries. + The Mempool Open Source Project™, mempool.space™, the mempool logo®, the mempool.space logos™, the mempool square logo®, and the mempool blocks logo™ are either registered trademarks or trademarks of Mempool Space K.K in Japan, the United States, and/or other countries.

While our software is available under an open source software license, the copyright license does not include an implied right or license to use our trademarks. See our Trademark Policy and Guidelines for more details, published on <https://mempool.space/trademark-policy>. diff --git a/frontend/src/app/components/trademark-policy/trademark-policy.component.html b/frontend/src/app/components/trademark-policy/trademark-policy.component.html index 4526a6c0c..4f9419642 100644 --- a/frontend/src/app/components/trademark-policy/trademark-policy.component.html +++ b/frontend/src/app/components/trademark-policy/trademark-policy.component.html @@ -304,7 +304,7 @@

Also, if you are using our Marks in a way described in the sections "Uses for Which We Are Granting a License," you must include the following trademark attribution at the foot of the webpage where you have used the Mark (or, if in a book, on the credits page), on any packaging or labeling, and on advertising or marketing materials:

-

“The Mempool Space K.K.™, The Mempool Open Source Project™, mempool.space™, the mempool logo™, the mempool.space logos™, the mempool square logo™, and the mempool blocks logo™ are either registered trademarks or trademarks of Mempool Space K.K in Japan, the United States, and/or other countries, and are used with permission. Mempool Space K.K. has no affiliation with and does not sponsor or endorse the information provided herein.”

+

“The Mempool Space K.K.™, The Mempool Open Source Project™, mempool.space™, the mempool logo®, the mempool.space logos™, the mempool square logo®, and the mempool blocks logo™ are either registered trademarks or trademarks of Mempool Space K.K in Japan, the United States, and/or other countries, and are used with permission. Mempool Space K.K. has no affiliation with and does not sponsor or endorse the information provided herein.”

  • What to Do When You See Abuse