From 51f91dc44d13ae2d7afcff25591c505ab63e1ae5 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sun, 14 May 2023 13:13:37 -0700 Subject: [PATCH 01/10] Use the same delimiter for all substitutions --- docker/backend/start.sh | 100 ++++++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/docker/backend/start.sh b/docker/backend/start.sh index a54f16ec6..54f0825d5 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -46,7 +46,7 @@ __ELECTRUM_TLS_ENABLED__=${ELECTRUM_TLS_ENABLED:=false} # ESPLORA __ESPLORA_REST_API_URL__=${ESPLORA_REST_API_URL:=http://127.0.0.1:3000} -__ESPLORA_UNIX_SOCKET_PATH__=${ESPLORA_UNIX_SOCKET_PATH:=null} +__ESPLORA_UNIX_SOCKET_PATH__=${ESPLORA_UNIX_SOCKET_PATH:="null"} __ESPLORA_RETRY_UNIX_SOCKET_AFTER__=${ESPLORA_RETRY_UNIX_SOCKET_AFTER:=30000} # SECOND_CORE_RPC @@ -127,28 +127,28 @@ __MAXMIND_GEOIP2_ISP__=${MAXMIND_GEOIP2_ISP:=""} mkdir -p "${__MEMPOOL_CACHE_DIR__}" -sed -i "s/__MEMPOOL_NETWORK__/${__MEMPOOL_NETWORK__}/g" mempool-config.json -sed -i "s/__MEMPOOL_BACKEND__/${__MEMPOOL_BACKEND__}/g" mempool-config.json -sed -i "s/__MEMPOOL_ENABLED__/${__MEMPOOL_ENABLED__}/g" mempool-config.json -sed -i "s/__MEMPOOL_HTTP_PORT__/${__MEMPOOL_HTTP_PORT__}/g" mempool-config.json -sed -i "s/__MEMPOOL_SPAWN_CLUSTER_PROCS__/${__MEMPOOL_SPAWN_CLUSTER_PROCS__}/g" mempool-config.json +sed -i "s!__MEMPOOL_NETWORK__!${__MEMPOOL_NETWORK__}!g" mempool-config.json +sed -i "s!__MEMPOOL_BACKEND__!${__MEMPOOL_BACKEND__}!g" mempool-config.json +sed -i "s!__MEMPOOL_ENABLED__!${__MEMPOOL_ENABLED__}!g" mempool-config.json +sed -i "s!__MEMPOOL_HTTP_PORT__!${__MEMPOOL_HTTP_PORT__}!g" mempool-config.json +sed -i "s!__MEMPOOL_SPAWN_CLUSTER_PROCS__!${__MEMPOOL_SPAWN_CLUSTER_PROCS__}!g" mempool-config.json sed -i "s!__MEMPOOL_API_URL_PREFIX__!${__MEMPOOL_API_URL_PREFIX__}!g" mempool-config.json -sed -i "s/__MEMPOOL_POLL_RATE_MS__/${__MEMPOOL_POLL_RATE_MS__}/g" mempool-config.json +sed -i "s!__MEMPOOL_POLL_RATE_MS__!${__MEMPOOL_POLL_RATE_MS__}!g" mempool-config.json sed -i "s!__MEMPOOL_CACHE_DIR__!${__MEMPOOL_CACHE_DIR__}!g" mempool-config.json -sed -i "s/__MEMPOOL_CLEAR_PROTECTION_MINUTES__/${__MEMPOOL_CLEAR_PROTECTION_MINUTES__}/g" mempool-config.json -sed -i "s/__MEMPOOL_RECOMMENDED_FEE_PERCENTILE__/${__MEMPOOL_RECOMMENDED_FEE_PERCENTILE__}/g" mempool-config.json -sed -i "s/__MEMPOOL_BLOCK_WEIGHT_UNITS__/${__MEMPOOL_BLOCK_WEIGHT_UNITS__}/g" mempool-config.json -sed -i "s/__MEMPOOL_INITIAL_BLOCKS_AMOUNT__/${__MEMPOOL_INITIAL_BLOCKS_AMOUNT__}/g" mempool-config.json -sed -i "s/__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__/${__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__}/g" mempool-config.json -sed -i "s/__MEMPOOL_INDEXING_BLOCKS_AMOUNT__/${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}/g" mempool-config.json -sed -i "s/__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__/${__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__}/g" mempool-config.json -sed -i "s/__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__/${__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__}/g" mempool-config.json +sed -i "s!__MEMPOOL_CLEAR_PROTECTION_MINUTES__!${__MEMPOOL_CLEAR_PROTECTION_MINUTES__}!g" mempool-config.json +sed -i "s!__MEMPOOL_RECOMMENDED_FEE_PERCENTILE__!${__MEMPOOL_RECOMMENDED_FEE_PERCENTILE__}!g" mempool-config.json +sed -i "s!__MEMPOOL_BLOCK_WEIGHT_UNITS__!${__MEMPOOL_BLOCK_WEIGHT_UNITS__}!g" mempool-config.json +sed -i "s!__MEMPOOL_INITIAL_BLOCKS_AMOUNT__!${__MEMPOOL_INITIAL_BLOCKS_AMOUNT__}!g" mempool-config.json +sed -i "s!__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__!${__MEMPOOL_MEMPOOL_BLOCKS_AMOUNT__}!g" mempool-config.json +sed -i "s!__MEMPOOL_INDEXING_BLOCKS_AMOUNT__!${__MEMPOOL_INDEXING_BLOCKS_AMOUNT__}!g" mempool-config.json +sed -i "s!__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__!${__MEMPOOL_BLOCKS_SUMMARIES_INDEXING__}!g" mempool-config.json +sed -i "s!__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__!${__MEMPOOL_USE_SECOND_NODE_FOR_MINFEE__}!g" mempool-config.json sed -i "s!__MEMPOOL_EXTERNAL_ASSETS__!${__MEMPOOL_EXTERNAL_ASSETS__}!g" mempool-config.json sed -i "s!__MEMPOOL_EXTERNAL_MAX_RETRY__!${__MEMPOOL_EXTERNAL_MAX_RETRY__}!g" mempool-config.json sed -i "s!__MEMPOOL_EXTERNAL_RETRY_INTERVAL__!${__MEMPOOL_EXTERNAL_RETRY_INTERVAL__}!g" mempool-config.json sed -i "s!__MEMPOOL_USER_AGENT__!${__MEMPOOL_USER_AGENT__}!g" mempool-config.json -sed -i "s/__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__/${__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__}/g" mempool-config.json -sed -i "s/__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__/${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}/g" mempool-config.json +sed -i "s!__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__!${__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__}!g" mempool-config.json +sed -i "s!__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__!${__MEMPOOL_AUTOMATIC_BLOCK_REINDEXING__}!g" mempool-config.json sed -i "s!__MEMPOOL_POOLS_JSON_URL__!${__MEMPOOL_POOLS_JSON_URL__}!g" mempool-config.json sed -i "s!__MEMPOOL_POOLS_JSON_TREE_URL__!${__MEMPOOL_POOLS_JSON_TREE_URL__}!g" mempool-config.json sed -i "s!__MEMPOOL_AUDIT__!${__MEMPOOL_AUDIT__}!g" mempool-config.json @@ -158,53 +158,53 @@ sed -i "s!__MEMPOOL_CPFP_INDEXING__!${__MEMPOOL_CPFP_INDEXING__}!g" mempool-conf sed -i "s!__MEMPOOL_MAX_BLOCKS_BULK_QUERY__!${__MEMPOOL_MAX_BLOCKS_BULK_QUERY__}!g" mempool-config.json sed -i "s!__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__!${__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__}!g" mempool-config.json -sed -i "s/__CORE_RPC_HOST__/${__CORE_RPC_HOST__}/g" mempool-config.json -sed -i "s/__CORE_RPC_PORT__/${__CORE_RPC_PORT__}/g" mempool-config.json -sed -i "s/__CORE_RPC_USERNAME__/${__CORE_RPC_USERNAME__}/g" mempool-config.json -sed -i "s/__CORE_RPC_PASSWORD__/${__CORE_RPC_PASSWORD__}/g" mempool-config.json -sed -i "s/__CORE_RPC_TIMEOUT__/${__CORE_RPC_TIMEOUT__}/g" mempool-config.json +sed -i "s!__CORE_RPC_HOST__!${__CORE_RPC_HOST__}!g" mempool-config.json +sed -i "s!__CORE_RPC_PORT__!${__CORE_RPC_PORT__}!g" mempool-config.json +sed -i "s!__CORE_RPC_USERNAME__!${__CORE_RPC_USERNAME__}!g" mempool-config.json +sed -i "s!__CORE_RPC_PASSWORD__!${__CORE_RPC_PASSWORD__}!g" mempool-config.json +sed -i "s!__CORE_RPC_TIMEOUT__!${__CORE_RPC_TIMEOUT__}!g" mempool-config.json -sed -i "s/__ELECTRUM_HOST__/${__ELECTRUM_HOST__}/g" mempool-config.json -sed -i "s/__ELECTRUM_PORT__/${__ELECTRUM_PORT__}/g" mempool-config.json -sed -i "s/__ELECTRUM_TLS_ENABLED__/${__ELECTRUM_TLS_ENABLED__}/g" mempool-config.json +sed -i "s!__ELECTRUM_HOST__!${__ELECTRUM_HOST__}!g" mempool-config.json +sed -i "s!__ELECTRUM_PORT__!${__ELECTRUM_PORT__}!g" mempool-config.json +sed -i "s!__ELECTRUM_TLS_ENABLED__!${__ELECTRUM_TLS_ENABLED__}!g" mempool-config.json sed -i "s!__ESPLORA_REST_API_URL__!${__ESPLORA_REST_API_URL__}!g" mempool-config.json sed -i "s!__ESPLORA_UNIX_SOCKET_PATH__!${__ESPLORA_UNIX_SOCKET_PATH__}!g" mempool-config.json sed -i "s!__ESPLORA_RETRY_UNIX_SOCKET_AFTER__!${__ESPLORA_RETRY_UNIX_SOCKET_AFTER__}!g" mempool-config.json -sed -i "s/__SECOND_CORE_RPC_HOST__/${__SECOND_CORE_RPC_HOST__}/g" mempool-config.json -sed -i "s/__SECOND_CORE_RPC_PORT__/${__SECOND_CORE_RPC_PORT__}/g" mempool-config.json -sed -i "s/__SECOND_CORE_RPC_USERNAME__/${__SECOND_CORE_RPC_USERNAME__}/g" mempool-config.json -sed -i "s/__SECOND_CORE_RPC_PASSWORD__/${__SECOND_CORE_RPC_PASSWORD__}/g" mempool-config.json -sed -i "s/__SECOND_CORE_RPC_TIMEOUT__/${__SECOND_CORE_RPC_TIMEOUT__}/g" mempool-config.json +sed -i "s!__SECOND_CORE_RPC_HOST__!${__SECOND_CORE_RPC_HOST__}!g" mempool-config.json +sed -i "s!__SECOND_CORE_RPC_PORT__!${__SECOND_CORE_RPC_PORT__}!g" mempool-config.json +sed -i "s!__SECOND_CORE_RPC_USERNAME__!${__SECOND_CORE_RPC_USERNAME__}!g" mempool-config.json +sed -i "s!__SECOND_CORE_RPC_PASSWORD__!${__SECOND_CORE_RPC_PASSWORD__}!g" mempool-config.json +sed -i "s!__SECOND_CORE_RPC_TIMEOUT__!${__SECOND_CORE_RPC_TIMEOUT__}!g" mempool-config.json -sed -i "s/__DATABASE_ENABLED__/${__DATABASE_ENABLED__}/g" mempool-config.json -sed -i "s/__DATABASE_HOST__/${__DATABASE_HOST__}/g" mempool-config.json +sed -i "s!__DATABASE_ENABLED__!${__DATABASE_ENABLED__}!g" mempool-config.json +sed -i "s!__DATABASE_HOST__!${__DATABASE_HOST__}!g" mempool-config.json sed -i "s!__DATABASE_SOCKET__!${__DATABASE_SOCKET__}!g" mempool-config.json -sed -i "s/__DATABASE_PORT__/${__DATABASE_PORT__}/g" mempool-config.json -sed -i "s/__DATABASE_DATABASE__/${__DATABASE_DATABASE__}/g" mempool-config.json -sed -i "s/__DATABASE_USERNAME__/${__DATABASE_USERNAME__}/g" mempool-config.json -sed -i "s/__DATABASE_PASSWORD__/${__DATABASE_PASSWORD__}/g" mempool-config.json +sed -i "s!__DATABASE_PORT__!${__DATABASE_PORT__}!g" mempool-config.json +sed -i "s!__DATABASE_DATABASE__!${__DATABASE_DATABASE__}!g" mempool-config.json +sed -i "s!__DATABASE_USERNAME__!${__DATABASE_USERNAME__}!g" mempool-config.json +sed -i "s!__DATABASE_PASSWORD__!${__DATABASE_PASSWORD__}!g" mempool-config.json -sed -i "s/__SYSLOG_ENABLED__/${__SYSLOG_ENABLED__}/g" mempool-config.json -sed -i "s/__SYSLOG_HOST__/${__SYSLOG_HOST__}/g" mempool-config.json -sed -i "s/__SYSLOG_PORT__/${__SYSLOG_PORT__}/g" mempool-config.json -sed -i "s/__SYSLOG_MIN_PRIORITY__/${__SYSLOG_MIN_PRIORITY__}/g" mempool-config.json -sed -i "s/__SYSLOG_FACILITY__/${__SYSLOG_FACILITY__}/g" mempool-config.json +sed -i "s!__SYSLOG_ENABLED__!${__SYSLOG_ENABLED__}!g" mempool-config.json +sed -i "s!__SYSLOG_HOST__!${__SYSLOG_HOST__}!g" mempool-config.json +sed -i "s!__SYSLOG_PORT__!${__SYSLOG_PORT__}!g" mempool-config.json +sed -i "s!__SYSLOG_MIN_PRIORITY__!${__SYSLOG_MIN_PRIORITY__}!g" mempool-config.json +sed -i "s!__SYSLOG_FACILITY__!${__SYSLOG_FACILITY__}!g" mempool-config.json -sed -i "s/__STATISTICS_ENABLED__/${__STATISTICS_ENABLED__}/g" mempool-config.json -sed -i "s/__STATISTICS_TX_PER_SECOND_SAMPLE_PERIOD__/${__STATISTICS_TX_PER_SECOND_SAMPLE_PERIOD__}/g" mempool-config.json +sed -i "s!__STATISTICS_ENABLED__!${__STATISTICS_ENABLED__}!g" mempool-config.json +sed -i "s!__STATISTICS_TX_PER_SECOND_SAMPLE_PERIOD__!${__STATISTICS_TX_PER_SECOND_SAMPLE_PERIOD__}!g" mempool-config.json -sed -i "s/__BISQ_ENABLED__/${__BISQ_ENABLED__}/g" mempool-config.json +sed -i "s!__BISQ_ENABLED__!${__BISQ_ENABLED__}!g" mempool-config.json sed -i "s!__BISQ_DATA_PATH__!${__BISQ_DATA_PATH__}!g" mempool-config.json -sed -i "s/__SOCKS5PROXY_ENABLED__/${__SOCKS5PROXY_ENABLED__}/g" mempool-config.json -sed -i "s/__SOCKS5PROXY_USE_ONION__/${__SOCKS5PROXY_USE_ONION__}/g" mempool-config.json -sed -i "s/__SOCKS5PROXY_HOST__/${__SOCKS5PROXY_HOST__}/g" mempool-config.json -sed -i "s/__SOCKS5PROXY_PORT__/${__SOCKS5PROXY_PORT__}/g" mempool-config.json -sed -i "s/__SOCKS5PROXY_USERNAME__/${__SOCKS5PROXY_USERNAME__}/g" mempool-config.json -sed -i "s/__SOCKS5PROXY_PASSWORD__/${__SOCKS5PROXY_PASSWORD__}/g" mempool-config.json +sed -i "s!__SOCKS5PROXY_ENABLED__!${__SOCKS5PROXY_ENABLED__}!g" mempool-config.json +sed -i "s!__SOCKS5PROXY_USE_ONION__!${__SOCKS5PROXY_USE_ONION__}!g" mempool-config.json +sed -i "s!__SOCKS5PROXY_HOST__!${__SOCKS5PROXY_HOST__}!g" mempool-config.json +sed -i "s!__SOCKS5PROXY_PORT__!${__SOCKS5PROXY_PORT__}!g" mempool-config.json +sed -i "s!__SOCKS5PROXY_USERNAME__!${__SOCKS5PROXY_USERNAME__}!g" mempool-config.json +sed -i "s!__SOCKS5PROXY_PASSWORD__!${__SOCKS5PROXY_PASSWORD__}!g" mempool-config.json sed -i "s!__PRICE_DATA_SERVER_TOR_URL__!${__PRICE_DATA_SERVER_TOR_URL__}!g" mempool-config.json sed -i "s!__PRICE_DATA_SERVER_CLEARNET_URL__!${__PRICE_DATA_SERVER_CLEARNET_URL__}!g" mempool-config.json From b54dbcd5f843f554c6daa8d778b7c51189f2fca1 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sun, 14 May 2023 13:14:44 -0700 Subject: [PATCH 02/10] Add a test to check if all the vars are on the Docker script --- backend/src/__tests__/config.test.ts | 48 ++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index aa287308b..4971403af 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -152,4 +152,52 @@ describe('Mempool Backend Config', () => { expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER); }); }); + + test('should ensure the docker start.sh script has default values', () => { + jest.isolateModules(() => { + const startSh = fs.readFileSync(`${__dirname}/../../../docker/backend/start.sh`, 'utf-8'); + const fixture = JSON.parse(fs.readFileSync(`${__dirname}/../__fixtures__/mempool-config.template.json`, 'utf8')); + + function parseJson(jsonObj, root?) { + for (const [key, value] of Object.entries(jsonObj)) { + // We have a few cases where we can't follow the pattern + if (root === 'MEMPOOL' && key === 'HTTP_PORT') { + console.log('skipping check for MEMPOOL_HTTP_PORT'); + return; + } + switch (typeof value) { + case 'object': { + if (Array.isArray(value)) { + return; + } else { + parseJson(value, key); + } + break; + } + default: { + //The flattened string, i.e, __MEMPOOL_ENABLED__ + const replaceStr = `${root ? '__' + root + '_' : '__'}${key}__`; + + //The string used as the environment variable, i.e, MEMPOOL_ENABLED + const envVarStr = `${root ? root : ''}_${key}`; + + //The string used as the default value, to be checked as a regex, i.e, __MEMPOOL_ENABLED__=${MEMPOOL_ENABLED:=(.*)} + const defaultEntry = replaceStr + '=' + '\\${' + envVarStr + ':=(.*)' + '}'; + + console.log(`looking for ${defaultEntry} in the start.sh script`); + const re = new RegExp(defaultEntry); + expect(startSh).toMatch(re); + + //The string that actually replaces the values in the config file + const sedStr = 'sed -i "s!' + replaceStr + '!${' + replaceStr + '}!g" mempool-config.json'; + console.log(`looking for ${sedStr} in the start.sh script`); + expect(startSh).toContain(sedStr); + break; + } + } + } + } + parseJson(fixture); + }); + }); }); From 730afb8f6864f67dbe28a97e94a7540c143bf8a7 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sun, 14 May 2023 13:20:18 -0700 Subject: [PATCH 03/10] Add missing DATABASE_TIMEOUT config --- docker/backend/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 54f0825d5..f1ddba3fd 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -181,11 +181,11 @@ sed -i "s!__SECOND_CORE_RPC_TIMEOUT__!${__SECOND_CORE_RPC_TIMEOUT__}!g" mempool- sed -i "s!__DATABASE_ENABLED__!${__DATABASE_ENABLED__}!g" mempool-config.json sed -i "s!__DATABASE_HOST__!${__DATABASE_HOST__}!g" mempool-config.json sed -i "s!__DATABASE_SOCKET__!${__DATABASE_SOCKET__}!g" mempool-config.json - sed -i "s!__DATABASE_PORT__!${__DATABASE_PORT__}!g" mempool-config.json sed -i "s!__DATABASE_DATABASE__!${__DATABASE_DATABASE__}!g" mempool-config.json sed -i "s!__DATABASE_USERNAME__!${__DATABASE_USERNAME__}!g" mempool-config.json sed -i "s!__DATABASE_PASSWORD__!${__DATABASE_PASSWORD__}!g" mempool-config.json +sed -i "s!__DATABASE_TIMEOUT__!${__DATABASE_TIMEOUT__}!g" mempool-config.json sed -i "s!__SYSLOG_ENABLED__!${__SYSLOG_ENABLED__}!g" mempool-config.json sed -i "s!__SYSLOG_HOST__!${__SYSLOG_HOST__}!g" mempool-config.json From e2f1ae9cf67b398424206004554d4a119276bf7d Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sun, 14 May 2023 13:26:35 -0700 Subject: [PATCH 04/10] Add missing Lightning config --- docker/backend/start.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/backend/start.sh b/docker/backend/start.sh index f1ddba3fd..cb1108a05 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -108,6 +108,8 @@ __LIGHTNING_TOPOLOGY_FOLDER__=${LIGHTNING_TOPOLOGY_FOLDER:=""} __LIGHTNING_STATS_REFRESH_INTERVAL__=${LIGHTNING_STATS_REFRESH_INTERVAL:=600} __LIGHTNING_GRAPH_REFRESH_INTERVAL__=${LIGHTNING_GRAPH_REFRESH_INTERVAL:=600} __LIGHTNING_LOGGER_UPDATE_INTERVAL__=${LIGHTNING_LOGGER_UPDATE_INTERVAL:=30} +__LIGHTNING_FORENSICS_INTERVAL__=${LIGHTNING_FORENSICS_INTERVAL:=43200} +__LIGHTNING_FORENSICS_RATE_LIMIT__=${LIGHTNING_FORENSICS_RATE_LIMIT:=20} # LND __LND_TLS_CERT_PATH__=${LND_TLS_CERT_PATH:=""} @@ -223,6 +225,8 @@ sed -i "s!__LIGHTNING_TOPOLOGY_FOLDER__!${__LIGHTNING_TOPOLOGY_FOLDER__}!g" memp sed -i "s!__LIGHTNING_STATS_REFRESH_INTERVAL__!${__LIGHTNING_STATS_REFRESH_INTERVAL__}!g" mempool-config.json sed -i "s!__LIGHTNING_GRAPH_REFRESH_INTERVAL__!${__LIGHTNING_GRAPH_REFRESH_INTERVAL__}!g" mempool-config.json sed -i "s!__LIGHTNING_LOGGER_UPDATE_INTERVAL__!${__LIGHTNING_LOGGER_UPDATE_INTERVAL__}!g" mempool-config.json +sed -i "s!__LIGHTNING_FORENSICS_INTERVAL__!${__LIGHTNING_FORENSICS_INTERVAL__}!g" mempool-config.json +sed -i "s!__LIGHTNING_FORENSICS_RATE_LIMIT__!${__LIGHTNING_FORENSICS_RATE_LIMIT__}!g" mempool-config.json # LND sed -i "s!__LND_TLS_CERT_PATH__!${__LND_TLS_CERT_PATH__}!g" mempool-config.json From 270bf760361c5c2b182733f5fb78c30818d8c4e1 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sun, 14 May 2023 14:17:53 -0700 Subject: [PATCH 05/10] Add a test to check that the docker json is in sync with the template --- backend/src/__tests__/config.test.ts | 42 ++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 4971403af..278d83f50 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -200,4 +200,46 @@ describe('Mempool Backend Config', () => { parseJson(fixture); }); }); + + test('should ensure that the mempool-config.json Docker template has all the keys', () => { + jest.isolateModules(() => { + const fixture = JSON.parse(fs.readFileSync(`${__dirname}/../__fixtures__/mempool-config.template.json`, 'utf8')); + const dockerJson = fs.readFileSync(`${__dirname}/../../../docker/backend/mempool-config.json`, 'utf-8'); + + function parseJson(jsonObj, root?) { + for (const [key, value] of Object.entries(jsonObj)) { + switch (typeof value) { + case 'object': { + if (Array.isArray(value)) { + // numbers, arrays and booleans won't be enclosed by quotes + const replaceStr = `${root ? '__' + root + '_' : '__'}${key}__`; + expect(dockerJson).toContain(`"${key}": ${replaceStr}`); + break; + } else { + //Check for top level config keys + expect(dockerJson).toContain(`"${key}"`); + parseJson(value, key); + break; + } + } + case 'string': { + // strings should be enclosed by quotes + const replaceStr = `${root ? '__' + root + '_' : '__'}${key}__`; + expect(dockerJson).toContain(`"${key}": "${replaceStr}"`); + break; + } + default: { + // numbers, arrays and booleans won't be enclosed by quotes + const replaceStr = `${root ? '__' + root + '_' : '__'}${key}__`; + expect(dockerJson).toContain(`"${key}": ${replaceStr}`); + break; + } + } + }; + } + parseJson(fixture); + }); + }); + + }); From 0253904b5a0dc00b9b88c392b262804eb6c3851e Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sun, 14 May 2023 14:25:23 -0700 Subject: [PATCH 06/10] Update the backend config fixture --- .../__fixtures__/mempool-config.template.json | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index eb082d89f..919784464 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -16,7 +16,7 @@ "INITIAL_BLOCKS_AMOUNT": 7, "MEMPOOL_BLOCKS_AMOUNT": 8, "USE_SECOND_NODE_FOR_MINFEE": 10, - "EXTERNAL_ASSETS": 11, + "EXTERNAL_ASSETS": [], "EXTERNAL_MAX_RETRY": 12, "EXTERNAL_RETRY_INTERVAL": 13, "USER_AGENT": "__MEMPOOL_USER_AGENT__", @@ -24,19 +24,19 @@ "INDEXING_BLOCKS_AMOUNT": 14, "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__", "POOLS_JSON_URL": "__POOLS_JSON_URL__", - "AUDIT": "__MEMPOOL_AUDIT__", - "ADVANCED_GBT_AUDIT": "__MEMPOOL_ADVANCED_GBT_AUDIT__", - "ADVANCED_GBT_MEMPOOL": "__MEMPOOL_ADVANCED_GBT_MEMPOOL__", - "CPFP_INDEXING": "__MEMPOOL_CPFP_INDEXING__", - "MAX_BLOCKS_BULK_QUERY": "__MEMPOOL_MAX_BLOCKS_BULK_QUERY__", - "DISK_CACHE_BLOCK_INTERVAL": "__MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__" + "AUDIT": true, + "ADVANCED_GBT_AUDIT": true, + "ADVANCED_GBT_MEMPOOL": true, + "CPFP_INDEXING": true, + "MAX_BLOCKS_BULK_QUERY": 999, + "DISK_CACHE_BLOCK_INTERVAL": 999 }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", "PORT": 15, "USERNAME": "__CORE_RPC_USERNAME__", "PASSWORD": "__CORE_RPC_PASSWORD__", - "TIMEOUT": "__CORE_RPC_TIMEOUT__" + "TIMEOUT": 1000 }, "ELECTRUM": { "HOST": "__ELECTRUM_HOST__", @@ -46,14 +46,14 @@ "ESPLORA": { "REST_API_URL": "__ESPLORA_REST_API_URL__", "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__", - "RETRY_UNIX_SOCKET_AFTER": "__ESPLORA_RETRY_UNIX_SOCKET_AFTER__" + "RETRY_UNIX_SOCKET_AFTER": 888 }, "SECOND_CORE_RPC": { "HOST": "__SECOND_CORE_RPC_HOST__", "PORT": 17, "USERNAME": "__SECOND_CORE_RPC_USERNAME__", "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__", - "TIMEOUT": "__SECOND_CORE_RPC_TIMEOUT__" + "TIMEOUT": 2000 }, "DATABASE": { "ENABLED": false, @@ -63,7 +63,7 @@ "DATABASE": "__DATABASE_DATABASE__", "USERNAME": "__DATABASE_USERNAME__", "PASSWORD": "__DATABASE_PASSWORD__", - "TIMEOUT": "__DATABASE_TIMEOUT__" + "TIMEOUT": 3000 }, "SYSLOG": { "ENABLED": false, @@ -101,14 +101,14 @@ "BISQ_ONION": "__EXTERNAL_DATA_SERVER_BISQ_ONION__" }, "LIGHTNING": { - "ENABLED": "__LIGHTNING_ENABLED__", + "ENABLED": true, "BACKEND": "__LIGHTNING_BACKEND__", "TOPOLOGY_FOLDER": "__LIGHTNING_TOPOLOGY_FOLDER__", "STATS_REFRESH_INTERVAL": 600, "GRAPH_REFRESH_INTERVAL": 600, "LOGGER_UPDATE_INTERVAL": 30, "FORENSICS_INTERVAL": 43200, - "FORENSICS_RATE_LIMIT": "__FORENSICS_RATE_LIMIT__" + "FORENSICS_RATE_LIMIT": 1234 }, "LND": { "TLS_CERT_PATH": "", @@ -119,4 +119,4 @@ "CLIGHTNING": { "SOCKET": "__CLIGHTNING_SOCKET__" } -} +} \ No newline at end of file From a79d715ad416634a60248134c21e7501cd9ffea8 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn <100320+knorrium@users.noreply.github.com> Date: Sun, 14 May 2023 14:26:38 -0700 Subject: [PATCH 07/10] Fix Docker config file to use the correct types --- docker/backend/mempool-config.json | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index fd8abaf02..06ed51f41 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -27,7 +27,9 @@ "ADVANCED_GBT_MEMPOOL": __MEMPOOL_ADVANCED_GBT_MEMPOOL__, "CPFP_INDEXING": __MEMPOOL_CPFP_INDEXING__, "MAX_BLOCKS_BULK_QUERY": __MEMPOOL_MAX_BLOCKS_BULK_QUERY__, - "DISK_CACHE_BLOCK_INTERVAL": __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__ + "DISK_CACHE_BLOCK_INTERVAL": __MEMPOOL_DISK_CACHE_BLOCK_INTERVAL__, + "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", + "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__" }, "CORE_RPC": { "HOST": "__CORE_RPC_HOST__", @@ -61,7 +63,7 @@ "DATABASE": "__DATABASE_DATABASE__", "USERNAME": "__DATABASE_USERNAME__", "PASSWORD": "__DATABASE_PASSWORD__", - "TIMEOUT": "__DATABASE_TIMEOUT__" + "TIMEOUT": __DATABASE_TIMEOUT__ }, "SYSLOG": { "ENABLED": __SYSLOG_ENABLED__, @@ -84,13 +86,15 @@ "STATS_REFRESH_INTERVAL": __LIGHTNING_STATS_REFRESH_INTERVAL__, "GRAPH_REFRESH_INTERVAL": __LIGHTNING_GRAPH_REFRESH_INTERVAL__, "LOGGER_UPDATE_INTERVAL": __LIGHTNING_LOGGER_UPDATE_INTERVAL__, - "TOPOLOGY_FOLDER": "__LIGHTNING_TOPOLOGY_FOLDER__" + "TOPOLOGY_FOLDER": "__LIGHTNING_TOPOLOGY_FOLDER__", + "FORENSICS_INTERVAL": __LIGHTNING_FORENSICS_INTERVAL__, + "FORENSICS_RATE_LIMIT": __LIGHTNING_FORENSICS_RATE_LIMIT__ }, "LND": { "TLS_CERT_PATH": "__LND_TLS_CERT_PATH__", "MACAROON_PATH": "__LND_MACAROON_PATH__", "REST_API_URL": "__LND_REST_API_URL__", - "TIMEOUT": "__LND_TIMEOUT__" + "TIMEOUT": __LND_TIMEOUT__ }, "CLIGHTNING": { "SOCKET": "__CLIGHTNING_SOCKET__" @@ -121,4 +125,4 @@ "GEOLITE2_ASN": "__MAXMIND_GEOLITE2_ASN__", "GEOIP2_ISP": "__MAXMIND_GEOIP2_ISP__" } -} +} \ No newline at end of file From 9b34d2245a105bd6cbb034a649d1ed842e1fe926 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Thu, 25 May 2023 01:29:55 -0400 Subject: [PATCH 08/10] Add bull bitcoin as enterprise sponsor --- frontend/src/app/components/about/about.component.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index e67facfe1..aa604f678 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -119,6 +119,10 @@ Gemini + + + Bull Bitcoin + From 748665d3f6041de1ad14ae7a5fa640815f2f5794 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Thu, 25 May 2023 15:58:48 -0400 Subject: [PATCH 09/10] Make svg pretty --- frontend/src/app/components/about/about.component.html | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index aa604f678..8655ab800 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -120,7 +120,15 @@ Gemini - + Bull Bitcoin From d51e94c93533220c0467aeb08df39be05d7add58 Mon Sep 17 00:00:00 2001 From: hunicus <93150691+hunicus@users.noreply.github.com> Date: Fri, 26 May 2023 17:25:52 -0400 Subject: [PATCH 10/10] Switch tm to r for registered trademarks --- frontend/src/app/components/about/about.component.html | 4 ++-- .../trademark-policy/trademark-policy.component.html | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index 9a778be44..c59f700a8 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -1,7 +1,7 @@
- ™ + ®
v{{ packetJsonVersion }} [{{ frontendGitCommitHash }}] @@ -396,7 +396,7 @@ Trademark Notice

- The Mempool Open Source Project™, mempool.space™, the mempool logo™, the mempool.space logos™, the mempool square logo™, and the mempool blocks logo™ are either registered trademarks or trademarks of Mempool Space K.K in Japan, the United States, and/or other countries. + The Mempool Open Source Project™, mempool.space™, the mempool logo®, the mempool.space logos™, the mempool square logo®, and the mempool blocks logo™ are either registered trademarks or trademarks of Mempool Space K.K in Japan, the United States, and/or other countries.

While our software is available under an open source software license, the copyright license does not include an implied right or license to use our trademarks. See our Trademark Policy and Guidelines for more details, published on <https://mempool.space/trademark-policy>. diff --git a/frontend/src/app/components/trademark-policy/trademark-policy.component.html b/frontend/src/app/components/trademark-policy/trademark-policy.component.html index 4526a6c0c..4f9419642 100644 --- a/frontend/src/app/components/trademark-policy/trademark-policy.component.html +++ b/frontend/src/app/components/trademark-policy/trademark-policy.component.html @@ -304,7 +304,7 @@

Also, if you are using our Marks in a way described in the sections "Uses for Which We Are Granting a License," you must include the following trademark attribution at the foot of the webpage where you have used the Mark (or, if in a book, on the credits page), on any packaging or labeling, and on advertising or marketing materials:

-

“The Mempool Space K.K.™, The Mempool Open Source Project™, mempool.space™, the mempool logo™, the mempool.space logos™, the mempool square logo™, and the mempool blocks logo™ are either registered trademarks or trademarks of Mempool Space K.K in Japan, the United States, and/or other countries, and are used with permission. Mempool Space K.K. has no affiliation with and does not sponsor or endorse the information provided herein.”

+

“The Mempool Space K.K.™, The Mempool Open Source Project™, mempool.space™, the mempool logo®, the mempool.space logos™, the mempool square logo®, and the mempool blocks logo™ are either registered trademarks or trademarks of Mempool Space K.K in Japan, the United States, and/or other countries, and are used with permission. Mempool Space K.K. has no affiliation with and does not sponsor or endorse the information provided herein.”

  • What to Do When You See Abuse