From 87a613e4dc6344106d441b3d8130303721aaaae0 Mon Sep 17 00:00:00 2001 From: wiz Date: Tue, 6 Feb 2024 13:53:03 -0500 Subject: [PATCH 1/5] ops: Fix broken nginx cache configuration --- production/nginx/http-proxy-cache.conf | 14 +++++++++----- production/nginx/location-api-v1-lightning.conf | 2 +- production/nginx/location-api.conf | 10 +++++----- production/nginx/location-liquid-api.conf | 8 ++++---- production/nginx/location-liquidtestnet-api.conf | 8 ++++---- .../nginx/location-signet-api-v1-lightning.conf | 2 +- production/nginx/location-signet-api.conf | 8 ++++---- .../nginx/location-testnet-api-v1-lightning.conf | 2 +- production/nginx/location-testnet-api.conf | 8 ++++---- 9 files changed, 33 insertions(+), 29 deletions(-) diff --git a/production/nginx/http-proxy-cache.conf b/production/nginx/http-proxy-cache.conf index 5505b094b..0024a3b30 100644 --- a/production/nginx/http-proxy-cache.conf +++ b/production/nginx/http-proxy-cache.conf @@ -1,8 +1,12 @@ # proxy cache -proxy_cache_path /var/cache/nginx/api keys_zone=api:20m levels=1:2 inactive=365d max_size=2000m; -proxy_cache_path /var/cache/nginx/unfurler keys_zone=unfurler:20m levels=1:2 inactive=365d max_size=2000m; -proxy_cache_path /var/cache/nginx/slurper keys_zone=slurper:20m levels=1:2 inactive=365d max_size=5000m; -proxy_cache_path /var/cache/nginx/services keys_zone=services:20m levels=1:2 inactive=365d max_size=100m; +proxy_cache_path /var/cache/nginx/services keys_zone=services:20m levels=1:2 inactive=30d max_size=200m; +proxy_cache_path /var/cache/nginx/apihot keys_zone=apihot:20m levels=1:2 inactive=60m max_size=20m; +proxy_cache_path /var/cache/nginx/apiwarm keys_zone=apiwarm:20m levels=1:2 inactive=24h max_size=200m; +proxy_cache_path /var/cache/nginx/apinormal keys_zone=apinormal:200m levels=1:2 inactive=30d max_size=2000m; +proxy_cache_path /var/cache/nginx/apicold keys_zone=apicold:200m levels=1:2 inactive=365d max_size=2000m; + +proxy_cache_path /var/cache/nginx/unfurler keys_zone=unfurler:200m levels=1:2 inactive=30d max_size=2000m; +proxy_cache_path /var/cache/nginx/slurper keys_zone=slurper:500m levels=1:2 inactive=365d max_size=5000m; proxy_cache_path /var/cache/nginx/markets keys_zone=markets:20m levels=1:2 inactive=365d max_size=100m; types_hash_max_size 4096; -proxy_buffer_size 8k; \ No newline at end of file +proxy_buffer_size 8k; diff --git a/production/nginx/location-api-v1-lightning.conf b/production/nginx/location-api-v1-lightning.conf index f90fd529a..625bebade 100644 --- a/production/nginx/location-api-v1-lightning.conf +++ b/production/nginx/location-api-v1-lightning.conf @@ -12,7 +12,7 @@ location @mempool-api-v1-lightning { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apiwarm; proxy_cache_valid 200 10s; proxy_redirect off; diff --git a/production/nginx/location-api.conf b/production/nginx/location-api.conf index f245154a0..fccdde532 100644 --- a/production/nginx/location-api.conf +++ b/production/nginx/location-api.conf @@ -77,7 +77,7 @@ location @mempool-api-v1-cache-forever { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apicold; proxy_cache_valid 200 30d; proxy_redirect off; @@ -94,7 +94,7 @@ location @mempool-api-v1-cache-hot { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apihot; proxy_cache_valid 200 1s; proxy_redirect off; } @@ -109,7 +109,7 @@ location @mempool-api-v1-cache-warm { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apiwarm; proxy_cache_valid 200 10s; proxy_redirect off; } @@ -122,7 +122,7 @@ location @mempool-api-v1-cache-normal { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_cache api; + proxy_cache apinormal; proxy_cache_valid 200 2s; proxy_redirect off; @@ -167,7 +167,7 @@ location @esplora-api-cache-forever { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apicold; proxy_cache_valid 200 30d; proxy_redirect off; diff --git a/production/nginx/location-liquid-api.conf b/production/nginx/location-liquid-api.conf index 6c222c469..5286c1a72 100644 --- a/production/nginx/location-liquid-api.conf +++ b/production/nginx/location-liquid-api.conf @@ -75,7 +75,7 @@ location @mempool-liquid-api-v1-cache-forever { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apicold; proxy_cache_valid 200 30d; proxy_redirect off; @@ -92,7 +92,7 @@ location @mempool-liquid-api-v1-cache-warm { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apiwarm; proxy_cache_valid 200 10s; proxy_redirect off; } @@ -105,7 +105,7 @@ location @mempool-liquid-api-v1-cache-normal { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_cache api; + proxy_cache apinormal; proxy_cache_valid 200 10s; proxy_redirect off; @@ -150,7 +150,7 @@ location @esplora-liquid-api-cache-forever { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apicold; proxy_cache_valid 200 30d; proxy_redirect off; diff --git a/production/nginx/location-liquidtestnet-api.conf b/production/nginx/location-liquidtestnet-api.conf index 5d5be5d43..4f6615b40 100644 --- a/production/nginx/location-liquidtestnet-api.conf +++ b/production/nginx/location-liquidtestnet-api.conf @@ -79,7 +79,7 @@ location @mempool-liquidtestnet-api-v1-cache-forever { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apicold; proxy_cache_valid 200 30d; proxy_redirect off; @@ -96,7 +96,7 @@ location @mempool-liquidtestnet-api-v1-cache-warm { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apiwarm; proxy_cache_valid 200 10s; proxy_redirect off; } @@ -109,7 +109,7 @@ location @mempool-liquidtestnet-api-v1-cache-normal { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_cache api; + proxy_cache apinormal; proxy_cache_valid 200 10s; proxy_redirect off; @@ -154,7 +154,7 @@ location @esplora-liquidtestnet-api-cache-forever { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apicold; proxy_cache_valid 200 30d; proxy_redirect off; diff --git a/production/nginx/location-signet-api-v1-lightning.conf b/production/nginx/location-signet-api-v1-lightning.conf index ab14a170b..4fe0b6ddd 100644 --- a/production/nginx/location-signet-api-v1-lightning.conf +++ b/production/nginx/location-signet-api-v1-lightning.conf @@ -13,7 +13,7 @@ location @mempool-signet-api-v1-lightning { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apiwarm; proxy_cache_valid 200 10s; proxy_redirect off; diff --git a/production/nginx/location-signet-api.conf b/production/nginx/location-signet-api.conf index 8469043a8..eb8cbeb6a 100644 --- a/production/nginx/location-signet-api.conf +++ b/production/nginx/location-signet-api.conf @@ -79,7 +79,7 @@ location @mempool-signet-api-v1-cache-forever { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apicold; proxy_cache_valid 200 30d; proxy_redirect off; @@ -96,7 +96,7 @@ location @mempool-signet-api-v1-cache-warm { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apiwarm; proxy_cache_valid 200 10s; proxy_redirect off; } @@ -109,7 +109,7 @@ location @mempool-signet-api-v1-cache-normal { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_cache api; + proxy_cache apinormal; proxy_cache_valid 200 10s; proxy_redirect off; @@ -154,7 +154,7 @@ location @esplora-signet-api-cache-forever { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apicold; proxy_cache_valid 200 30d; proxy_redirect off; diff --git a/production/nginx/location-testnet-api-v1-lightning.conf b/production/nginx/location-testnet-api-v1-lightning.conf index cc7c617a6..316b37cc3 100644 --- a/production/nginx/location-testnet-api-v1-lightning.conf +++ b/production/nginx/location-testnet-api-v1-lightning.conf @@ -13,7 +13,7 @@ location @mempool-testnet-api-v1-lightning { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apiwarm; proxy_cache_valid 200 10s; proxy_redirect off; diff --git a/production/nginx/location-testnet-api.conf b/production/nginx/location-testnet-api.conf index 9f0c41147..cce3e585c 100644 --- a/production/nginx/location-testnet-api.conf +++ b/production/nginx/location-testnet-api.conf @@ -79,7 +79,7 @@ location @mempool-testnet-api-v1-cache-forever { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apicold; proxy_cache_valid 200 30d; proxy_redirect off; @@ -96,7 +96,7 @@ location @mempool-testnet-api-v1-cache-warm { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apiwarm; proxy_cache_valid 200 10s; proxy_redirect off; } @@ -109,7 +109,7 @@ location @mempool-testnet-api-v1-cache-normal { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_cache api; + proxy_cache apinormal; proxy_cache_valid 200 10s; proxy_redirect off; @@ -154,7 +154,7 @@ location @esplora-testnet-api-cache-forever { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apicold; proxy_cache_valid 200 30d; proxy_redirect off; From f00470589695811435405afc3690c9cd45ae0531 Mon Sep 17 00:00:00 2001 From: wiz Date: Tue, 6 Feb 2024 13:55:44 -0500 Subject: [PATCH 2/5] ops: Add missing HTTP Expires header for fees API --- production/nginx/location-api.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/production/nginx/location-api.conf b/production/nginx/location-api.conf index fccdde532..bee4ce50d 100644 --- a/production/nginx/location-api.conf +++ b/production/nginx/location-api.conf @@ -97,6 +97,8 @@ location @mempool-api-v1-cache-hot { proxy_cache apihot; proxy_cache_valid 200 1s; proxy_redirect off; + + expires 1s; } location @mempool-api-v1-cache-warm { From fea115bcbca69980b4688da12256115130e3216f Mon Sep 17 00:00:00 2001 From: wiz Date: Tue, 6 Feb 2024 14:03:44 -0500 Subject: [PATCH 3/5] ops: Fix obvious bug in cache heater script --- production/nginx-cache-heater | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/nginx-cache-heater b/production/nginx-cache-heater index 00fc6ea7a..4bbe8ee15 100755 --- a/production/nginx-cache-heater +++ b/production/nginx-cache-heater @@ -1,5 +1,5 @@ #!/usr/bin/env zsh -hostname=mempool.space +hostname=$(hostname) heat() { From 1547c2c4775330b639a8ad072e87df6e6db615b0 Mon Sep 17 00:00:00 2001 From: James Blacklock Date: Tue, 6 Feb 2024 22:13:10 -0500 Subject: [PATCH 4/5] fix syntax error in mempool-config.json --- docker/backend/mempool-config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index aefb095cf..8f69fd0c1 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -35,7 +35,7 @@ "ALLOW_UNREACHABLE": __MEMPOOL_ALLOW_UNREACHABLE__, "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", - "PRICE_UPDATES_PER_HOUR": __MEMPOOL_PRICE_UPDATES_PER_HOUR__ + "PRICE_UPDATES_PER_HOUR": __MEMPOOL_PRICE_UPDATES_PER_HOUR__, "MAX_TRACKED_ADDRESSES": __MEMPOOL_MAX_TRACKED_ADDRESSES__ }, "CORE_RPC": { From 624cbf05c02036fa79ae0b4a5dd7769b08633ef6 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Tue, 6 Feb 2024 19:47:09 -0800 Subject: [PATCH 5/5] Add a step to validate the JSON config generated for Docker --- .github/workflows/ci.yml | 30 +++++++++++++++++++++++++++++- .gitignore | 1 + 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c435d6ea5..0961f1c88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -329,4 +329,32 @@ jobs: CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }} - \ No newline at end of file + + validate_docker_json: + if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')" + runs-on: "ubuntu-latest" + name: Validate generated backend Docker JSON + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + path: docker + + - name: Install jq + run: sudo apt-get install jq -y + + - name: Create new start script to run on CI + run: | + sed '$d' start.sh > start_ci.sh + working-directory: docker/docker/backend + + - name: Run the script to generate the sample JSON + run: | + sh start_ci.sh + working-directory: docker/docker/backend + + - name: Validate JSON syntax + run: | + cat mempool-config.json | jq + working-directory: docker/docker/backend diff --git a/.gitignore b/.gitignore index 4f19f2522..381f2187c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ backend/mempool-config.json frontend/src/resources/config.template.js frontend/src/resources/config.js target +docker/backend/start_ci.sh \ No newline at end of file