From 5f3f48354622901e13269ba639cce083922a2880 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 Jan 2024 02:07:07 +0000 Subject: [PATCH 01/10] Bump mysql2 from 3.7.0 to 3.9.1 in /backend Bumps [mysql2](https://github.com/sidorares/node-mysql2) from 3.7.0 to 3.9.1. - [Release notes](https://github.com/sidorares/node-mysql2/releases) - [Changelog](https://github.com/sidorares/node-mysql2/blob/master/Changelog.md) - [Commits](https://github.com/sidorares/node-mysql2/compare/v3.7.0...v3.9.1) --- updated-dependencies: - dependency-name: mysql2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- backend/package-lock.json | 14 +++++++------- backend/package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/backend/package-lock.json b/backend/package-lock.json index 4d16fa012..1794934d2 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -17,7 +17,7 @@ "crypto-js": "~4.2.0", "express": "~4.18.2", "maxmind": "~4.3.11", - "mysql2": "~3.7.0", + "mysql2": "~3.9.1", "redis": "^4.6.6", "rust-gbt": "file:./rust-gbt", "socks-proxy-agent": "~7.0.0", @@ -6110,9 +6110,9 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "node_modules/mysql2": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.7.0.tgz", - "integrity": "sha512-c45jA3Jc1X8yJKzrWu1GpplBKGwv/wIV6ITZTlCSY7npF2YfJR+6nMP5e+NTQhUeJPSyOQAbGDCGEHbAl8HN9w==", + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.9.1.tgz", + "integrity": "sha512-3njoWAAhGBYy0tWBabqUQcLtczZUxrmmtc2vszQUekg3kTJyZ5/IeLC3Fo04u6y6Iy5Sba7pIIa2P/gs8D3ZeQ==", "dependencies": { "denque": "^2.1.0", "generate-function": "^2.3.1", @@ -12230,9 +12230,9 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, "mysql2": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.7.0.tgz", - "integrity": "sha512-c45jA3Jc1X8yJKzrWu1GpplBKGwv/wIV6ITZTlCSY7npF2YfJR+6nMP5e+NTQhUeJPSyOQAbGDCGEHbAl8HN9w==", + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/mysql2/-/mysql2-3.9.1.tgz", + "integrity": "sha512-3njoWAAhGBYy0tWBabqUQcLtczZUxrmmtc2vszQUekg3kTJyZ5/IeLC3Fo04u6y6Iy5Sba7pIIa2P/gs8D3ZeQ==", "requires": { "denque": "^2.1.0", "generate-function": "^2.3.1", diff --git a/backend/package.json b/backend/package.json index cd1255392..c42f455e5 100644 --- a/backend/package.json +++ b/backend/package.json @@ -47,7 +47,7 @@ "crypto-js": "~4.2.0", "express": "~4.18.2", "maxmind": "~4.3.11", - "mysql2": "~3.7.0", + "mysql2": "~3.9.1", "rust-gbt": "file:./rust-gbt", "redis": "^4.6.6", "socks-proxy-agent": "~7.0.0", From 933bc47b445688977de709b1d8f5a6473a28db30 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Sun, 4 Feb 2024 09:17:42 -0800 Subject: [PATCH 02/10] Add DRY_RUN and MEMPOOL_CDN options --- frontend/sync-assets.js | 147 +++++++++++++++++++++++++++++++--------- 1 file changed, 116 insertions(+), 31 deletions(-) diff --git a/frontend/sync-assets.js b/frontend/sync-assets.js index b15b5e315..2ff642e27 100644 --- a/frontend/sync-assets.js +++ b/frontend/sync-assets.js @@ -4,6 +4,8 @@ var crypto = require('crypto'); var path = require('node:path'); const LOG_TAG = '[sync-assets]'; let verbose = false; +let MEMPOOL_CDN = false; +let DRY_RUN = false; if (parseInt(process.env.SKIP_SYNC) === 1) { console.log(`${LOG_TAG} SKIP_SYNC is set, not checking any assets`); @@ -15,6 +17,18 @@ if (parseInt(process.env.VERBOSE) === 1) { verbose = true; } +if (parseInt(process.env.MEMPOOL_CDN) === 1) { + console.log(`${LOG_TAG} MEMPOOL_CDN is set, assets will be downloaded from mempool.space`); + MEMPOOL_CDN = true; +} + +if (parseInt(process.env.DRY_RUN) === 1) { + console.log(`${LOG_TAG} DRY_RUN is set, not downloading any assets`); + DRY_RUN = true; +} + +const githubSecret = process.env.GITHUB_TOKEN; + const CONFIG_FILE_NAME = 'mempool-frontend-config.json'; let configContent = {}; @@ -46,8 +60,6 @@ try { } } -const githubSecret = process.env.GITHUB_TOKEN; - function download(filename, url) { https.get(url, (response) => { if (response.statusCode < 200 || response.statusCode > 299) { @@ -60,7 +72,7 @@ function download(filename, url) { }) .on('finish', () => { if (verbose) { - console.log(`${LOG_TAG} Finished downloading ${url} to ${filename}`); + console.log(`${LOG_TAG} \tFinished downloading ${url} to ${filename}`); } }); } @@ -72,7 +84,7 @@ function getLocalHash(filePath) { const hash = crypto.createHash('sha1').update(bufferWithHeader).digest('hex'); if (verbose) { - console.log(`${LOG_TAG} \tgetLocalHash ${filePath} ${hash}`); + console.log(`${LOG_TAG} \t\tgetLocalHash ${filePath} ${hash}`); } return hash; @@ -80,7 +92,7 @@ function getLocalHash(filePath) { function downloadMiningPoolLogos$() { return new Promise((resolve, reject) => { - console.log(`${LOG_TAG} Checking if mining pool logos needs downloading or updating...`); + console.log(`${LOG_TAG} \tChecking if mining pool logos needs downloading or updating...`); const options = { host: 'api.github.com', path: '/repos/mempool/mining-pool-logos/contents/', @@ -110,29 +122,54 @@ function downloadMiningPoolLogos$() { } let downloadedCount = 0; for (const poolLogo of poolLogos) { + if (verbose) { + console.log(`${LOG_TAG} Processing ${poolLogo.name}`); + } const filePath = `${PATH}/mining-pools/${poolLogo.name}`; if (fs.existsSync(filePath)) { const localHash = getLocalHash(filePath); if (verbose) { - console.log(`${LOG_TAG} Remote ${poolLogo.name} logo hash ${poolLogo.sha}`); - console.log(`${LOG_TAG} \tchecking if ${filePath} exists: ${fs.existsSync(filePath)}`); + console.log(`${LOG_TAG} \t\tremote ${poolLogo.name} logo hash ${poolLogo.sha}`); + console.log(`${LOG_TAG} \t\t\tchecking if ${filePath} exists: ${fs.existsSync(filePath)}`); } if (localHash !== poolLogo.sha) { - console.log(`${LOG_TAG} \t\t${poolLogo.name} is different on the remote, downloading...`); - download(filePath, poolLogo.download_url); - downloadedCount++; + console.log(`${LOG_TAG} \t\t\t\t${poolLogo.name} is different on the remote, downloading...`); + let download_url = poolLogo.download_url; + if (MEMPOOL_CDN) { + download_url = download_url.replace("raw.githubusercontent.com/mempool/mining-pool-logos/master", "mempool.space/resources/mining-pools"); + } + if (DRY_RUN) { + console.log(`${LOG_TAG} \t\tDRY_RUN is set, not downloading ${poolLogo.name} but we should`); + } else { + if (verbose) { + console.log(`${LOG_TAG} \t\tDownloading ${download_url} to ${filePath}`); + } + download(filePath, download_url); + downloadedCount++; + } + } else { + console.log(`${LOG_TAG} \t\t${poolLogo.name} is already up to date. Skipping.`); } } else { - console.log(`${LOG_TAG} ${poolLogo.name} is missing, downloading...`); + console.log(`${LOG_TAG} \t\t${poolLogo.name} is missing, downloading...`); const miningPoolsDir = `${PATH}/mining-pools/`; if (!fs.existsSync(miningPoolsDir)){ fs.mkdirSync(miningPoolsDir, { recursive: true }); } - download(filePath, poolLogo.download_url); - downloadedCount++; + let download_url = poolLogo.download_url; + if (MEMPOOL_CDN) { + download_url = download_url.replace("raw.githubusercontent.com/mempool/mining-pool-logos/master", "mempool.space/resources/mining-pools"); + } + if (DRY_RUN) { + console.log(`${LOG_TAG} DRY_RUN is set, not downloading ${poolLogo.name} but it should`); + } else { + console.log(`${LOG_TAG} \tDownloading ${download_url} to ${filePath}`); + download(filePath, download_url); + downloadedCount++; + } } } - console.log(`${LOG_TAG} Downloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`); + console.log(`${LOG_TAG} \t\tDownloaded ${downloadedCount} and skipped ${poolLogos.length - downloadedCount} existing mining pool logos`); resolve(); } catch (e) { reject(`Unable to download mining pool logos. Trying again at next restart. Reason: ${e instanceof Error ? e.message : e}`); @@ -148,7 +185,7 @@ function downloadMiningPoolLogos$() { function downloadPromoVideoSubtiles$() { return new Promise((resolve, reject) => { - console.log(`${LOG_TAG} Checking if promo video subtitles needs downloading or updating...`); + console.log(`${LOG_TAG} \tChecking if promo video subtitles needs downloading or updating...`); const options = { host: 'api.github.com', path: '/repos/mempool/mempool-promo/contents/subtitles', @@ -157,7 +194,7 @@ function downloadPromoVideoSubtiles$() { }; if (githubSecret) { - console.log(`${LOG_TAG} Downloading the promo video subtitles with authentication`); + console.log(`${LOG_TAG} \tDownloading the promo video subtitles with authentication`); options.headers['authorization'] = `Bearer ${githubSecret}`; options.headers['X-GitHub-Api-Version'] = '2022-11-28'; } @@ -179,27 +216,53 @@ function downloadPromoVideoSubtiles$() { } let downloadedCount = 0; for (const language of videoLanguages) { + if (verbose) { + console.log(`${LOG_TAG} Processing ${language.name}`); + } const filePath = `${PATH}/promo-video/${language.name}`; if (fs.existsSync(filePath)) { if (verbose) { - console.log(`${LOG_TAG} ${language.name} remote promo video hash ${language.sha}`); + console.log(`${LOG_TAG} \t${language.name} remote promo video hash ${language.sha}`); } const localHash = getLocalHash(filePath); - if (localHash !== language.sha) { - console.log(`${LOG_TAG} ${language.name} is different on the remote, updating`); - download(filePath, language.download_url); - downloadedCount++; + console.log(`${LOG_TAG} \t\t${language.name} is different on the remote, updating`); + let download_url = language.download_url; + if (MEMPOOL_CDN) { + download_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/subtitles", "mempool.space/resources/promo-video"); + } + if (DRY_RUN) { + console.log(`${LOG_TAG} \t\tDRY_RUN is set, not downloading ${language.name} but we should`); + } else { + if (verbose) { + console.log(`${LOG_TAG} \t\tdownloading ${download_url} to ${filePath}`); + } + download(filePath, download_url); + downloadedCount++; + } + } else { + console.log(`${LOG_TAG} \t\t${language.name} is already up to date. Skipping.`); } } else { - console.log(`${LOG_TAG} ${language.name} is missing, downloading`); + console.log(`${LOG_TAG} \t\t${language.name} is missing, downloading`); const promoVideosDir = `${PATH}/promo-video/`; if (!fs.existsSync(promoVideosDir)){ fs.mkdirSync(promoVideosDir, { recursive: true }); } - download(filePath, language.download_url); - downloadedCount++; + let download_url = language.download_url; + if (MEMPOOL_CDN) { + download_url = downloadownload_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/subtitles", "mempool.space/resources/promo-video"); + } + if (DRY_RUN) { + console.log(`${LOG_TAG} \tDRY_RUN is set, not downloading ${language.name} but we should`); + } else { + if (verbose) { + console.log(`${LOG_TAG} downloading ${download_url} to ${filePath}`); + } + download(filePath, download_url); + downloadedCount++; + } } } console.log(`${LOG_TAG} Downloaded ${downloadedCount} and skipped ${videoLanguages.length - downloadedCount} existing video subtitles`); @@ -218,7 +281,7 @@ function downloadPromoVideoSubtiles$() { function downloadPromoVideo$() { return new Promise((resolve, reject) => { - console.log(`${LOG_TAG} Checking if promo video needs downloading or updating...`); + console.log(`${LOG_TAG} \tChecking if promo video needs downloading or updating...`); const options = { host: 'api.github.com', path: '/repos/mempool/mempool-promo/contents', @@ -227,7 +290,7 @@ function downloadPromoVideo$() { }; if (githubSecret) { - console.log(`${LOG_TAG} Downloading the promo video with authentication`); + console.log(`${LOG_TAG} \tDownloading the promo video with authentication`); options.headers['authorization'] = `Bearer ${githubSecret}`; options.headers['X-GitHub-Api-Version'] = '2022-11-28'; } @@ -256,14 +319,36 @@ function downloadPromoVideo$() { if (localHash !== item.sha) { console.log(`${LOG_TAG} \tmempool-promo.mp4 is different on the remote, updating`); - download(filePath, item.download_url); - console.log(`${LOG_TAG} \tmempool-promo.mp4 downloaded.`); + let download_url = item.download_url; + if (MEMPOOL_CDN) { + download_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/promo.mp4", "mempool.space/resources/promo-video/mempool-promo.mp4"); + } + if (DRY_RUN) { + console.log(`${LOG_TAG} DRY_RUN is set, not downloading mempool-promo.mp4 but we should`); + } else { + if (verbose) { + console.log(`${LOG_TAG} downloading ${download_url} to ${filePath}`); + } + download(filePath, download_url); + console.log(`${LOG_TAG} \tmempool-promo.mp4 downloaded.`); + } } else { - console.log(`${LOG_TAG} \tmempool-promo.mp4 is already up to date. Skipping.`); + console.log(`${LOG_TAG} \t\tmempool-promo.mp4 is already up to date. Skipping.`); } } else { console.log(`${LOG_TAG} \tmempool-promo.mp4 is missing, downloading`); - download(filePath, item.download_url); + let download_url = item.download_url; + if (MEMPOOL_CDN) { + download_url = download_url.replace("raw.githubusercontent.com/mempool/mempool-promo/master/promo.mp4", "mempool.space/resources/promo-video/mempool-promo.mp4"); + } + if (DRY_RUN) { + console.log(`${LOG_TAG} DRY_RUN is set, not downloading mempool-promo.mp4 but we should`); + } else { + if (verbose) { + console.log(`${LOG_TAG} downloading ${download_url} to ${filePath}`); + } + download(filePath, download_url); + } } } resolve(); @@ -300,7 +385,7 @@ if (configContent.BASE_MODULE && configContent.BASE_MODULE === 'liquid') { download(`${PATH}/assets-testnet.minimal.json`, testnetAssetsMinimalJsonUrl); } else { if (verbose) { - console.log(`${LOG_TAG} BASE_MODULE is not set to Liquid (${configContent.BASE_MODULE}), skipping downloading assets`); + console.log(`${LOG_TAG} BASE_MODULE is not set to Liquid (currently ${configContent.BASE_MODULE}), skipping downloading assets`); } } From 9c72d1df3b1e5463c45360a43c97cd5de5d5d822 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Sun, 4 Feb 2024 09:18:50 -0800 Subject: [PATCH 03/10] Sync assets from the mempool CDN --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c435d6ea5..82fe307b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,6 +115,10 @@ jobs: - name: Sync-assets run: npm run sync-assets-dev + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MEMPOOL_CDN: 1 + VERBOSE: 1 working-directory: assets/frontend - name: Zip mining-pool assets @@ -237,6 +241,8 @@ jobs: working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/frontend env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + MEMPOOL_CDN: 1 + VERBOSE: 1 e2e: if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')" From 517d36783fb3c7bd3e4d2a96c1eb434cb8b62cd6 Mon Sep 17 00:00:00 2001 From: natsoni Date: Tue, 6 Feb 2024 11:37:51 +0100 Subject: [PATCH 04/10] Add P2WSH Liquid peg out to input label --- .../app/components/address-labels/address-labels.component.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/components/address-labels/address-labels.component.ts b/frontend/src/app/components/address-labels/address-labels.component.ts index f4b3d0ca5..2365c167f 100644 --- a/frontend/src/app/components/address-labels/address-labels.component.ts +++ b/frontend/src/app/components/address-labels/address-labels.component.ts @@ -43,7 +43,7 @@ export class AddressLabelsComponent implements OnChanges { handleVin() { if (this.vin.inner_witnessscript_asm) { - if (this.vin.inner_witnessscript_asm.indexOf('OP_DEPTH OP_PUSHNUM_12 OP_EQUAL OP_IF OP_PUSHNUM_11') === 0) { + if (this.vin.inner_witnessscript_asm.indexOf('OP_DEPTH OP_PUSHNUM_12 OP_EQUAL OP_IF OP_PUSHNUM_11') === 0 || this.vin.inner_witnessscript_asm.indexOf('OP_PUSHNUM_15 OP_CHECKMULTISIG OP_IFDUP OP_NOTIF OP_PUSHBYTES_2') === 1259) { if (this.vin.witness.length > 11) { this.label = 'Liquid Peg Out'; } else { From 87a613e4dc6344106d441b3d8130303721aaaae0 Mon Sep 17 00:00:00 2001 From: wiz Date: Tue, 6 Feb 2024 13:53:03 -0500 Subject: [PATCH 05/10] ops: Fix broken nginx cache configuration --- production/nginx/http-proxy-cache.conf | 14 +++++++++----- production/nginx/location-api-v1-lightning.conf | 2 +- production/nginx/location-api.conf | 10 +++++----- production/nginx/location-liquid-api.conf | 8 ++++---- production/nginx/location-liquidtestnet-api.conf | 8 ++++---- .../nginx/location-signet-api-v1-lightning.conf | 2 +- production/nginx/location-signet-api.conf | 8 ++++---- .../nginx/location-testnet-api-v1-lightning.conf | 2 +- production/nginx/location-testnet-api.conf | 8 ++++---- 9 files changed, 33 insertions(+), 29 deletions(-) diff --git a/production/nginx/http-proxy-cache.conf b/production/nginx/http-proxy-cache.conf index 5505b094b..0024a3b30 100644 --- a/production/nginx/http-proxy-cache.conf +++ b/production/nginx/http-proxy-cache.conf @@ -1,8 +1,12 @@ # proxy cache -proxy_cache_path /var/cache/nginx/api keys_zone=api:20m levels=1:2 inactive=365d max_size=2000m; -proxy_cache_path /var/cache/nginx/unfurler keys_zone=unfurler:20m levels=1:2 inactive=365d max_size=2000m; -proxy_cache_path /var/cache/nginx/slurper keys_zone=slurper:20m levels=1:2 inactive=365d max_size=5000m; -proxy_cache_path /var/cache/nginx/services keys_zone=services:20m levels=1:2 inactive=365d max_size=100m; +proxy_cache_path /var/cache/nginx/services keys_zone=services:20m levels=1:2 inactive=30d max_size=200m; +proxy_cache_path /var/cache/nginx/apihot keys_zone=apihot:20m levels=1:2 inactive=60m max_size=20m; +proxy_cache_path /var/cache/nginx/apiwarm keys_zone=apiwarm:20m levels=1:2 inactive=24h max_size=200m; +proxy_cache_path /var/cache/nginx/apinormal keys_zone=apinormal:200m levels=1:2 inactive=30d max_size=2000m; +proxy_cache_path /var/cache/nginx/apicold keys_zone=apicold:200m levels=1:2 inactive=365d max_size=2000m; + +proxy_cache_path /var/cache/nginx/unfurler keys_zone=unfurler:200m levels=1:2 inactive=30d max_size=2000m; +proxy_cache_path /var/cache/nginx/slurper keys_zone=slurper:500m levels=1:2 inactive=365d max_size=5000m; proxy_cache_path /var/cache/nginx/markets keys_zone=markets:20m levels=1:2 inactive=365d max_size=100m; types_hash_max_size 4096; -proxy_buffer_size 8k; \ No newline at end of file +proxy_buffer_size 8k; diff --git a/production/nginx/location-api-v1-lightning.conf b/production/nginx/location-api-v1-lightning.conf index f90fd529a..625bebade 100644 --- a/production/nginx/location-api-v1-lightning.conf +++ b/production/nginx/location-api-v1-lightning.conf @@ -12,7 +12,7 @@ location @mempool-api-v1-lightning { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apiwarm; proxy_cache_valid 200 10s; proxy_redirect off; diff --git a/production/nginx/location-api.conf b/production/nginx/location-api.conf index f245154a0..fccdde532 100644 --- a/production/nginx/location-api.conf +++ b/production/nginx/location-api.conf @@ -77,7 +77,7 @@ location @mempool-api-v1-cache-forever { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apicold; proxy_cache_valid 200 30d; proxy_redirect off; @@ -94,7 +94,7 @@ location @mempool-api-v1-cache-hot { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apihot; proxy_cache_valid 200 1s; proxy_redirect off; } @@ -109,7 +109,7 @@ location @mempool-api-v1-cache-warm { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apiwarm; proxy_cache_valid 200 10s; proxy_redirect off; } @@ -122,7 +122,7 @@ location @mempool-api-v1-cache-normal { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_cache api; + proxy_cache apinormal; proxy_cache_valid 200 2s; proxy_redirect off; @@ -167,7 +167,7 @@ location @esplora-api-cache-forever { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apicold; proxy_cache_valid 200 30d; proxy_redirect off; diff --git a/production/nginx/location-liquid-api.conf b/production/nginx/location-liquid-api.conf index 6c222c469..5286c1a72 100644 --- a/production/nginx/location-liquid-api.conf +++ b/production/nginx/location-liquid-api.conf @@ -75,7 +75,7 @@ location @mempool-liquid-api-v1-cache-forever { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apicold; proxy_cache_valid 200 30d; proxy_redirect off; @@ -92,7 +92,7 @@ location @mempool-liquid-api-v1-cache-warm { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apiwarm; proxy_cache_valid 200 10s; proxy_redirect off; } @@ -105,7 +105,7 @@ location @mempool-liquid-api-v1-cache-normal { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_cache api; + proxy_cache apinormal; proxy_cache_valid 200 10s; proxy_redirect off; @@ -150,7 +150,7 @@ location @esplora-liquid-api-cache-forever { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apicold; proxy_cache_valid 200 30d; proxy_redirect off; diff --git a/production/nginx/location-liquidtestnet-api.conf b/production/nginx/location-liquidtestnet-api.conf index 5d5be5d43..4f6615b40 100644 --- a/production/nginx/location-liquidtestnet-api.conf +++ b/production/nginx/location-liquidtestnet-api.conf @@ -79,7 +79,7 @@ location @mempool-liquidtestnet-api-v1-cache-forever { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apicold; proxy_cache_valid 200 30d; proxy_redirect off; @@ -96,7 +96,7 @@ location @mempool-liquidtestnet-api-v1-cache-warm { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apiwarm; proxy_cache_valid 200 10s; proxy_redirect off; } @@ -109,7 +109,7 @@ location @mempool-liquidtestnet-api-v1-cache-normal { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_cache api; + proxy_cache apinormal; proxy_cache_valid 200 10s; proxy_redirect off; @@ -154,7 +154,7 @@ location @esplora-liquidtestnet-api-cache-forever { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apicold; proxy_cache_valid 200 30d; proxy_redirect off; diff --git a/production/nginx/location-signet-api-v1-lightning.conf b/production/nginx/location-signet-api-v1-lightning.conf index ab14a170b..4fe0b6ddd 100644 --- a/production/nginx/location-signet-api-v1-lightning.conf +++ b/production/nginx/location-signet-api-v1-lightning.conf @@ -13,7 +13,7 @@ location @mempool-signet-api-v1-lightning { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apiwarm; proxy_cache_valid 200 10s; proxy_redirect off; diff --git a/production/nginx/location-signet-api.conf b/production/nginx/location-signet-api.conf index 8469043a8..eb8cbeb6a 100644 --- a/production/nginx/location-signet-api.conf +++ b/production/nginx/location-signet-api.conf @@ -79,7 +79,7 @@ location @mempool-signet-api-v1-cache-forever { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apicold; proxy_cache_valid 200 30d; proxy_redirect off; @@ -96,7 +96,7 @@ location @mempool-signet-api-v1-cache-warm { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apiwarm; proxy_cache_valid 200 10s; proxy_redirect off; } @@ -109,7 +109,7 @@ location @mempool-signet-api-v1-cache-normal { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_cache api; + proxy_cache apinormal; proxy_cache_valid 200 10s; proxy_redirect off; @@ -154,7 +154,7 @@ location @esplora-signet-api-cache-forever { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apicold; proxy_cache_valid 200 30d; proxy_redirect off; diff --git a/production/nginx/location-testnet-api-v1-lightning.conf b/production/nginx/location-testnet-api-v1-lightning.conf index cc7c617a6..316b37cc3 100644 --- a/production/nginx/location-testnet-api-v1-lightning.conf +++ b/production/nginx/location-testnet-api-v1-lightning.conf @@ -13,7 +13,7 @@ location @mempool-testnet-api-v1-lightning { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apiwarm; proxy_cache_valid 200 10s; proxy_redirect off; diff --git a/production/nginx/location-testnet-api.conf b/production/nginx/location-testnet-api.conf index 9f0c41147..cce3e585c 100644 --- a/production/nginx/location-testnet-api.conf +++ b/production/nginx/location-testnet-api.conf @@ -79,7 +79,7 @@ location @mempool-testnet-api-v1-cache-forever { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apicold; proxy_cache_valid 200 30d; proxy_redirect off; @@ -96,7 +96,7 @@ location @mempool-testnet-api-v1-cache-warm { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apiwarm; proxy_cache_valid 200 10s; proxy_redirect off; } @@ -109,7 +109,7 @@ location @mempool-testnet-api-v1-cache-normal { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_cache api; + proxy_cache apinormal; proxy_cache_valid 200 10s; proxy_redirect off; @@ -154,7 +154,7 @@ location @esplora-testnet-api-cache-forever { proxy_cache_background_update on; proxy_cache_use_stale updating; - proxy_cache api; + proxy_cache apicold; proxy_cache_valid 200 30d; proxy_redirect off; From f00470589695811435405afc3690c9cd45ae0531 Mon Sep 17 00:00:00 2001 From: wiz Date: Tue, 6 Feb 2024 13:55:44 -0500 Subject: [PATCH 06/10] ops: Add missing HTTP Expires header for fees API --- production/nginx/location-api.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/production/nginx/location-api.conf b/production/nginx/location-api.conf index fccdde532..bee4ce50d 100644 --- a/production/nginx/location-api.conf +++ b/production/nginx/location-api.conf @@ -97,6 +97,8 @@ location @mempool-api-v1-cache-hot { proxy_cache apihot; proxy_cache_valid 200 1s; proxy_redirect off; + + expires 1s; } location @mempool-api-v1-cache-warm { From fea115bcbca69980b4688da12256115130e3216f Mon Sep 17 00:00:00 2001 From: wiz Date: Tue, 6 Feb 2024 14:03:44 -0500 Subject: [PATCH 07/10] ops: Fix obvious bug in cache heater script --- production/nginx-cache-heater | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/production/nginx-cache-heater b/production/nginx-cache-heater index 00fc6ea7a..4bbe8ee15 100755 --- a/production/nginx-cache-heater +++ b/production/nginx-cache-heater @@ -1,5 +1,5 @@ #!/usr/bin/env zsh -hostname=mempool.space +hostname=$(hostname) heat() { From 1547c2c4775330b639a8ad072e87df6e6db615b0 Mon Sep 17 00:00:00 2001 From: James Blacklock Date: Tue, 6 Feb 2024 22:13:10 -0500 Subject: [PATCH 08/10] fix syntax error in mempool-config.json --- docker/backend/mempool-config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index aefb095cf..8f69fd0c1 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -35,7 +35,7 @@ "ALLOW_UNREACHABLE": __MEMPOOL_ALLOW_UNREACHABLE__, "POOLS_JSON_TREE_URL": "__MEMPOOL_POOLS_JSON_TREE_URL__", "POOLS_JSON_URL": "__MEMPOOL_POOLS_JSON_URL__", - "PRICE_UPDATES_PER_HOUR": __MEMPOOL_PRICE_UPDATES_PER_HOUR__ + "PRICE_UPDATES_PER_HOUR": __MEMPOOL_PRICE_UPDATES_PER_HOUR__, "MAX_TRACKED_ADDRESSES": __MEMPOOL_MAX_TRACKED_ADDRESSES__ }, "CORE_RPC": { From 624cbf05c02036fa79ae0b4a5dd7769b08633ef6 Mon Sep 17 00:00:00 2001 From: Felipe Knorr Kuhn Date: Tue, 6 Feb 2024 19:47:09 -0800 Subject: [PATCH 09/10] Add a step to validate the JSON config generated for Docker --- .github/workflows/ci.yml | 30 +++++++++++++++++++++++++++++- .gitignore | 1 + 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c435d6ea5..0961f1c88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -329,4 +329,32 @@ jobs: CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} CYPRESS_PROJECT_ID: ${{ secrets.CYPRESS_PROJECT_ID }} - \ No newline at end of file + + validate_docker_json: + if: "!contains(github.event.pull_request.labels.*.name, 'ops') && !contains(github.head_ref, 'ops/')" + runs-on: "ubuntu-latest" + name: Validate generated backend Docker JSON + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + path: docker + + - name: Install jq + run: sudo apt-get install jq -y + + - name: Create new start script to run on CI + run: | + sed '$d' start.sh > start_ci.sh + working-directory: docker/docker/backend + + - name: Run the script to generate the sample JSON + run: | + sh start_ci.sh + working-directory: docker/docker/backend + + - name: Validate JSON syntax + run: | + cat mempool-config.json | jq + working-directory: docker/docker/backend diff --git a/.gitignore b/.gitignore index 4f19f2522..381f2187c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ backend/mempool-config.json frontend/src/resources/config.template.js frontend/src/resources/config.js target +docker/backend/start_ci.sh \ No newline at end of file From 1e568e5b4c4308e8e830a9fe1ad21165ad938c62 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 7 Feb 2024 11:53:08 +0000 Subject: [PATCH 10/10] Bump follow-redirects from 1.15.2 to 1.15.5 in /backend Bumps [follow-redirects](https://github.com/follow-redirects/follow-redirects) from 1.15.2 to 1.15.5. - [Release notes](https://github.com/follow-redirects/follow-redirects/releases) - [Commits](https://github.com/follow-redirects/follow-redirects/compare/v1.15.2...v1.15.5) --- updated-dependencies: - dependency-name: follow-redirects dependency-type: indirect ... Signed-off-by: dependabot[bot] --- backend/package-lock.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/package-lock.json b/backend/package-lock.json index 1794934d2..db71881cc 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -3673,9 +3673,9 @@ "dev": true }, "node_modules/follow-redirects": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", - "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==", + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.5.tgz", + "integrity": "sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==", "funding": [ { "type": "individual", @@ -10440,9 +10440,9 @@ "dev": true }, "follow-redirects": { - "version": "1.15.2", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.2.tgz", - "integrity": "sha512-VQLG33o04KaQ8uYi2tVNbdrWp1QWxNNea+nmIB4EVM28v0hmP17z7aG1+wAkNzVq4KeXTq3221ye5qTJP91JwA==" + "version": "1.15.5", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.5.tgz", + "integrity": "sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==" }, "form-data": { "version": "4.0.0",