diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a824900f4..8352de555 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -16,6 +16,20 @@ updates: - package-ecosystem: npm directory: "/frontend" versioning-strategy: increase + groups: + frontend-angular-dependencies: + patterns: + - "@angular*" + - "@ng-*" + - "ngx-*" + frontend-jest-dependencies: + patterns: + - "@types/jest" + - "jest" + frontend-eslint-dependencies: + patterns: + - "@typescript-eslint*" + - "eslint" schedule: interval: daily open-pull-requests-limit: 10 diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index 2370fe7a1..8097a2465 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -191,30 +191,31 @@ describe('Mempool Backend Config', () => { } continue; } - switch (typeof value) { - case 'object': { - if (Array.isArray(value)) { - continue; - } else { - parseJson(value, key); - } - break; - } - default: { + + if (root) { //The flattened string, i.e, __MEMPOOL_ENABLED__ const replaceStr = `${root ? '__' + root + '_' : '__'}${key}__`; //The string used as the environment variable, i.e, MEMPOOL_ENABLED const envVarStr = `${root ? root : ''}_${key}`; + let defaultEntry; //The string used as the default value, to be checked as a regex, i.e, __MEMPOOL_ENABLED__=${MEMPOOL_ENABLED:=(.*)} - const defaultEntry = replaceStr + '=' + '\\${' + envVarStr + ':=(.*)' + '}'; - - if (process.env.CI) { - console.log(`looking for ${defaultEntry} in the start.sh script`); + if (Array.isArray(value)) { + defaultEntry = `${replaceStr}=\${${envVarStr}:=[]}`; + if (process.env.CI) { + console.log(`looking for ${defaultEntry} in the start.sh script`); + } + //Regex matching does not work with the array values + expect(startSh).toContain(defaultEntry); + } else { + defaultEntry = replaceStr + '=' + '\\${' + envVarStr + ':=(.*)' + '}'; + if (process.env.CI) { + console.log(`looking for ${defaultEntry} in the start.sh script`); + } + const re = new RegExp(defaultEntry); + expect(startSh).toMatch(re); } - const re = new RegExp(defaultEntry); - expect(startSh).toMatch(re); //The string that actually replaces the values in the config file const sedStr = 'sed -i "s!' + replaceStr + '!${' + replaceStr + '}!g" mempool-config.json'; @@ -222,11 +223,13 @@ describe('Mempool Backend Config', () => { console.log(`looking for ${sedStr} in the start.sh script`); } expect(startSh).toContain(sedStr); - break; } + else { + parseJson(value, key); } } } + parseJson(fixture); }); }); diff --git a/docker/backend/mempool-config.json b/docker/backend/mempool-config.json index e283d1171..aa084133f 100644 --- a/docker/backend/mempool-config.json +++ b/docker/backend/mempool-config.json @@ -52,7 +52,7 @@ "REST_API_URL": "__ESPLORA_REST_API_URL__", "UNIX_SOCKET_PATH": "__ESPLORA_UNIX_SOCKET_PATH__", "RETRY_UNIX_SOCKET_AFTER": __ESPLORA_RETRY_UNIX_SOCKET_AFTER__, - "FALLBACK": __ESPLORA_FALLBACK__, + "FALLBACK": __ESPLORA_FALLBACK__ }, "SECOND_CORE_RPC": { "HOST": "__SECOND_CORE_RPC_HOST__", diff --git a/docker/backend/start.sh b/docker/backend/start.sh index 681872681..2e293ce34 100755 --- a/docker/backend/start.sh +++ b/docker/backend/start.sh @@ -53,6 +53,7 @@ __ELECTRUM_TLS_ENABLED__=${ELECTRUM_TLS_ENABLED:=false} __ESPLORA_REST_API_URL__=${ESPLORA_REST_API_URL:=http://127.0.0.1:3000} __ESPLORA_UNIX_SOCKET_PATH__=${ESPLORA_UNIX_SOCKET_PATH:="null"} __ESPLORA_RETRY_UNIX_SOCKET_AFTER__=${ESPLORA_RETRY_UNIX_SOCKET_AFTER:=30000} +__ESPLORA_FALLBACK__=${ESPLORA_FALLBACK:=[]} # SECOND_CORE_RPC __SECOND_CORE_RPC_HOST__=${SECOND_CORE_RPC_HOST:=127.0.0.1} @@ -192,6 +193,7 @@ sed -i "s!__ELECTRUM_TLS_ENABLED__!${__ELECTRUM_TLS_ENABLED__}!g" mempool-config sed -i "s!__ESPLORA_REST_API_URL__!${__ESPLORA_REST_API_URL__}!g" mempool-config.json sed -i "s!__ESPLORA_UNIX_SOCKET_PATH__!${__ESPLORA_UNIX_SOCKET_PATH__}!g" mempool-config.json sed -i "s!__ESPLORA_RETRY_UNIX_SOCKET_AFTER__!${__ESPLORA_RETRY_UNIX_SOCKET_AFTER__}!g" mempool-config.json +sed -i "s!__ESPLORA_FALLBACK__!${__ESPLORA_FALLBACK__}!g" mempool-config.json sed -i "s!__SECOND_CORE_RPC_HOST__!${__SECOND_CORE_RPC_HOST__}!g" mempool-config.json sed -i "s!__SECOND_CORE_RPC_PORT__!${__SECOND_CORE_RPC_PORT__}!g" mempool-config.json diff --git a/frontend/src/app/components/about/about.component.html b/frontend/src/app/components/about/about.component.html index e9d5ec3b2..d5c82b784 100644 --- a/frontend/src/app/components/about/about.component.html +++ b/frontend/src/app/components/about/about.component.html @@ -181,21 +181,6 @@ Exodus - - - - - - - - - - Luminex - diff --git a/production/mempool-start-all b/production/mempool-start-all index 306156660..7efbf7dd6 100755 --- a/production/mempool-start-all +++ b/production/mempool-start-all @@ -10,12 +10,9 @@ for site in mainnet mainnet-lightning testnet testnet-lightning signet signet-li screen -dmS "${site}" sh -c 'while true;do npm run start-production;sleep 1;done' done -# only start xorg if GPU present -if pciconf -lv|grep -i nvidia >/dev/null 2>&1;then - export DISPLAY=:0 - screen -dmS x startx - sleep 3 -fi +export DISPLAY=:0 +screen -dmS x startx +sleep 3 # start unfurlers for each frontend for site in mainnet liquid bisq;do diff --git a/production/nginx-cache-warmer b/production/nginx-cache-warmer index 6a82871a5..f02091747 100755 --- a/production/nginx-cache-warmer +++ b/production/nginx-cache-warmer @@ -1,140 +1,192 @@ #!/usr/bin/env zsh +delay=0.15 hostname=$(hostname) slugs=(`curl -sSL https://${hostname}/api/v1/mining/pools/3y|jq -r -S '(.pools[].slug)'`) +warmSlurp() +{ + echo "$1" + curl -i -s -H 'User-Agent: Googlebot' "$1" | head -1 +} + +warmUnfurl() +{ + echo "$1" + curl -i -s -H 'User-Agent: Twitterbot' "$1" | head -1 +} + warm() { echo "$1" curl -i -s "$1" | head -1 } -while true -do for url in / \ - '/api/v1/blocks' \ - '/api/v1/statistics/2h' \ - '/api/v1/statistics/24h' \ - '/api/v1/statistics/1w' \ - '/api/v1/statistics/1m' \ - '/api/v1/statistics/3m' \ - '/api/v1/statistics/6m' \ - '/api/v1/statistics/1y' \ - '/api/v1/statistics/2y' \ - '/api/v1/statistics/3y' \ - '/api/v1/statistics/4y' \ - '/api/v1/statistics/all' \ - '/api/v1/mining/pools/24h' \ - '/api/v1/mining/pools/3d' \ - '/api/v1/mining/pools/1w' \ - '/api/v1/mining/pools/1m' \ - '/api/v1/mining/pools/3m' \ - '/api/v1/mining/pools/6m' \ - '/api/v1/mining/pools/1y' \ - '/api/v1/mining/pools/2y' \ - '/api/v1/mining/pools/3y' \ - '/api/v1/mining/pools/all' \ - '/api/v1/mining/hashrate/3m' \ - '/api/v1/mining/hashrate/6m' \ - '/api/v1/mining/hashrate/1y' \ - '/api/v1/mining/hashrate/2y' \ - '/api/v1/mining/hashrate/3y' \ - '/api/v1/mining/hashrate/all' \ - '/api/v1/mining/hashrate/pools/3m' \ - '/api/v1/mining/hashrate/pools/6m' \ - '/api/v1/mining/hashrate/pools/1y' \ - '/api/v1/mining/hashrate/pools/2y' \ - '/api/v1/mining/hashrate/pools/3y' \ - '/api/v1/mining/hashrate/pools/all' \ - '/api/v1/mining/reward-stats/144' \ - '/api/v1/mining/blocks/fees/24h' \ - '/api/v1/mining/blocks/fees/3d' \ - '/api/v1/mining/blocks/fees/1w' \ - '/api/v1/mining/blocks/fees/1m' \ - '/api/v1/mining/blocks/fees/3m' \ - '/api/v1/mining/blocks/fees/6m' \ - '/api/v1/mining/blocks/fees/1y' \ - '/api/v1/mining/blocks/fees/2y' \ - '/api/v1/mining/blocks/fees/3y' \ - '/api/v1/mining/blocks/fees/all' \ - '/api/v1/mining/blocks/rewards/24h' \ - '/api/v1/mining/blocks/rewards/3d' \ - '/api/v1/mining/blocks/rewards/1w' \ - '/api/v1/mining/blocks/rewards/1m' \ - '/api/v1/mining/blocks/rewards/3m' \ - '/api/v1/mining/blocks/rewards/6m' \ - '/api/v1/mining/blocks/rewards/1y' \ - '/api/v1/mining/blocks/rewards/2y' \ - '/api/v1/mining/blocks/rewards/3y' \ - '/api/v1/mining/blocks/rewards/all' \ - '/api/v1/mining/blocks/fee-rates/24h' \ - '/api/v1/mining/blocks/fee-rates/3d' \ - '/api/v1/mining/blocks/fee-rates/1w' \ - '/api/v1/mining/blocks/fee-rates/1m' \ - '/api/v1/mining/blocks/fee-rates/3m' \ - '/api/v1/mining/blocks/fee-rates/6m' \ - '/api/v1/mining/blocks/fee-rates/1y' \ - '/api/v1/mining/blocks/fee-rates/2y' \ - '/api/v1/mining/blocks/fee-rates/3y' \ - '/api/v1/mining/blocks/fee-rates/all' \ - '/api/v1/mining/difficulty-adjustments/24h' \ - '/api/v1/mining/difficulty-adjustments/3d' \ - '/api/v1/mining/difficulty-adjustments/1w' \ - '/api/v1/mining/difficulty-adjustments/1m' \ - '/api/v1/mining/difficulty-adjustments/3m' \ - '/api/v1/mining/difficulty-adjustments/6m' \ - '/api/v1/mining/difficulty-adjustments/1y' \ - '/api/v1/mining/difficulty-adjustments/2y' \ - '/api/v1/mining/difficulty-adjustments/3y' \ - '/api/v1/mining/difficulty-adjustments/all' \ - '/api/v1/lightning/channels-geo?style=widget' \ - '/api/v1/lightning/channels-geo?style=graph' \ - '/api/v1/lightning/statistics/latest' \ - '/api/v1/lightning/statistics/1m' \ - '/api/v1/lightning/statistics/3m' \ - '/api/v1/lightning/statistics/6m' \ - '/api/v1/lightning/statistics/1y' \ - '/api/v1/lightning/statistics/2y' \ - '/api/v1/lightning/statistics/3y' \ - '/api/v1/lightning/statistics/all' \ - '/api/v1/lightning/nodes/isp-ranking' \ - '/api/v1/lightning/nodes/isp/15169,396982' `# Google` \ - '/api/v1/lightning/nodes/isp/14618,16509' `# Amazon` \ - '/api/v1/lightning/nodes/isp/39572' `# DataWeb` \ - '/api/v1/lightning/nodes/isp/14061' `# Digital Ocean` \ - '/api/v1/lightning/nodes/isp/24940,213230' `# Hetzner` \ - '/api/v1/lightning/nodes/isp/394745' `# LunaNode` \ - '/api/v1/lightning/nodes/isp/45102' `# Alibaba` \ - '/api/v1/lightning/nodes/isp/3209' `# Vodafone Germany` \ - '/api/v1/lightning/nodes/isp/7922' `# Comcast Cable` \ - '/api/v1/lightning/nodes/isp/34197' `# SHRD SARL` \ - '/api/v1/lightning/nodes/isp/42275' `# Three Fourteen SASU` \ - '/api/v1/lightning/nodes/isp/16276' `# OVH SAS` \ - '/api/v1/lightning/nodes/isp/10796,11351,11426,11427,12271,20001,20115,33363' `# Spectrum` \ - '/api/v1/lightning/nodes/isp/701' `# Verizon` \ - '/api/v1/lightning/nodes/isp/12876' `# Scaleway` \ - '/api/v1/lightning/nodes/isp/33915' `# Ziggo` \ - '/api/v1/lightning/nodes/isp/3320' `# Deutsche Telekom AG` \ - '/api/v1/lightning/nodes/isp/8075' `# Microsoft Azure` \ - '/api/v1/lightning/nodes/isp/212531', `# UAB Interneto vizija` \ - '/api/v1/lightning/nodes/isp/63949', `# Linode` \ - '/api/v1/lightning/nodes/isp/51167', `# Contabo GmbH` \ - '/api/v1/lightning/nodes/countries' \ - '/api/v1/lightning/nodes/rankings' \ - '/api/v1/lightning/nodes/rankings/liquidity' \ - '/api/v1/lightning/nodes/rankings/connectivity' \ - '/api/v1/lightning/nodes/rankings/age' \ +warmSlurpURLs=( + '/' + '/mining' + '/lightning' + '/graphs' + '/docs' + '/docs/faq' + '/docs/api/rest' + '/about' +) +warmUnfurlURLs=( + '/' + '/mining' + '/lightning' +) + +warmURLs=( + '/api/v1/blocks' + '/api/v1/statistics/2h' + '/api/v1/statistics/24h' + '/api/v1/statistics/1w' + '/api/v1/statistics/1m' + '/api/v1/statistics/3m' + '/api/v1/statistics/6m' + '/api/v1/statistics/1y' + '/api/v1/statistics/2y' + '/api/v1/statistics/3y' + '/api/v1/statistics/4y' + '/api/v1/statistics/all' + '/api/v1/mining/pools/24h' + '/api/v1/mining/pools/3d' + '/api/v1/mining/pools/1w' + '/api/v1/mining/pools/1m' + '/api/v1/mining/pools/3m' + '/api/v1/mining/pools/6m' + '/api/v1/mining/pools/1y' + '/api/v1/mining/pools/2y' + '/api/v1/mining/pools/3y' + '/api/v1/mining/pools/all' + '/api/v1/mining/hashrate/3m' + '/api/v1/mining/hashrate/6m' + '/api/v1/mining/hashrate/1y' + '/api/v1/mining/hashrate/2y' + '/api/v1/mining/hashrate/3y' + '/api/v1/mining/hashrate/all' + '/api/v1/mining/hashrate/pools/3m' + '/api/v1/mining/hashrate/pools/6m' + '/api/v1/mining/hashrate/pools/1y' + '/api/v1/mining/hashrate/pools/2y' + '/api/v1/mining/hashrate/pools/3y' + '/api/v1/mining/hashrate/pools/all' + '/api/v1/mining/reward-stats/144' + '/api/v1/mining/blocks/fees/24h' + '/api/v1/mining/blocks/fees/3d' + '/api/v1/mining/blocks/fees/1w' + '/api/v1/mining/blocks/fees/1m' + '/api/v1/mining/blocks/fees/3m' + '/api/v1/mining/blocks/fees/6m' + '/api/v1/mining/blocks/fees/1y' + '/api/v1/mining/blocks/fees/2y' + '/api/v1/mining/blocks/fees/3y' + '/api/v1/mining/blocks/fees/all' + '/api/v1/mining/blocks/rewards/24h' + '/api/v1/mining/blocks/rewards/3d' + '/api/v1/mining/blocks/rewards/1w' + '/api/v1/mining/blocks/rewards/1m' + '/api/v1/mining/blocks/rewards/3m' + '/api/v1/mining/blocks/rewards/6m' + '/api/v1/mining/blocks/rewards/1y' + '/api/v1/mining/blocks/rewards/2y' + '/api/v1/mining/blocks/rewards/3y' + '/api/v1/mining/blocks/rewards/all' + '/api/v1/mining/blocks/fee-rates/24h' + '/api/v1/mining/blocks/fee-rates/3d' + '/api/v1/mining/blocks/fee-rates/1w' + '/api/v1/mining/blocks/fee-rates/1m' + '/api/v1/mining/blocks/fee-rates/3m' + '/api/v1/mining/blocks/fee-rates/6m' + '/api/v1/mining/blocks/fee-rates/1y' + '/api/v1/mining/blocks/fee-rates/2y' + '/api/v1/mining/blocks/fee-rates/3y' + '/api/v1/mining/blocks/fee-rates/all' + '/api/v1/mining/difficulty-adjustments/24h' + '/api/v1/mining/difficulty-adjustments/3d' + '/api/v1/mining/difficulty-adjustments/1w' + '/api/v1/mining/difficulty-adjustments/1m' + '/api/v1/mining/difficulty-adjustments/3m' + '/api/v1/mining/difficulty-adjustments/6m' + '/api/v1/mining/difficulty-adjustments/1y' + '/api/v1/mining/difficulty-adjustments/2y' + '/api/v1/mining/difficulty-adjustments/3y' + '/api/v1/mining/difficulty-adjustments/all' + '/api/v1/lightning/channels-geo?style=widget' + '/api/v1/lightning/channels-geo?style=graph' + '/api/v1/lightning/statistics/latest' + '/api/v1/lightning/statistics/1m' + '/api/v1/lightning/statistics/3m' + '/api/v1/lightning/statistics/6m' + '/api/v1/lightning/statistics/1y' + '/api/v1/lightning/statistics/2y' + '/api/v1/lightning/statistics/3y' + '/api/v1/lightning/statistics/all' + '/api/v1/lightning/nodes/isp-ranking' + '/api/v1/lightning/nodes/isp/15169,396982' `# Google` + '/api/v1/lightning/nodes/isp/14618,16509' `# Amazon` + '/api/v1/lightning/nodes/isp/39572' `# DataWeb` + '/api/v1/lightning/nodes/isp/14061' `# Digital Ocean` + '/api/v1/lightning/nodes/isp/24940,213230' `# Hetzner` + '/api/v1/lightning/nodes/isp/394745' `# LunaNode` + '/api/v1/lightning/nodes/isp/45102' `# Alibaba` + '/api/v1/lightning/nodes/isp/3209' `# Vodafone Germany` + '/api/v1/lightning/nodes/isp/7922' `# Comcast Cable` + '/api/v1/lightning/nodes/isp/34197' `# SHRD SARL` + '/api/v1/lightning/nodes/isp/42275' `# Three Fourteen SASU` + '/api/v1/lightning/nodes/isp/16276' `# OVH SAS` + '/api/v1/lightning/nodes/isp/10796,11351,11426,11427,12271,20001,2$delay,33363' `# Spectrum` + '/api/v1/lightning/nodes/isp/701' `# Verizon` + '/api/v1/lightning/nodes/isp/12876' `# Scaleway` + '/api/v1/lightning/nodes/isp/33915' `# Ziggo` + '/api/v1/lightning/nodes/isp/3320' `# Deutsche Telekom AG` + '/api/v1/lightning/nodes/isp/8075' `# Microsoft Azure` + '/api/v1/lightning/nodes/isp/212531' `# UAB Interneto vizija` + '/api/v1/lightning/nodes/isp/63949' `# Linode` + '/api/v1/lightning/nodes/isp/51167' `# Contabo GmbH` + '/api/v1/lightning/nodes/countries' + '/api/v1/lightning/nodes/rankings' + '/api/v1/lightning/nodes/rankings/liquidity' + '/api/v1/lightning/nodes/rankings/connectivity' + '/api/v1/lightning/nodes/rankings/age' +) + +echo "waiting for mempool backend to start..." +sleep 60 + +while true +do + echo "starting warm cache cycle..." + + for url in $warmSlurpURLs + do + warmSlurp "https://${hostname}${url}" + sleep $delay # delay between queries to not DoS unfurler + done + + for url in $warmUnfurlURLs + do + warmUnfurl "https://${hostname}${url}" + sleep $delay # delay between queries to not DoS unfurler + done + + for url in $warmURLs do warm "https://${hostname}${url}" - sleep 0.25 # wait 250ms between queries to not DoS mariadb + sleep $delay # delay between queries to not DoS mariadb done for slug in $slugs do warm "https://${hostname}/api/v1/mining/pool/${slug}" - sleep 0.25 # wait 250ms between queries to not DoS mariadb + sleep $delay # delay between queries to not DoS mariadb warm "https://${hostname}/api/v1/mining/pool/${slug}/hashrate" - sleep 0.25 # wait 250ms between queries to not DoS mariadb + sleep $delay # delay between queries to not DoS mariadb warm "https://${hostname}/api/v1/mining/pool/${slug}/blocks" - sleep 0.25 # wait 250ms between queries to not DoS mariadb + sleep $delay # delay between queries to not DoS mariadb done + + sleep 1 done diff --git a/production/nginx/http-proxy-cache.conf b/production/nginx/http-proxy-cache.conf index f870939b3..4d0e6614e 100644 --- a/production/nginx/http-proxy-cache.conf +++ b/production/nginx/http-proxy-cache.conf @@ -1,5 +1,7 @@ # proxy cache -proxy_cache_path /var/cache/nginx/api keys_zone=api:20m levels=1:2 inactive=600s max_size=100m; -proxy_cache_path /var/cache/nginx/services keys_zone=services:20m levels=1:2 inactive=600s max_size=100m; -proxy_cache_path /var/cache/nginx/markets keys_zone=markets:20m levels=1:2 inactive=600s max_size=100m; +proxy_cache_path /var/cache/nginx/api keys_zone=api:20m levels=1:2 inactive=600s max_size=200m; +proxy_cache_path /var/cache/nginx/services keys_zone=services:20m levels=1:2 inactive=600s max_size=200m; +proxy_cache_path /var/cache/nginx/markets keys_zone=markets:20m levels=1:2 inactive=600s max_size=200m; +proxy_cache_path /var/cache/nginx/unfurler keys_zone=unfurler:20m levels=1:2 inactive=600s max_size=200m; +proxy_cache_path /var/cache/nginx/slurper keys_zone=slurper:20m levels=1:2 inactive=600s max_size=200m; types_hash_max_size 2048; diff --git a/production/nginx/nginx.conf b/production/nginx/nginx.conf index 5861067e8..c08005d14 100644 --- a/production/nginx/nginx.conf +++ b/production/nginx/nginx.conf @@ -23,8 +23,8 @@ http { include mempool/production/nginx/http-language.conf; # match preview/unfurl bot user-agents - map $http_user_agent $unfurlbot { - default 0; + map $http_user_agent $unfurlprefix { + default ""; } # mempool configuration diff --git a/production/nginx/server-common.conf b/production/nginx/server-common.conf index 68f3b6971..3406e3450 100644 --- a/production/nginx/server-common.conf +++ b/production/nginx/server-common.conf @@ -48,8 +48,8 @@ add_header Vary Cookie; # for exact / requests, redirect based on $lang # cache redirect for 5 minutes location = / { - if ($unfurlbot) { - proxy_pass $mempoolSpaceUnfurler; + if ($unfurlprefix != '') { + rewrite ^(.*)$ $unfurlprefix$1 last; } if ($lang != '') { return 302 $scheme://$host/$lang/; @@ -65,15 +65,15 @@ location ~ ^/([a-z][a-z])/(.+\..+\.(js|css))$ { } # cache everything else for 5 minutes location ~ ^/([a-z][a-z])$ { - if ($unfurlbot) { - proxy_pass $mempoolSpaceUnfurler; + if ($unfurlprefix != '') { + rewrite ^(.*)$ $unfurlprefix$1 last; } try_files $uri /$1/index.html /en-US/index.html =404; expires 5m; } location ~ ^/([a-z][a-z])/ { - if ($unfurlbot) { - proxy_pass $mempoolSpaceUnfurler; + if ($unfurlprefix != '') { + rewrite ^(.*)$ $unfurlprefix$1 last; } try_files $uri /$1/index.html /en-US/index.html =404; expires 5m; @@ -104,16 +104,53 @@ location /preview { } # unfurl renderer location ^~ /render { - proxy_pass $mempoolSpaceUnfurler; + try_files /dev/null @mempool-space-unfurler; expires 10m; } +# unfurl handler +location /unfurl/ { + try_files /dev/null @mempool-space-unfurler; +} +location /slurp/ { + try_files /dev/null @mempool-space-slurper; +} # catch-all for all URLs i.e. /address/foo /tx/foo /block/000 # cache 5 minutes since they change frequently location / { - if ($unfurlbot) { - proxy_pass $mempoolSpaceUnfurler; + if ($unfurlprefix != '') { + rewrite ^(.*)$ $unfurlprefix$1 last; } try_files /$lang/$uri $uri /en-US/$uri /en-US/index.html =404; expires 5m; } + +location @mempool-space-unfurler { + proxy_pass $mempoolSpaceUnfurler; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache_background_update on; + proxy_cache_use_stale updating; + proxy_cache unfurler; + proxy_cache_valid 200 10m; + proxy_redirect off; +} + +location @mempool-space-slurper { + proxy_pass $mempoolSpaceUnfurler; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_cache_background_update on; + proxy_cache_use_stale updating; + proxy_cache slurper; + proxy_cache_valid 200 10m; + proxy_redirect off; +} diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index f3acd61bf..69882579f 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -109,11 +109,15 @@ class Server { setUpRoutes() { if (puppeteerEnabled) { + this.app.get('/unfurl/render*', async (req, res) => { return this.renderPreview(req, res) }) this.app.get('/render*', async (req, res) => { return this.renderPreview(req, res) }) } else { + this.app.get('/unfurl/render*', async (req, res) => { return this.renderDisabled(req, res) }) this.app.get('/render*', async (req, res) => { return this.renderDisabled(req, res) }) } - this.app.get('*', (req, res) => { return this.renderHTML(req, res) }) + this.app.get('/unfurl*', (req, res) => { return this.renderHTML(req, res, true) }) + this.app.get('/slurp*', (req, res) => { return this.renderHTML(req, res, false) }) + this.app.get('*', (req, res) => { return this.renderHTML(req, res, false) }) } async clusterTask({ page, data: { url, path, action } }) { @@ -240,7 +244,7 @@ class Server { } } - async renderHTML(req, res) { + async renderHTML(req, res, unfurl: boolean = false) { // drop requests for static files const rawPath = req.params[0]; const match = rawPath.match(/\.[\w]+$/); @@ -250,25 +254,25 @@ class Server { || rawPath.startsWith('/api/v1/translators/images') || rawPath.startsWith('/resources/profile') ) { - if (isSearchCrawler(req.headers['user-agent'])) { + if (unfurl) { + res.status(404).send(); + return; + } else { if (this.secureHost) { https.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res)); } else { http.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res)); } return; - } else { - res.status(404).send(); - return; } } let result = ''; try { - if (isSearchCrawler(req.headers['user-agent'])) { - result = await this.renderSEOPage(rawPath); - } else { + if (unfurl) { result = await this.renderUnfurlMeta(rawPath); + } else { + result = await this.renderSEOPage(rawPath); } if (result && result.length) { if (result === '404') { @@ -348,7 +352,3 @@ function capitalize(str) { return str; } } - -function isSearchCrawler(useragent: string): boolean { - return /googlebot|applebot|bingbot/i.test(useragent); -}