From 13f6f9f9e5a5db7eae0197b1bf3f079324c01cf0 Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Mon, 14 Aug 2023 17:52:15 +0900
Subject: [PATCH 1/5] Select unfurl mode with X-Unfurl-Type header

---
 unfurler/src/index.ts | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts
index f3acd61bf..ae751d1f5 100644
--- a/unfurler/src/index.ts
+++ b/unfurler/src/index.ts
@@ -250,25 +250,25 @@ class Server {
       || rawPath.startsWith('/api/v1/translators/images')
       || rawPath.startsWith('/resources/profile')
     ) {
-      if (isSearchCrawler(req.headers['user-agent'])) {
+      if (isPreviewCrawler(req)) {
+        res.status(404).send();
+        return;
+      } else {
         if (this.secureHost) {
           https.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res));
         } else {
           http.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res));
         }
         return;
-      } else {
-        res.status(404).send();
-        return;
       }
     }
 
     let result = '';
     try {
-      if (isSearchCrawler(req.headers['user-agent'])) {
-        result = await this.renderSEOPage(rawPath);
-      } else {
+      if (isPreviewCrawler(req)) {
         result = await this.renderUnfurlMeta(rawPath);
+      } else {
+        result = await this.renderSEOPage(rawPath);
       }
       if (result && result.length) {
         if (result === '404') {
@@ -349,6 +349,6 @@ function capitalize(str) {
   }
 }
 
-function isSearchCrawler(useragent: string): boolean {
-  return /googlebot|applebot|bingbot/i.test(useragent);
+function isPreviewCrawler(req: Request): boolean {
+  return req?.header('X-Unfurl-Type') === 'preview';
 }

From 83a92a7e3a709c9e192e195f25b0e6c370cc5b55 Mon Sep 17 00:00:00 2001
From: wiz <j@wiz.biz>
Date: Wed, 16 Aug 2023 03:27:09 +0900
Subject: [PATCH 2/5] ops: Add nginx config for /unfurler and /slurper prefixes

---
 production/nginx/http-proxy-cache.conf |  8 ++--
 production/nginx/nginx.conf            |  4 +-
 production/nginx/server-common.conf    | 55 +++++++++++++++++++++-----
 3 files changed, 53 insertions(+), 14 deletions(-)

diff --git a/production/nginx/http-proxy-cache.conf b/production/nginx/http-proxy-cache.conf
index f870939b3..4d0e6614e 100644
--- a/production/nginx/http-proxy-cache.conf
+++ b/production/nginx/http-proxy-cache.conf
@@ -1,5 +1,7 @@
 # proxy cache
-proxy_cache_path /var/cache/nginx/api keys_zone=api:20m levels=1:2 inactive=600s max_size=100m;
-proxy_cache_path /var/cache/nginx/services keys_zone=services:20m levels=1:2 inactive=600s max_size=100m;
-proxy_cache_path /var/cache/nginx/markets keys_zone=markets:20m levels=1:2 inactive=600s max_size=100m;
+proxy_cache_path /var/cache/nginx/api keys_zone=api:20m levels=1:2 inactive=600s max_size=200m;
+proxy_cache_path /var/cache/nginx/services keys_zone=services:20m levels=1:2 inactive=600s max_size=200m;
+proxy_cache_path /var/cache/nginx/markets keys_zone=markets:20m levels=1:2 inactive=600s max_size=200m;
+proxy_cache_path /var/cache/nginx/unfurler keys_zone=unfurler:20m levels=1:2 inactive=600s max_size=200m;
+proxy_cache_path /var/cache/nginx/slurper keys_zone=slurper:20m levels=1:2 inactive=600s max_size=200m;
 types_hash_max_size 2048;
diff --git a/production/nginx/nginx.conf b/production/nginx/nginx.conf
index 5861067e8..c08005d14 100644
--- a/production/nginx/nginx.conf
+++ b/production/nginx/nginx.conf
@@ -23,8 +23,8 @@ http {
 	include mempool/production/nginx/http-language.conf;
 
 	# match preview/unfurl bot user-agents
-	map $http_user_agent $unfurlbot {
-		default 0;
+	map $http_user_agent $unfurlprefix {
+		default "";
 	}
 
 	# mempool configuration
diff --git a/production/nginx/server-common.conf b/production/nginx/server-common.conf
index 68f3b6971..3406e3450 100644
--- a/production/nginx/server-common.conf
+++ b/production/nginx/server-common.conf
@@ -48,8 +48,8 @@ add_header Vary Cookie;
 # for exact / requests, redirect based on $lang
 # cache redirect for 5 minutes
 location = / {
-	if ($unfurlbot) {
-		proxy_pass $mempoolSpaceUnfurler;
+	if ($unfurlprefix != '') {
+		rewrite ^(.*)$ $unfurlprefix$1 last;
 	}
 	if ($lang != '') {
 		return 302 $scheme://$host/$lang/;
@@ -65,15 +65,15 @@ location ~ ^/([a-z][a-z])/(.+\..+\.(js|css))$ {
 }
 # cache everything else for 5 minutes
 location ~ ^/([a-z][a-z])$ {
-	if ($unfurlbot) {
-		proxy_pass $mempoolSpaceUnfurler;
+	if ($unfurlprefix != '') {
+		rewrite ^(.*)$ $unfurlprefix$1 last;
 	}
 	try_files $uri /$1/index.html /en-US/index.html =404;
 	expires 5m;
 }
 location ~ ^/([a-z][a-z])/ {
-	if ($unfurlbot) {
-		proxy_pass $mempoolSpaceUnfurler;
+	if ($unfurlprefix != '') {
+		rewrite ^(.*)$ $unfurlprefix$1 last;
 	}
 	try_files $uri /$1/index.html /en-US/index.html =404;
 	expires 5m;
@@ -104,16 +104,53 @@ location /preview {
 }
 # unfurl renderer
 location ^~ /render {
-	proxy_pass $mempoolSpaceUnfurler;
+	try_files /dev/null @mempool-space-unfurler;
 	expires 10m;
 }
+# unfurl handler
+location /unfurl/ {
+	try_files /dev/null @mempool-space-unfurler;
+}
+location /slurp/ {
+	try_files /dev/null @mempool-space-slurper;
+}
 
 # catch-all for all URLs i.e. /address/foo /tx/foo /block/000
 # cache 5 minutes since they change frequently
 location / {
-	if ($unfurlbot) {
-		proxy_pass $mempoolSpaceUnfurler;
+	if ($unfurlprefix != '') {
+		rewrite ^(.*)$ $unfurlprefix$1 last;
 	}
 	try_files /$lang/$uri $uri /en-US/$uri /en-US/index.html =404;
 	expires 5m;
 }
+
+location @mempool-space-unfurler {
+	proxy_pass $mempoolSpaceUnfurler;
+
+	proxy_set_header Host $host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_cache_background_update on;
+	proxy_cache_use_stale updating;
+	proxy_cache unfurler;
+	proxy_cache_valid 200 10m;
+	proxy_redirect off;
+}
+
+location @mempool-space-slurper {
+	proxy_pass $mempoolSpaceUnfurler;
+
+	proxy_set_header Host $host;
+	proxy_set_header X-Real-IP $remote_addr;
+	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto $scheme;
+
+	proxy_cache_background_update on;
+	proxy_cache_use_stale updating;
+	proxy_cache slurper;
+	proxy_cache_valid 200 10m;
+	proxy_redirect off;
+}

From 0bfd01e732cf104b3a457016648219cff2ec748b Mon Sep 17 00:00:00 2001
From: wiz <j@wiz.biz>
Date: Wed, 16 Aug 2023 03:51:10 +0900
Subject: [PATCH 3/5] ops: Add unfurl/slurp queries to nginx warm cache script

---
 production/nginx-cache-warmer | 285 ++++++++++++++++++++--------------
 1 file changed, 167 insertions(+), 118 deletions(-)

diff --git a/production/nginx-cache-warmer b/production/nginx-cache-warmer
index 6a82871a5..fe3587c93 100755
--- a/production/nginx-cache-warmer
+++ b/production/nginx-cache-warmer
@@ -1,140 +1,189 @@
 #!/usr/bin/env zsh
+delay=0.15
 hostname=$(hostname)
 slugs=(`curl -sSL https://${hostname}/api/v1/mining/pools/3y|jq -r -S '(.pools[].slug)'`)
 
+warmSlurp()
+{
+	echo "$1"
+	curl -i -s -H 'User-Agent: Googlebot' "$1" | head -1
+}
+
+warmUnfurl()
+{
+	echo "$1"
+	curl -i -s -H 'User-Agent: Twitterbot' "$1" | head -1
+}
+
 warm()
 {
 	echo "$1"
 	curl -i -s "$1" | head -1
 }
 
-while true
-do for url in / \
-	'/api/v1/blocks' \
-	'/api/v1/statistics/2h' \
-	'/api/v1/statistics/24h' \
-	'/api/v1/statistics/1w' \
-	'/api/v1/statistics/1m' \
-	'/api/v1/statistics/3m' \
-	'/api/v1/statistics/6m' \
-	'/api/v1/statistics/1y' \
-	'/api/v1/statistics/2y' \
-	'/api/v1/statistics/3y' \
-	'/api/v1/statistics/4y' \
-	'/api/v1/statistics/all' \
-	'/api/v1/mining/pools/24h' \
-	'/api/v1/mining/pools/3d' \
-	'/api/v1/mining/pools/1w' \
-	'/api/v1/mining/pools/1m' \
-	'/api/v1/mining/pools/3m' \
-	'/api/v1/mining/pools/6m' \
-	'/api/v1/mining/pools/1y' \
-	'/api/v1/mining/pools/2y' \
-	'/api/v1/mining/pools/3y' \
-	'/api/v1/mining/pools/all' \
-	'/api/v1/mining/hashrate/3m' \
-	'/api/v1/mining/hashrate/6m' \
-	'/api/v1/mining/hashrate/1y' \
-	'/api/v1/mining/hashrate/2y' \
-	'/api/v1/mining/hashrate/3y' \
-	'/api/v1/mining/hashrate/all' \
-	'/api/v1/mining/hashrate/pools/3m' \
-	'/api/v1/mining/hashrate/pools/6m' \
-	'/api/v1/mining/hashrate/pools/1y' \
-	'/api/v1/mining/hashrate/pools/2y' \
-	'/api/v1/mining/hashrate/pools/3y' \
-	'/api/v1/mining/hashrate/pools/all' \
-	'/api/v1/mining/reward-stats/144' \
-	'/api/v1/mining/blocks/fees/24h' \
-	'/api/v1/mining/blocks/fees/3d' \
-	'/api/v1/mining/blocks/fees/1w' \
-	'/api/v1/mining/blocks/fees/1m' \
-	'/api/v1/mining/blocks/fees/3m' \
-	'/api/v1/mining/blocks/fees/6m' \
-	'/api/v1/mining/blocks/fees/1y' \
-	'/api/v1/mining/blocks/fees/2y' \
-	'/api/v1/mining/blocks/fees/3y' \
-	'/api/v1/mining/blocks/fees/all' \
-	'/api/v1/mining/blocks/rewards/24h' \
-	'/api/v1/mining/blocks/rewards/3d' \
-	'/api/v1/mining/blocks/rewards/1w' \
-	'/api/v1/mining/blocks/rewards/1m' \
-	'/api/v1/mining/blocks/rewards/3m' \
-	'/api/v1/mining/blocks/rewards/6m' \
-	'/api/v1/mining/blocks/rewards/1y' \
-	'/api/v1/mining/blocks/rewards/2y' \
-	'/api/v1/mining/blocks/rewards/3y' \
-	'/api/v1/mining/blocks/rewards/all' \
-	'/api/v1/mining/blocks/fee-rates/24h' \
-	'/api/v1/mining/blocks/fee-rates/3d' \
-	'/api/v1/mining/blocks/fee-rates/1w' \
-	'/api/v1/mining/blocks/fee-rates/1m' \
-	'/api/v1/mining/blocks/fee-rates/3m' \
-	'/api/v1/mining/blocks/fee-rates/6m' \
-	'/api/v1/mining/blocks/fee-rates/1y' \
-	'/api/v1/mining/blocks/fee-rates/2y' \
-	'/api/v1/mining/blocks/fee-rates/3y' \
-	'/api/v1/mining/blocks/fee-rates/all' \
-	'/api/v1/mining/difficulty-adjustments/24h' \
-	'/api/v1/mining/difficulty-adjustments/3d' \
-	'/api/v1/mining/difficulty-adjustments/1w' \
-	'/api/v1/mining/difficulty-adjustments/1m' \
-	'/api/v1/mining/difficulty-adjustments/3m' \
-	'/api/v1/mining/difficulty-adjustments/6m' \
-	'/api/v1/mining/difficulty-adjustments/1y' \
-	'/api/v1/mining/difficulty-adjustments/2y' \
-	'/api/v1/mining/difficulty-adjustments/3y' \
-	'/api/v1/mining/difficulty-adjustments/all' \
-	'/api/v1/lightning/channels-geo?style=widget' \
-	'/api/v1/lightning/channels-geo?style=graph' \
-	'/api/v1/lightning/statistics/latest' \
-	'/api/v1/lightning/statistics/1m' \
-	'/api/v1/lightning/statistics/3m' \
-	'/api/v1/lightning/statistics/6m' \
-	'/api/v1/lightning/statistics/1y' \
-	'/api/v1/lightning/statistics/2y' \
-	'/api/v1/lightning/statistics/3y' \
-	'/api/v1/lightning/statistics/all' \
-	'/api/v1/lightning/nodes/isp-ranking' \
-	'/api/v1/lightning/nodes/isp/15169,396982' `# Google` \
-	'/api/v1/lightning/nodes/isp/14618,16509' `# Amazon` \
-	'/api/v1/lightning/nodes/isp/39572' `# DataWeb` \
-	'/api/v1/lightning/nodes/isp/14061' `# Digital Ocean` \
-	'/api/v1/lightning/nodes/isp/24940,213230' `# Hetzner` \
-	'/api/v1/lightning/nodes/isp/394745' `# LunaNode` \
-	'/api/v1/lightning/nodes/isp/45102' `# Alibaba` \
-	'/api/v1/lightning/nodes/isp/3209' `# Vodafone Germany` \
-	'/api/v1/lightning/nodes/isp/7922' `# Comcast Cable` \
-	'/api/v1/lightning/nodes/isp/34197' `# SHRD SARL` \
-	'/api/v1/lightning/nodes/isp/42275' `# Three Fourteen SASU` \
-	'/api/v1/lightning/nodes/isp/16276' `# OVH SAS` \
-	'/api/v1/lightning/nodes/isp/10796,11351,11426,11427,12271,20001,20115,33363' `# Spectrum` \
-	'/api/v1/lightning/nodes/isp/701' `# Verizon` \
-	'/api/v1/lightning/nodes/isp/12876' `# Scaleway` \
-	'/api/v1/lightning/nodes/isp/33915' `# Ziggo` \
-	'/api/v1/lightning/nodes/isp/3320' `# Deutsche Telekom AG` \
-	'/api/v1/lightning/nodes/isp/8075' `# Microsoft Azure` \
-	'/api/v1/lightning/nodes/isp/212531', `# UAB Interneto vizija` \
-	'/api/v1/lightning/nodes/isp/63949', `# Linode` \
-	'/api/v1/lightning/nodes/isp/51167', `# Contabo GmbH` \
-	'/api/v1/lightning/nodes/countries' \
-	'/api/v1/lightning/nodes/rankings' \
-	'/api/v1/lightning/nodes/rankings/liquidity' \
-	'/api/v1/lightning/nodes/rankings/connectivity' \
-	'/api/v1/lightning/nodes/rankings/age' \
+warmSlurpURLs=(
+	'/'
+	'/mining'
+	'/lightning'
+	'/graphs'
+	'/docs'
+	'/docs/faq'
+	'/docs/api/rest'
+	'/about'
+)
 
+warmUnfurlURLs=(
+	'/'
+	'/mining'
+	'/lightning'
+)
+
+warmURLs=(
+	'/api/v1/blocks'
+	'/api/v1/statistics/2h'
+	'/api/v1/statistics/24h'
+	'/api/v1/statistics/1w'
+	'/api/v1/statistics/1m'
+	'/api/v1/statistics/3m'
+	'/api/v1/statistics/6m'
+	'/api/v1/statistics/1y'
+	'/api/v1/statistics/2y'
+	'/api/v1/statistics/3y'
+	'/api/v1/statistics/4y'
+	'/api/v1/statistics/all'
+	'/api/v1/mining/pools/24h'
+	'/api/v1/mining/pools/3d'
+	'/api/v1/mining/pools/1w'
+	'/api/v1/mining/pools/1m'
+	'/api/v1/mining/pools/3m'
+	'/api/v1/mining/pools/6m'
+	'/api/v1/mining/pools/1y'
+	'/api/v1/mining/pools/2y'
+	'/api/v1/mining/pools/3y'
+	'/api/v1/mining/pools/all'
+	'/api/v1/mining/hashrate/3m'
+	'/api/v1/mining/hashrate/6m'
+	'/api/v1/mining/hashrate/1y'
+	'/api/v1/mining/hashrate/2y'
+	'/api/v1/mining/hashrate/3y'
+	'/api/v1/mining/hashrate/all'
+	'/api/v1/mining/hashrate/pools/3m'
+	'/api/v1/mining/hashrate/pools/6m'
+	'/api/v1/mining/hashrate/pools/1y'
+	'/api/v1/mining/hashrate/pools/2y'
+	'/api/v1/mining/hashrate/pools/3y'
+	'/api/v1/mining/hashrate/pools/all'
+	'/api/v1/mining/reward-stats/144'
+	'/api/v1/mining/blocks/fees/24h'
+	'/api/v1/mining/blocks/fees/3d'
+	'/api/v1/mining/blocks/fees/1w'
+	'/api/v1/mining/blocks/fees/1m'
+	'/api/v1/mining/blocks/fees/3m'
+	'/api/v1/mining/blocks/fees/6m'
+	'/api/v1/mining/blocks/fees/1y'
+	'/api/v1/mining/blocks/fees/2y'
+	'/api/v1/mining/blocks/fees/3y'
+	'/api/v1/mining/blocks/fees/all'
+	'/api/v1/mining/blocks/rewards/24h'
+	'/api/v1/mining/blocks/rewards/3d'
+	'/api/v1/mining/blocks/rewards/1w'
+	'/api/v1/mining/blocks/rewards/1m'
+	'/api/v1/mining/blocks/rewards/3m'
+	'/api/v1/mining/blocks/rewards/6m'
+	'/api/v1/mining/blocks/rewards/1y'
+	'/api/v1/mining/blocks/rewards/2y'
+	'/api/v1/mining/blocks/rewards/3y'
+	'/api/v1/mining/blocks/rewards/all'
+	'/api/v1/mining/blocks/fee-rates/24h'
+	'/api/v1/mining/blocks/fee-rates/3d'
+	'/api/v1/mining/blocks/fee-rates/1w'
+	'/api/v1/mining/blocks/fee-rates/1m'
+	'/api/v1/mining/blocks/fee-rates/3m'
+	'/api/v1/mining/blocks/fee-rates/6m'
+	'/api/v1/mining/blocks/fee-rates/1y'
+	'/api/v1/mining/blocks/fee-rates/2y'
+	'/api/v1/mining/blocks/fee-rates/3y'
+	'/api/v1/mining/blocks/fee-rates/all'
+	'/api/v1/mining/difficulty-adjustments/24h'
+	'/api/v1/mining/difficulty-adjustments/3d'
+	'/api/v1/mining/difficulty-adjustments/1w'
+	'/api/v1/mining/difficulty-adjustments/1m'
+	'/api/v1/mining/difficulty-adjustments/3m'
+	'/api/v1/mining/difficulty-adjustments/6m'
+	'/api/v1/mining/difficulty-adjustments/1y'
+	'/api/v1/mining/difficulty-adjustments/2y'
+	'/api/v1/mining/difficulty-adjustments/3y'
+	'/api/v1/mining/difficulty-adjustments/all'
+	'/api/v1/lightning/channels-geo?style=widget'
+	'/api/v1/lightning/channels-geo?style=graph'
+	'/api/v1/lightning/statistics/latest'
+	'/api/v1/lightning/statistics/1m'
+	'/api/v1/lightning/statistics/3m'
+	'/api/v1/lightning/statistics/6m'
+	'/api/v1/lightning/statistics/1y'
+	'/api/v1/lightning/statistics/2y'
+	'/api/v1/lightning/statistics/3y'
+	'/api/v1/lightning/statistics/all'
+	'/api/v1/lightning/nodes/isp-ranking'
+	'/api/v1/lightning/nodes/isp/15169,396982' `# Google`
+	'/api/v1/lightning/nodes/isp/14618,16509' `# Amazon`
+	'/api/v1/lightning/nodes/isp/39572' `# DataWeb`
+	'/api/v1/lightning/nodes/isp/14061' `# Digital Ocean`
+	'/api/v1/lightning/nodes/isp/24940,213230' `# Hetzner`
+	'/api/v1/lightning/nodes/isp/394745' `# LunaNode`
+	'/api/v1/lightning/nodes/isp/45102' `# Alibaba`
+	'/api/v1/lightning/nodes/isp/3209' `# Vodafone Germany`
+	'/api/v1/lightning/nodes/isp/7922' `# Comcast Cable`
+	'/api/v1/lightning/nodes/isp/34197' `# SHRD SARL`
+	'/api/v1/lightning/nodes/isp/42275' `# Three Fourteen SASU`
+	'/api/v1/lightning/nodes/isp/16276' `# OVH SAS`
+	'/api/v1/lightning/nodes/isp/10796,11351,11426,11427,12271,20001,2$delay,33363' `# Spectrum`
+	'/api/v1/lightning/nodes/isp/701' `# Verizon`
+	'/api/v1/lightning/nodes/isp/12876' `# Scaleway`
+	'/api/v1/lightning/nodes/isp/33915' `# Ziggo`
+	'/api/v1/lightning/nodes/isp/3320' `# Deutsche Telekom AG`
+	'/api/v1/lightning/nodes/isp/8075' `# Microsoft Azure`
+	'/api/v1/lightning/nodes/isp/212531' `# UAB Interneto vizija`
+	'/api/v1/lightning/nodes/isp/63949' `# Linode`
+	'/api/v1/lightning/nodes/isp/51167' `# Contabo GmbH`
+	'/api/v1/lightning/nodes/countries'
+	'/api/v1/lightning/nodes/rankings'
+	'/api/v1/lightning/nodes/rankings/liquidity'
+	'/api/v1/lightning/nodes/rankings/connectivity'
+	'/api/v1/lightning/nodes/rankings/age'
+)
+
+while true
+do
+	echo "starting warm cache cycle..."
+
+	for url in $warmSlurpURLs
+	do
+		warmSlurp "https://${hostname}${url}"
+		sleep $delay # delay between queries to not DoS unfurler
+	done
+
+	for url in $warmUnfurlURLs
+	do
+		warmUnfurl "https://${hostname}${url}"
+		sleep $delay # delay between queries to not DoS unfurler
+	done
+
+	for url in $warmURLs
 	do
 		warm "https://${hostname}${url}"
-		sleep 0.25 # wait 250ms between queries to not DoS mariadb
+		sleep $delay # delay between queries to not DoS mariadb
 	done
 
 	for slug in $slugs
 	do
 		warm "https://${hostname}/api/v1/mining/pool/${slug}"
-		sleep 0.25 # wait 250ms between queries to not DoS mariadb
+		sleep $delay # delay between queries to not DoS mariadb
 		warm "https://${hostname}/api/v1/mining/pool/${slug}/hashrate"
-		sleep 0.25 # wait 250ms between queries to not DoS mariadb
+		sleep $delay # delay between queries to not DoS mariadb
 		warm "https://${hostname}/api/v1/mining/pool/${slug}/blocks"
-		sleep 0.25 # wait 250ms between queries to not DoS mariadb
+		sleep $delay # delay between queries to not DoS mariadb
 	done
+
+	sleep 1
 done

From 8f2a2d1c8187e5db5787aa74083667189cf2af33 Mon Sep 17 00:00:00 2001
From: wiz <j@wiz.biz>
Date: Wed, 16 Aug 2023 04:03:17 +0900
Subject: [PATCH 4/5] ops: Add delay before warm cache loop begins

---
 production/nginx-cache-warmer | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/production/nginx-cache-warmer b/production/nginx-cache-warmer
index fe3587c93..f02091747 100755
--- a/production/nginx-cache-warmer
+++ b/production/nginx-cache-warmer
@@ -153,6 +153,9 @@ warmURLs=(
 	'/api/v1/lightning/nodes/rankings/age'
 )
 
+echo "waiting for mempool backend to start..."
+sleep 60
+
 while true
 do
 	echo "starting warm cache cycle..."

From b1e1601ccf661913f928c7435c8bd568922019e1 Mon Sep 17 00:00:00 2001
From: Mononaut <github@monospace.live>
Date: Wed, 16 Aug 2023 21:55:03 +0900
Subject: [PATCH 5/5] Switch unfurl proxy header to url prefix

---
 unfurler/src/index.ts | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts
index ae751d1f5..69882579f 100644
--- a/unfurler/src/index.ts
+++ b/unfurler/src/index.ts
@@ -109,11 +109,15 @@ class Server {
 
   setUpRoutes() {
     if (puppeteerEnabled) {
+      this.app.get('/unfurl/render*', async (req, res) => { return this.renderPreview(req, res) })
       this.app.get('/render*', async (req, res) => { return this.renderPreview(req, res) })
     } else {
+      this.app.get('/unfurl/render*', async (req, res) => { return this.renderDisabled(req, res) })
       this.app.get('/render*', async (req, res) => { return this.renderDisabled(req, res) })
     }
-    this.app.get('*', (req, res) => { return this.renderHTML(req, res) })
+    this.app.get('/unfurl*', (req, res) => { return this.renderHTML(req, res, true) })
+    this.app.get('/slurp*', (req, res) => { return this.renderHTML(req, res, false) })
+    this.app.get('*', (req, res) => { return this.renderHTML(req, res, false) })
   }
 
   async clusterTask({ page, data: { url, path, action } }) {
@@ -240,7 +244,7 @@ class Server {
     }
   }
 
-  async renderHTML(req, res) {
+  async renderHTML(req, res, unfurl: boolean = false) {
     // drop requests for static files
     const rawPath = req.params[0];
     const match = rawPath.match(/\.[\w]+$/);
@@ -250,7 +254,7 @@ class Server {
       || rawPath.startsWith('/api/v1/translators/images')
       || rawPath.startsWith('/resources/profile')
     ) {
-      if (isPreviewCrawler(req)) {
+      if (unfurl) {
         res.status(404).send();
         return;
       } else {
@@ -265,7 +269,7 @@ class Server {
 
     let result = '';
     try {
-      if (isPreviewCrawler(req)) {
+      if (unfurl) {
         result = await this.renderUnfurlMeta(rawPath);
       } else {
         result = await this.renderSEOPage(rawPath);
@@ -348,7 +352,3 @@ function capitalize(str) {
     return str;
   }
 }
-
-function isPreviewCrawler(req: Request): boolean {
-  return req?.header('X-Unfurl-Type') === 'preview';
-}