machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #4152 from mempool/mononaut/unfurler-mode-headers

Browse Source 
Select unfurl mode with route prefix
This commit is contained in:
wiz 2023-08-17 16:48:26 +09:00 committed by GitHub
commit 487d5de0ef
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 236 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

288
production/nginx-cache-warmer
View File

@ -1,140 +1,192 @@
#!/usr/bin/env zsh #!/usr/bin/env zsh
delay=0.15
hostname=$(hostname) hostname=$(hostname)
slugs=(`curl -sSL https://${hostname}/api/v1/mining/pools/3y|jq -r -S '(.pools[].slug)'`) slugs=(`curl -sSL https://${hostname}/api/v1/mining/pools/3y|jq -r -S '(.pools[].slug)'`)
warmSlurp()
{
echo "$1"
curl -i -s -H 'User-Agent: Googlebot' "$1" | head -1
}
warmUnfurl()
{
echo "$1"
curl -i -s -H 'User-Agent: Twitterbot' "$1" | head -1
}
warm() warm()
{ {
echo "$1" echo "$1"
curl -i -s "$1" | head -1 curl -i -s "$1" | head -1
} }
while true warmSlurpURLs=(
do for url in / \ '/'
'/api/v1/blocks' \ '/mining'
'/api/v1/statistics/2h' \ '/lightning'
'/api/v1/statistics/24h' \ '/graphs'
'/api/v1/statistics/1w' \ '/docs'
'/api/v1/statistics/1m' \ '/docs/faq'
'/api/v1/statistics/3m' \ '/docs/api/rest'
'/api/v1/statistics/6m' \ '/about'
'/api/v1/statistics/1y' \ )
'/api/v1/statistics/2y' \
'/api/v1/statistics/3y' \
'/api/v1/statistics/4y' \
'/api/v1/statistics/all' \
'/api/v1/mining/pools/24h' \
'/api/v1/mining/pools/3d' \
'/api/v1/mining/pools/1w' \
'/api/v1/mining/pools/1m' \
'/api/v1/mining/pools/3m' \
'/api/v1/mining/pools/6m' \
'/api/v1/mining/pools/1y' \
'/api/v1/mining/pools/2y' \
'/api/v1/mining/pools/3y' \
'/api/v1/mining/pools/all' \
'/api/v1/mining/hashrate/3m' \
'/api/v1/mining/hashrate/6m' \
'/api/v1/mining/hashrate/1y' \
'/api/v1/mining/hashrate/2y' \
'/api/v1/mining/hashrate/3y' \
'/api/v1/mining/hashrate/all' \
'/api/v1/mining/hashrate/pools/3m' \
'/api/v1/mining/hashrate/pools/6m' \
'/api/v1/mining/hashrate/pools/1y' \
'/api/v1/mining/hashrate/pools/2y' \
'/api/v1/mining/hashrate/pools/3y' \
'/api/v1/mining/hashrate/pools/all' \
'/api/v1/mining/reward-stats/144' \
'/api/v1/mining/blocks/fees/24h' \
'/api/v1/mining/blocks/fees/3d' \
'/api/v1/mining/blocks/fees/1w' \
'/api/v1/mining/blocks/fees/1m' \
'/api/v1/mining/blocks/fees/3m' \
'/api/v1/mining/blocks/fees/6m' \
'/api/v1/mining/blocks/fees/1y' \
'/api/v1/mining/blocks/fees/2y' \
'/api/v1/mining/blocks/fees/3y' \
'/api/v1/mining/blocks/fees/all' \
'/api/v1/mining/blocks/rewards/24h' \
'/api/v1/mining/blocks/rewards/3d' \
'/api/v1/mining/blocks/rewards/1w' \
'/api/v1/mining/blocks/rewards/1m' \
'/api/v1/mining/blocks/rewards/3m' \
'/api/v1/mining/blocks/rewards/6m' \
'/api/v1/mining/blocks/rewards/1y' \
'/api/v1/mining/blocks/rewards/2y' \
'/api/v1/mining/blocks/rewards/3y' \
'/api/v1/mining/blocks/rewards/all' \
'/api/v1/mining/blocks/fee-rates/24h' \
'/api/v1/mining/blocks/fee-rates/3d' \
'/api/v1/mining/blocks/fee-rates/1w' \
'/api/v1/mining/blocks/fee-rates/1m' \
'/api/v1/mining/blocks/fee-rates/3m' \
'/api/v1/mining/blocks/fee-rates/6m' \
'/api/v1/mining/blocks/fee-rates/1y' \
'/api/v1/mining/blocks/fee-rates/2y' \
'/api/v1/mining/blocks/fee-rates/3y' \
'/api/v1/mining/blocks/fee-rates/all' \
'/api/v1/mining/difficulty-adjustments/24h' \
'/api/v1/mining/difficulty-adjustments/3d' \
'/api/v1/mining/difficulty-adjustments/1w' \
'/api/v1/mining/difficulty-adjustments/1m' \
'/api/v1/mining/difficulty-adjustments/3m' \
'/api/v1/mining/difficulty-adjustments/6m' \
'/api/v1/mining/difficulty-adjustments/1y' \
'/api/v1/mining/difficulty-adjustments/2y' \
'/api/v1/mining/difficulty-adjustments/3y' \
'/api/v1/mining/difficulty-adjustments/all' \
'/api/v1/lightning/channels-geo?style=widget' \
'/api/v1/lightning/channels-geo?style=graph' \
'/api/v1/lightning/statistics/latest' \
'/api/v1/lightning/statistics/1m' \
'/api/v1/lightning/statistics/3m' \
'/api/v1/lightning/statistics/6m' \
'/api/v1/lightning/statistics/1y' \
'/api/v1/lightning/statistics/2y' \
'/api/v1/lightning/statistics/3y' \
'/api/v1/lightning/statistics/all' \
'/api/v1/lightning/nodes/isp-ranking' \
'/api/v1/lightning/nodes/isp/15169,396982' `# Google` \
'/api/v1/lightning/nodes/isp/14618,16509' `# Amazon` \
'/api/v1/lightning/nodes/isp/39572' `# DataWeb` \
'/api/v1/lightning/nodes/isp/14061' `# Digital Ocean` \
'/api/v1/lightning/nodes/isp/24940,213230' `# Hetzner` \
'/api/v1/lightning/nodes/isp/394745' `# LunaNode` \
'/api/v1/lightning/nodes/isp/45102' `# Alibaba` \
'/api/v1/lightning/nodes/isp/3209' `# Vodafone Germany` \
'/api/v1/lightning/nodes/isp/7922' `# Comcast Cable` \
'/api/v1/lightning/nodes/isp/34197' `# SHRD SARL` \
'/api/v1/lightning/nodes/isp/42275' `# Three Fourteen SASU` \
'/api/v1/lightning/nodes/isp/16276' `# OVH SAS` \
'/api/v1/lightning/nodes/isp/10796,11351,11426,11427,12271,20001,20115,33363' `# Spectrum` \
'/api/v1/lightning/nodes/isp/701' `# Verizon` \
'/api/v1/lightning/nodes/isp/12876' `# Scaleway` \
'/api/v1/lightning/nodes/isp/33915' `# Ziggo` \
'/api/v1/lightning/nodes/isp/3320' `# Deutsche Telekom AG` \
'/api/v1/lightning/nodes/isp/8075' `# Microsoft Azure` \
'/api/v1/lightning/nodes/isp/212531', `# UAB Interneto vizija` \
'/api/v1/lightning/nodes/isp/63949', `# Linode` \
'/api/v1/lightning/nodes/isp/51167', `# Contabo GmbH` \
'/api/v1/lightning/nodes/countries' \
'/api/v1/lightning/nodes/rankings' \
'/api/v1/lightning/nodes/rankings/liquidity' \
'/api/v1/lightning/nodes/rankings/connectivity' \
'/api/v1/lightning/nodes/rankings/age' \
warmUnfurlURLs=(
'/'
'/mining'
'/lightning'
)
warmURLs=(
'/api/v1/blocks'
'/api/v1/statistics/2h'
'/api/v1/statistics/24h'
'/api/v1/statistics/1w'
'/api/v1/statistics/1m'
'/api/v1/statistics/3m'
'/api/v1/statistics/6m'
'/api/v1/statistics/1y'
'/api/v1/statistics/2y'
'/api/v1/statistics/3y'
'/api/v1/statistics/4y'
'/api/v1/statistics/all'
'/api/v1/mining/pools/24h'
'/api/v1/mining/pools/3d'
'/api/v1/mining/pools/1w'
'/api/v1/mining/pools/1m'
'/api/v1/mining/pools/3m'
'/api/v1/mining/pools/6m'
'/api/v1/mining/pools/1y'
'/api/v1/mining/pools/2y'
'/api/v1/mining/pools/3y'
'/api/v1/mining/pools/all'
'/api/v1/mining/hashrate/3m'
'/api/v1/mining/hashrate/6m'
'/api/v1/mining/hashrate/1y'
'/api/v1/mining/hashrate/2y'
'/api/v1/mining/hashrate/3y'
'/api/v1/mining/hashrate/all'
'/api/v1/mining/hashrate/pools/3m'
'/api/v1/mining/hashrate/pools/6m'
'/api/v1/mining/hashrate/pools/1y'
'/api/v1/mining/hashrate/pools/2y'
'/api/v1/mining/hashrate/pools/3y'
'/api/v1/mining/hashrate/pools/all'
'/api/v1/mining/reward-stats/144'
'/api/v1/mining/blocks/fees/24h'
'/api/v1/mining/blocks/fees/3d'
'/api/v1/mining/blocks/fees/1w'
'/api/v1/mining/blocks/fees/1m'
'/api/v1/mining/blocks/fees/3m'
'/api/v1/mining/blocks/fees/6m'
'/api/v1/mining/blocks/fees/1y'
'/api/v1/mining/blocks/fees/2y'
'/api/v1/mining/blocks/fees/3y'
'/api/v1/mining/blocks/fees/all'
'/api/v1/mining/blocks/rewards/24h'
'/api/v1/mining/blocks/rewards/3d'
'/api/v1/mining/blocks/rewards/1w'
'/api/v1/mining/blocks/rewards/1m'
'/api/v1/mining/blocks/rewards/3m'
'/api/v1/mining/blocks/rewards/6m'
'/api/v1/mining/blocks/rewards/1y'
'/api/v1/mining/blocks/rewards/2y'
'/api/v1/mining/blocks/rewards/3y'
'/api/v1/mining/blocks/rewards/all'
'/api/v1/mining/blocks/fee-rates/24h'
'/api/v1/mining/blocks/fee-rates/3d'
'/api/v1/mining/blocks/fee-rates/1w'
'/api/v1/mining/blocks/fee-rates/1m'
'/api/v1/mining/blocks/fee-rates/3m'
'/api/v1/mining/blocks/fee-rates/6m'
'/api/v1/mining/blocks/fee-rates/1y'
'/api/v1/mining/blocks/fee-rates/2y'
'/api/v1/mining/blocks/fee-rates/3y'
'/api/v1/mining/blocks/fee-rates/all'
'/api/v1/mining/difficulty-adjustments/24h'
'/api/v1/mining/difficulty-adjustments/3d'
'/api/v1/mining/difficulty-adjustments/1w'
'/api/v1/mining/difficulty-adjustments/1m'
'/api/v1/mining/difficulty-adjustments/3m'
'/api/v1/mining/difficulty-adjustments/6m'
'/api/v1/mining/difficulty-adjustments/1y'
'/api/v1/mining/difficulty-adjustments/2y'
'/api/v1/mining/difficulty-adjustments/3y'
'/api/v1/mining/difficulty-adjustments/all'
'/api/v1/lightning/channels-geo?style=widget'
'/api/v1/lightning/channels-geo?style=graph'
'/api/v1/lightning/statistics/latest'
'/api/v1/lightning/statistics/1m'
'/api/v1/lightning/statistics/3m'
'/api/v1/lightning/statistics/6m'
'/api/v1/lightning/statistics/1y'
'/api/v1/lightning/statistics/2y'
'/api/v1/lightning/statistics/3y'
'/api/v1/lightning/statistics/all'
'/api/v1/lightning/nodes/isp-ranking'
'/api/v1/lightning/nodes/isp/15169,396982' `# Google`
'/api/v1/lightning/nodes/isp/14618,16509' `# Amazon`
'/api/v1/lightning/nodes/isp/39572' `# DataWeb`
'/api/v1/lightning/nodes/isp/14061' `# Digital Ocean`
'/api/v1/lightning/nodes/isp/24940,213230' `# Hetzner`
'/api/v1/lightning/nodes/isp/394745' `# LunaNode`
'/api/v1/lightning/nodes/isp/45102' `# Alibaba`
'/api/v1/lightning/nodes/isp/3209' `# Vodafone Germany`
'/api/v1/lightning/nodes/isp/7922' `# Comcast Cable`
'/api/v1/lightning/nodes/isp/34197' `# SHRD SARL`
'/api/v1/lightning/nodes/isp/42275' `# Three Fourteen SASU`
'/api/v1/lightning/nodes/isp/16276' `# OVH SAS`
'/api/v1/lightning/nodes/isp/10796,11351,11426,11427,12271,20001,2$delay,33363' `# Spectrum`
'/api/v1/lightning/nodes/isp/701' `# Verizon`
'/api/v1/lightning/nodes/isp/12876' `# Scaleway`
'/api/v1/lightning/nodes/isp/33915' `# Ziggo`
'/api/v1/lightning/nodes/isp/3320' `# Deutsche Telekom AG`
'/api/v1/lightning/nodes/isp/8075' `# Microsoft Azure`
'/api/v1/lightning/nodes/isp/212531' `# UAB Interneto vizija`
'/api/v1/lightning/nodes/isp/63949' `# Linode`
'/api/v1/lightning/nodes/isp/51167' `# Contabo GmbH`
'/api/v1/lightning/nodes/countries'
'/api/v1/lightning/nodes/rankings'
'/api/v1/lightning/nodes/rankings/liquidity'
'/api/v1/lightning/nodes/rankings/connectivity'
'/api/v1/lightning/nodes/rankings/age'
)
echo "waiting for mempool backend to start..."
sleep 60
while true
do
echo "starting warm cache cycle..."
for url in $warmSlurpURLs
do
warmSlurp "https://${hostname}${url}"
sleep $delay # delay between queries to not DoS unfurler
done
for url in $warmUnfurlURLs
do
warmUnfurl "https://${hostname}${url}"
sleep $delay # delay between queries to not DoS unfurler
done
for url in $warmURLs
do do
warm "https://${hostname}${url}" warm "https://${hostname}${url}"
sleep 0.25 # wait 250ms between queries to not DoS mariadb sleep $delay # delay between queries to not DoS mariadb
done done
for slug in $slugs for slug in $slugs
do do
warm "https://${hostname}/api/v1/mining/pool/${slug}" warm "https://${hostname}/api/v1/mining/pool/${slug}"
sleep 0.25 # wait 250ms between queries to not DoS mariadb sleep $delay # delay between queries to not DoS mariadb
warm "https://${hostname}/api/v1/mining/pool/${slug}/hashrate" warm "https://${hostname}/api/v1/mining/pool/${slug}/hashrate"
sleep 0.25 # wait 250ms between queries to not DoS mariadb sleep $delay # delay between queries to not DoS mariadb
warm "https://${hostname}/api/v1/mining/pool/${slug}/blocks" warm "https://${hostname}/api/v1/mining/pool/${slug}/blocks"
sleep 0.25 # wait 250ms between queries to not DoS mariadb sleep $delay # delay between queries to not DoS mariadb
done done
sleep 1
done done

8
production/nginx/http-proxy-cache.conf
View File

@ -1,5 +1,7 @@
# proxy cache # proxy cache
proxy_cache_path /var/cache/nginx/api keys_zone=api:20m levels=1:2 inactive=600s max_size=100m; proxy_cache_path /var/cache/nginx/api keys_zone=api:20m levels=1:2 inactive=600s max_size=200m;
proxy_cache_path /var/cache/nginx/services keys_zone=services:20m levels=1:2 inactive=600s max_size=100m; proxy_cache_path /var/cache/nginx/services keys_zone=services:20m levels=1:2 inactive=600s max_size=200m;
proxy_cache_path /var/cache/nginx/markets keys_zone=markets:20m levels=1:2 inactive=600s max_size=100m; proxy_cache_path /var/cache/nginx/markets keys_zone=markets:20m levels=1:2 inactive=600s max_size=200m;
proxy_cache_path /var/cache/nginx/unfurler keys_zone=unfurler:20m levels=1:2 inactive=600s max_size=200m;
proxy_cache_path /var/cache/nginx/slurper keys_zone=slurper:20m levels=1:2 inactive=600s max_size=200m;
types_hash_max_size 2048; types_hash_max_size 2048;

4
production/nginx/nginx.conf
View File

@ -23,8 +23,8 @@ http {
include mempool/production/nginx/http-language.conf; include mempool/production/nginx/http-language.conf;
# match preview/unfurl bot user-agents # match preview/unfurl bot user-agents
map $http_user_agent $unfurlbot { map $http_user_agent $unfurlprefix {
default 0; default "";
} }
# mempool configuration # mempool configuration

55
production/nginx/server-common.conf
View File

@ -48,8 +48,8 @@ add_header Vary Cookie;
# for exact / requests, redirect based on $lang # for exact / requests, redirect based on $lang
# cache redirect for 5 minutes # cache redirect for 5 minutes
location = / { location = / {
if ($unfurlbot) { if ($unfurlprefix != '') {
proxy_pass $mempoolSpaceUnfurler; rewrite ^(.*)$ $unfurlprefix$1 last;
} }
if ($lang != '') { if ($lang != '') {
return 302 $scheme://$host/$lang/; return 302 $scheme://$host/$lang/;
@ -65,15 +65,15 @@ location ~ ^/([a-z][a-z])/(.+\..+\.(js|css))$ {
} }
# cache everything else for 5 minutes # cache everything else for 5 minutes
location ~ ^/([a-z][a-z])$ { location ~ ^/([a-z][a-z])$ {
if ($unfurlbot) { if ($unfurlprefix != '') {
proxy_pass $mempoolSpaceUnfurler; rewrite ^(.*)$ $unfurlprefix$1 last;
} }
try_files $uri /$1/index.html /en-US/index.html =404; try_files $uri /$1/index.html /en-US/index.html =404;
expires 5m; expires 5m;
} }
location ~ ^/([a-z][a-z])/ { location ~ ^/([a-z][a-z])/ {
if ($unfurlbot) { if ($unfurlprefix != '') {
proxy_pass $mempoolSpaceUnfurler; rewrite ^(.*)$ $unfurlprefix$1 last;
} }
try_files $uri /$1/index.html /en-US/index.html =404; try_files $uri /$1/index.html /en-US/index.html =404;
expires 5m; expires 5m;
@ -104,16 +104,53 @@ location /preview {
} }
# unfurl renderer # unfurl renderer
location ^~ /render { location ^~ /render {
proxy_pass $mempoolSpaceUnfurler; try_files /dev/null @mempool-space-unfurler;
expires 10m; expires 10m;
} }
# unfurl handler
location /unfurl/ {
try_files /dev/null @mempool-space-unfurler;
}
location /slurp/ {
try_files /dev/null @mempool-space-slurper;
}
# catch-all for all URLs i.e. /address/foo /tx/foo /block/000 # catch-all for all URLs i.e. /address/foo /tx/foo /block/000
# cache 5 minutes since they change frequently # cache 5 minutes since they change frequently
location / { location / {
if ($unfurlbot) { if ($unfurlprefix != '') {
proxy_pass $mempoolSpaceUnfurler; rewrite ^(.*)$ $unfurlprefix$1 last;
} }
try_files /$lang/$uri $uri /en-US/$uri /en-US/index.html =404; try_files /$lang/$uri $uri /en-US/$uri /en-US/index.html =404;
expires 5m; expires 5m;
} }
location @mempool-space-unfurler {
proxy_pass $mempoolSpaceUnfurler;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_cache_background_update on;
proxy_cache_use_stale updating;
proxy_cache unfurler;
proxy_cache_valid 200 10m;
proxy_redirect off;
}
location @mempool-space-slurper {
proxy_pass $mempoolSpaceUnfurler;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_cache_background_update on;
proxy_cache_use_stale updating;
proxy_cache slurper;
proxy_cache_valid 200 10m;
proxy_redirect off;
}

26
unfurler/src/index.ts
View File

@ -109,11 +109,15 @@ class Server {
setUpRoutes() { setUpRoutes() {
if (puppeteerEnabled) { if (puppeteerEnabled) {
this.app.get('/unfurl/render*', async (req, res) => { return this.renderPreview(req, res) })
this.app.get('/render*', async (req, res) => { return this.renderPreview(req, res) }) this.app.get('/render*', async (req, res) => { return this.renderPreview(req, res) })
} else { } else {
this.app.get('/unfurl/render*', async (req, res) => { return this.renderDisabled(req, res) })
this.app.get('/render*', async (req, res) => { return this.renderDisabled(req, res) }) this.app.get('/render*', async (req, res) => { return this.renderDisabled(req, res) })
} }
this.app.get('*', (req, res) => { return this.renderHTML(req, res) }) this.app.get('/unfurl*', (req, res) => { return this.renderHTML(req, res, true) })
this.app.get('/slurp*', (req, res) => { return this.renderHTML(req, res, false) })
this.app.get('*', (req, res) => { return this.renderHTML(req, res, false) })
} }
async clusterTask({ page, data: { url, path, action } }) { async clusterTask({ page, data: { url, path, action } }) {
@ -240,7 +244,7 @@ class Server {
} }
} }
async renderHTML(req, res) { async renderHTML(req, res, unfurl: boolean = false) {
// drop requests for static files // drop requests for static files
const rawPath = req.params[0]; const rawPath = req.params[0];
const match = rawPath.match(/\.[\w]+$/); const match = rawPath.match(/\.[\w]+$/);
@ -250,25 +254,25 @@ class Server {
|| rawPath.startsWith('/api/v1/translators/images') || rawPath.startsWith('/api/v1/translators/images')
|| rawPath.startsWith('/resources/profile') || rawPath.startsWith('/resources/profile')
) { ) {
if (isSearchCrawler(req.headers['user-agent'])) { if (unfurl) {
res.status(404).send();
return;
} else {
if (this.secureHost) { if (this.secureHost) {
https.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res)); https.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res));
} else { } else {
http.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res)); http.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res));
} }
return; return;
} else {
res.status(404).send();
return;
} }
} }
let result = ''; let result = '';
try { try {
if (isSearchCrawler(req.headers['user-agent'])) { if (unfurl) {
result = await this.renderSEOPage(rawPath);
} else {
result = await this.renderUnfurlMeta(rawPath); result = await this.renderUnfurlMeta(rawPath);
} else {
result = await this.renderSEOPage(rawPath);
} }
if (result && result.length) { if (result && result.length) {
if (result === '404') { if (result === '404') {
@ -348,7 +352,3 @@ function capitalize(str) {
return str; return str;
} }
} }
function isSearchCrawler(useragent: string): boolean {
return /googlebot|applebot|bingbot/i.test(useragent);
}