From 2f27d9279d4551576343b6c62cc981804a48b01e Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 24 Feb 2023 20:21:57 -0600 Subject: [PATCH 1/7] extend unfurler to dynamically render search crawler requests --- frontend/src/app/services/seo.service.ts | 2 +- unfurler/src/concurrency/ReusableSSRPage.ts | 65 ++++++++++++ unfurler/src/index.ts | 107 ++++++++++++++++++-- 3 files changed, 167 insertions(+), 7 deletions(-) create mode 100644 unfurler/src/concurrency/ReusableSSRPage.ts diff --git a/frontend/src/app/services/seo.service.ts b/frontend/src/app/services/seo.service.ts index 5f5d15c89..78c7afc4c 100644 --- a/frontend/src/app/services/seo.service.ts +++ b/frontend/src/app/services/seo.service.ts @@ -7,7 +7,7 @@ import { StateService } from './state.service'; }) export class SeoService { network = ''; - baseTitle = 'mempool'; + baseTitle = 'Mempool'; constructor( private titleService: Title, diff --git a/unfurler/src/concurrency/ReusableSSRPage.ts b/unfurler/src/concurrency/ReusableSSRPage.ts new file mode 100644 index 000000000..c68514a16 --- /dev/null +++ b/unfurler/src/concurrency/ReusableSSRPage.ts @@ -0,0 +1,65 @@ +import * as puppeteer from 'puppeteer'; +import { timeoutExecute } from 'puppeteer-cluster/dist/util'; +import logger from '../logger'; +import config from '../config'; +import ReusablePage from './ReusablePage'; +const mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : ''); + +const mockImageBuffer = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVQYV2NgYAAAAAMAAWgmWQ0AAAAASUVORK5CYII=", 'base64'); + +interface RepairablePage extends puppeteer.Page { + repairRequested?: boolean; + language?: string | null; + createdAt?: number; + free?: boolean; + index?: number; +} + +export default class ReusableSSRPage extends ReusablePage { + + public constructor(options: puppeteer.LaunchOptions, puppeteer: any) { + super(options, puppeteer); + } + + public async close() { + await (this.browser as puppeteer.Browser).close(); + } + + protected async initPage(): Promise { + const page = await (this.browser as puppeteer.Browser).newPage() as RepairablePage; + page.language = null; + page.createdAt = Date.now(); + const defaultUrl = mempoolHost + '/about'; + + page.on('pageerror', (err) => { + console.log(err); + // page.repairRequested = true; + }); + await page.setRequestInterception(true); + page.on('request', req => { + if (req.isInterceptResolutionHandled()) { + return req.continue(); + } + if (req.resourceType() === 'image') { + return req.respond({ + contentType: 'image/png', + headers: {"Access-Control-Allow-Origin": "*"}, + body: mockImageBuffer + }); + } else if (!['document', 'script', 'xhr', 'fetch'].includes(req.resourceType())) { + return req.abort(); + } else { + return req.continue(); + } + }); + try { + await page.goto(defaultUrl, { waitUntil: "networkidle0" }); + await page.waitForSelector('meta[property="og:meta:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }); + } catch (e) { + logger.err(`failed to load frontend during ssr page initialization: ` + (e instanceof Error ? e.message : `${e}`)); + page.repairRequested = true; + } + page.free = true; + return page + } +} diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index 0b423ff92..c24cd65af 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -5,9 +5,11 @@ import * as https from 'https'; import config from './config'; import { Cluster } from 'puppeteer-cluster'; import ReusablePage from './concurrency/ReusablePage'; +import ReusableSSRPage from './concurrency/ReusablePage'; import { parseLanguageUrl } from './language/lang'; import { matchRoute } from './routes'; import logger from './logger'; +import { TimeoutError } from "puppeteer"; const puppeteerConfig = require('../puppeteer.config.json'); if (config.PUPPETEER.EXEC_PATH) { @@ -20,13 +22,16 @@ class Server { private server: http.Server | undefined; private app: Application; cluster?: Cluster; + ssrCluster?: Cluster; mempoolHost: string; + mempoolUrl: URL; network: string; secureHost = true; constructor() { this.app = express(); this.mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : ''); + this.mempoolUrl = new URL(this.mempoolHost); this.secureHost = config.SERVER.HOST.startsWith('https'); this.network = config.MEMPOOL.NETWORK || 'bitcoin'; this.startServer(); @@ -49,6 +54,12 @@ class Server { puppeteerOptions: puppeteerConfig, }); await this.cluster?.task(async (args) => { return this.clusterTask(args) }); + this.ssrCluster = await Cluster.launch({ + concurrency: ReusableSSRPage, + maxConcurrency: config.PUPPETEER.CLUSTER_SIZE, + puppeteerOptions: puppeteerConfig, + }); + await this.ssrCluster?.task(async (args) => { return this.ssrClusterTask(args) }); } this.setUpRoutes(); @@ -65,6 +76,10 @@ class Server { await this.cluster.idle(); await this.cluster.close(); } + if (this.ssrCluster) { + await this.ssrCluster.idle(); + await this.ssrCluster.close(); + } if (this.server) { await this.server.close(); } @@ -102,8 +117,8 @@ class Server { } // wait for preview component to initialize - await page.waitForSelector('meta[property="og:preview:loading"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }) let success; + await page.waitForSelector('meta[property="og:preview:loading"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }) success = await Promise.race([ page.waitForSelector('meta[property="og:preview:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => true), page.waitForSelector('meta[property="og:preview:fail"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => false) @@ -124,6 +139,44 @@ class Server { } } + async ssrClusterTask({ page, data: { url, path, action } }) { + try { + const urlParts = parseLanguageUrl(path); + if (page.language !== urlParts.lang) { + // switch language + page.language = urlParts.lang; + const localizedUrl = urlParts.lang ? `${this.mempoolHost}/${urlParts.lang}${urlParts.path}` : `${this.mempoolHost}${urlParts.path}` ; + await page.goto(localizedUrl, { waitUntil: "load" }); + } else { + const loaded = await page.evaluate(async (path) => { + if (window['ogService']) { + window['ogService'].loadPage(path); + return true; + } else { + return false; + } + }, urlParts.path); + if (!loaded) { + throw new Error('failed to access open graph service'); + } + } + + await page.waitForNetworkIdle({ + timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000, + }); + let html = await page.content(); + return html; + } catch (e) { + if (e instanceof TimeoutError) { + let html = await page.content(); + return html; + } else { + logger.err(`failed to render ${path} for ${action}: ` + (e instanceof Error ? e.message : `${e}`)); + page.repairRequested = true; + } + } + } + async renderDisabled(req, res) { res.status(500).send("preview rendering disabled"); } @@ -163,11 +216,44 @@ class Server { // drop requests for static files const rawPath = req.params[0]; const match = rawPath.match(/\.[\w]+$/); - if (match?.length && match[0] !== '.html') { - res.status(404).send(); - return; + if (match?.length && match[0] !== '.html' + || rawPath.startsWith('/api/v1/donations/images') + || rawPath.startsWith('/api/v1/contributors/images') + || rawPath.startsWith('/api/v1/translators/images') + || rawPath.startsWith('/resources/profile') + ) { + if (req.headers['user-agent'] === 'googlebot') { + if (this.secureHost) { + https.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res)); + } else { + http.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res)); + } + return; + } else { + res.status(404).send(); + return; + } } + let result = ''; + try { + if (req.headers['user-agent'] === 'googlebot') { + result = await this.renderSEOPage(rawPath); + } else { + result = await this.renderUnfurlMeta(rawPath); + } + if (result && result.length) { + res.send(result); + } else { + res.status(500).send(); + } + } catch (e) { + logger.err(e instanceof Error ? e.message : `${e} ${req.params[0]}`); + res.status(500).send(e instanceof Error ? e.message : e); + } + } + + async renderUnfurlMeta(rawPath: string): Promise { const { lang, path } = parseLanguageUrl(rawPath); const matchedRoute = matchRoute(this.network, path); let ogImageUrl = config.SERVER.HOST + (matchedRoute.staticImg || matchedRoute.fallbackImg); @@ -178,7 +264,7 @@ class Server { ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${matchedRoute.networkMode !== 'mainnet' ? capitalize(matchedRoute.networkMode) + ' ' : ''}${matchedRoute.title}`; } - res.send(` + return ` @@ -199,7 +285,16 @@ class Server { - `); + `; + } + + async renderSEOPage(rawPath: string): Promise { + let html = await this.ssrCluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'ssr' }); + // remove javascript to prevent double hydration + if (html && html.length) { + html = html.replace(//g, ""); + } + return html; } } From b1e32ed55f413741ae95244bac8741766d78fa52 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 27 Feb 2023 10:48:13 -0600 Subject: [PATCH 2/7] Fix googlebot user-agent detection --- unfurler/src/index.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index c24cd65af..0bbcb32bc 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -222,7 +222,7 @@ class Server { || rawPath.startsWith('/api/v1/translators/images') || rawPath.startsWith('/resources/profile') ) { - if (req.headers['user-agent'] === 'googlebot') { + if (isSearchCrawler(req.headers['user-agent'])) { if (this.secureHost) { https.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res)); } else { @@ -237,7 +237,7 @@ class Server { let result = ''; try { - if (req.headers['user-agent'] === 'googlebot') { + if (isSearchCrawler(req.headers['user-agent'])) { result = await this.renderSEOPage(rawPath); } else { result = await this.renderUnfurlMeta(rawPath); @@ -313,3 +313,7 @@ function capitalize(str) { return str; } } + +function isSearchCrawler(useragent: string): boolean { + return /googlebot/i.test(useragent); +} From f6cae729a76f2166f563369a3190b765bdbf7930 Mon Sep 17 00:00:00 2001 From: mononaut <83316221+mononaut@users.noreply.github.com> Date: Tue, 28 Feb 2023 18:40:59 -0600 Subject: [PATCH 3/7] revert capitalization in title tag Co-authored-by: wiz --- frontend/src/app/services/seo.service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/services/seo.service.ts b/frontend/src/app/services/seo.service.ts index 78c7afc4c..5f5d15c89 100644 --- a/frontend/src/app/services/seo.service.ts +++ b/frontend/src/app/services/seo.service.ts @@ -7,7 +7,7 @@ import { StateService } from './state.service'; }) export class SeoService { network = ''; - baseTitle = 'Mempool'; + baseTitle = 'mempool'; constructor( private titleService: Title, From 477f3bd70a7583cc80b62b73239baeba6ea4c268 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 8 Mar 2023 01:11:54 -0600 Subject: [PATCH 4/7] add applebot & bingbot to seo user-agent detection --- unfurler/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index 0bbcb32bc..6d209b140 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -315,5 +315,5 @@ function capitalize(str) { } function isSearchCrawler(useragent: string): boolean { - return /googlebot/i.test(useragent); + return /googlebot|applebot|bingbot/i.test(useragent); } From 82a808529b58172585fd196bd8eecdf7545321c4 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Wed, 8 Mar 2023 01:47:35 -0600 Subject: [PATCH 5/7] clean up unfurler meta html template --- unfurler/src/index.ts | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index 6d209b140..90938a5a8 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -264,28 +264,27 @@ class Server { ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${matchedRoute.networkMode !== 'mainnet' ? capitalize(matchedRoute.networkMode) + ' ' : ''}${matchedRoute.title}`; } - return ` - - - - - ${ogTitle} - - - - - - - - - - - - - - - - `; + return ` + + + + ${ogTitle} + + + + + + + + + + + + + + + +`; } async renderSEOPage(rawPath: string): Promise { From 2f3e49890628318f5f9261a2bbd12bda63f1789f Mon Sep 17 00:00:00 2001 From: Mononaut Date: Thu, 9 Mar 2023 00:26:28 -0600 Subject: [PATCH 6/7] fix canonical/meta tags in unfurler --- unfurler/src/index.ts | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index 90938a5a8..412dd0f4c 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -27,6 +27,7 @@ class Server { mempoolUrl: URL; network: string; secureHost = true; + canonicalHost: string; constructor() { this.app = express(); @@ -34,6 +35,20 @@ class Server { this.mempoolUrl = new URL(this.mempoolHost); this.secureHost = config.SERVER.HOST.startsWith('https'); this.network = config.MEMPOOL.NETWORK || 'bitcoin'; + + let canonical; + switch(config.MEMPOOL.NETWORK) { + case "liquid": + canonical = "https://liquid.network" + break; + case "bisq": + canonical = "https://bisq.markets" + break; + default: + canonical = "https://mempool.space" + } + this.canonicalHost = canonical; + this.startServer(); } @@ -259,6 +274,8 @@ class Server { let ogImageUrl = config.SERVER.HOST + (matchedRoute.staticImg || matchedRoute.fallbackImg); let ogTitle = 'The Mempool Open Source Project™'; + const canonical = this.canonicalHost + rawPath; + if (matchedRoute.render) { ogImageUrl = `${config.SERVER.HOST}/render/${lang || 'en'}/preview${path}`; ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${matchedRoute.networkMode !== 'mainnet' ? capitalize(matchedRoute.networkMode) + ' ' : ''}${matchedRoute.title}`; @@ -269,6 +286,7 @@ class Server { ${ogTitle} + @@ -291,7 +309,8 @@ class Server { let html = await this.ssrCluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'ssr' }); // remove javascript to prevent double hydration if (html && html.length) { - html = html.replace(//g, ""); + html = html.replaceAll(//g, ""); + html = html.replaceAll(this.mempoolHost, this.canonicalHost); } return html; } From 105cccf9b0fd6b2dc2b4229eb2c51d148f6467c7 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Thu, 9 Mar 2023 02:34:21 -0600 Subject: [PATCH 7/7] convert soft 404s to hard 404s in unfurler ssr --- .../bisq-address/bisq-address.component.ts | 2 ++ .../bisq/bisq-block/bisq-block.component.ts | 2 ++ .../bisq-transaction.component.ts | 3 +++ .../components/address/address.component.ts | 2 ++ .../app/components/asset/asset.component.ts | 2 ++ .../block/block-preview.component.ts | 2 ++ .../app/components/block/block.component.ts | 3 +++ .../components/pool/pool-preview.component.ts | 2 ++ .../src/app/components/pool/pool.component.ts | 17 +++++++++--- .../transaction-preview.component.ts | 2 ++ .../transaction/transaction.component.ts | 12 ++++++--- .../channel/channel-preview.component.ts | 1 + .../lightning/channel/channel.component.ts | 1 + .../group/group-preview.component.ts | 2 ++ .../lightning/node/node-preview.component.ts | 1 + .../src/app/lightning/node/node.component.ts | 1 + .../nodes-per-isp-preview.component.ts | 1 + frontend/src/app/services/seo.service.ts | 26 +++++++++++++++++++ unfurler/src/index.ts | 17 +++++++++--- 19 files changed, 90 insertions(+), 9 deletions(-) diff --git a/frontend/src/app/bisq/bisq-address/bisq-address.component.ts b/frontend/src/app/bisq/bisq-address/bisq-address.component.ts index eccc88bc7..7711c4d3c 100644 --- a/frontend/src/app/bisq/bisq-address/bisq-address.component.ts +++ b/frontend/src/app/bisq/bisq-address/bisq-address.component.ts @@ -47,6 +47,7 @@ export class BisqAddressComponent implements OnInit, OnDestroy { catchError((err) => { this.isLoadingAddress = false; this.error = err; + this.seoService.logSoft404(); console.log(err); return of(null); }) @@ -62,6 +63,7 @@ export class BisqAddressComponent implements OnInit, OnDestroy { (error) => { console.log(error); this.error = error; + this.seoService.logSoft404(); this.isLoadingAddress = false; }); } diff --git a/frontend/src/app/bisq/bisq-block/bisq-block.component.ts b/frontend/src/app/bisq/bisq-block/bisq-block.component.ts index 1bb3a24ab..206a18031 100644 --- a/frontend/src/app/bisq/bisq-block/bisq-block.component.ts +++ b/frontend/src/app/bisq/bisq-block/bisq-block.component.ts @@ -82,6 +82,7 @@ export class BisqBlockComponent implements OnInit, OnDestroy { ) .subscribe((block: BisqBlock) => { if (!block) { + this.seoService.logSoft404(); return; } this.isLoading = false; @@ -97,6 +98,7 @@ export class BisqBlockComponent implements OnInit, OnDestroy { caughtHttpError(err: HttpErrorResponse){ this.error = err; + this.seoService.logSoft404(); return of(null); } } diff --git a/frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.ts b/frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.ts index fb30fc59f..78e2c0bb0 100644 --- a/frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.ts +++ b/frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.ts @@ -70,11 +70,13 @@ export class BisqTransactionComponent implements OnInit, OnDestroy { catchError((txError: HttpErrorResponse) => { console.log(txError); this.error = txError; + this.seoService.logSoft404(); return of(null); }) ); } this.error = bisqTxError; + this.seoService.logSoft404(); return of(null); }) ); @@ -103,6 +105,7 @@ export class BisqTransactionComponent implements OnInit, OnDestroy { this.isLoadingTx = false; if (!tx) { + this.seoService.logSoft404(); return; } diff --git a/frontend/src/app/components/address/address.component.ts b/frontend/src/app/components/address/address.component.ts index 2ae9a962b..b7752228d 100644 --- a/frontend/src/app/components/address/address.component.ts +++ b/frontend/src/app/components/address/address.component.ts @@ -88,6 +88,7 @@ export class AddressComponent implements OnInit, OnDestroy { catchError((err) => { this.isLoadingAddress = false; this.error = err; + this.seoService.logSoft404(); console.log(err); return of(null); }) @@ -157,6 +158,7 @@ export class AddressComponent implements OnInit, OnDestroy { (error) => { console.log(error); this.error = error; + this.seoService.logSoft404(); this.isLoadingAddress = false; }); diff --git a/frontend/src/app/components/asset/asset.component.ts b/frontend/src/app/components/asset/asset.component.ts index 0e642063a..562ebff53 100644 --- a/frontend/src/app/components/asset/asset.component.ts +++ b/frontend/src/app/components/asset/asset.component.ts @@ -86,6 +86,7 @@ export class AssetComponent implements OnInit, OnDestroy { catchError((err) => { this.isLoadingAsset = false; this.error = err; + this.seoService.logSoft404(); console.log(err); return of(null); }) @@ -153,6 +154,7 @@ export class AssetComponent implements OnInit, OnDestroy { (error) => { console.log(error); this.error = error; + this.seoService.logSoft404(); this.isLoadingAsset = false; }); diff --git a/frontend/src/app/components/block/block-preview.component.ts b/frontend/src/app/components/block/block-preview.component.ts index d0fec960a..7c10dab6f 100644 --- a/frontend/src/app/components/block/block-preview.component.ts +++ b/frontend/src/app/components/block/block-preview.component.ts @@ -82,6 +82,7 @@ export class BlockPreviewComponent implements OnInit, OnDestroy { }), catchError((err) => { this.error = err; + this.seoService.logSoft404(); this.openGraphService.fail('block-data-' + this.rawId); this.openGraphService.fail('block-viz-' + this.rawId); return of(null); @@ -138,6 +139,7 @@ export class BlockPreviewComponent implements OnInit, OnDestroy { (error) => { this.error = error; this.isLoadingOverview = false; + this.seoService.logSoft404(); this.openGraphService.fail('block-viz-' + this.rawId); this.openGraphService.fail('block-data-' + this.rawId); if (this.blockGraph) { diff --git a/frontend/src/app/components/block/block.component.ts b/frontend/src/app/components/block/block.component.ts index f5a0c93b0..6494d2f70 100644 --- a/frontend/src/app/components/block/block.component.ts +++ b/frontend/src/app/components/block/block.component.ts @@ -192,6 +192,7 @@ export class BlockComponent implements OnInit, OnDestroy { this.error = err; this.isLoadingBlock = false; this.isLoadingOverview = false; + this.seoService.logSoft404(); return EMPTY; }) ); @@ -200,6 +201,7 @@ export class BlockComponent implements OnInit, OnDestroy { this.error = err; this.isLoadingBlock = false; this.isLoadingOverview = false; + this.seoService.logSoft404(); return EMPTY; }), ); @@ -215,6 +217,7 @@ export class BlockComponent implements OnInit, OnDestroy { this.error = err; this.isLoadingBlock = false; this.isLoadingOverview = false; + this.seoService.logSoft404(); return EMPTY; }) ); diff --git a/frontend/src/app/components/pool/pool-preview.component.ts b/frontend/src/app/components/pool/pool-preview.component.ts index 277bacb33..aa83537fe 100644 --- a/frontend/src/app/components/pool/pool-preview.component.ts +++ b/frontend/src/app/components/pool/pool-preview.component.ts @@ -61,6 +61,7 @@ export class PoolPreviewComponent implements OnInit { }), catchError(() => { this.isLoading = false; + this.seoService.logSoft404(); this.openGraphService.fail('pool-hash-' + this.slug); return of([slug]); }) @@ -70,6 +71,7 @@ export class PoolPreviewComponent implements OnInit { return this.apiService.getPoolStats$(slug).pipe( catchError(() => { this.isLoading = false; + this.seoService.logSoft404(); this.openGraphService.fail('pool-stats-' + this.slug); return of(null); }) diff --git a/frontend/src/app/components/pool/pool.component.ts b/frontend/src/app/components/pool/pool.component.ts index 56b8bd392..1c2ce5c1f 100644 --- a/frontend/src/app/components/pool/pool.component.ts +++ b/frontend/src/app/components/pool/pool.component.ts @@ -1,8 +1,8 @@ import { ChangeDetectionStrategy, Component, Inject, Input, LOCALE_ID, OnInit } from '@angular/core'; import { ActivatedRoute } from '@angular/router'; import { EChartsOption, graphic } from 'echarts'; -import { BehaviorSubject, Observable, timer } from 'rxjs'; -import { distinctUntilChanged, map, share, switchMap, tap } from 'rxjs/operators'; +import { BehaviorSubject, Observable, of, timer } from 'rxjs'; +import { catchError, distinctUntilChanged, map, share, switchMap, tap } from 'rxjs/operators'; import { BlockExtended, PoolStat } from '../../interfaces/node-api.interface'; import { ApiService } from '../../services/api.service'; import { StateService } from '../../services/state.service'; @@ -59,10 +59,21 @@ export class PoolComponent implements OnInit { this.prepareChartOptions(data.map(val => [val.timestamp * 1000, val.avgHashrate])); return [slug]; }), + catchError(() => { + this.isLoading = false; + this.seoService.logSoft404(); + return of([slug]); + }) ); }), switchMap((slug) => { - return this.apiService.getPoolStats$(slug); + return this.apiService.getPoolStats$(slug).pipe( + catchError(() => { + this.isLoading = false; + this.seoService.logSoft404(); + return of(null); + }) + ); }), tap(() => { this.loadMoreSubject.next(this.blocks[this.blocks.length - 1]?.height); diff --git a/frontend/src/app/components/transaction/transaction-preview.component.ts b/frontend/src/app/components/transaction/transaction-preview.component.ts index 6db0e588c..07843cc57 100644 --- a/frontend/src/app/components/transaction/transaction-preview.component.ts +++ b/frontend/src/app/components/transaction/transaction-preview.component.ts @@ -133,6 +133,7 @@ export class TransactionPreviewComponent implements OnInit, OnDestroy { ) .subscribe((tx: Transaction) => { if (!tx) { + this.seoService.logSoft404(); this.openGraphService.fail('tx-data-' + this.txId); return; } @@ -182,6 +183,7 @@ export class TransactionPreviewComponent implements OnInit, OnDestroy { this.openGraphService.waitOver('tx-data-' + this.txId); }, (error) => { + this.seoService.logSoft404(); this.openGraphService.fail('tx-data-' + this.txId); this.error = error; this.isLoadingTx = false; diff --git a/frontend/src/app/components/transaction/transaction.component.ts b/frontend/src/app/components/transaction/transaction.component.ts index 673743344..0754f6d18 100644 --- a/frontend/src/app/components/transaction/transaction.component.ts +++ b/frontend/src/app/components/transaction/transaction.component.ts @@ -193,8 +193,10 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { }) ).subscribe((tx) => { if (!tx) { + this.seoService.logSoft404(); return; } + this.seoService.clearSoft404(); this.tx = tx; this.isCached = true; @@ -286,9 +288,11 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { }) ) .subscribe((tx: Transaction) => { - if (!tx) { - return; - } + if (!tx) { + this.seoService.logSoft404(); + return; + } + this.seoService.clearSoft404(); this.tx = tx; this.isCached = false; @@ -340,6 +344,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { }, (error) => { this.error = error; + this.seoService.logSoft404(); this.isLoadingTx = false; } ); @@ -394,6 +399,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy { this.waitingForTransaction = true; } this.error = error; + this.seoService.logSoft404(); this.isLoadingTx = false; return of(false); } diff --git a/frontend/src/app/lightning/channel/channel-preview.component.ts b/frontend/src/app/lightning/channel/channel-preview.component.ts index 9f1bea4b8..9c7fdc1d6 100644 --- a/frontend/src/app/lightning/channel/channel-preview.component.ts +++ b/frontend/src/app/lightning/channel/channel-preview.component.ts @@ -54,6 +54,7 @@ export class ChannelPreviewComponent implements OnInit { }), catchError((err) => { this.error = err; + this.seoService.logSoft404(); this.openGraphService.fail('channel-map-' + this.shortId); this.openGraphService.fail('channel-data-' + this.shortId); return of(null); diff --git a/frontend/src/app/lightning/channel/channel.component.ts b/frontend/src/app/lightning/channel/channel.component.ts index d57aa3f01..052225cc3 100644 --- a/frontend/src/app/lightning/channel/channel.component.ts +++ b/frontend/src/app/lightning/channel/channel.component.ts @@ -38,6 +38,7 @@ export class ChannelComponent implements OnInit { }), catchError((err) => { this.error = err; + this.seoService.logSoft404(); return [{ short_id: params.get('short_id') }]; diff --git a/frontend/src/app/lightning/group/group-preview.component.ts b/frontend/src/app/lightning/group/group-preview.component.ts index be23e6178..5fd730931 100644 --- a/frontend/src/app/lightning/group/group-preview.component.ts +++ b/frontend/src/app/lightning/group/group-preview.component.ts @@ -50,6 +50,7 @@ export class GroupPreviewComponent implements OnInit { name: this.slug.replace(/-/gi, ' '), description: '', }; + this.seoService.logSoft404(); this.openGraphService.fail('ln-group-map-' + this.slug); this.openGraphService.fail('ln-group-data-' + this.slug); return of(null); @@ -106,6 +107,7 @@ export class GroupPreviewComponent implements OnInit { }; }), catchError(() => { + this.seoService.logSoft404(); this.openGraphService.fail('ln-group-map-' + this.slug); this.openGraphService.fail('ln-group-data-' + this.slug); return of({ diff --git a/frontend/src/app/lightning/node/node-preview.component.ts b/frontend/src/app/lightning/node/node-preview.component.ts index 4c2782c2d..56753b18b 100644 --- a/frontend/src/app/lightning/node/node-preview.component.ts +++ b/frontend/src/app/lightning/node/node-preview.component.ts @@ -81,6 +81,7 @@ export class NodePreviewComponent implements OnInit { }), catchError(err => { this.error = err; + this.seoService.logSoft404(); this.openGraphService.fail('node-map-' + this.publicKey); this.openGraphService.fail('node-data-' + this.publicKey); return [{ diff --git a/frontend/src/app/lightning/node/node.component.ts b/frontend/src/app/lightning/node/node.component.ts index 47f65007f..6f222688d 100644 --- a/frontend/src/app/lightning/node/node.component.ts +++ b/frontend/src/app/lightning/node/node.component.ts @@ -121,6 +121,7 @@ export class NodeComponent implements OnInit { }), catchError(err => { this.error = err; + this.seoService.logSoft404(); return [{ alias: this.publicKey, public_key: this.publicKey, diff --git a/frontend/src/app/lightning/nodes-per-isp/nodes-per-isp-preview.component.ts b/frontend/src/app/lightning/nodes-per-isp/nodes-per-isp-preview.component.ts index 759606372..b823a5188 100644 --- a/frontend/src/app/lightning/nodes-per-isp/nodes-per-isp-preview.component.ts +++ b/frontend/src/app/lightning/nodes-per-isp/nodes-per-isp-preview.component.ts @@ -85,6 +85,7 @@ export class NodesPerISPPreview implements OnInit { }), catchError(err => { this.error = err; + this.seoService.logSoft404(); this.openGraphService.fail('isp-map-' + this.id); this.openGraphService.fail('isp-data-' + this.id); return of({ diff --git a/frontend/src/app/services/seo.service.ts b/frontend/src/app/services/seo.service.ts index 5f5d15c89..d536b4938 100644 --- a/frontend/src/app/services/seo.service.ts +++ b/frontend/src/app/services/seo.service.ts @@ -1,5 +1,7 @@ import { Injectable } from '@angular/core'; import { Title, Meta } from '@angular/platform-browser'; +import { ActivatedRoute, NavigationEnd, Router } from '@angular/router'; +import { filter, map, switchMap } from 'rxjs'; import { StateService } from './state.service'; @Injectable({ @@ -13,8 +15,22 @@ export class SeoService { private titleService: Title, private metaService: Meta, private stateService: StateService, + private router: Router, + private activatedRoute: ActivatedRoute, ) { this.stateService.networkChanged$.subscribe((network) => this.network = network); + this.router.events.pipe( + filter(event => event instanceof NavigationEnd), + map(() => this.activatedRoute), + map(route => { + while (route.firstChild) route = route.firstChild; + return route; + }), + filter(route => route.outlet === 'primary'), + switchMap(route => route.data), + ).subscribe((data) => { + this.clearSoft404(); + }); } setTitle(newTitle: string): void { @@ -53,4 +69,14 @@ export class SeoService { ucfirst(str: string) { return str.charAt(0).toUpperCase() + str.slice(1); } + + clearSoft404() { + window['soft404'] = false; + console.log('cleared soft 404'); + } + + logSoft404() { + window['soft404'] = true; + console.log('set soft 404'); + } } diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index 412dd0f4c..d7589797d 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -179,8 +179,15 @@ class Server { await page.waitForNetworkIdle({ timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000, }); - let html = await page.content(); - return html; + const is404 = await page.evaluate(async () => { + return !!window['soft404']; + }); + if (is404) { + return '404'; + } else { + let html = await page.content(); + return html; + } } catch (e) { if (e instanceof TimeoutError) { let html = await page.content(); @@ -258,7 +265,11 @@ class Server { result = await this.renderUnfurlMeta(rawPath); } if (result && result.length) { - res.send(result); + if (result === '404') { + res.status(404).send(); + } else { + res.send(result); + } } else { res.status(500).send(); }