machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3169 from mempool/mononaut/seo-ssr

Browse Source 
dynamically render search crawler requests
This commit is contained in:
wiz 2023-08-05 18:34:45 +09:00 committed by GitHub
commit 794a4ded9c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 292 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
frontend/src/app/bisq/bisq-address/bisq-address.component.ts
View File

@ -47,6 +47,7 @@ export class BisqAddressComponent implements OnInit, OnDestroy {
catchError((err) => {
this.isLoadingAddress = false;
this.error = err;
this.seoService.logSoft404();
console.log(err);
return of(null);
})
@ -62,6 +63,7 @@ export class BisqAddressComponent implements OnInit, OnDestroy {
(error) => {
console.log(error);
this.error = error;
this.seoService.logSoft404();
this.isLoadingAddress = false;
});
}

2
frontend/src/app/bisq/bisq-block/bisq-block.component.ts
View File

@ -82,6 +82,7 @@ export class BisqBlockComponent implements OnInit, OnDestroy {
)
.subscribe((block: BisqBlock) => {
if (!block) {
this.seoService.logSoft404();
return;
}
this.isLoading = false;
@ -97,6 +98,7 @@ export class BisqBlockComponent implements OnInit, OnDestroy {
caughtHttpError(err: HttpErrorResponse){
this.error = err;
this.seoService.logSoft404();
return of(null);
}
}

3
frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.ts
View File

@ -70,11 +70,13 @@ export class BisqTransactionComponent implements OnInit, OnDestroy {
catchError((txError: HttpErrorResponse) => {
console.log(txError);
this.error = txError;
this.seoService.logSoft404();
return of(null);
})
);
}
this.error = bisqTxError;
this.seoService.logSoft404();
return of(null);
})
);
@ -103,6 +105,7 @@ export class BisqTransactionComponent implements OnInit, OnDestroy {
this.isLoadingTx = false;
if (!tx) {
this.seoService.logSoft404();
return;
}

2
frontend/src/app/components/address/address.component.ts
View File

@ -91,6 +91,7 @@ export class AddressComponent implements OnInit, OnDestroy {
catchError((err) => {
this.isLoadingAddress = false;
this.error = err;
this.seoService.logSoft404();
console.log(err);
return of(null);
})
@ -162,6 +163,7 @@ export class AddressComponent implements OnInit, OnDestroy {
(error) => {
console.log(error);
this.error = error;
this.seoService.logSoft404();
this.isLoadingAddress = false;
});

2
frontend/src/app/components/asset/asset.component.ts
View File

@ -86,6 +86,7 @@ export class AssetComponent implements OnInit, OnDestroy {
catchError((err) => {
this.isLoadingAsset = false;
this.error = err;
this.seoService.logSoft404();
console.log(err);
return of(null);
})
@ -153,6 +154,7 @@ export class AssetComponent implements OnInit, OnDestroy {
(error) => {
console.log(error);
this.error = error;
this.seoService.logSoft404();
this.isLoadingAsset = false;
});

2
frontend/src/app/components/block/block-preview.component.ts
View File

@ -82,6 +82,7 @@ export class BlockPreviewComponent implements OnInit, OnDestroy {
}),
catchError((err) => {
this.error = err;
this.seoService.logSoft404();
this.openGraphService.fail('block-data-' + this.rawId);
this.openGraphService.fail('block-viz-' + this.rawId);
return of(null);
@ -138,6 +139,7 @@ export class BlockPreviewComponent implements OnInit, OnDestroy {
(error) => {
this.error = error;
this.isLoadingOverview = false;
this.seoService.logSoft404();
this.openGraphService.fail('block-viz-' + this.rawId);
this.openGraphService.fail('block-data-' + this.rawId);
if (this.blockGraph) {

3
frontend/src/app/components/block/block.component.ts
View File

@ -206,6 +206,7 @@ export class BlockComponent implements OnInit, OnDestroy {
this.error = err;
this.isLoadingBlock = false;
this.isLoadingOverview = false;
this.seoService.logSoft404();
return EMPTY;
})
);
@ -214,6 +215,7 @@ export class BlockComponent implements OnInit, OnDestroy {
this.error = err;
this.isLoadingBlock = false;
this.isLoadingOverview = false;
this.seoService.logSoft404();
return EMPTY;
}),
);
@ -229,6 +231,7 @@ export class BlockComponent implements OnInit, OnDestroy {
this.error = err;
this.isLoadingBlock = false;
this.isLoadingOverview = false;
this.seoService.logSoft404();
return EMPTY;
})
);

2
frontend/src/app/components/pool/pool-preview.component.ts
View File

@ -61,6 +61,7 @@ export class PoolPreviewComponent implements OnInit {
}),
catchError(() => {
this.isLoading = false;
this.seoService.logSoft404();
this.openGraphService.fail('pool-hash-' + this.slug);
return of([slug]);
})
@ -70,6 +71,7 @@ export class PoolPreviewComponent implements OnInit {
return this.apiService.getPoolStats$(slug).pipe(
catchError(() => {
this.isLoading = false;
this.seoService.logSoft404();
this.openGraphService.fail('pool-stats-' + this.slug);
return of(null);
})

17
frontend/src/app/components/pool/pool.component.ts
View File

@ -1,8 +1,8 @@
import { ChangeDetectionStrategy, Component, Inject, Input, LOCALE_ID, OnInit } from '@angular/core';
import { ActivatedRoute } from '@angular/router';
import { EChartsOption, graphic } from 'echarts';
import { BehaviorSubject, Observable } from 'rxjs';
import { distinctUntilChanged, map, share, switchMap, tap } from 'rxjs/operators';
import { BehaviorSubject, Observable, of, timer } from 'rxjs';
import { catchError, distinctUntilChanged, map, share, switchMap, tap } from 'rxjs/operators';
import { BlockExtended, PoolStat } from '../../interfaces/node-api.interface';
import { ApiService } from '../../services/api.service';
import { StateService } from '../../services/state.service';
@ -62,10 +62,21 @@ export class PoolComponent implements OnInit {
this.prepareChartOptions(data.map(val => [val.timestamp * 1000, val.avgHashrate]));
return [slug];
}),
catchError(() => {
this.isLoading = false;
this.seoService.logSoft404();
return of([slug]);
})
);
}),
switchMap((slug) => {
return this.apiService.getPoolStats$(slug);
return this.apiService.getPoolStats$(slug).pipe(
catchError(() => {
this.isLoading = false;
this.seoService.logSoft404();
return of(null);
})
);
}),
tap(() => {
this.loadMoreSubject.next(this.blocks[0]?.height);

2
frontend/src/app/components/transaction/transaction-preview.component.ts
View File

@ -133,6 +133,7 @@ export class TransactionPreviewComponent implements OnInit, OnDestroy {
)
.subscribe((tx: Transaction) => {
if (!tx) {
this.seoService.logSoft404();
this.openGraphService.fail('tx-data-' + this.txId);
return;
}
@ -182,6 +183,7 @@ export class TransactionPreviewComponent implements OnInit, OnDestroy {
this.openGraphService.waitOver('tx-data-' + this.txId);
},
(error) => {
this.seoService.logSoft404();
this.openGraphService.fail('tx-data-' + this.txId);
this.error = error;
this.isLoadingTx = false;

6
frontend/src/app/components/transaction/transaction.component.ts
View File

@ -220,8 +220,10 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
).subscribe((tx) => {
this.loadingCachedTx = false;
if (!tx) {
this.seoService.logSoft404();
return;
}
this.seoService.clearSoft404();
if (!this.tx) {
this.tx = tx;
@ -338,8 +340,10 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
.subscribe((tx: Transaction) => {
if (!tx) {
this.fetchCachedTx$.next(this.txId);
this.seoService.logSoft404();
return;
}
this.seoService.clearSoft404();
this.tx = tx;
this.setFeatures();
@ -400,6 +404,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
},
(error) => {
this.error = error;
this.seoService.logSoft404();
this.isLoadingTx = false;
}
);
@ -487,6 +492,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
this.waitingForTransaction = true;
}
this.error = error;
this.seoService.logSoft404();
this.isLoadingTx = false;
return of(false);
}

1
frontend/src/app/lightning/channel/channel-preview.component.ts
View File

@ -54,6 +54,7 @@ export class ChannelPreviewComponent implements OnInit {
}),
catchError((err) => {
this.error = err;
this.seoService.logSoft404();
this.openGraphService.fail('channel-map-' + this.shortId);
this.openGraphService.fail('channel-data-' + this.shortId);
return of(null);

1
frontend/src/app/lightning/channel/channel.component.ts
View File

@ -38,6 +38,7 @@ export class ChannelComponent implements OnInit {
}),
catchError((err) => {
this.error = err;
this.seoService.logSoft404();
return [{
short_id: params.get('short_id')
}];

2
frontend/src/app/lightning/group/group-preview.component.ts
View File

@ -50,6 +50,7 @@ export class GroupPreviewComponent implements OnInit {
name: this.slug.replace(/-/gi, ' '),
description: '',
};
this.seoService.logSoft404();
this.openGraphService.fail('ln-group-map-' + this.slug);
this.openGraphService.fail('ln-group-data-' + this.slug);
return of(null);
@ -106,6 +107,7 @@ export class GroupPreviewComponent implements OnInit {
};
}),
catchError(() => {
this.seoService.logSoft404();
this.openGraphService.fail('ln-group-map-' + this.slug);
this.openGraphService.fail('ln-group-data-' + this.slug);
return of({

1
frontend/src/app/lightning/node/node-preview.component.ts
View File

@ -81,6 +81,7 @@ export class NodePreviewComponent implements OnInit {
}),
catchError(err => {
this.error = err;
this.seoService.logSoft404();
this.openGraphService.fail('node-map-' + this.publicKey);
this.openGraphService.fail('node-data-' + this.publicKey);
return [{

1
frontend/src/app/lightning/node/node.component.ts
View File

@ -123,6 +123,7 @@ export class NodeComponent implements OnInit {
}),
catchError(err => {
this.error = err;
this.seoService.logSoft404();
return [{
alias: this.publicKey,
public_key: this.publicKey,

1
frontend/src/app/lightning/nodes-per-isp/nodes-per-isp-preview.component.ts
View File

@ -85,6 +85,7 @@ export class NodesPerISPPreview implements OnInit {
}),
catchError(err => {
this.error = err;
this.seoService.logSoft404();
this.openGraphService.fail('isp-map-' + this.id);
this.openGraphService.fail('isp-data-' + this.id);
return of({

26
frontend/src/app/services/seo.service.ts
View File

@ -1,5 +1,7 @@
import { Injectable } from '@angular/core';
import { Title, Meta } from '@angular/platform-browser';
import { ActivatedRoute, NavigationEnd, Router } from '@angular/router';
import { filter, map, switchMap } from 'rxjs';
import { StateService } from './state.service';
@Injectable({
@ -13,8 +15,22 @@ export class SeoService {
private titleService: Title,
private metaService: Meta,
private stateService: StateService,
private router: Router,
private activatedRoute: ActivatedRoute,
) {
this.stateService.networkChanged$.subscribe((network) => this.network = network);
this.router.events.pipe(
filter(event => event instanceof NavigationEnd),
map(() => this.activatedRoute),
map(route => {
while (route.firstChild) route = route.firstChild;
return route;
}),
filter(route => route.outlet === 'primary'),
switchMap(route => route.data),
).subscribe((data) => {
this.clearSoft404();
});
}
setTitle(newTitle: string): void {
@ -53,4 +69,14 @@ export class SeoService {
ucfirst(str: string) {
return str.charAt(0).toUpperCase() + str.slice(1);
}
clearSoft404() {
window['soft404'] = false;
console.log('cleared soft 404');
}
logSoft404() {
window['soft404'] = true;
console.log('set soft 404');
}
}

65
unfurler/src/concurrency/ReusableSSRPage.ts Normal file
View File

@ -0,0 +1,65 @@
import * as puppeteer from 'puppeteer';
import { timeoutExecute } from 'puppeteer-cluster/dist/util';
import logger from '../logger';
import config from '../config';
import ReusablePage from './ReusablePage';
const mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : '');
const mockImageBuffer = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVQYV2NgYAAAAAMAAWgmWQ0AAAAASUVORK5CYII=", 'base64');
interface RepairablePage extends puppeteer.Page {
repairRequested?: boolean;
language?: string | null;
createdAt?: number;
free?: boolean;
index?: number;
}
export default class ReusableSSRPage extends ReusablePage {
public constructor(options: puppeteer.LaunchOptions, puppeteer: any) {
super(options, puppeteer);
}
public async close() {
await (this.browser as puppeteer.Browser).close();
}
protected async initPage(): Promise<RepairablePage> {
const page = await (this.browser as puppeteer.Browser).newPage() as RepairablePage;
page.language = null;
page.createdAt = Date.now();
const defaultUrl = mempoolHost + '/about';
page.on('pageerror', (err) => {
console.log(err);
// page.repairRequested = true;
});
await page.setRequestInterception(true);
page.on('request', req => {
if (req.isInterceptResolutionHandled()) {
return req.continue();
}
if (req.resourceType() === 'image') {
return req.respond({
contentType: 'image/png',
headers: {"Access-Control-Allow-Origin": "*"},
body: mockImageBuffer
});
} else if (!['document', 'script', 'xhr', 'fetch'].includes(req.resourceType())) {
return req.abort();
} else {
return req.continue();
}
});
try {
await page.goto(defaultUrl, { waitUntil: "networkidle0" });
await page.waitForSelector('meta[property="og:meta:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 });
} catch (e) {
logger.err(`failed to load frontend during ssr page initialization: ` + (e instanceof Error ? e.message : `${e}`));
page.repairRequested = true;
}
page.free = true;
return page
}
}

180
unfurler/src/index.ts
View File

@ -5,9 +5,11 @@ import * as https from 'https';
import config from './config';
import { Cluster } from 'puppeteer-cluster';
import ReusablePage from './concurrency/ReusablePage';
import ReusableSSRPage from './concurrency/ReusablePage';
import { parseLanguageUrl } from './language/lang';
import { matchRoute } from './routes';
import logger from './logger';
import { TimeoutError } from "puppeteer";
const puppeteerConfig = require('../puppeteer.config.json');
if (config.PUPPETEER.EXEC_PATH) {
@ -20,15 +22,33 @@ class Server {
private server: http.Server | undefined;
private app: Application;
cluster?: Cluster;
ssrCluster?: Cluster;
mempoolHost: string;
mempoolUrl: URL;
network: string;
secureHost = true;
canonicalHost: string;
constructor() {
this.app = express();
this.mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : '');
this.mempoolUrl = new URL(this.mempoolHost);
this.secureHost = config.SERVER.HOST.startsWith('https');
this.network = config.MEMPOOL.NETWORK || 'bitcoin';
let canonical;
switch(config.MEMPOOL.NETWORK) {
case "liquid":
canonical = "https://liquid.network"
break;
case "bisq":
canonical = "https://bisq.markets"
break;
default:
canonical = "https://mempool.space"
}
this.canonicalHost = canonical;
this.startServer();
}
@ -49,6 +69,12 @@ class Server {
puppeteerOptions: puppeteerConfig,
});
await this.cluster?.task(async (args) => { return this.clusterTask(args) });
this.ssrCluster = await Cluster.launch({
concurrency: ReusableSSRPage,
maxConcurrency: config.PUPPETEER.CLUSTER_SIZE,
puppeteerOptions: puppeteerConfig,
});
await this.ssrCluster?.task(async (args) => { return this.ssrClusterTask(args) });
}
this.setUpRoutes();
@ -65,6 +91,10 @@ class Server {
await this.cluster.idle();
await this.cluster.close();
}
if (this.ssrCluster) {
await this.ssrCluster.idle();
await this.ssrCluster.close();
}
if (this.server) {
await this.server.close();
}
@ -102,8 +132,8 @@ class Server {
}
// wait for preview component to initialize
await page.waitForSelector('meta[property="og:preview:loading"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 })
let success;
await page.waitForSelector('meta[property="og:preview:loading"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 })
success = await Promise.race([
page.waitForSelector('meta[property="og:preview:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => true),
page.waitForSelector('meta[property="og:preview:fail"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => false)
@ -127,6 +157,51 @@ class Server {
}
}
async ssrClusterTask({ page, data: { url, path, action } }) {
try {
const urlParts = parseLanguageUrl(path);
if (page.language !== urlParts.lang) {
// switch language
page.language = urlParts.lang;
const localizedUrl = urlParts.lang ? `${this.mempoolHost}/${urlParts.lang}${urlParts.path}` : `${this.mempoolHost}${urlParts.path}` ;
await page.goto(localizedUrl, { waitUntil: "load" });
} else {
const loaded = await page.evaluate(async (path) => {
if (window['ogService']) {
window['ogService'].loadPage(path);
return true;
} else {
return false;
}
}, urlParts.path);
if (!loaded) {
throw new Error('failed to access open graph service');
}
}
await page.waitForNetworkIdle({
timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000,
});
const is404 = await page.evaluate(async () => {
return !!window['soft404'];
});
if (is404) {
return '404';
} else {
let html = await page.content();
return html;
}
} catch (e) {
if (e instanceof TimeoutError) {
let html = await page.content();
return html;
} else {
logger.err(`failed to render ${path} for ${action}: ` + (e instanceof Error ? e.message : `${e}`));
page.repairRequested = true;
}
}
}
async renderDisabled(req, res) {
res.status(500).send("preview rendering disabled");
}
@ -166,43 +241,92 @@ class Server {
// drop requests for static files
const rawPath = req.params[0];
const match = rawPath.match(/\.[\w]+$/);
if (match?.length && match[0] !== '.html') {
res.status(404).send();
return;
if (match?.length && match[0] !== '.html'
|| rawPath.startsWith('/api/v1/donations/images')
|| rawPath.startsWith('/api/v1/contributors/images')
|| rawPath.startsWith('/api/v1/translators/images')
|| rawPath.startsWith('/resources/profile')
) {
if (isSearchCrawler(req.headers['user-agent'])) {
if (this.secureHost) {
https.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res));
} else {
http.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res));
}
return;
} else {
res.status(404).send();
return;
}
}
let result = '';
try {
if (isSearchCrawler(req.headers['user-agent'])) {
result = await this.renderSEOPage(rawPath);
} else {
result = await this.renderUnfurlMeta(rawPath);
}
if (result && result.length) {
if (result === '404') {
res.status(404).send();
} else {
res.send(result);
}
} else {
res.status(500).send();
}
} catch (e) {
logger.err(e instanceof Error ? e.message : `${e} ${req.params[0]}`);
res.status(500).send(e instanceof Error ? e.message : e);
}
}
async renderUnfurlMeta(rawPath: string): Promise<string> {
const { lang, path } = parseLanguageUrl(rawPath);
const matchedRoute = matchRoute(this.network, path);
let ogImageUrl = config.SERVER.HOST + (matchedRoute.staticImg || matchedRoute.fallbackImg);
let ogTitle = 'The Mempool Open Source Project®';
const canonical = this.canonicalHost + rawPath;
if (matchedRoute.render) {
ogImageUrl = `${config.SERVER.HOST}/render/${lang || 'en'}/preview${path}`;
ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${matchedRoute.networkMode !== 'mainnet' ? capitalize(matchedRoute.networkMode) + ' ' : ''}${matchedRoute.title}`;
}
res.send(`
<!doctype html>
<html lang="en-US" dir="ltr">
<head>
<meta charset="utf-8">
<title>${ogTitle}</title>
<meta name="description" content="The Mempool Open Source Project® - Explore the full Bitcoin ecosystem with mempool.space™"/>
<meta property="og:image" content="${ogImageUrl}"/>
<meta property="og:image:type" content="image/png"/>
<meta property="og:image:width" content="${matchedRoute.render ? 1200 : 1000}"/>
<meta property="og:image:height" content="${matchedRoute.render ? 600 : 500}"/>
<meta property="og:title" content="${ogTitle}">
<meta property="twitter:card" content="summary_large_image">
<meta property="twitter:site" content="@mempool">
<meta property="twitter:creator" content="@mempool">
<meta property="twitter:title" content="${ogTitle}">
<meta property="twitter:description" content="Explore the full Bitcoin ecosystem with mempool.space"/>
<meta property="twitter:image:src" content="${ogImageUrl}"/>
<meta property="twitter:domain" content="mempool.space">
<body></body>
</html>
`);
return `<!doctype html>
<html lang="en-US" dir="ltr">
<head>
<meta charset="utf-8">
<title>${ogTitle}</title>
<link rel="canonical" href="${canonical}" />
<meta name="description" content="The Mempool Open Source Project® - Explore the full Bitcoin ecosystem with mempool.space™"/>
<meta property="og:image" content="${ogImageUrl}"/>
<meta property="og:image:type" content="image/png"/>
<meta property="og:image:width" content="${matchedRoute.render ? 1200 : 1000}"/>
<meta property="og:image:height" content="${matchedRoute.render ? 600 : 500}"/>
<meta property="og:title" content="${ogTitle}">
<meta property="twitter:card" content="summary_large_image">
<meta property="twitter:site" content="@mempool">
<meta property="twitter:creator" content="@mempool">
<meta property="twitter:title" content="${ogTitle}">
<meta property="twitter:description" content="Explore the full Bitcoin ecosystem with mempool.space"/>
<meta property="twitter:image:src" content="${ogImageUrl}"/>
<meta property="twitter:domain" content="mempool.space">
</head>
<body></body>
</html>`;
}
async renderSEOPage(rawPath: string): Promise<string> {
let html = await this.ssrCluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'ssr' });
// remove javascript to prevent double hydration
if (html && html.length) {
html = html.replaceAll(/<script.*<\/script>/g, "");
html = html.replaceAll(this.mempoolHost, this.canonicalHost);
}
return html;
}
}
@ -221,3 +345,7 @@ function capitalize(str) {
return str;
}
}
function isSearchCrawler(useragent: string): boolean {
return /googlebot|applebot|bingbot/i.test(useragent);
}