Merge pull request #3169 from mempool/mononaut/seo-ssr

dynamically render search crawler requests

frontend/src/app/bisq/bisq-address/bisq-address.component.ts

@@ -47,6 +47,7 @@ export class BisqAddressComponent implements OnInit, OnDestroy {
               catchError((err) => {
                 this.isLoadingAddress = false;
                 this.error = err;
+                this.seoService.logSoft404();
                 console.log(err);
                 return of(null);
               })
@@ -62,6 +63,7 @@ export class BisqAddressComponent implements OnInit, OnDestroy {
       (error) => {
         console.log(error);
         this.error = error;
+        this.seoService.logSoft404();
         this.isLoadingAddress = false;
       });
   }

frontend/src/app/bisq/bisq-block/bisq-block.component.ts

@@ -82,6 +82,7 @@ export class BisqBlockComponent implements OnInit, OnDestroy {
       )
       .subscribe((block: BisqBlock) => {
         if (!block) {
+          this.seoService.logSoft404();
           return;
         }
         this.isLoading = false;
@@ -97,6 +98,7 @@ export class BisqBlockComponent implements OnInit, OnDestroy {
 
   caughtHttpError(err: HttpErrorResponse){
     this.error = err;
+    this.seoService.logSoft404();
     return of(null);
   }
 }

frontend/src/app/bisq/bisq-transaction/bisq-transaction.component.ts

@@ -70,11 +70,13 @@ export class BisqTransactionComponent implements OnInit, OnDestroy {
                     catchError((txError: HttpErrorResponse) => {
                       console.log(txError);
                       this.error = txError;
+                      this.seoService.logSoft404();
                       return of(null);
                     })
                   );
               }
               this.error = bisqTxError;
+              this.seoService.logSoft404();
               return of(null);
             })
           );
@@ -103,6 +105,7 @@ export class BisqTransactionComponent implements OnInit, OnDestroy {
       this.isLoadingTx = false;
 
       if (!tx) {
+        this.seoService.logSoft404();
         return;
       }
 

frontend/src/app/components/address/address.component.ts

@@ -91,6 +91,7 @@ export class AddressComponent implements OnInit, OnDestroy {
                 catchError((err) => {
                   this.isLoadingAddress = false;
                   this.error = err;
+                  this.seoService.logSoft404();
                   console.log(err);
                   return of(null);
                 })
@@ -162,6 +163,7 @@ export class AddressComponent implements OnInit, OnDestroy {
       (error) => {
         console.log(error);
         this.error = error;
+        this.seoService.logSoft404();
         this.isLoadingAddress = false;
       });
 

frontend/src/app/components/asset/asset.component.ts

@@ -86,6 +86,7 @@ export class AssetComponent implements OnInit, OnDestroy {
                   catchError((err) => {
                     this.isLoadingAsset = false;
                     this.error = err;
+                    this.seoService.logSoft404();
                     console.log(err);
                     return of(null);
                   })
@@ -153,6 +154,7 @@ export class AssetComponent implements OnInit, OnDestroy {
       (error) => {
         console.log(error);
         this.error = error;
+        this.seoService.logSoft404();
         this.isLoadingAsset = false;
       });
 

frontend/src/app/components/block/block-preview.component.ts

@@ -82,6 +82,7 @@ export class BlockPreviewComponent implements OnInit, OnDestroy {
               }),
               catchError((err) => {
                 this.error = err;
+                this.seoService.logSoft404();
                 this.openGraphService.fail('block-data-' + this.rawId);
                 this.openGraphService.fail('block-viz-' + this.rawId);
                 return of(null);
@@ -138,6 +139,7 @@ export class BlockPreviewComponent implements OnInit, OnDestroy {
     (error) => {
       this.error = error;
       this.isLoadingOverview = false;
+      this.seoService.logSoft404();
       this.openGraphService.fail('block-viz-' + this.rawId);
       this.openGraphService.fail('block-data-' + this.rawId);
       if (this.blockGraph) {

frontend/src/app/components/block/block.component.ts

@@ -206,6 +206,7 @@ export class BlockComponent implements OnInit, OnDestroy {
                       this.error = err;
                       this.isLoadingBlock = false;
                       this.isLoadingOverview = false;
+                      this.seoService.logSoft404();
                       return EMPTY;
                     })
                   );
@@ -214,6 +215,7 @@ export class BlockComponent implements OnInit, OnDestroy {
                   this.error = err;
                   this.isLoadingBlock = false;
                   this.isLoadingOverview = false;
+                  this.seoService.logSoft404();
                   return EMPTY;
                 }),
               );
@@ -229,6 +231,7 @@ export class BlockComponent implements OnInit, OnDestroy {
               this.error = err;
               this.isLoadingBlock = false;
               this.isLoadingOverview = false;
+              this.seoService.logSoft404();
               return EMPTY;
             })
           );

frontend/src/app/components/pool/pool-preview.component.ts

@@ -61,6 +61,7 @@ export class PoolPreviewComponent implements OnInit {
               }),
               catchError(() => {
                 this.isLoading = false;
+                this.seoService.logSoft404();
                 this.openGraphService.fail('pool-hash-' + this.slug);
                 return of([slug]);
               })
@@ -70,6 +71,7 @@ export class PoolPreviewComponent implements OnInit {
           return this.apiService.getPoolStats$(slug).pipe(
             catchError(() => {
               this.isLoading = false;
+              this.seoService.logSoft404();
               this.openGraphService.fail('pool-stats-' + this.slug);
               return of(null);
             })

frontend/src/app/components/pool/pool.component.ts

@@ -1,8 +1,8 @@
 import { ChangeDetectionStrategy, Component, Inject, Input, LOCALE_ID, OnInit } from '@angular/core';
 import { ActivatedRoute } from '@angular/router';
 import { EChartsOption, graphic } from 'echarts';
-import { BehaviorSubject, Observable } from 'rxjs';
+import { BehaviorSubject, Observable, of, timer } from 'rxjs';
-import { distinctUntilChanged, map, share, switchMap, tap } from 'rxjs/operators';
+import { catchError, distinctUntilChanged, map, share, switchMap, tap } from 'rxjs/operators';
 import { BlockExtended, PoolStat } from '../../interfaces/node-api.interface';
 import { ApiService } from '../../services/api.service';
 import { StateService } from '../../services/state.service';
@@ -62,10 +62,21 @@ export class PoolComponent implements OnInit {
                 this.prepareChartOptions(data.map(val => [val.timestamp * 1000, val.avgHashrate]));
                 return [slug];
               }),
+              catchError(() => {
+                this.isLoading = false;
+                this.seoService.logSoft404();
+                return of([slug]);
+              })
             );
         }),
         switchMap((slug) => {
-          return this.apiService.getPoolStats$(slug);
+          return this.apiService.getPoolStats$(slug).pipe(
+            catchError(() => {
+              this.isLoading = false;
+              this.seoService.logSoft404();
+              return of(null);
+            })
+          );
         }),
         tap(() => {
           this.loadMoreSubject.next(this.blocks[0]?.height);

frontend/src/app/components/transaction/transaction-preview.component.ts

@@ -133,6 +133,7 @@ export class TransactionPreviewComponent implements OnInit, OnDestroy {
       )
       .subscribe((tx: Transaction) => {
           if (!tx) {
+            this.seoService.logSoft404();
             this.openGraphService.fail('tx-data-' + this.txId);
             return;
           }
@@ -182,6 +183,7 @@ export class TransactionPreviewComponent implements OnInit, OnDestroy {
           this.openGraphService.waitOver('tx-data-' + this.txId);
         },
         (error) => {
+          this.seoService.logSoft404();
           this.openGraphService.fail('tx-data-' + this.txId);
           this.error = error;
           this.isLoadingTx = false;

frontend/src/app/components/transaction/transaction.component.ts

@@ -220,8 +220,10 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
     ).subscribe((tx) => {
       this.loadingCachedTx = false;
       if (!tx) {
+        this.seoService.logSoft404();
         return;
       }
+      this.seoService.clearSoft404();
 
       if (!this.tx) {
         this.tx = tx;
@@ -338,8 +340,10 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
       .subscribe((tx: Transaction) => {
           if (!tx) {
             this.fetchCachedTx$.next(this.txId);
+            this.seoService.logSoft404();
             return;
           }
+          this.seoService.clearSoft404();
 
           this.tx = tx;
           this.setFeatures();
@@ -400,6 +404,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
         },
         (error) => {
           this.error = error;
+          this.seoService.logSoft404();
           this.isLoadingTx = false;
         }
       );
@@ -487,6 +492,7 @@ export class TransactionComponent implements OnInit, AfterViewInit, OnDestroy {
       this.waitingForTransaction = true;
     }
     this.error = error;
+    this.seoService.logSoft404();
     this.isLoadingTx = false;
     return of(false);
   }

frontend/src/app/lightning/channel/channel-preview.component.ts

@@ -54,6 +54,7 @@ export class ChannelPreviewComponent implements OnInit {
               }),
               catchError((err) => {
                 this.error = err;
+                this.seoService.logSoft404();
                 this.openGraphService.fail('channel-map-' + this.shortId);
                 this.openGraphService.fail('channel-data-' + this.shortId);
                 return of(null);

frontend/src/app/lightning/channel/channel.component.ts

@@ -38,6 +38,7 @@ export class ChannelComponent implements OnInit {
               }),
               catchError((err) => {
                 this.error = err;
+                this.seoService.logSoft404();
                 return [{
                   short_id: params.get('short_id')
                 }];

frontend/src/app/lightning/group/group-preview.component.ts

@@ -50,6 +50,7 @@ export class GroupPreviewComponent implements OnInit {
               name: this.slug.replace(/-/gi, ' '),
               description: '',
             };
+            this.seoService.logSoft404();
             this.openGraphService.fail('ln-group-map-' + this.slug);
             this.openGraphService.fail('ln-group-data-' + this.slug);
             return of(null);
@@ -106,6 +107,7 @@ export class GroupPreviewComponent implements OnInit {
           };
         }),
         catchError(() => {
+          this.seoService.logSoft404();
           this.openGraphService.fail('ln-group-map-' + this.slug);
           this.openGraphService.fail('ln-group-data-' + this.slug);
           return of({

frontend/src/app/lightning/node/node-preview.component.ts

@@ -81,6 +81,7 @@ export class NodePreviewComponent implements OnInit {
         }),
         catchError(err => {
           this.error = err;
+          this.seoService.logSoft404();
           this.openGraphService.fail('node-map-' + this.publicKey);
           this.openGraphService.fail('node-data-' + this.publicKey);
           return [{

frontend/src/app/lightning/node/node.component.ts

@@ -123,6 +123,7 @@ export class NodeComponent implements OnInit {
         }),
         catchError(err => {
           this.error = err;
+          this.seoService.logSoft404();
           return [{
             alias: this.publicKey,
             public_key: this.publicKey,

frontend/src/app/lightning/nodes-per-isp/nodes-per-isp-preview.component.ts

@@ -85,6 +85,7 @@ export class NodesPerISPPreview implements OnInit {
         }),
         catchError(err => {
           this.error = err;
+          this.seoService.logSoft404();
           this.openGraphService.fail('isp-map-' + this.id);
           this.openGraphService.fail('isp-data-' + this.id);
           return of({

frontend/src/app/services/seo.service.ts

@@ -1,5 +1,7 @@
 import { Injectable } from '@angular/core';
 import { Title, Meta } from '@angular/platform-browser';
+import { ActivatedRoute, NavigationEnd, Router } from '@angular/router';
+import { filter, map, switchMap } from 'rxjs';
 import { StateService } from './state.service';
 
 @Injectable({
@@ -13,8 +15,22 @@ export class SeoService {
     private titleService: Title,
     private metaService: Meta,
     private stateService: StateService,
+    private router: Router,
+    private activatedRoute: ActivatedRoute,
   ) {
     this.stateService.networkChanged$.subscribe((network) => this.network = network);
+    this.router.events.pipe(
+      filter(event => event instanceof NavigationEnd),
+      map(() => this.activatedRoute),
+      map(route => {
+        while (route.firstChild) route = route.firstChild;
+        return route;
+      }),
+      filter(route => route.outlet === 'primary'),
+      switchMap(route => route.data),
+    ).subscribe((data) => {
+      this.clearSoft404();
+    });
   }
 
   setTitle(newTitle: string): void {
@@ -53,4 +69,14 @@ export class SeoService {
   ucfirst(str: string) {
     return str.charAt(0).toUpperCase() + str.slice(1);
   }
+
+  clearSoft404() {
+    window['soft404'] = false;
+    console.log('cleared soft 404');
+  }
+
+  logSoft404() {
+    window['soft404'] = true;
+    console.log('set soft 404');
+  }
 }

unfurler/src/concurrency/ReusableSSRPage.ts

@@ -0,0 +1,65 @@
+import * as puppeteer from 'puppeteer';
+import { timeoutExecute } from 'puppeteer-cluster/dist/util';
+import logger from '../logger';
+import config from '../config';
+import ReusablePage from './ReusablePage';
+const mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : '');
+
+const mockImageBuffer = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVQYV2NgYAAAAAMAAWgmWQ0AAAAASUVORK5CYII=", 'base64');
+
+interface RepairablePage extends puppeteer.Page {
+  repairRequested?: boolean;
+  language?: string | null;
+  createdAt?: number;
+  free?: boolean;
+  index?: number;
+}
+
+export default class ReusableSSRPage extends ReusablePage {
+
+  public constructor(options: puppeteer.LaunchOptions, puppeteer: any) {
+    super(options, puppeteer);
+  }
+
+  public async close() {
+    await (this.browser as puppeteer.Browser).close();
+  }
+
+  protected async initPage(): Promise<RepairablePage> {
+    const page = await (this.browser as puppeteer.Browser).newPage() as RepairablePage;
+    page.language = null;
+    page.createdAt = Date.now();
+    const defaultUrl = mempoolHost + '/about';
+
+    page.on('pageerror', (err) => {
+      console.log(err);
+      // page.repairRequested = true;
+    });
+    await page.setRequestInterception(true);
+      page.on('request', req => {
+        if (req.isInterceptResolutionHandled()) {
+          return req.continue();
+        }
+        if (req.resourceType() === 'image') {
+          return req.respond({
+            contentType: 'image/png',
+            headers: {"Access-Control-Allow-Origin": "*"},
+            body: mockImageBuffer
+          });
+        } else if (!['document', 'script', 'xhr', 'fetch'].includes(req.resourceType())) {
+          return req.abort();
+        } else {
+          return req.continue();
+        }
+      });
+    try {
+      await page.goto(defaultUrl, { waitUntil: "networkidle0" });
+      await page.waitForSelector('meta[property="og:meta:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 });
+    } catch (e) {
+      logger.err(`failed to load frontend during ssr page initialization: ` + (e instanceof Error ? e.message : `${e}`));
+      page.repairRequested = true;
+    }
+    page.free = true;
+    return page
+  }
+}

unfurler/src/index.ts

@@ -5,9 +5,11 @@ import * as https from 'https';
 import config from './config';
 import { Cluster } from 'puppeteer-cluster';
 import ReusablePage from './concurrency/ReusablePage';
+import ReusableSSRPage from './concurrency/ReusablePage';
 import { parseLanguageUrl } from './language/lang';
 import { matchRoute } from './routes';
 import logger from './logger';
+import { TimeoutError } from "puppeteer";
 const puppeteerConfig = require('../puppeteer.config.json');
 
 if (config.PUPPETEER.EXEC_PATH) {
@@ -20,15 +22,33 @@ class Server {
   private server: http.Server | undefined;
   private app: Application;
   cluster?: Cluster;
+  ssrCluster?: Cluster;
   mempoolHost: string;
+  mempoolUrl: URL;
   network: string;
   secureHost = true;
+  canonicalHost: string;
 
   constructor() {
     this.app = express();
     this.mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : '');
+    this.mempoolUrl = new URL(this.mempoolHost);
     this.secureHost = config.SERVER.HOST.startsWith('https');
     this.network = config.MEMPOOL.NETWORK || 'bitcoin';
+
+    let canonical;
+    switch(config.MEMPOOL.NETWORK) {
+      case "liquid":
+        canonical = "https://liquid.network"
+        break;
+      case "bisq":
+        canonical = "https://bisq.markets"
+        break;
+      default:
+        canonical = "https://mempool.space"
+    }
+    this.canonicalHost = canonical;
+
     this.startServer();
   }
 
@@ -49,6 +69,12 @@ class Server {
           puppeteerOptions: puppeteerConfig,
       });
       await this.cluster?.task(async (args) => { return this.clusterTask(args) });
+      this.ssrCluster = await Cluster.launch({
+        concurrency: ReusableSSRPage,
+        maxConcurrency: config.PUPPETEER.CLUSTER_SIZE,
+        puppeteerOptions: puppeteerConfig,
+      });
+      await this.ssrCluster?.task(async (args) => { return this.ssrClusterTask(args) });
     }
 
     this.setUpRoutes();
@@ -65,6 +91,10 @@ class Server {
       await this.cluster.idle();
       await this.cluster.close();
     }
+    if (this.ssrCluster) {
+      await this.ssrCluster.idle();
+      await this.ssrCluster.close();
+    }
     if (this.server) {
       await this.server.close();
     }
@@ -102,8 +132,8 @@ class Server {
       }
 
       // wait for preview component to initialize
-      await page.waitForSelector('meta[property="og:preview:loading"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 })
       let success;
+      await page.waitForSelector('meta[property="og:preview:loading"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 })
       success = await Promise.race([
         page.waitForSelector('meta[property="og:preview:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => true),
         page.waitForSelector('meta[property="og:preview:fail"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => false)
@@ -127,6 +157,51 @@ class Server {
     }
   }
 
+  async ssrClusterTask({ page, data: { url, path, action } }) {
+    try {
+      const urlParts = parseLanguageUrl(path);
+      if (page.language !== urlParts.lang) {
+        // switch language
+        page.language = urlParts.lang;
+        const localizedUrl = urlParts.lang ? `${this.mempoolHost}/${urlParts.lang}${urlParts.path}` : `${this.mempoolHost}${urlParts.path}` ;
+        await page.goto(localizedUrl, { waitUntil: "load" });
+      } else {
+        const loaded = await page.evaluate(async (path) => {
+          if (window['ogService']) {
+            window['ogService'].loadPage(path);
+            return true;
+          } else {
+            return false;
+          }
+        }, urlParts.path);
+        if (!loaded) {
+          throw new Error('failed to access open graph service');
+        }
+      }
+
+      await page.waitForNetworkIdle({
+        timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000,
+      });
+      const is404 = await page.evaluate(async () => {
+        return !!window['soft404'];
+      });
+      if (is404) {
+        return '404';
+      } else {
+        let html = await page.content();
+        return html;
+      }
+    } catch (e) {
+      if (e instanceof TimeoutError) {
+        let html = await page.content();
+        return html;
+      } else {
+        logger.err(`failed to render ${path} for ${action}: ` + (e instanceof Error ? e.message : `${e}`));
+        page.repairRequested = true;
+      }
+    }
+  }
+
   async renderDisabled(req, res) {
     res.status(500).send("preview rendering disabled");
   }
@@ -166,43 +241,92 @@ class Server {
     // drop requests for static files
     const rawPath = req.params[0];
     const match = rawPath.match(/\.[\w]+$/);
-    if (match?.length && match[0] !== '.html') {
+    if (match?.length && match[0] !== '.html'
-      res.status(404).send();
+      || rawPath.startsWith('/api/v1/donations/images')
-      return;
+      || rawPath.startsWith('/api/v1/contributors/images')
+      || rawPath.startsWith('/api/v1/translators/images')
+      || rawPath.startsWith('/resources/profile')
+    ) {
+      if (isSearchCrawler(req.headers['user-agent'])) {
+        if (this.secureHost) {
+          https.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res));
+        } else {
+          http.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res));
+        }
+        return;
+      } else {
+        res.status(404).send();
+        return;
+      }
     }
 
+    let result = '';
+    try {
+      if (isSearchCrawler(req.headers['user-agent'])) {
+        result = await this.renderSEOPage(rawPath);
+      } else {
+        result = await this.renderUnfurlMeta(rawPath);
+      }
+      if (result && result.length) {
+        if (result === '404') {
+          res.status(404).send();
+        } else {
+          res.send(result);
+        }
+      } else {
+        res.status(500).send();
+      }
+    } catch (e) {
+      logger.err(e instanceof Error ? e.message : `${e} ${req.params[0]}`);
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  async renderUnfurlMeta(rawPath: string): Promise<string> {
     const { lang, path } = parseLanguageUrl(rawPath);
     const matchedRoute = matchRoute(this.network, path);
     let ogImageUrl = config.SERVER.HOST + (matchedRoute.staticImg || matchedRoute.fallbackImg);
     let ogTitle = 'The Mempool Open Source Project®';
 
+    const canonical = this.canonicalHost + rawPath;
+
     if (matchedRoute.render) {
       ogImageUrl = `${config.SERVER.HOST}/render/${lang || 'en'}/preview${path}`;
       ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${matchedRoute.networkMode !== 'mainnet' ? capitalize(matchedRoute.networkMode) + ' ' : ''}${matchedRoute.title}`;
     }
 
-    res.send(`
+    return `<!doctype html>
-      <!doctype html>
+<html lang="en-US" dir="ltr">
-      <html lang="en-US" dir="ltr">
+  <head>
-      <head>
+    <meta charset="utf-8">
-        <meta charset="utf-8">
+    <title>${ogTitle}</title>
-        <title>${ogTitle}</title>
+    <link rel="canonical" href="${canonical}" />
-        <meta name="description" content="The Mempool Open Source Project® - Explore the full Bitcoin ecosystem with mempool.space™"/>
+    <meta name="description" content="The Mempool Open Source Project® - Explore the full Bitcoin ecosystem with mempool.space™"/>
-        <meta property="og:image" content="${ogImageUrl}"/>
+    <meta property="og:image" content="${ogImageUrl}"/>
-        <meta property="og:image:type" content="image/png"/>
+    <meta property="og:image:type" content="image/png"/>
-        <meta property="og:image:width" content="${matchedRoute.render ? 1200 : 1000}"/>
+    <meta property="og:image:width" content="${matchedRoute.render ? 1200 : 1000}"/>
-        <meta property="og:image:height" content="${matchedRoute.render ? 600 : 500}"/>
+    <meta property="og:image:height" content="${matchedRoute.render ? 600 : 500}"/>
-        <meta property="og:title" content="${ogTitle}">
+    <meta property="og:title" content="${ogTitle}">
-        <meta property="twitter:card" content="summary_large_image">
+    <meta property="twitter:card" content="summary_large_image">
-        <meta property="twitter:site" content="@mempool">
+    <meta property="twitter:site" content="@mempool">
-        <meta property="twitter:creator" content="@mempool">
+    <meta property="twitter:creator" content="@mempool">
-        <meta property="twitter:title" content="${ogTitle}">
+    <meta property="twitter:title" content="${ogTitle}">
-        <meta property="twitter:description" content="Explore the full Bitcoin ecosystem with mempool.space"/>
+    <meta property="twitter:description" content="Explore the full Bitcoin ecosystem with mempool.space"/>
-        <meta property="twitter:image:src" content="${ogImageUrl}"/>
+    <meta property="twitter:image:src" content="${ogImageUrl}"/>
-        <meta property="twitter:domain" content="mempool.space">
+    <meta property="twitter:domain" content="mempool.space">
-      <body></body>
+  </head>
-      </html>
+  <body></body>
-    `);
+</html>`;
+  }
+
+  async renderSEOPage(rawPath: string): Promise<string> {
+    let html = await this.ssrCluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'ssr' });
+    // remove javascript to prevent double hydration
+    if (html && html.length) {
+      html = html.replaceAll(/<script.*<\/script>/g, "");
+      html = html.replaceAll(this.mempoolHost, this.canonicalHost);
+    }
+    return html;
   }
 }
 
@@ -221,3 +345,7 @@ function capitalize(str) {
     return str;
   }
 }
+
+function isSearchCrawler(useragent: string): boolean {
+  return /googlebot|applebot|bingbot/i.test(useragent);
+}