import express from "express"; import { Application, Request, Response, NextFunction } from 'express'; import * as http from 'http'; import * as https from 'https'; import config from './config'; import { Cluster } from 'puppeteer-cluster'; import ReusablePage from './concurrency/ReusablePage'; import ReusableSSRPage from './concurrency/ReusablePage'; import { parseLanguageUrl } from './language/lang'; import { matchRoute } from './routes'; import logger from './logger'; import { TimeoutError } from "puppeteer"; const puppeteerConfig = require('../puppeteer.config.json'); if (config.PUPPETEER.EXEC_PATH) { puppeteerConfig.executablePath = config.PUPPETEER.EXEC_PATH; } const puppeteerEnabled = config.PUPPETEER.ENABLED && (config.PUPPETEER.CLUSTER_SIZE > 0); class Server { private server: http.Server | undefined; private app: Application; cluster?: Cluster; ssrCluster?: Cluster; mempoolHost: string; mempoolUrl: URL; network: string; secureHost = true; canonicalHost: string; constructor() { this.app = express(); this.mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : ''); this.mempoolUrl = new URL(this.mempoolHost); this.secureHost = config.SERVER.HOST.startsWith('https'); this.network = config.MEMPOOL.NETWORK || 'bitcoin'; let canonical; switch(config.MEMPOOL.NETWORK) { case "liquid": canonical = "https://liquid.network" break; case "bisq": canonical = "https://bisq.markets" break; default: canonical = "https://mempool.space" } this.canonicalHost = canonical; this.startServer(); } async startServer() { this.app .use((req: Request, res: Response, next: NextFunction) => { res.setHeader('Access-Control-Allow-Origin', '*'); next(); }) .use(express.urlencoded({ extended: true })) .use(express.text()) ; if (puppeteerEnabled) { this.cluster = await Cluster.launch({ concurrency: ReusablePage, maxConcurrency: config.PUPPETEER.CLUSTER_SIZE, puppeteerOptions: puppeteerConfig, }); await this.cluster?.task(async (args) => { return this.clusterTask(args) }); this.ssrCluster = await Cluster.launch({ concurrency: ReusableSSRPage, maxConcurrency: config.PUPPETEER.CLUSTER_SIZE, puppeteerOptions: puppeteerConfig, }); await this.ssrCluster?.task(async (args) => { return this.ssrClusterTask(args) }); } this.setUpRoutes(); this.server = http.createServer(this.app); this.server.listen(config.SERVER.HTTP_PORT, () => { logger.info(`Mempool Unfurl Server is running on port ${config.SERVER.HTTP_PORT}`); }); } async stopServer() { if (this.cluster) { await this.cluster.idle(); await this.cluster.close(); } if (this.ssrCluster) { await this.ssrCluster.idle(); await this.ssrCluster.close(); } if (this.server) { await this.server.close(); } } setUpRoutes() { if (puppeteerEnabled) { this.app.get('/render*', async (req, res) => { return this.renderPreview(req, res) }) } else { this.app.get('/render*', async (req, res) => { return this.renderDisabled(req, res) }) } this.app.get('*', (req, res) => { return this.renderHTML(req, res) }) } async clusterTask({ page, data: { url, path, action } }) { try { const urlParts = parseLanguageUrl(path); if (page.language !== urlParts.lang) { // switch language page.language = urlParts.lang; const localizedUrl = urlParts.lang ? `${this.mempoolHost}/${urlParts.lang}${urlParts.path}` : `${this.mempoolHost}${urlParts.path}` ; await page.goto(localizedUrl, { waitUntil: "load" }); } else { const loaded = await page.evaluate(async (path) => { if (window['ogService']) { window['ogService'].loadPage(path); return true; } else { return false; } }, urlParts.path); if (!loaded) { throw new Error('failed to access open graph service'); } } // wait for preview component to initialize let success; await page.waitForSelector('meta[property="og:preview:loading"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }) success = await Promise.race([ page.waitForSelector('meta[property="og:preview:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => true), page.waitForSelector('meta[property="og:preview:fail"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => false) ]) if (success === true) { const screenshot = await page.screenshot(); return screenshot; } else if (success === false) { logger.warn(`failed to render ${path} for ${action} due to client-side error, e.g. requested an invalid txid`); page.repairRequested = true; } else { logger.warn(`failed to render ${path} for ${action} due to puppeteer timeout`); page.repairRequested = true; } } catch (e) { logger.err(`failed to render ${path} for ${action}: ` + (e instanceof Error ? e.message : `${e}`)); page.repairRequested = true; } } async ssrClusterTask({ page, data: { url, path, action } }) { try { const urlParts = parseLanguageUrl(path); if (page.language !== urlParts.lang) { // switch language page.language = urlParts.lang; const localizedUrl = urlParts.lang ? `${this.mempoolHost}/${urlParts.lang}${urlParts.path}` : `${this.mempoolHost}${urlParts.path}` ; await page.goto(localizedUrl, { waitUntil: "load" }); } else { const loaded = await page.evaluate(async (path) => { if (window['ogService']) { window['ogService'].loadPage(path); return true; } else { return false; } }, urlParts.path); if (!loaded) { throw new Error('failed to access open graph service'); } } await page.waitForNetworkIdle({ timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000, }); let html = await page.content(); return html; } catch (e) { if (e instanceof TimeoutError) { let html = await page.content(); return html; } else { logger.err(`failed to render ${path} for ${action}: ` + (e instanceof Error ? e.message : `${e}`)); page.repairRequested = true; } } } async renderDisabled(req, res) { res.status(500).send("preview rendering disabled"); } async renderPreview(req, res) { try { const rawPath = req.params[0]; let img = null; const { lang, path } = parseLanguageUrl(rawPath); const matchedRoute = matchRoute(this.network, path); // don't bother unless the route is definitely renderable if (rawPath.includes('/preview/') && matchedRoute.render) { img = await this.cluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'screenshot' }); } if (!img) { // proxy fallback image from the frontend if (this.secureHost) { https.get(config.SERVER.HOST + matchedRoute.fallbackImg, (got) => got.pipe(res)); } else { http.get(config.SERVER.HOST + matchedRoute.fallbackImg, (got) => got.pipe(res)); } } else { res.contentType('image/png'); res.send(img); } } catch (e) { logger.err(e instanceof Error ? e.message : `${e} ${req.params[0]}`); res.status(500).send(e instanceof Error ? e.message : e); } } async renderHTML(req, res) { // drop requests for static files const rawPath = req.params[0]; const match = rawPath.match(/\.[\w]+$/); if (match?.length && match[0] !== '.html' || rawPath.startsWith('/api/v1/donations/images') || rawPath.startsWith('/api/v1/contributors/images') || rawPath.startsWith('/api/v1/translators/images') || rawPath.startsWith('/resources/profile') ) { if (isSearchCrawler(req.headers['user-agent'])) { if (this.secureHost) { https.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res)); } else { http.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res)); } return; } else { res.status(404).send(); return; } } let result = ''; try { if (isSearchCrawler(req.headers['user-agent'])) { result = await this.renderSEOPage(rawPath); } else { result = await this.renderUnfurlMeta(rawPath); } if (result && result.length) { res.send(result); } else { res.status(500).send(); } } catch (e) { logger.err(e instanceof Error ? e.message : `${e} ${req.params[0]}`); res.status(500).send(e instanceof Error ? e.message : e); } } async renderUnfurlMeta(rawPath: string): Promise { const { lang, path } = parseLanguageUrl(rawPath); const matchedRoute = matchRoute(this.network, path); let ogImageUrl = config.SERVER.HOST + (matchedRoute.staticImg || matchedRoute.fallbackImg); let ogTitle = 'The Mempool Open Source Project™'; const canonical = this.canonicalHost + rawPath; if (matchedRoute.render) { ogImageUrl = `${config.SERVER.HOST}/render/${lang || 'en'}/preview${path}`; ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${matchedRoute.networkMode !== 'mainnet' ? capitalize(matchedRoute.networkMode) + ' ' : ''}${matchedRoute.title}`; } return ` ${ogTitle} `; } async renderSEOPage(rawPath: string): Promise { let html = await this.ssrCluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'ssr' }); // remove javascript to prevent double hydration if (html && html.length) { html = html.replaceAll(//g, ""); html = html.replaceAll(this.mempoolHost, this.canonicalHost); } return html; } } const server = new Server(); process.on('SIGTERM', async () => { logger.info('Shutting down Mempool Unfurl Server'); await server.stopServer(); process.exit(0); }); function capitalize(str) { if (str && str.length) { return str[0].toUpperCase() + str.slice(1); } else { return str; } } function isSearchCrawler(useragent: string): boolean { return /googlebot|applebot|bingbot/i.test(useragent); }