Merge pull request #2173 from mononaut/unfurl-daemon

Open Graph link unfurler service

unfurler/.editorconfig

@@ -0,0 +1,17 @@
+# Editor configuration, see https://editorconfig.org
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 2
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.ts]
+quote_type = single
+
+[*.md]
+max_line_length = off
+trim_trailing_whitespace = false
+

unfurler/.eslintignore

@@ -0,0 +1,2 @@
+node_modules
+dist

unfurler/.eslintrc

@@ -0,0 +1,33 @@
+{
+  "root": true,
+  "parser": "@typescript-eslint/parser",
+  "plugins": [
+    "@typescript-eslint"
+  ],
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/eslint-recommended",
+    "plugin:@typescript-eslint/recommended",
+    "prettier"
+  ],
+  "rules": {
+    "@typescript-eslint/ban-ts-comment": 1,
+    "@typescript-eslint/ban-types": 1,
+    "@typescript-eslint/no-empty-function": 1,
+    "@typescript-eslint/no-explicit-any": 1,
+    "@typescript-eslint/no-inferrable-types": 1,
+    "@typescript-eslint/no-namespace": 1,
+    "@typescript-eslint/no-this-alias": 1,
+    "@typescript-eslint/no-var-requires": 1,
+    "no-console": 1,
+    "no-constant-condition": 1,
+    "no-dupe-else-if": 1,
+    "no-empty": 1,
+    "no-prototype-builtins": 1,
+    "no-self-assign": 1,
+    "no-useless-catch": 1,
+    "no-var": 1,
+    "prefer-const": 1,
+    "prefer-rest-params": 1
+  }
+}

unfurler/.gitignore

@@ -0,0 +1,38 @@
+# See http://help.github.com/ignore-files/ for more about ignoring files.
+
+# production config and external assets
+config.json
+
+# compiled output
+/dist
+/tmp
+
+# dependencies
+/node_modules
+
+# IDEs and editors
+/.idea
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+
+# IDE - VSCode
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+
+# misc
+/.sass-cache
+/connect.lock
+/libpeerconnection.log
+npm-debug.log
+testem.log
+/typings
+
+#System Files
+.DS_Store
+Thumbs.db

unfurler/.prettierignore

@@ -0,0 +1,2 @@
+node_modules
+package-lock.json

unfurler/.prettierrc

@@ -0,0 +1,6 @@
+{
+  "endOfLine": "lf",
+  "printWidth": 80,
+  "tabWidth": 2,
+  "trailingComma": "es5"
+}

unfurler/README.md

@@ -0,0 +1,107 @@
+# Mempool Link Unfurler Service
+
+This is a standalone nodejs service which implements the [Open Graph protocol](https://ogp.me/) for Mempool instances. It performs two main tasks:
+
+1. Serving Open Graph html meta tags to social media link crawler bots.
+2. Rendering link preview images for social media sharing.
+
+Some additional server configuration is required to properly route requests (see section 4 below).
+
+## Setup
+
+### 0. Install deps
+
+For Linux, in addition to NodeJS/npm you'll need at least:
+* nginx
+* cups
+* chromium-bsu
+* libatk1.0
+* libatk-bridge2.0
+* libxkbcommon-dev
+* libxcomposite-dev
+* libxdamage-dev
+* libxrandr-dev
+* libgbm-dev
+* libpango1.0-dev
+* libasound-dev
+
+### 1. Clone Mempool Repository
+
+Get the latest Mempool code:
+
+```
+git clone https://github.com/mempool/mempool
+cd mempool
+```
+
+Check out the latest release:
+
+```
+latestrelease=$(curl -s https://api.github.com/repos/mempool/mempool/releases/latest|grep tag_name|head -1|cut -d '"' -f4)
+git checkout $latestrelease
+```
+
+### 2. Prepare the Mempool Unfurler
+
+#### Install
+
+Install dependencies with `npm` and build the backend:
+
+```
+cd unfurler
+npm install
+```
+
+The npm install may fail if your system does not support automatic installation of Chromium for Puppeteer. In that case, manually install Puppeteer without Chromium first:
+```
+PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true npm install puppeteer
+npm install
+```
+
+#### Configure
+
+In the `unfurler` folder, make a copy of the sample config file:
+
+```
+cp config.sample.json config.json
+```
+
+Edit `config.json` as needed:
+
+| variable | usage |
+|---|---|
+| SERVER.HOST | the host where **this** service will be served |
+| SERVER.HTTP_PORT | the port on which **this** service should run |
+| MEMPOOL.HTTP_HOST | the host where **the Mempool frontend** is being served |
+| MEMPOOL.HTTP_PORT | the port on which **the Mempool frontend** is running (or `null`) |
+| PUPPETEER.CLUSTER_SIZE | the maximum number of Chromium browser instances to run in parallel, for rendering link previews |
+| PUPPETEER.EXEC_PATH | (optional) an absolute path to the Chromium browser executable, e.g. `/usr/local/bin/chrome`. Only required when using a manual installation of Chromium |
+
+#### Build
+
+```
+npm run build
+```
+
+### 3. Run the Mempool Unfurler
+
+```
+npm run start
+```
+
+### 4. Server configuration
+
+To enable social media link previews, the system serving the Mempool frontend should detect requests from social media crawler bots and proxy those requests to this service instead.
+
+Precise implementation is left as an exercise to the reader, but the following snippet may be of some help for Nginx users:
+```Nginx
+map $http_user_agent $crawler {
+    default 0;
+    ~*facebookexternalhit 1;
+    ~*twitterbot 1;
+    ~*slackbot 1;
+    ~*redditbot 1;
+    ~*linkedinbot 1;
+    ~*pinterestbot 1;
+}
+```

unfurler/config.sample.json

@@ -0,0 +1,14 @@
+{
+  "SERVER": {
+    "HOST": "http://localhost",
+    "HTTP_PORT": 4201
+  },
+  "MEMPOOL": {
+    "HTTP_HOST": "http://localhost",
+    "HTTP_PORT": 4200
+  },
+  "PUPPETEER": {
+    "CLUSTER_SIZE": 2,
+    "EXEC_PATH": "/usr/local/bin/chrome" // optional
+  }
+}

unfurler/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "mempool-unfurl",
+  "version": "0.0.1",
+  "description": "Renderer for mempool open graph link preview images",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/mempool/mempool"
+  },
+  "main": "index.ts",
+  "scripts": {
+    "tsc": "./node_modules/typescript/bin/tsc",
+    "build": "npm run tsc",
+    "start": "node --max-old-space-size=2048 dist/index.js",
+    "start-production": "node --max-old-space-size=4096 dist/index.js",
+    "lint": "./node_modules/.bin/eslint . --ext .ts",
+    "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",
+    "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""
+  },
+  "dependencies": {
+    "@types/node": "^16.11.41",
+    "express": "^4.18.0",
+    "puppeteer": "^15.3.2",
+    "puppeteer-cluster": "^0.23.0",
+    "typescript": "~4.7.4"
+  },
+  "devDependencies": {
+    "@typescript-eslint/eslint-plugin": "^5.30.5",
+    "@typescript-eslint/parser": "^5.30.5",
+    "eslint": "^8.19.0",
+    "eslint-config-prettier": "^8.5.0",
+    "prettier": "^2.7.1"
+  }
+}

unfurler/puppeteer.config.json

@@ -0,0 +1,46 @@
+{
+  "headless": true,
+  "defaultViewport": {
+    "width": 1024,
+    "height": 512
+  },
+  "args": [
+    "--window-size=1024,512",
+    "--autoplay-policy=user-gesture-required",
+    "--disable-background-networking",
+    "--disable-background-timer-throttling",
+    "--disable-backgrounding-occluded-windows",
+    "--disable-breakpad",
+    "--disable-client-side-phishing-detection",
+    "--disable-component-update",
+    "--disable-default-apps",
+    "--disable-dev-shm-usage",
+    "--disable-domain-reliability",
+    "--disable-extensions",
+    "--disable-features=AudioServiceOutOfProcess",
+    "--disable-hang-monitor",
+    "--disable-ipc-flooding-protection",
+    "--disable-notifications",
+    "--disable-offer-store-unmasked-wallet-cards",
+    "--disable-popup-blocking",
+    "--disable-print-preview",
+    "--disable-prompt-on-repost",
+    "--disable-renderer-backgrounding",
+    "--disable-setuid-sandbox",
+    "--disable-speech-api",
+    "--disable-sync",
+    "--hide-scrollbars",
+    "--metrics-recording-only",
+    "--mute-audio",
+    "--no-default-browser-check",
+    "--no-first-run",
+    "--no-pings",
+    "--no-sandbox",
+    "--no-zygote",
+    "--password-store=basic",
+    "--use-mock-keychain",
+    "--ignore-gpu-blacklist",
+    "--ignore-gpu-blocklist",
+    "--use-gl=swiftshader"
+  ]
+}

unfurler/src/config.ts

@@ -0,0 +1,55 @@
+const configFile = require('../config.json');
+
+interface IConfig {
+  SERVER: {
+    HOST: 'http://localhost';
+    HTTP_PORT: number;
+  };
+  MEMPOOL: {
+    HTTP_HOST: string;
+    HTTP_PORT: number;
+  };
+  PUPPETEER: {
+    CLUSTER_SIZE: number;
+    EXEC_PATH?: string;
+  };
+}
+
+const defaults: IConfig = {
+  'SERVER': {
+    'HOST': 'http://localhost',
+    'HTTP_PORT': 4201,
+  },
+  'MEMPOOL': {
+    'HTTP_HOST': 'http://localhost',
+    'HTTP_PORT': 4200,
+  },
+  'PUPPETEER': {
+    'CLUSTER_SIZE': 1,
+  },
+};
+
+class Config implements IConfig {
+  SERVER: IConfig['SERVER'];
+  MEMPOOL: IConfig['MEMPOOL'];
+  PUPPETEER: IConfig['PUPPETEER'];
+
+  constructor() {
+    const configs = this.merge(configFile, defaults);
+    this.SERVER = configs.SERVER;
+    this.MEMPOOL = configs.MEMPOOL;
+    this.PUPPETEER = configs.PUPPETEER;
+  }
+
+  merge = (...objects: object[]): IConfig => {
+    // @ts-ignore
+    return objects.reduce((prev, next) => {
+      Object.keys(prev).forEach(key => {
+        next[key] = { ...next[key], ...prev[key] };
+      });
+      return next;
+    });
+  }
+}
+
+export default new Config();

unfurler/src/index.ts

@@ -0,0 +1,125 @@
+import express from "express";
+import { Application, Request, Response, NextFunction } from 'express';
+import * as http from 'http';
+import config from './config';
+import { Cluster } from 'puppeteer-cluster';
+const puppeteerConfig = require('../puppeteer.config.json');
+
+if (config.PUPPETEER.EXEC_PATH) {
+  puppeteerConfig.executablePath = config.PUPPETEER.EXEC_PATH;
+}
+
+class Server {
+  private server: http.Server | undefined;
+  private app: Application;
+  cluster?: Cluster;
+  mempoolHost: string;
+
+  constructor() {
+    this.app = express();
+    this.mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : '');
+    this.startServer();
+  }
+
+  async startServer() {
+    this.app
+      .use((req: Request, res: Response, next: NextFunction) => {
+        res.setHeader('Access-Control-Allow-Origin', '*');
+        next();
+      })
+      .use(express.urlencoded({ extended: true }))
+      .use(express.text())
+      ;
+
+    this.cluster = await Cluster.launch({
+        concurrency: Cluster.CONCURRENCY_CONTEXT,
+        maxConcurrency: config.PUPPETEER.CLUSTER_SIZE,
+        puppeteerOptions: puppeteerConfig,
+    });
+    await this.cluster?.task(async (args) => { return this.renderPreviewTask(args) });
+
+    this.setUpRoutes();
+
+    this.server = http.createServer(this.app);
+
+    this.server.listen(config.SERVER.HTTP_PORT, () => {
+      console.log(`Mempool Unfurl Server is running on port ${config.SERVER.HTTP_PORT}`);
+    });
+  }
+
+  setUpRoutes() {
+    this.app.get('/render*', async (req, res) => { return this.renderPreview(req, res) })
+    this.app.get('*', (req, res) => { return this.renderHTML(req, res) })
+  }
+
+  async renderPreviewTask({ page, data: url }) {
+    await page.goto(url, { waitUntil: "domcontentloaded" });
+    await page.evaluate(async () => {
+      // wait for all images to finish loading
+      const imgs = Array.from(document.querySelectorAll("img"));
+      await Promise.all([
+        document.fonts.ready,
+        ...imgs.map((img) => {
+          if (img.complete) {
+            if (img.naturalHeight !== 0) return;
+            throw new Error("Image failed to load");
+          }
+          return new Promise((resolve, reject) => {
+            img.addEventListener("load", resolve);
+            img.addEventListener("error", reject);
+          });
+        }),
+      ]);
+    });
+    return page.screenshot();
+  }
+
+  async renderPreview(req, res) {
+    try {
+      const img = await this.cluster?.execute(this.mempoolHost + req.params[0]);
+
+      res.contentType('image/png');
+      res.send(img);
+    } catch (e) {
+      console.log(e);
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
+  renderHTML(req, res) {
+    let lang = '';
+    let path = req.originalUrl
+    // extract the language setting (if any)
+    const parts = path.split(/^\/(ar|bg|bs|ca|cs|da|de|et|el|es|eo|eu|fa|fr|gl|ko|hr|id|it|he|ka|lv|lt|hu|mk|ms|nl|ja|nb|nn|pl|pt|pt-BR|ro|ru|sk|sl|sr|sh|fi|sv|th|tr|uk|vi|zh|hi)\//)
+    if (parts.length > 1) {
+      lang = "/" + parts[1];
+      path = "/" + parts[2];
+    }
+    const ogImageUrl = config.SERVER.HOST + '/render' + lang + "/preview" + path;
+    res.send(`
+      <!doctype html>
+      <html lang="en-US" dir="ltr">
+      <head>
+        <meta charset="utf-8">
+        <title>mempool - Bitcoin Explorer</title>
+
+        <meta name="description" content="The Mempool Open Source Project™ - our self-hosted explorer for the Bitcoin community."/>
+        <meta property="og:image" content="${ogImageUrl}"/>
+        <meta property="og:image:type" content="image/png"/>
+        <meta property="og:image:width" content="1024"/>
+        <meta property="og:image:height" content="512"/>
+        <meta property="twitter:card" content="summary_large_image">
+        <meta property="twitter:site" content="@mempool">
+        <meta property="twitter:creator" content="@mempool">
+        <meta property="twitter:title" content="The Mempool Open Source Project™">
+        <meta property="twitter:description" content="Our self-hosted mempool explorer for the Bitcoin community."/>
+        <meta property="twitter:image:src" content="${ogImageUrl}"/>
+        <meta property="twitter:domain" content="mempool.space">
+
+      <body></body>
+      </html>
+    `);
+  }
+}
+
+const server = new Server();

unfurler/tsconfig.json

@@ -0,0 +1,24 @@
+{
+  "compilerOptions": {
+    "types": ["node"],
+    "module": "commonjs",
+    "target": "esnext",
+    "lib": ["es2019", "dom"],
+    "strict": true,
+    "noImplicitAny": false,
+    "sourceMap": false,
+    "outDir": "dist",
+    "moduleResolution": "node",
+    "typeRoots": [
+      "node_modules/@types"
+    ],
+    "allowSyntheticDefaultImports": true,
+    "esModuleInterop": true
+  },
+  "include": [
+    "src/**/*.ts"
+  ],
+  "exclude": [
+    "dist/**"
+  ]
+}