Merge branch 'master' into nymkappa/menu

frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts

@@ -70,9 +70,11 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
     this.canvas.nativeElement.addEventListener('webglcontextlost', this.handleContextLost, false);
     this.canvas.nativeElement.addEventListener('webglcontextrestored', this.handleContextRestored, false);
     this.gl = this.canvas.nativeElement.getContext('webgl');
-    this.initCanvas();
 
-    this.resizeCanvas();
+    if (this.gl) {
+      this.initCanvas();
+      this.resizeCanvas();
+    }
   }
 
   ngOnChanges(changes): void {
@@ -195,10 +197,16 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
     cancelAnimationFrame(this.animationFrameRequest);
     this.animationFrameRequest = null;
     this.running = false;
+    this.gl = null;
   }
 
   handleContextRestored(event): void {
-    this.initCanvas();
+    if (this.canvas?.nativeElement) {
+      this.gl = this.canvas.nativeElement.getContext('webgl');
+      if (this.gl) {
+        this.initCanvas();
+      }
+    }
   }
 
   @HostListener('window:resize', ['$event'])
@@ -224,6 +232,9 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
   }
 
   compileShader(src, type): WebGLShader {
+    if (!this.gl) {
+      return;
+    }
     const shader = this.gl.createShader(type);
 
     this.gl.shaderSource(shader, src);
@@ -237,6 +248,9 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
   }
 
   buildShaderProgram(shaderInfo): WebGLProgram {
+    if (!this.gl) {
+      return;
+    }
     const program = this.gl.createProgram();
 
     shaderInfo.forEach((desc) => {
@@ -273,7 +287,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
       now = performance.now();
     }
     // skip re-render if there's no change to the scene
-    if (this.scene) {
+    if (this.scene && this.gl) {
       /* SET UP SHADER UNIFORMS */
       // screen dimensions
       this.gl.uniform2f(this.gl.getUniformLocation(this.shaderProgram, 'screenSize'), this.displayWidth, this.displayHeight);

frontend/src/app/shared/graphs.utils.ts

@@ -90,7 +90,7 @@ export const download = (href, name) => {
 
 export function detectWebGL(): boolean {
   const canvas = document.createElement('canvas');
-  const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
+  const gl = canvas.getContext('webgl');
   return !!(gl && gl instanceof WebGLRenderingContext);
 }
 

production/nginx/server-common.conf

@@ -97,6 +97,14 @@ location ~* ^/.+\..+\.(js|css)$ {
 	expires 1y;
 }
 
+# old stuff is gone
+location /explorer/ {
+	return 410;
+}
+location /sitemap/ {
+	return 410;
+}
+
 # unfurl preview
 location /preview {
 	try_files /$lang/$uri $uri /en-US/$uri /en-US/index.html =404;
@@ -105,7 +113,6 @@ location /preview {
 # unfurl renderer
 location ^~ /render {
 	try_files /dev/null @mempool-space-unfurler;
-	expires 10m;
 }
 # unfurl handler
 location /unfurl/ {

unfurler/src/concurrency/ReusablePage.ts

@@ -110,7 +110,7 @@ export default class ReusablePage extends ConcurrencyImplementation {
           page.waitForSelector('meta[property="og:preview:fail"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => false)
         ])
       } catch (e) {
-        logger.err(`failed to load frontend during page initialization: ` + (e instanceof Error ? e.message : `${e}`));
+        logger.err(`failed to load frontend during page initialization  ${page.clusterGroup}:${page.index}: ` + (e instanceof Error ? e.message : `${e}`));
         page.repairRequested = true;
       }
     }
@@ -131,7 +131,7 @@ export default class ReusablePage extends ConcurrencyImplementation {
 
   protected async repairPage(page) {
     // create a new page
-    logger.debug(`Repairing page ${page.clusterGroup}:${page.index}`);
+    logger.info(`Repairing page ${page.clusterGroup}:${page.index}`);
     const newPage = await this.initPage();
     newPage.free = true;
     // replace the old page
@@ -142,8 +142,9 @@ export default class ReusablePage extends ConcurrencyImplementation {
       await page.goto('about:blank', {timeout: 200}); // prevents memory leak (maybe?)
     } catch (e) {
       logger.err(`unexpected page repair error ${page.clusterGroup}:${page.index}`);
+    } finally {
+      await page.close();
     }
-    await page.close();
     return newPage;
   }
 

unfurler/src/concurrency/ReusableSSRPage.ts

@@ -22,7 +22,7 @@ export default class ReusableSSRPage extends ReusablePage {
     page.clusterGroup = 'slurper';
     page.language = null;
     page.createdAt = Date.now();
-    const defaultUrl = mempoolHost + '/about';
+    const defaultUrl = mempoolHost + '/preview/block/1';
 
     page.on('pageerror', (err) => {
       console.log(err);
@@ -39,7 +39,7 @@ export default class ReusableSSRPage extends ReusablePage {
           headers: {"Access-Control-Allow-Origin": "*"},
           body: mockImageBuffer
         });
-      } else if (!['document', 'script', 'xhr', 'fetch'].includes(req.resourceType())) {
+      } else if (req.resourceType() === 'media') {
         return req.abort();
       } else {
         return req.continue();
@@ -49,7 +49,7 @@ export default class ReusableSSRPage extends ReusablePage {
       await page.goto(defaultUrl, { waitUntil: "networkidle0" });
       await page.waitForSelector('meta[property="og:meta:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 });
     } catch (e) {
-      logger.err(`failed to load frontend during ssr page initialization: ` + (e instanceof Error ? e.message : `${e}`));
+      logger.err(`failed to load frontend during ssr page initialization ${page.clusterGroup}:${page.index}: ` + (e instanceof Error ? e.message : `${e}`));
       page.repairRequested = true;
     }
     page.free = true;

unfurler/src/index.ts

@@ -28,13 +28,18 @@ class Server {
   mempoolUrl: URL;
   network: string;
   secureHost = true;
+  secureMempoolHost = true;
   canonicalHost: string;
 
+  seoQueueLength: number = 0;
+  unfurlQueueLength: number = 0;
+
   constructor() {
     this.app = express();
     this.mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : '');
     this.mempoolUrl = new URL(this.mempoolHost);
     this.secureHost = config.SERVER.HOST.startsWith('https');
+    this.secureMempoolHost = config.MEMPOOL.HTTP_HOST.startsWith('https');
     this.network = config.MEMPOOL.NETWORK || 'bitcoin';
 
     let canonical;
@@ -121,6 +126,7 @@ class Server {
   }
 
   async clusterTask({ page, data: { url, path, action, reqUrl } }) {
+    const start = Date.now();
     try {
       logger.info(`rendering "${reqUrl}" on tab ${page.clusterGroup}:${page.index}`);
       const urlParts = parseLanguageUrl(path);
@@ -155,6 +161,7 @@ class Server {
           captureBeyondViewport: false,
           clip: { width: 1200, height: 600, x: 0, y: 0, scale: 1 },
         });
+        logger.info(`rendered unfurl img in ${Date.now() - start}ms for "${reqUrl}" on tab ${page.clusterGroup}:${page.index}`);
         return screenshot;
       } else if (success === false) {
         logger.warn(`failed to render ${reqUrl} for ${action} due to client-side error, e.g. requested an invalid txid`);
@@ -170,13 +177,14 @@ class Server {
   }
 
   async ssrClusterTask({ page, data: { url, path, action, reqUrl } }) {
+    const start = Date.now();
     try {
       logger.info(`slurping "${reqUrl}" on tab ${page.clusterGroup}:${page.index}`);
       const urlParts = parseLanguageUrl(path);
       if (page.language !== urlParts.lang) {
         // switch language
         page.language = urlParts.lang;
-        const localizedUrl = urlParts.lang ? `${this.mempoolHost}/${urlParts.lang}${urlParts.path}` : `${this.mempoolHost}${urlParts.path}` ;
+        const localizedUrl = urlParts.lang ? `${this.mempoolHost}/${urlParts.lang}${urlParts.path}` : `${this.mempoolHost}${urlParts.path}`;
         await page.goto(localizedUrl, { waitUntil: "load" });
       } else {
         const loaded = await page.evaluate(async (path) => {
@@ -199,14 +207,17 @@ class Server {
         return !!window['soft404'];
       });
       if (is404) {
+        logger.info(`slurp 404 in ${Date.now() - start}ms for "${reqUrl}" on tab ${page.clusterGroup}:${page.index}`);
         return '404';
       } else {
         let html = await page.content();
+        logger.info(`rendered slurp in ${Date.now() - start}ms for "${reqUrl}" on tab ${page.clusterGroup}:${page.index}`);
         return html;
       }
     } catch (e) {
       if (e instanceof TimeoutError) {
         let html = await page.content();
+        logger.info(`rendered partial slurp in ${Date.now() - start}ms for "${reqUrl}" on tab ${page.clusterGroup}:${page.index}`);
         return html;
       } else {
         logger.err(`failed to render ${reqUrl} for ${action}: ` + (e instanceof Error ? e.message : `${e}`));
@@ -221,6 +232,8 @@ class Server {
 
   async renderPreview(req, res) {
     try {
+      this.unfurlQueueLength++;
+      const start = Date.now();
       const rawPath = req.params[0];
 
       let img = null;
@@ -231,13 +244,14 @@ class Server {
       // don't bother unless the route is definitely renderable
       if (rawPath.includes('/preview/') && matchedRoute.render) {
         img = await this.cluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'screenshot', reqUrl: req.url });
+        logger.info(`unfurl returned "${req.url}" in ${Date.now() - start}ms | ${this.unfurlQueueLength - 1} tasks in queue`);
       } else {
         logger.info('rendering not enabled for page "' + req.url + '"');
       }
 
       if (!img) {
-        // proxy fallback image from the frontend
+        // send local fallback image file
-        res.sendFile(nodejsPath.join(__dirname, matchedRoute.fallbackImg));
+        res.sendFile(nodejsPath.join(__dirname, matchedRoute.fallbackFile));
       } else {
         res.contentType('image/png');
         res.send(img);
@@ -245,6 +259,8 @@ class Server {
     } catch (e) {
       logger.err(e instanceof Error ? e.message : `${e} ${req.params[0]}`);
       res.status(500).send(e instanceof Error ? e.message : e);
+    } finally {
+      this.unfurlQueueLength--;
     }
   }
 
@@ -263,13 +279,13 @@ class Server {
         return;
       } else {
         logger.info('proxying resource "' + req.url + '"');
-        if (this.secureHost) {
+        if (this.secureMempoolHost) {
-          https.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => {
+          https.get(this.mempoolHost + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => {
             res.writeHead(got.statusCode, got.headers);
             return got.pipe(res);
           });
         } else {
-          http.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => {
+          http.get(this.mempoolHost + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => {
             res.writeHead(got.statusCode, got.headers);
             return got.pipe(res);
           });
@@ -284,7 +300,10 @@ class Server {
         logger.info('unfurling "' + req.url + '"');
         result = await this.renderUnfurlMeta(rawPath);
       } else {
+        this.seoQueueLength++;
+        const start = Date.now();
         result = await this.renderSEOPage(rawPath, req.url);
+        logger.info(`slurp returned "${req.url}" in ${Date.now() - start}ms | ${this.seoQueueLength - 1} tasks in queue`);
       }
       if (result && result.length) {
         if (result === '404') {
@@ -298,6 +317,10 @@ class Server {
     } catch (e) {
       logger.err(e instanceof Error ? e.message : `${e} ${req.params[0]}`);
       res.status(500).send(e instanceof Error ? e.message : e);
+    } finally {
+      if (!unfurl) {
+        this.seoQueueLength--;
+      }
     }
   }
 

unfurler/src/routes.ts

@@ -2,6 +2,7 @@ interface Match {
   render: boolean;
   title: string;
   fallbackImg: string;
+  fallbackFile: string;
   staticImg?: string;
   networkMode: string;
 }
@@ -30,7 +31,8 @@ const routes = {
   },
   lightning: {
     title: "Lightning",
-    fallbackImg: '/resources/img/lightning.png',
+    fallbackImg: '/resources/previews/lightning.png',
+    fallbackFile: '/resources/img/lightning.png',
     routes: {
       node: {
         render: true,
@@ -68,7 +70,8 @@ const routes = {
   },
   mining: {
     title: "Mining",
-    fallbackImg: '/resources/img/mining.png',
+    fallbackImg: '/resources/previews/mining.png',
+    fallbackFile: '/resources/img/mining.png',
     routes: {
       pool: {
         render: true,
@@ -83,13 +86,15 @@ const routes = {
 
 const networks = {
   bitcoin: {
-    fallbackImg: '/resources/img/dashboard.png',
+    fallbackImg: '/resources/previews/dashboard.png',
+    fallbackFile: '/resources/img/dashboard.png',
     routes: {
       ...routes // all routes supported
     }
   },
   liquid: {
-    fallbackImg: '/resources/img/liquid.png',
+    fallbackImg: '/resources/liquid/liquid-network-preview.png',
+    fallbackFile: '/resources/img/liquid',
     routes: { // only block, address & tx routes supported
       block: routes.block,
       address: routes.address,
@@ -97,7 +102,8 @@ const networks = {
     }
   },
   bisq: {
-    fallbackImg: '/resources/img/bisq.png',
+    fallbackImg: '/resources/bisq/bisq-markets-preview.png',
+    fallbackFile: '/resources/img/bisq.png',
     routes: {} // no routes supported
   }
 };
@@ -107,6 +113,7 @@ export function matchRoute(network: string, path: string): Match {
     render: false,
     title: '',
     fallbackImg: '',
+    fallbackFile: '',
     networkMode: 'mainnet'
   }
 
@@ -121,6 +128,7 @@ export function matchRoute(network: string, path: string): Match {
 
   let route = networks[network] || networks.bitcoin;
   match.fallbackImg = route.fallbackImg;
+  match.fallbackFile = route.fallbackFile;
 
   // traverse the route tree until we run out of route or tree, or hit a renderable match
   while (!route.render && route.routes && parts.length && route.routes[parts[0]]) {
@@ -128,6 +136,7 @@ export function matchRoute(network: string, path: string): Match {
     parts.shift();
     if (route.fallbackImg) {
       match.fallbackImg = route.fallbackImg;
+      match.fallbackFile = route.fallbackFile;
     }
   }