diff --git a/frontend/src/app/components/address/address-preview.component.ts b/frontend/src/app/components/address/address-preview.component.ts index c661c29db..b762d7c9e 100644 --- a/frontend/src/app/components/address/address-preview.component.ts +++ b/frontend/src/app/components/address/address-preview.component.ts @@ -44,7 +44,6 @@ export class AddressPreviewComponent implements OnInit, OnDestroy { ) { } ngOnInit() { - this.openGraphService.setPreviewLoading(); this.stateService.networkChanged$.subscribe((network) => this.network = network); this.addressLoadingStatus$ = this.route.paramMap @@ -56,6 +55,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy { this.mainSubscription = this.route.paramMap .pipe( switchMap((params: ParamMap) => { + this.openGraphService.waitFor('address-data'); this.error = undefined; this.isLoadingAddress = true; this.loadedConfirmedTxCount = 0; @@ -90,7 +90,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy { this.address = address; this.updateChainStats(); this.isLoadingAddress = false; - this.openGraphService.setPreviewReady(); + this.openGraphService.waitOver('address-data'); }) ) .subscribe(() => {}, diff --git a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts index 6e62a2fd0..7309a0a85 100644 --- a/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts +++ b/frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts @@ -18,6 +18,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy { @Input() orientation = 'left'; @Input() flip = true; @Output() txClickEvent = new EventEmitter(); + @Output() readyEvent = new EventEmitter(); @ViewChild('blockCanvas') canvas: ElementRef; @@ -37,6 +38,8 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy { selectedTx: TxView | void; tooltipPosition: Position; + readyNextFrame = false; + constructor( readonly ngZone: NgZone, readonly elRef: ElementRef, @@ -78,6 +81,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy { setup(transactions: TransactionStripped[]): void { if (this.scene) { this.scene.setup(transactions); + this.readyNextFrame = true; this.start(); } } @@ -258,6 +262,11 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy { this.gl.drawArrays(this.gl.TRIANGLES, 0, pointArray.length / TxSprite.vertexSize); } } + + if (this.readyNextFrame) { + this.readyNextFrame = false; + this.readyEvent.emit(); + } } /* LOOP */ diff --git a/frontend/src/app/components/block/block-preview.component.html b/frontend/src/app/components/block/block-preview.component.html index c47ea236e..768bc3da3 100644 --- a/frontend/src/app/components/block/block-preview.component.html +++ b/frontend/src/app/components/block/block-preview.component.html @@ -30,44 +30,42 @@ Weight - - - Median fee - ~{{ block?.extras?.medianFee | number:'1.0-0' }} sat/vB - - - - Total fees - - + + Median fee + ~{{ block?.extras?.medianFee | number:'1.0-0' }} sat/vB + + + + Total fees + + + + + + - - - - - - - - - Miner - - - {{ block?.extras.pool.name }} - - - - - {{ block?.extras.pool.name }} - - + + + Miner + + + {{ block?.extras.pool.name }} + + + + + {{ block?.extras.pool.name }} + + + -
+
diff --git a/frontend/src/app/components/block/block-preview.component.ts b/frontend/src/app/components/block/block-preview.component.ts index bab4e0489..e59bc9c6c 100644 --- a/frontend/src/app/components/block/block-preview.component.ts +++ b/frontend/src/app/components/block/block-preview.component.ts @@ -1,11 +1,153 @@ -import { Component } from '@angular/core'; -import { BlockComponent } from './block.component'; +import { Component, OnInit, OnDestroy, ViewChild, ElementRef } from '@angular/core'; +import { ActivatedRoute, ParamMap } from '@angular/router'; +import { ElectrsApiService } from '../../services/electrs-api.service'; +import { switchMap, tap, throttleTime, catchError, shareReplay, startWith, pairwise } from 'rxjs/operators'; +import { of, Subscription, asyncScheduler } from 'rxjs'; +import { StateService } from '../../services/state.service'; +import { SeoService } from 'src/app/services/seo.service'; +import { OpenGraphService } from 'src/app/services/opengraph.service'; +import { BlockExtended, TransactionStripped } from 'src/app/interfaces/node-api.interface'; +import { ApiService } from 'src/app/services/api.service'; +import { BlockOverviewGraphComponent } from 'src/app/components/block-overview-graph/block-overview-graph.component'; @Component({ selector: 'app-block-preview', templateUrl: './block-preview.component.html', - styleUrls: ['./block.component.scss', './block-preview.component.scss'] + styleUrls: ['./block-preview.component.scss'] }) -export class BlockPreviewComponent extends BlockComponent { - +export class BlockPreviewComponent implements OnInit, OnDestroy { + network = ''; + block: BlockExtended; + blockHeight: number; + blockHash: string; + isLoadingBlock = true; + strippedTransactions: TransactionStripped[]; + overviewTransitionDirection: string; + isLoadingOverview = true; + error: any; + blockSubsidy: number; + fees: number; + overviewError: any = null; + + overviewSubscription: Subscription; + networkChangedSubscription: Subscription; + + @ViewChild('blockGraph') blockGraph: BlockOverviewGraphComponent; + + constructor( + private route: ActivatedRoute, + private electrsApiService: ElectrsApiService, + public stateService: StateService, + private seoService: SeoService, + private openGraphService: OpenGraphService, + private apiService: ApiService + ) { } + + ngOnInit() { + this.network = this.stateService.network; + + const block$ = this.route.paramMap.pipe( + switchMap((params: ParamMap) => { + this.openGraphService.waitFor('block-viz'); + this.openGraphService.waitFor('block-data'); + + const blockHash: string = params.get('id') || ''; + this.block = undefined; + this.error = undefined; + this.fees = undefined; + + let isBlockHeight = false; + if (/^[0-9]+$/.test(blockHash)) { + isBlockHeight = true; + } else { + this.blockHash = blockHash; + } + + this.isLoadingBlock = true; + this.isLoadingOverview = true; + + if (isBlockHeight) { + return this.electrsApiService.getBlockHashFromHeight$(parseInt(blockHash, 10)) + .pipe( + switchMap((hash) => { + this.blockHash = hash; + return this.apiService.getBlock$(hash); + }) + ); + } + return this.apiService.getBlock$(blockHash); + }), + tap((block: BlockExtended) => { + this.block = block; + this.blockHeight = block.height; + + this.seoService.setTitle($localize`:@@block.component.browser-title:Block ${block.height}:BLOCK_HEIGHT:: ${block.id}:BLOCK_ID:`); + this.isLoadingBlock = false; + this.setBlockSubsidy(); + if (block?.extras?.reward !== undefined) { + this.fees = block.extras.reward / 100000000 - this.blockSubsidy; + } + this.stateService.markBlock$.next({ blockHeight: this.blockHeight }); + this.isLoadingOverview = true; + this.overviewError = null; + + this.openGraphService.waitOver('block-data'); + }), + throttleTime(50, asyncScheduler, { leading: true, trailing: true }), + shareReplay(1) + ); + + this.overviewSubscription = block$.pipe( + startWith(null), + pairwise(), + switchMap(([prevBlock, block]) => this.apiService.getStrippedBlockTransactions$(block.id) + .pipe( + catchError((err) => { + this.overviewError = err; + return of([]); + }), + switchMap((transactions) => { + return of({ transactions, direction: 'down' }); + }) + ) + ), + ) + .subscribe(({transactions, direction}: {transactions: TransactionStripped[], direction: string}) => { + this.strippedTransactions = transactions; + this.isLoadingOverview = false; + if (this.blockGraph) { + this.blockGraph.destroy(); + this.blockGraph.setup(this.strippedTransactions); + } + }, + (error) => { + this.error = error; + this.isLoadingOverview = false; + if (this.blockGraph) { + this.blockGraph.destroy(); + } + }); + + this.networkChangedSubscription = this.stateService.networkChanged$ + .subscribe((network) => this.network = network); + } + + ngOnDestroy() { + if (this.overviewSubscription) { + this.overviewSubscription.unsubscribe(); + } + if (this.networkChangedSubscription) { + this.networkChangedSubscription.unsubscribe(); + } + } + + // TODO - Refactor this.fees/this.reward for liquid because it is not + // used anymore on Bitcoin networks (we use block.extras directly) + setBlockSubsidy() { + this.blockSubsidy = 0; + } + + onGraphReady(): void { + this.openGraphService.waitOver('block-viz'); + } } diff --git a/frontend/src/app/services/opengraph.service.ts b/frontend/src/app/services/opengraph.service.ts index ad62a889c..12c74efcb 100644 --- a/frontend/src/app/services/opengraph.service.ts +++ b/frontend/src/app/services/opengraph.service.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@angular/core'; +import { Injectable, NgZone } from '@angular/core'; import { Meta } from '@angular/platform-browser'; import { Router, ActivatedRoute, NavigationEnd } from '@angular/router'; import { filter, map, switchMap } from 'rxjs/operators'; @@ -12,8 +12,11 @@ import { LanguageService } from './language.service'; export class OpenGraphService { network = ''; defaultImageUrl = ''; + previewLoadingEvents = {}; + previewLoadingCount = 0; constructor( + private ngZone: NgZone, private metaService: Meta, private stateService: StateService, private LanguageService: LanguageService, @@ -39,6 +42,9 @@ export class OpenGraphService { this.clearOgImage(); } }); + + // expose this service to global scope, so we can access it from the unfurler + window['ogService'] = this; } setOgImage() { @@ -59,13 +65,44 @@ export class OpenGraphService { this.metaService.updateTag({ property: 'og:image:height', content: '500' }); } - /// signal that the unfurler should wait for a 'ready' signal before taking a screenshot - setPreviewLoading() { - this.metaService.updateTag({ property: 'og:loading', content: 'loading'}); + /// register an event that needs to resolve before we can take a screenshot + waitFor(event) { + if (!this.previewLoadingEvents[event]) { + this.previewLoadingEvents[event] = 1; + this.previewLoadingCount++; + } else { + this.previewLoadingEvents[event]++; + } + this.metaService.updateTag({ property: 'og:preview:loading', content: 'loading'}); } - // signal to the unfurler that the page is ready for a screenshot - setPreviewReady() { - this.metaService.updateTag({ property: 'og:ready', content: 'ready'}); + // signal that an event has resolved + // if all registered events have resolved, signal we are ready for a screenshot + waitOver(event) { + if (this.previewLoadingEvents[event]) { + this.previewLoadingEvents[event]--; + if (this.previewLoadingEvents[event] === 0) { + delete this.previewLoadingEvents[event] + this.previewLoadingCount--; + } + } + if (this.previewLoadingCount === 0) { + this.metaService.updateTag({ property: 'og:preview:ready', content: 'ready'}); + } + } + + resetLoading() { + this.previewLoadingEvents = {}; + this.previewLoadingCount = 0; + this.metaService.removeTag("property='og:preview:loading'"); + this.metaService.removeTag("property='og:preview:ready'"); + this.metaService.removeTag("property='og:meta:ready'"); + } + + loadPage(path) { + this.resetLoading(); + this.ngZone.run(() => { + this.router.navigateByUrl(path); + }) } } diff --git a/frontend/src/app/services/seo.service.ts b/frontend/src/app/services/seo.service.ts index 01ed7ae8c..5f5d15c89 100644 --- a/frontend/src/app/services/seo.service.ts +++ b/frontend/src/app/services/seo.service.ts @@ -21,12 +21,14 @@ export class SeoService { this.titleService.setTitle(newTitle + ' - ' + this.getTitle()); this.metaService.updateTag({ property: 'og:title', content: newTitle}); this.metaService.updateTag({ property: 'twitter:title', content: newTitle}); + this.metaService.updateTag({ property: 'og:meta:ready', content: 'ready'}); } resetTitle(): void { this.titleService.setTitle(this.getTitle()); this.metaService.updateTag({ property: 'og:title', content: this.getTitle()}); this.metaService.updateTag({ property: 'twitter:title', content: this.getTitle()}); + this.metaService.updateTag({ property: 'og:meta:ready', content: 'ready'}); } setEnterpriseTitle(title: string) { diff --git a/unfurler/config.sample.json b/unfurler/config.sample.json index 02f2b78f0..c48f6f5b2 100644 --- a/unfurler/config.sample.json +++ b/unfurler/config.sample.json @@ -9,6 +9,7 @@ }, "PUPPETEER": { "CLUSTER_SIZE": 2, - "EXEC_PATH": "/usr/local/bin/chrome" // optional + "EXEC_PATH": "/usr/local/bin/chrome", // optional + "MAX_PAGE_AGE": 86400 // maximum lifetime of a page session (in seconds) } } diff --git a/unfurler/src/concurrency/ReusablePage.ts b/unfurler/src/concurrency/ReusablePage.ts new file mode 100644 index 000000000..98cdadc4d --- /dev/null +++ b/unfurler/src/concurrency/ReusablePage.ts @@ -0,0 +1,119 @@ +import * as puppeteer from 'puppeteer'; +import ConcurrencyImplementation, { ResourceData } from 'puppeteer-cluster/dist/concurrency/ConcurrencyImplementation'; +import { timeoutExecute } from 'puppeteer-cluster/dist/util'; + +import config from '../config'; +const mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : ''); + +const BROWSER_TIMEOUT = 5000; +// maximum lifetime of a single page session +const maxAgeMs = (config.PUPPETEER.MAX_PAGE_AGE || (24 * 60 * 60)) * 1000; + +interface repairablePage extends puppeteer.Page { + repairRequested?: boolean; +} + +export default class ReusablePage extends ConcurrencyImplementation { + + protected browser: puppeteer.Browser | null = null; + protected currentPage: repairablePage | null = null; + protected pageCreatedAt: number = 0; + private repairing: boolean = false; + private repairRequested: boolean = false; + private openInstances: number = 0; + private waitingForRepairResolvers: (() => void)[] = []; + + public constructor(options: puppeteer.LaunchOptions, puppeteer: any) { + super(options, puppeteer); + } + + private async repair() { + if (this.openInstances !== 0 || this.repairing) { + // already repairing or there are still pages open? wait for start/finish + await new Promise(resolve => this.waitingForRepairResolvers.push(resolve)); + return; + } + + this.repairing = true; + console.log('Starting repair'); + + try { + // will probably fail, but just in case the repair was not necessary + await (this.browser).close(); + } catch (e) { + console.log('Unable to close browser.'); + } + + try { + this.browser = await this.puppeteer.launch(this.options) as puppeteer.Browser; + } catch (err) { + throw new Error('Unable to restart chrome.'); + } + this.currentPage = null; + this.repairRequested = false; + this.repairing = false; + this.waitingForRepairResolvers.forEach(resolve => resolve()); + this.waitingForRepairResolvers = []; + await this.createResources(); + } + + public async init() { + this.browser = await this.puppeteer.launch(this.options); + } + + public async close() { + await (this.browser as puppeteer.Browser).close(); + } + + protected async createResources(): Promise { + if (!this.currentPage) { + this.currentPage = await (this.browser as puppeteer.Browser).newPage(); + this.pageCreatedAt = Date.now(); + const defaultUrl = mempoolHost + '/preview/block/1'; + this.currentPage.on('pageerror', (err) => { + this.repairRequested = true; + }); + await this.currentPage.goto(defaultUrl, { waitUntil: "load" }); + } + return { + page: this.currentPage + } + } + + public async workerInstance() { + let resources: ResourceData; + + return { + jobInstance: async () => { + if (this.repairRequested || this.currentPage?.repairRequested) { + await this.repair(); + } + + await timeoutExecute(BROWSER_TIMEOUT, (async () => { + resources = await this.createResources(); + })()); + this.openInstances += 1; + + return { + resources, + + close: async () => { + this.openInstances -= 1; // decrement first in case of error + + if (this.repairRequested || this.currentPage?.repairRequested || (Date.now() - this.pageCreatedAt > maxAgeMs)) { + await this.repair(); + } + }, + }; + }, + + close: async () => {}, + + repair: async () => { + console.log('Repair requested'); + this.repairRequested = true; + await this.repair(); + }, + }; + } +} diff --git a/unfurler/src/config.ts b/unfurler/src/config.ts index 1df60ce98..dd77eae56 100644 --- a/unfurler/src/config.ts +++ b/unfurler/src/config.ts @@ -12,6 +12,7 @@ interface IConfig { PUPPETEER: { CLUSTER_SIZE: number; EXEC_PATH?: string; + MAX_PAGE_AGE?: number; }; } diff --git a/unfurler/src/index.ts b/unfurler/src/index.ts index 49815fcb1..d84ce883a 100644 --- a/unfurler/src/index.ts +++ b/unfurler/src/index.ts @@ -3,6 +3,7 @@ import { Application, Request, Response, NextFunction } from 'express'; import * as http from 'http'; import config from './config'; import { Cluster } from 'puppeteer-cluster'; +import ReusablePage from './concurrency/ReusablePage'; const puppeteerConfig = require('../puppeteer.config.json'); if (config.PUPPETEER.EXEC_PATH) { @@ -32,7 +33,7 @@ class Server { ; this.cluster = await Cluster.launch({ - concurrency: Cluster.CONCURRENCY_CONTEXT, + concurrency: ReusablePage, maxConcurrency: config.PUPPETEER.CLUSTER_SIZE, puppeteerOptions: puppeteerConfig, }); @@ -52,47 +53,40 @@ class Server { this.app.get('*', (req, res) => { return this.renderHTML(req, res) }) } - async clusterTask({ page, data: { url, action } }) { - await page.goto(url, { waitUntil: "networkidle0" }); - switch (action) { - case 'screenshot': { - await page.evaluate(async () => { - // wait for all images to finish loading - const imgs = Array.from(document.querySelectorAll("img")); - await Promise.all([ - document.fonts.ready, - ...imgs.map((img) => { - if (img.complete) { - if (img.naturalHeight !== 0) return; - throw new Error("Image failed to load"); - } - return new Promise((resolve, reject) => { - img.addEventListener("load", resolve); - img.addEventListener("error", reject); - }); - }), - ]); - }); - const waitForReady = await page.$('meta[property="og:loading"]'); - const alreadyReady = await page.$('meta[property="og:ready"]'); - if (waitForReady != null && alreadyReady == null) { - try { - await page.waitForSelector('meta[property="og:ready]"', { timeout: 10000 }); - } catch (e) { - // probably timed out + async clusterTask({ page, data: { url, path, action } }) { + try { + if (action === 'screenshot' || action === 'html') { + const loaded = await page.evaluate(async (path) => { + if (window['ogService']) { + window['ogService'].loadPage(path); + return true; + } else { + return false; } + }, path) + + if (!loaded) { + throw new Error('failed to access open graph service'); } - return page.screenshot(); - } break; - default: { - try { - await page.waitForSelector('meta[property="og:title"]', { timeout: 10000 }) - const tag = await page.$('meta[property="og:title"]'); - } catch (e) { - // probably timed out + + if (action === 'screenshot') { + const waitForReady = await page.$('meta[property="og:preview:loading"]'); + const alreadyReady = await page.$('meta[property="og:preview:ready"]'); + if (waitForReady != null && alreadyReady == null) { + await page.waitForSelector('meta[property="og:preview:ready"]', { timeout: 8000 }); + } + return page.screenshot(); + } else if (action === 'html') { + const alreadyReady = await page.$('meta[property="og:meta:ready"]'); + if (alreadyReady == null) { + await page.waitForSelector('meta[property="og:meta:ready"]', { timeout: 8000 }); + } + return page.content(); } - return page.content(); } + } catch (e) { + console.log(`failed to render page for ${action}`, e instanceof Error ? e.message : e); + page.repairRequested = true; } } @@ -100,8 +94,11 @@ class Server { try { // strip default language code for compatibility const path = req.params[0].replace('/en/', '/'); - const img = await this.cluster?.execute({ url: this.mempoolHost + path, action: 'screenshot' }); + const img = await this.cluster?.execute({ url: this.mempoolHost + path, path: path, action: 'screenshot' }); + if (!img) { + throw new Error('failed to render preview image'); + } res.contentType('image/png'); res.send(img); } catch (e) { @@ -120,9 +117,14 @@ class Server { } try { - let html = await this.cluster?.execute({ url: this.mempoolHost + req.params[0], action: 'html' }); + // strip default language code for compatibility + const path = req.params[0].replace('/en/', '/'); - res.send(html) + let html = await this.cluster?.execute({ url: this.mempoolHost + req.params[0], path: req.params[0], action: 'html' }); + if (!html) { + throw new Error('failed to render preview image'); + } + res.send(html); } catch (e) { console.log(e); res.status(500).send(e instanceof Error ? e.message : e);