Merge pull request #4198 from mempool/mononaut/unfurler-fixes
More unfurler fixes
This commit is contained in:
commit
063d7e96a1
@ -70,9 +70,11 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
|
|||||||
this.canvas.nativeElement.addEventListener('webglcontextlost', this.handleContextLost, false);
|
this.canvas.nativeElement.addEventListener('webglcontextlost', this.handleContextLost, false);
|
||||||
this.canvas.nativeElement.addEventListener('webglcontextrestored', this.handleContextRestored, false);
|
this.canvas.nativeElement.addEventListener('webglcontextrestored', this.handleContextRestored, false);
|
||||||
this.gl = this.canvas.nativeElement.getContext('webgl');
|
this.gl = this.canvas.nativeElement.getContext('webgl');
|
||||||
this.initCanvas();
|
|
||||||
|
|
||||||
this.resizeCanvas();
|
if (this.gl) {
|
||||||
|
this.initCanvas();
|
||||||
|
this.resizeCanvas();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ngOnChanges(changes): void {
|
ngOnChanges(changes): void {
|
||||||
@ -195,10 +197,16 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
|
|||||||
cancelAnimationFrame(this.animationFrameRequest);
|
cancelAnimationFrame(this.animationFrameRequest);
|
||||||
this.animationFrameRequest = null;
|
this.animationFrameRequest = null;
|
||||||
this.running = false;
|
this.running = false;
|
||||||
|
this.gl = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
handleContextRestored(event): void {
|
handleContextRestored(event): void {
|
||||||
this.initCanvas();
|
if (this.canvas?.nativeElement) {
|
||||||
|
this.gl = this.canvas.nativeElement.getContext('webgl');
|
||||||
|
if (this.gl) {
|
||||||
|
this.initCanvas();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@HostListener('window:resize', ['$event'])
|
@HostListener('window:resize', ['$event'])
|
||||||
@ -224,6 +232,9 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
|
|||||||
}
|
}
|
||||||
|
|
||||||
compileShader(src, type): WebGLShader {
|
compileShader(src, type): WebGLShader {
|
||||||
|
if (!this.gl) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
const shader = this.gl.createShader(type);
|
const shader = this.gl.createShader(type);
|
||||||
|
|
||||||
this.gl.shaderSource(shader, src);
|
this.gl.shaderSource(shader, src);
|
||||||
@ -237,6 +248,9 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
|
|||||||
}
|
}
|
||||||
|
|
||||||
buildShaderProgram(shaderInfo): WebGLProgram {
|
buildShaderProgram(shaderInfo): WebGLProgram {
|
||||||
|
if (!this.gl) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
const program = this.gl.createProgram();
|
const program = this.gl.createProgram();
|
||||||
|
|
||||||
shaderInfo.forEach((desc) => {
|
shaderInfo.forEach((desc) => {
|
||||||
@ -273,7 +287,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy, On
|
|||||||
now = performance.now();
|
now = performance.now();
|
||||||
}
|
}
|
||||||
// skip re-render if there's no change to the scene
|
// skip re-render if there's no change to the scene
|
||||||
if (this.scene) {
|
if (this.scene && this.gl) {
|
||||||
/* SET UP SHADER UNIFORMS */
|
/* SET UP SHADER UNIFORMS */
|
||||||
// screen dimensions
|
// screen dimensions
|
||||||
this.gl.uniform2f(this.gl.getUniformLocation(this.shaderProgram, 'screenSize'), this.displayWidth, this.displayHeight);
|
this.gl.uniform2f(this.gl.getUniformLocation(this.shaderProgram, 'screenSize'), this.displayWidth, this.displayHeight);
|
||||||
|
@ -90,7 +90,7 @@ export const download = (href, name) => {
|
|||||||
|
|
||||||
export function detectWebGL(): boolean {
|
export function detectWebGL(): boolean {
|
||||||
const canvas = document.createElement('canvas');
|
const canvas = document.createElement('canvas');
|
||||||
const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
|
const gl = canvas.getContext('webgl');
|
||||||
return !!(gl && gl instanceof WebGLRenderingContext);
|
return !!(gl && gl instanceof WebGLRenderingContext);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -110,7 +110,7 @@ export default class ReusablePage extends ConcurrencyImplementation {
|
|||||||
page.waitForSelector('meta[property="og:preview:fail"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => false)
|
page.waitForSelector('meta[property="og:preview:fail"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => false)
|
||||||
])
|
])
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
logger.err(`failed to load frontend during page initialization: ` + (e instanceof Error ? e.message : `${e}`));
|
logger.err(`failed to load frontend during page initialization ${page.clusterGroup}:${page.index}: ` + (e instanceof Error ? e.message : `${e}`));
|
||||||
page.repairRequested = true;
|
page.repairRequested = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -131,7 +131,7 @@ export default class ReusablePage extends ConcurrencyImplementation {
|
|||||||
|
|
||||||
protected async repairPage(page) {
|
protected async repairPage(page) {
|
||||||
// create a new page
|
// create a new page
|
||||||
logger.debug(`Repairing page ${page.clusterGroup}:${page.index}`);
|
logger.info(`Repairing page ${page.clusterGroup}:${page.index}`);
|
||||||
const newPage = await this.initPage();
|
const newPage = await this.initPage();
|
||||||
newPage.free = true;
|
newPage.free = true;
|
||||||
// replace the old page
|
// replace the old page
|
||||||
@ -142,8 +142,9 @@ export default class ReusablePage extends ConcurrencyImplementation {
|
|||||||
await page.goto('about:blank', {timeout: 200}); // prevents memory leak (maybe?)
|
await page.goto('about:blank', {timeout: 200}); // prevents memory leak (maybe?)
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
logger.err(`unexpected page repair error ${page.clusterGroup}:${page.index}`);
|
logger.err(`unexpected page repair error ${page.clusterGroup}:${page.index}`);
|
||||||
|
} finally {
|
||||||
|
await page.close();
|
||||||
}
|
}
|
||||||
await page.close();
|
|
||||||
return newPage;
|
return newPage;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -22,7 +22,7 @@ export default class ReusableSSRPage extends ReusablePage {
|
|||||||
page.clusterGroup = 'slurper';
|
page.clusterGroup = 'slurper';
|
||||||
page.language = null;
|
page.language = null;
|
||||||
page.createdAt = Date.now();
|
page.createdAt = Date.now();
|
||||||
const defaultUrl = mempoolHost + '/about';
|
const defaultUrl = mempoolHost + '/preview/block/1';
|
||||||
|
|
||||||
page.on('pageerror', (err) => {
|
page.on('pageerror', (err) => {
|
||||||
console.log(err);
|
console.log(err);
|
||||||
@ -39,7 +39,7 @@ export default class ReusableSSRPage extends ReusablePage {
|
|||||||
headers: {"Access-Control-Allow-Origin": "*"},
|
headers: {"Access-Control-Allow-Origin": "*"},
|
||||||
body: mockImageBuffer
|
body: mockImageBuffer
|
||||||
});
|
});
|
||||||
} else if (!['document', 'script', 'xhr', 'fetch'].includes(req.resourceType())) {
|
} else if (req.resourceType() === 'media') {
|
||||||
return req.abort();
|
return req.abort();
|
||||||
} else {
|
} else {
|
||||||
return req.continue();
|
return req.continue();
|
||||||
@ -49,7 +49,7 @@ export default class ReusableSSRPage extends ReusablePage {
|
|||||||
await page.goto(defaultUrl, { waitUntil: "networkidle0" });
|
await page.goto(defaultUrl, { waitUntil: "networkidle0" });
|
||||||
await page.waitForSelector('meta[property="og:meta:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 });
|
await page.waitForSelector('meta[property="og:meta:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 });
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
logger.err(`failed to load frontend during ssr page initialization: ` + (e instanceof Error ? e.message : `${e}`));
|
logger.err(`failed to load frontend during ssr page initialization ${page.clusterGroup}:${page.index}: ` + (e instanceof Error ? e.message : `${e}`));
|
||||||
page.repairRequested = true;
|
page.repairRequested = true;
|
||||||
}
|
}
|
||||||
page.free = true;
|
page.free = true;
|
||||||
|
@ -28,13 +28,18 @@ class Server {
|
|||||||
mempoolUrl: URL;
|
mempoolUrl: URL;
|
||||||
network: string;
|
network: string;
|
||||||
secureHost = true;
|
secureHost = true;
|
||||||
|
secureMempoolHost = true;
|
||||||
canonicalHost: string;
|
canonicalHost: string;
|
||||||
|
|
||||||
|
seoQueueLength: number = 0;
|
||||||
|
unfurlQueueLength: number = 0;
|
||||||
|
|
||||||
constructor() {
|
constructor() {
|
||||||
this.app = express();
|
this.app = express();
|
||||||
this.mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : '');
|
this.mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : '');
|
||||||
this.mempoolUrl = new URL(this.mempoolHost);
|
this.mempoolUrl = new URL(this.mempoolHost);
|
||||||
this.secureHost = config.SERVER.HOST.startsWith('https');
|
this.secureHost = config.SERVER.HOST.startsWith('https');
|
||||||
|
this.secureMempoolHost = config.MEMPOOL.HTTP_HOST.startsWith('https');
|
||||||
this.network = config.MEMPOOL.NETWORK || 'bitcoin';
|
this.network = config.MEMPOOL.NETWORK || 'bitcoin';
|
||||||
|
|
||||||
let canonical;
|
let canonical;
|
||||||
@ -121,6 +126,7 @@ class Server {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async clusterTask({ page, data: { url, path, action, reqUrl } }) {
|
async clusterTask({ page, data: { url, path, action, reqUrl } }) {
|
||||||
|
const start = Date.now();
|
||||||
try {
|
try {
|
||||||
logger.info(`rendering "${reqUrl}" on tab ${page.clusterGroup}:${page.index}`);
|
logger.info(`rendering "${reqUrl}" on tab ${page.clusterGroup}:${page.index}`);
|
||||||
const urlParts = parseLanguageUrl(path);
|
const urlParts = parseLanguageUrl(path);
|
||||||
@ -155,6 +161,7 @@ class Server {
|
|||||||
captureBeyondViewport: false,
|
captureBeyondViewport: false,
|
||||||
clip: { width: 1200, height: 600, x: 0, y: 0, scale: 1 },
|
clip: { width: 1200, height: 600, x: 0, y: 0, scale: 1 },
|
||||||
});
|
});
|
||||||
|
logger.info(`rendered unfurl img in ${Date.now() - start}ms for "${reqUrl}" on tab ${page.clusterGroup}:${page.index}`);
|
||||||
return screenshot;
|
return screenshot;
|
||||||
} else if (success === false) {
|
} else if (success === false) {
|
||||||
logger.warn(`failed to render ${reqUrl} for ${action} due to client-side error, e.g. requested an invalid txid`);
|
logger.warn(`failed to render ${reqUrl} for ${action} due to client-side error, e.g. requested an invalid txid`);
|
||||||
@ -170,13 +177,14 @@ class Server {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async ssrClusterTask({ page, data: { url, path, action, reqUrl } }) {
|
async ssrClusterTask({ page, data: { url, path, action, reqUrl } }) {
|
||||||
|
const start = Date.now();
|
||||||
try {
|
try {
|
||||||
logger.info(`slurping "${reqUrl}" on tab ${page.clusterGroup}:${page.index}`);
|
logger.info(`slurping "${reqUrl}" on tab ${page.clusterGroup}:${page.index}`);
|
||||||
const urlParts = parseLanguageUrl(path);
|
const urlParts = parseLanguageUrl(path);
|
||||||
if (page.language !== urlParts.lang) {
|
if (page.language !== urlParts.lang) {
|
||||||
// switch language
|
// switch language
|
||||||
page.language = urlParts.lang;
|
page.language = urlParts.lang;
|
||||||
const localizedUrl = urlParts.lang ? `${this.mempoolHost}/${urlParts.lang}${urlParts.path}` : `${this.mempoolHost}${urlParts.path}` ;
|
const localizedUrl = urlParts.lang ? `${this.mempoolHost}/${urlParts.lang}${urlParts.path}` : `${this.mempoolHost}${urlParts.path}`;
|
||||||
await page.goto(localizedUrl, { waitUntil: "load" });
|
await page.goto(localizedUrl, { waitUntil: "load" });
|
||||||
} else {
|
} else {
|
||||||
const loaded = await page.evaluate(async (path) => {
|
const loaded = await page.evaluate(async (path) => {
|
||||||
@ -199,14 +207,17 @@ class Server {
|
|||||||
return !!window['soft404'];
|
return !!window['soft404'];
|
||||||
});
|
});
|
||||||
if (is404) {
|
if (is404) {
|
||||||
|
logger.info(`slurp 404 in ${Date.now() - start}ms for "${reqUrl}" on tab ${page.clusterGroup}:${page.index}`);
|
||||||
return '404';
|
return '404';
|
||||||
} else {
|
} else {
|
||||||
let html = await page.content();
|
let html = await page.content();
|
||||||
|
logger.info(`rendered slurp in ${Date.now() - start}ms for "${reqUrl}" on tab ${page.clusterGroup}:${page.index}`);
|
||||||
return html;
|
return html;
|
||||||
}
|
}
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
if (e instanceof TimeoutError) {
|
if (e instanceof TimeoutError) {
|
||||||
let html = await page.content();
|
let html = await page.content();
|
||||||
|
logger.info(`rendered partial slurp in ${Date.now() - start}ms for "${reqUrl}" on tab ${page.clusterGroup}:${page.index}`);
|
||||||
return html;
|
return html;
|
||||||
} else {
|
} else {
|
||||||
logger.err(`failed to render ${reqUrl} for ${action}: ` + (e instanceof Error ? e.message : `${e}`));
|
logger.err(`failed to render ${reqUrl} for ${action}: ` + (e instanceof Error ? e.message : `${e}`));
|
||||||
@ -221,6 +232,8 @@ class Server {
|
|||||||
|
|
||||||
async renderPreview(req, res) {
|
async renderPreview(req, res) {
|
||||||
try {
|
try {
|
||||||
|
this.unfurlQueueLength++;
|
||||||
|
const start = Date.now();
|
||||||
const rawPath = req.params[0];
|
const rawPath = req.params[0];
|
||||||
|
|
||||||
let img = null;
|
let img = null;
|
||||||
@ -231,6 +244,7 @@ class Server {
|
|||||||
// don't bother unless the route is definitely renderable
|
// don't bother unless the route is definitely renderable
|
||||||
if (rawPath.includes('/preview/') && matchedRoute.render) {
|
if (rawPath.includes('/preview/') && matchedRoute.render) {
|
||||||
img = await this.cluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'screenshot', reqUrl: req.url });
|
img = await this.cluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'screenshot', reqUrl: req.url });
|
||||||
|
logger.info(`unfurl returned "${req.url}" in ${Date.now() - start}ms | ${this.unfurlQueueLength - 1} tasks in queue`);
|
||||||
} else {
|
} else {
|
||||||
logger.info('rendering not enabled for page "' + req.url + '"');
|
logger.info('rendering not enabled for page "' + req.url + '"');
|
||||||
}
|
}
|
||||||
@ -245,6 +259,8 @@ class Server {
|
|||||||
} catch (e) {
|
} catch (e) {
|
||||||
logger.err(e instanceof Error ? e.message : `${e} ${req.params[0]}`);
|
logger.err(e instanceof Error ? e.message : `${e} ${req.params[0]}`);
|
||||||
res.status(500).send(e instanceof Error ? e.message : e);
|
res.status(500).send(e instanceof Error ? e.message : e);
|
||||||
|
} finally {
|
||||||
|
this.unfurlQueueLength--;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -263,13 +279,13 @@ class Server {
|
|||||||
return;
|
return;
|
||||||
} else {
|
} else {
|
||||||
logger.info('proxying resource "' + req.url + '"');
|
logger.info('proxying resource "' + req.url + '"');
|
||||||
if (this.secureHost) {
|
if (this.secureMempoolHost) {
|
||||||
https.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => {
|
https.get(this.mempoolHost + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => {
|
||||||
res.writeHead(got.statusCode, got.headers);
|
res.writeHead(got.statusCode, got.headers);
|
||||||
return got.pipe(res);
|
return got.pipe(res);
|
||||||
});
|
});
|
||||||
} else {
|
} else {
|
||||||
http.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => {
|
http.get(this.mempoolHost + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => {
|
||||||
res.writeHead(got.statusCode, got.headers);
|
res.writeHead(got.statusCode, got.headers);
|
||||||
return got.pipe(res);
|
return got.pipe(res);
|
||||||
});
|
});
|
||||||
@ -284,7 +300,10 @@ class Server {
|
|||||||
logger.info('unfurling "' + req.url + '"');
|
logger.info('unfurling "' + req.url + '"');
|
||||||
result = await this.renderUnfurlMeta(rawPath);
|
result = await this.renderUnfurlMeta(rawPath);
|
||||||
} else {
|
} else {
|
||||||
|
this.seoQueueLength++;
|
||||||
|
const start = Date.now();
|
||||||
result = await this.renderSEOPage(rawPath, req.url);
|
result = await this.renderSEOPage(rawPath, req.url);
|
||||||
|
logger.info(`slurp returned "${req.url}" in ${Date.now() - start}ms | ${this.seoQueueLength - 1} tasks in queue`);
|
||||||
}
|
}
|
||||||
if (result && result.length) {
|
if (result && result.length) {
|
||||||
if (result === '404') {
|
if (result === '404') {
|
||||||
@ -298,6 +317,10 @@ class Server {
|
|||||||
} catch (e) {
|
} catch (e) {
|
||||||
logger.err(e instanceof Error ? e.message : `${e} ${req.params[0]}`);
|
logger.err(e instanceof Error ? e.message : `${e} ${req.params[0]}`);
|
||||||
res.status(500).send(e instanceof Error ? e.message : e);
|
res.status(500).send(e instanceof Error ? e.message : e);
|
||||||
|
} finally {
|
||||||
|
if (!unfurl) {
|
||||||
|
this.seoQueueLength--;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user