machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add cluster/tab to unfurler logs

Browse Source
This commit is contained in:
Mononaut 2023-08-19 21:07:10 +09:00
parent 7f2a459575
commit ed9d31686e
No known key found for this signature in database
GPG Key ID: A3F058E41374C04E
3 changed files with 19 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
unfurler/src/concurrency/ReusablePage.ts
View File

@ -17,6 +17,7 @@ export interface RepairablePage extends puppeteer.Page {
createdAt?: number;
free?: boolean;
index?: number;
clusterGroup?: string;
}
interface ResourceData {
@ -76,7 +77,7 @@ export default class ReusablePage extends ConcurrencyImplementation {
for (let i = 0; i < maxConcurrency; i++) {
const newPage = await this.initPage();
newPage.index = this.pages.length;
logger.info(`initialized page ${newPage.index}`);
logger.info(`initialized page ${newPage.clusterGroup}:${newPage.index}`);
this.pages.push(newPage);
}
}
@ -87,6 +88,7 @@ export default class ReusablePage extends ConcurrencyImplementation {
protected async initPage(): Promise<RepairablePage> {
const page = await (this.browser as puppeteer.Browser).newPage() as RepairablePage;
page.clusterGroup = 'unfurler';
page.language = null;
page.createdAt = Date.now();
let defaultUrl
@ -129,6 +131,7 @@ export default class ReusablePage extends ConcurrencyImplementation {
protected async repairPage(page) {
// create a new page
logger.debug(`Repairing page ${page.clusterGroup}:${page.index}`);
const newPage = await this.initPage();
newPage.free = true;
// replace the old page
@ -138,7 +141,7 @@ export default class ReusablePage extends ConcurrencyImplementation {
try {
await page.goto('about:blank', {timeout: 200}); // prevents memory leak (maybe?)
} catch (e) {
logger.err('unexpected page repair error');
logger.err(`unexpected page repair error ${page.clusterGroup}:${page.index}`);
}
await page.close();
return newPage;

1
unfurler/src/concurrency/ReusableSSRPage.ts
View File

@ -19,6 +19,7 @@ export default class ReusableSSRPage extends ReusablePage {
protected async initPage(): Promise<RepairablePage> {
const page = await (this.browser as puppeteer.Browser).newPage() as RepairablePage;
page.clusterGroup = 'slurper';
page.language = null;
page.createdAt = Date.now();
const defaultUrl = mempoolHost + '/about';

26
unfurler/src/index.ts
View File

@ -120,8 +120,9 @@ class Server {
this.app.get('*', (req, res) => { return this.renderHTML(req, res, false) })
}
async clusterTask({ page, data: { url, path, action } }) {
async clusterTask({ page, data: { url, path, action, reqUrl } }) {
try {
logger.info(`rendering "${reqUrl}" on tab ${page.clusterGroup}:${page.index}`);
const urlParts = parseLanguageUrl(path);
if (page.language !== urlParts.lang) {
// switch language
@ -156,20 +157,21 @@ class Server {
});
return screenshot;
} else if (success === false) {
logger.warn(`failed to render ${path} for ${action} due to client-side error, e.g. requested an invalid txid`);
logger.warn(`failed to render ${reqUrl} for ${action} due to client-side error, e.g. requested an invalid txid`);
page.repairRequested = true;
} else {
logger.warn(`failed to render ${path} for ${action} due to puppeteer timeout`);
logger.warn(`failed to render ${reqUrl} for ${action} due to puppeteer timeout`);
page.repairRequested = true;
}
} catch (e) {
logger.err(`failed to render ${path} for ${action}: ` + (e instanceof Error ? e.message : `${e}`));
logger.err(`failed to render ${reqUrl} for ${action}: ` + (e instanceof Error ? e.message : `${e}`));
page.repairRequested = true;
}
}
async ssrClusterTask({ page, data: { url, path, action } }) {
async ssrClusterTask({ page, data: { url, path, action, reqUrl } }) {
try {
logger.info(`slurping "${reqUrl}" on tab ${page.clusterGroup}:${page.index}`);
const urlParts = parseLanguageUrl(path);
if (page.language !== urlParts.lang) {
// switch language
@ -207,7 +209,7 @@ class Server {
let html = await page.content();
return html;
} else {
logger.err(`failed to render ${path} for ${action}: ` + (e instanceof Error ? e.message : `${e}`));
logger.err(`failed to render ${reqUrl} for ${action}: ` + (e instanceof Error ? e.message : `${e}`));
page.repairRequested = true;
}
}
@ -228,8 +230,7 @@ class Server {
// don't bother unless the route is definitely renderable
if (rawPath.includes('/preview/') && matchedRoute.render) {
logger.info('rendering "' + req.url + '"');
img = await this.cluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'screenshot' });
img = await this.cluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'screenshot', reqUrl: req.url });
} else {
logger.info('rendering not enabled for page "' + req.url + '"');
}
@ -277,14 +278,13 @@ class Server {
}
}
logger.info((unfurl ? 'unfurling ' : 'slurping "') + req.url + '"');
let result = '';
try {
if (unfurl) {
logger.info('unfurling "' + req.url + '"');
result = await this.renderUnfurlMeta(rawPath);
} else {
result = await this.renderSEOPage(rawPath);
result = await this.renderSEOPage(rawPath, req.url);
}
if (result && result.length) {
if (result === '404') {
@ -338,8 +338,8 @@ class Server {
</html>`;
}
async renderSEOPage(rawPath: string): Promise<string> {
let html = await this.ssrCluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'ssr' });
async renderSEOPage(rawPath: string, reqUrl: string): Promise<string> {
let html = await this.ssrCluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'ssr', reqUrl });
// remove javascript to prevent double hydration
if (html && html.length) {
html = html.replaceAll(/<script.*<\/script>/g, "");