machankura/mempool
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into nymkappa/menu

Browse Source
This commit is contained in:
nymkappa 2023-08-20 08:21:38 +02:00 committed by GitHub
commit ef554ad67b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 76 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend/src/api/difficulty-adjustment.ts
View File

@ -32,13 +32,13 @@ export interface DifficultyAdjustment {
export function calcBitsDifference(oldBits: number, newBits: number): number { export function calcBitsDifference(oldBits: number, newBits: number): number {
// Must be // Must be
// - integer // - integer
// - highest exponent is 0x1f, so max value (as integer) is 0x1f0000ff // - highest exponent is 0x20, so max value (as integer) is 0x207fffff
// - min value is 1 (exponent = 0) // - min value is 1 (exponent = 0)
// - highest bit of the number-part is +- sign, it must not be 1 // - highest bit of the number-part is +- sign, it must not be 1
const verifyBits = (bits: number): void => { const verifyBits = (bits: number): void => {
if ( if (
Math.floor(bits) !== bits || Math.floor(bits) !== bits ||
bits > 0x1f0000ff || bits > 0x207fffff ||
bits < 1 || bits < 1 ||
(bits & 0x00800000) !== 0 || (bits & 0x00800000) !== 0 ||
(bits & 0x007fffff) === 0 (bits & 0x007fffff) === 0

23
frontend/src/app/components/mempool-blocks/mempool-blocks.component.ts
View File

@ -31,6 +31,7 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy {
@Input() spotlight: number = 0; @Input() spotlight: number = 0;
@Input() getHref?: (index) => string = (index) => `/mempool-block/${index}`; @Input() getHref?: (index) => string = (index) => `/mempool-block/${index}`;
@Input() allBlocks: boolean = false; @Input() allBlocks: boolean = false;
@Input() forceRtl: boolean = false;
mempoolWidth: number = 0; mempoolWidth: number = 0;
@Output() widthChange: EventEmitter<number> = new EventEmitter(); @Output() widthChange: EventEmitter<number> = new EventEmitter();
@ -102,7 +103,7 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy {
} }
this.timeLtrSubscription = this.stateService.timeLtr.subscribe((ltr) => { this.timeLtrSubscription = this.stateService.timeLtr.subscribe((ltr) => {
this.timeLtr = !!ltr; this.timeLtr = !this.forceRtl && !!ltr;
this.cd.markForCheck(); this.cd.markForCheck();
}); });
@ -114,11 +115,6 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy {
}); });
this.reduceEmptyBlocksToFitScreen(this.mempoolEmptyBlocks); this.reduceEmptyBlocksToFitScreen(this.mempoolEmptyBlocks);
this.mempoolBlocks.map(() => {
this.updateMempoolBlockStyles();
this.calculateTransactionPosition();
});
this.reduceMempoolBlocksToFitScreen(this.mempoolBlocks);
this.isTabHiddenSubscription = this.stateService.isTabHidden$.subscribe((tabHidden) => this.tabHidden = tabHidden); this.isTabHiddenSubscription = this.stateService.isTabHidden$.subscribe((tabHidden) => this.tabHidden = tabHidden);
this.loadingBlocks$ = combineLatest([ this.loadingBlocks$ = combineLatest([
this.stateService.isLoadingWebSocket$, this.stateService.isLoadingWebSocket$,
@ -206,14 +202,17 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy {
if (!block) { if (!block) {
return; return;
} }
const isNewBlock = block.height > this.chainTip;
if (this.chainTip === -1) { if (this.chainTip === -1) {
this.animateEntry = block.height === this.stateService.latestBlockHeight; this.animateEntry = block.height === this.stateService.latestBlockHeight;
} else { } else {
this.animateEntry = block.height > this.chainTip; this.animateEntry = isNewBlock;
} }
this.chainTip = this.stateService.latestBlockHeight; this.chainTip = this.stateService.latestBlockHeight;
if ((block?.extras?.similarity == null || block?.extras?.similarity > 0.5) && !this.tabHidden) { if (isNewBlock && (block?.extras?.similarity == null || block?.extras?.similarity > 0.5) && !this.tabHidden) {
this.blockIndex++; this.blockIndex++;
} }
}); });
@ -283,7 +282,7 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy {
} }
reduceEmptyBlocksToFitScreen(blocks: MempoolBlock[]): MempoolBlock[] { reduceEmptyBlocksToFitScreen(blocks: MempoolBlock[]): MempoolBlock[] {
const innerWidth = this.stateService.env.BASE_MODULE !== 'liquid' && window.innerWidth <= 767.98 ? window.innerWidth : window.innerWidth / 2; const innerWidth = this.containerWidth || (this.stateService.env.BASE_MODULE !== 'liquid' && window.innerWidth <= 767.98 ? window.innerWidth : window.innerWidth / 2);
let blocksAmount = this.stateService.env.MEMPOOL_BLOCKS_AMOUNT; let blocksAmount = this.stateService.env.MEMPOOL_BLOCKS_AMOUNT;
if (!this.allBlocks) { if (!this.allBlocks) {
blocksAmount = Math.min(this.stateService.env.MEMPOOL_BLOCKS_AMOUNT, Math.floor(innerWidth / (this.blockWidth + this.blockPadding))); blocksAmount = Math.min(this.stateService.env.MEMPOOL_BLOCKS_AMOUNT, Math.floor(innerWidth / (this.blockWidth + this.blockPadding)));
@ -306,7 +305,7 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy {
} }
reduceMempoolBlocksToFitScreen(blocks: MempoolBlock[]): MempoolBlock[] { reduceMempoolBlocksToFitScreen(blocks: MempoolBlock[]): MempoolBlock[] {
const innerWidth = this.stateService.env.BASE_MODULE !== 'liquid' && window.innerWidth <= 767.98 ? window.innerWidth : window.innerWidth / 2; const innerWidth = this.containerWidth || (this.stateService.env.BASE_MODULE !== 'liquid' && window.innerWidth <= 767.98 ? window.innerWidth : window.innerWidth / 2);
let blocksAmount = this.stateService.env.MEMPOOL_BLOCKS_AMOUNT; let blocksAmount = this.stateService.env.MEMPOOL_BLOCKS_AMOUNT;
if (this.count) { if (this.count) {
blocksAmount = 8; blocksAmount = 8;
@ -316,7 +315,7 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy {
while (blocks.length > blocksAmount) { while (blocks.length > blocksAmount) {
const block = blocks.pop(); const block = blocks.pop();
if (!this.count) { if (!this.count) {
const lastBlock = blocks[0]; const lastBlock = blocks[blocks.length - 1];
lastBlock.blockSize += block.blockSize; lastBlock.blockSize += block.blockSize;
lastBlock.blockVSize += block.blockVSize; lastBlock.blockVSize += block.blockVSize;
lastBlock.nTx += block.nTx; lastBlock.nTx += block.nTx;
@ -327,7 +326,7 @@ export class MempoolBlocksComponent implements OnInit, OnChanges, OnDestroy {
} }
} }
if (blocks.length) { if (blocks.length) {
blocks[0].isStack = blocks[0].blockVSize > this.stateService.blockVSize; blocks[blocks.length - 1].isStack = blocks[blocks.length - 1].blockVSize > this.stateService.blockVSize;
} }
return blocks; return blocks;
} }

11
production/install
View File

@ -530,6 +530,7 @@ osCertbotDryRun()
zfsCreateFilesystems() zfsCreateFilesystems()
{ {
zfs create -o "mountpoint=/backup" "${ZPOOL}/backup" zfs create -o "mountpoint=/backup" "${ZPOOL}/backup"
zfs create -o "mountpoint=/var/cache/nginx" "${ZPOOL}/cache"
zfs create -o "mountpoint=${ELEMENTS_HOME}" "${ZPOOL}/elements" zfs create -o "mountpoint=${ELEMENTS_HOME}" "${ZPOOL}/elements"
zfs create -o "mountpoint=${BITCOIN_HOME}" "${ZPOOL}/bitcoin" zfs create -o "mountpoint=${BITCOIN_HOME}" "${ZPOOL}/bitcoin"
@ -1641,19 +1642,11 @@ fi
################################################ ################################################
if [ "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then if [ "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then
echo "[*] Installing Elements Liquid Testnet electrs start script"
osSudo "${ROOT_USER}" install -c -o "${ELEMENTS_USER}" -g "${ELEMENTS_GROUP}" -m 755 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/electrs-start-liquidtestnet" "${ELEMENTS_ELECTRS_HOME}"
echo "[*] Installing Elements Liquid Testnet RPC credentials" echo "[*] Installing Elements Liquid Testnet RPC credentials"
osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_USER__/${BITCOIN_RPC_USER}/" "${ELEMENTS_HOME}/elements.conf" osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_USER__/${BITCOIN_RPC_USER}/" "${ELEMENTS_HOME}/elements.conf"
osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_PASS__/${BITCOIN_RPC_PASS}/" "${ELEMENTS_HOME}/elements.conf" osSudo "${ROOT_USER}" sed -i.orig "s/__BITCOIN_RPC_PASS__/${BITCOIN_RPC_PASS}/" "${ELEMENTS_HOME}/elements.conf"
osSudo "${ROOT_USER}" sed -i.orig "s/__ELEMENTS_RPC_USER__/${ELEMENTS_RPC_USER}/" "${ELEMENTS_HOME}/elements.conf" osSudo "${ROOT_USER}" sed -i.orig "s/__ELEMENTS_RPC_USER__/${ELEMENTS_RPC_USER}/" "${ELEMENTS_HOME}/elements.conf"
osSudo "${ROOT_USER}" sed -i.orig "s/__ELEMENTS_RPC_PASS__/${ELEMENTS_RPC_PASS}/" "${ELEMENTS_HOME}/elements.conf" osSudo "${ROOT_USER}" sed -i.orig "s/__ELEMENTS_RPC_PASS__/${ELEMENTS_RPC_PASS}/" "${ELEMENTS_HOME}/elements.conf"
echo "[*] Configuring Elements LiquidTestnet RPC credentials in electrs start script"
osSudo "${ROOT_USER}" sed -i.orig "s/__ELEMENTS_RPC_USER__/${ELEMENTS_RPC_USER}/" "${ELEMENTS_ELECTRS_HOME}/electrs-start-liquidtestnet"
osSudo "${ROOT_USER}" sed -i.orig "s/__ELEMENTS_RPC_PASS__/${ELEMENTS_RPC_PASS}/" "${ELEMENTS_ELECTRS_HOME}/electrs-start-liquidtestnet"
osSudo "${ROOT_USER}" sed -i.orig "s!__ELECTRS_DATA_ROOT__!${ELECTRS_DATA_ROOT}!" "${ELEMENTS_ELECTRS_HOME}/electrs-start-liquidtestnet"
fi fi
################################ ################################
@ -1860,8 +1853,6 @@ chown "${MEMPOOL_USER}:${MEMPOOL_GROUP}" "${MEMPOOL_MYSQL_CREDENTIALS}"
echo "[*] Adding Nginx configuration" echo "[*] Adding Nginx configuration"
osSudo "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/nginx/nginx.conf" "${NGINX_CONFIGURATION}" osSudo "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/nginx/nginx.conf" "${NGINX_CONFIGURATION}"
mkdir -p /var/cache/nginx/services /var/cache/nginx/api
chown "${NGINX_USER}:${NGINX_GROUP}" /var/cache/nginx/services /var/cache/nginx/api
ln -s "${MEMPOOL_HOME}/mempool" "${NGINX_ETC_FOLDER}/mempool" ln -s "${MEMPOOL_HOME}/mempool" "${NGINX_ETC_FOLDER}/mempool"
osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_USER__!${NGINX_USER}!" "${NGINX_CONFIGURATION}" osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_USER__!${NGINX_USER}!" "${NGINX_CONFIGURATION}"
osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_ETC_FOLDER__!${NGINX_ETC_FOLDER}!" "${NGINX_CONFIGURATION}" osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_ETC_FOLDER__!${NGINX_ETC_FOLDER}!" "${NGINX_CONFIGURATION}"

12
production/nginx/http-proxy-cache.conf
View File

@ -1,7 +1,7 @@
# proxy cache # proxy cache
proxy_cache_path /var/cache/nginx/api keys_zone=api:20m levels=1:2 inactive=600s max_size=200m; proxy_cache_path /var/cache/nginx/api keys_zone=api:20m levels=1:2 inactive=365d max_size=2000m;
proxy_cache_path /var/cache/nginx/services keys_zone=services:20m levels=1:2 inactive=600s max_size=200m; proxy_cache_path /var/cache/nginx/unfurler keys_zone=unfurler:20m levels=1:2 inactive=365d max_size=2000m;
proxy_cache_path /var/cache/nginx/markets keys_zone=markets:20m levels=1:2 inactive=600s max_size=200m; proxy_cache_path /var/cache/nginx/slurper keys_zone=slurper:20m levels=1:2 inactive=365d max_size=5000m;
proxy_cache_path /var/cache/nginx/unfurler keys_zone=unfurler:20m levels=1:2 inactive=600s max_size=200m; proxy_cache_path /var/cache/nginx/services keys_zone=services:20m levels=1:2 inactive=365d max_size=100m;
proxy_cache_path /var/cache/nginx/slurper keys_zone=slurper:20m levels=1:2 inactive=600s max_size=200m; proxy_cache_path /var/cache/nginx/markets keys_zone=markets:20m levels=1:2 inactive=365d max_size=100m;
types_hash_max_size 2048; types_hash_max_size 4096;

8
production/nginx/server-common.conf
View File

@ -136,8 +136,10 @@ location @mempool-space-unfurler {
proxy_cache_background_update on; proxy_cache_background_update on;
proxy_cache_use_stale updating; proxy_cache_use_stale updating;
proxy_cache unfurler; proxy_cache unfurler;
proxy_cache_valid 200 10m; proxy_cache_valid 200 1h; # will re-render page if older than this
proxy_redirect off; proxy_redirect off;
expires 1d;
} }
location @mempool-space-slurper { location @mempool-space-slurper {
@ -151,6 +153,8 @@ location @mempool-space-slurper {
proxy_cache_background_update on; proxy_cache_background_update on;
proxy_cache_use_stale updating; proxy_cache_use_stale updating;
proxy_cache slurper; proxy_cache slurper;
proxy_cache_valid 200 10m; proxy_cache_valid 200 1h; # will re-render page if older than this
proxy_redirect off; proxy_redirect off;
expires 10d;
} }

9
unfurler/src/concurrency/ReusablePage.ts
View File

@ -11,12 +11,13 @@ const BROWSER_TIMEOUT = 8000;
const maxAgeMs = (config.PUPPETEER.MAX_PAGE_AGE || (24 * 60 * 60)) * 1000; const maxAgeMs = (config.PUPPETEER.MAX_PAGE_AGE || (24 * 60 * 60)) * 1000;
const maxConcurrency = config.PUPPETEER.CLUSTER_SIZE; const maxConcurrency = config.PUPPETEER.CLUSTER_SIZE;
interface RepairablePage extends puppeteer.Page { export interface RepairablePage extends puppeteer.Page {
repairRequested?: boolean; repairRequested?: boolean;
language?: string | null; language?: string | null;
createdAt?: number; createdAt?: number;
free?: boolean; free?: boolean;
index?: number; index?: number;
clusterGroup?: string;
} }
interface ResourceData { interface ResourceData {
@ -76,7 +77,7 @@ export default class ReusablePage extends ConcurrencyImplementation {
for (let i = 0; i < maxConcurrency; i++) { for (let i = 0; i < maxConcurrency; i++) {
const newPage = await this.initPage(); const newPage = await this.initPage();
newPage.index = this.pages.length; newPage.index = this.pages.length;
logger.info(`initialized page ${newPage.index}`); logger.info(`initialized page ${newPage.clusterGroup}:${newPage.index}`);
this.pages.push(newPage); this.pages.push(newPage);
} }
} }
@ -87,6 +88,7 @@ export default class ReusablePage extends ConcurrencyImplementation {
protected async initPage(): Promise<RepairablePage> { protected async initPage(): Promise<RepairablePage> {
const page = await (this.browser as puppeteer.Browser).newPage() as RepairablePage; const page = await (this.browser as puppeteer.Browser).newPage() as RepairablePage;
page.clusterGroup = 'unfurler';
page.language = null; page.language = null;
page.createdAt = Date.now(); page.createdAt = Date.now();
let defaultUrl let defaultUrl
@ -129,6 +131,7 @@ export default class ReusablePage extends ConcurrencyImplementation {
protected async repairPage(page) { protected async repairPage(page) {
// create a new page // create a new page
logger.debug(`Repairing page ${page.clusterGroup}:${page.index}`);
const newPage = await this.initPage(); const newPage = await this.initPage();
newPage.free = true; newPage.free = true;
// replace the old page // replace the old page
@ -138,7 +141,7 @@ export default class ReusablePage extends ConcurrencyImplementation {
try { try {
await page.goto('about:blank', {timeout: 200}); // prevents memory leak (maybe?) await page.goto('about:blank', {timeout: 200}); // prevents memory leak (maybe?)
} catch (e) { } catch (e) {
logger.err('unexpected page repair error'); logger.err(`unexpected page repair error ${page.clusterGroup}:${page.index}`);
} }
await page.close(); await page.close();
return newPage; return newPage;

45
unfurler/src/concurrency/ReusableSSRPage.ts
View File

@ -2,19 +2,11 @@ import * as puppeteer from 'puppeteer';
import { timeoutExecute } from 'puppeteer-cluster/dist/util'; import { timeoutExecute } from 'puppeteer-cluster/dist/util';
import logger from '../logger'; import logger from '../logger';
import config from '../config'; import config from '../config';
import ReusablePage from './ReusablePage'; import ReusablePage, { RepairablePage } from './ReusablePage';
const mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : ''); const mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : '');
const mockImageBuffer = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVQYV2NgYAAAAAMAAWgmWQ0AAAAASUVORK5CYII=", 'base64'); const mockImageBuffer = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVQYV2NgYAAAAAMAAWgmWQ0AAAAASUVORK5CYII=", 'base64');
interface RepairablePage extends puppeteer.Page {
repairRequested?: boolean;
language?: string | null;
createdAt?: number;
free?: boolean;
index?: number;
}
export default class ReusableSSRPage extends ReusablePage { export default class ReusableSSRPage extends ReusablePage {
public constructor(options: puppeteer.LaunchOptions, puppeteer: any) { public constructor(options: puppeteer.LaunchOptions, puppeteer: any) {
@ -27,31 +19,32 @@ export default class ReusableSSRPage extends ReusablePage {
protected async initPage(): Promise<RepairablePage> { protected async initPage(): Promise<RepairablePage> {
const page = await (this.browser as puppeteer.Browser).newPage() as RepairablePage; const page = await (this.browser as puppeteer.Browser).newPage() as RepairablePage;
page.clusterGroup = 'slurper';
page.language = null; page.language = null;
page.createdAt = Date.now(); page.createdAt = Date.now();
const defaultUrl = mempoolHost + '/about'; const defaultUrl = mempoolHost + '/about';
page.on('pageerror', (err) => { page.on('pageerror', (err) => {
console.log(err); console.log(err);
// page.repairRequested = true; page.repairRequested = true;
}); });
await page.setRequestInterception(true); await page.setRequestInterception(true);
page.on('request', req => { page.on('request', req => {
if (req.isInterceptResolutionHandled()) { if (req.isInterceptResolutionHandled()) {
return req.continue(); return req.continue();
} }
if (req.resourceType() === 'image') { if (req.resourceType() === 'image') {
return req.respond({ return req.respond({
contentType: 'image/png', contentType: 'image/png',
headers: {"Access-Control-Allow-Origin": "*"}, headers: {"Access-Control-Allow-Origin": "*"},
body: mockImageBuffer body: mockImageBuffer
}); });
} else if (!['document', 'script', 'xhr', 'fetch'].includes(req.resourceType())) { } else if (!['document', 'script', 'xhr', 'fetch'].includes(req.resourceType())) {
return req.abort(); return req.abort();
} else { } else {
return req.continue(); return req.continue();
} }
}); });
try { try {
await page.goto(defaultUrl, { waitUntil: "networkidle0" }); await page.goto(defaultUrl, { waitUntil: "networkidle0" });
await page.waitForSelector('meta[property="og:meta:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }); await page.waitForSelector('meta[property="og:meta:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 });

38
unfurler/src/index.ts
View File

@ -5,7 +5,7 @@ import * as https from 'https';
import config from './config'; import config from './config';
import { Cluster } from 'puppeteer-cluster'; import { Cluster } from 'puppeteer-cluster';
import ReusablePage from './concurrency/ReusablePage'; import ReusablePage from './concurrency/ReusablePage';
import ReusableSSRPage from './concurrency/ReusablePage'; import ReusableSSRPage from './concurrency/ReusableSSRPage';
import { parseLanguageUrl } from './language/lang'; import { parseLanguageUrl } from './language/lang';
import { matchRoute } from './routes'; import { matchRoute } from './routes';
import nodejsPath from 'path'; import nodejsPath from 'path';
@ -120,8 +120,9 @@ class Server {
this.app.get('*', (req, res) => { return this.renderHTML(req, res, false) }) this.app.get('*', (req, res) => { return this.renderHTML(req, res, false) })
} }
async clusterTask({ page, data: { url, path, action } }) { async clusterTask({ page, data: { url, path, action, reqUrl } }) {
try { try {
logger.info(`rendering "${reqUrl}" on tab ${page.clusterGroup}:${page.index}`);
const urlParts = parseLanguageUrl(path); const urlParts = parseLanguageUrl(path);
if (page.language !== urlParts.lang) { if (page.language !== urlParts.lang) {
// switch language // switch language
@ -156,20 +157,21 @@ class Server {
}); });
return screenshot; return screenshot;
} else if (success === false) { } else if (success === false) {
logger.warn(`failed to render ${path} for ${action} due to client-side error, e.g. requested an invalid txid`); logger.warn(`failed to render ${reqUrl} for ${action} due to client-side error, e.g. requested an invalid txid`);
page.repairRequested = true; page.repairRequested = true;
} else { } else {
logger.warn(`failed to render ${path} for ${action} due to puppeteer timeout`); logger.warn(`failed to render ${reqUrl} for ${action} due to puppeteer timeout`);
page.repairRequested = true; page.repairRequested = true;
} }
} catch (e) { } catch (e) {
logger.err(`failed to render ${path} for ${action}: ` + (e instanceof Error ? e.message : `${e}`)); logger.err(`failed to render ${reqUrl} for ${action}: ` + (e instanceof Error ? e.message : `${e}`));
page.repairRequested = true; page.repairRequested = true;
} }
} }
async ssrClusterTask({ page, data: { url, path, action } }) { async ssrClusterTask({ page, data: { url, path, action, reqUrl } }) {
try { try {
logger.info(`slurping "${reqUrl}" on tab ${page.clusterGroup}:${page.index}`);
const urlParts = parseLanguageUrl(path); const urlParts = parseLanguageUrl(path);
if (page.language !== urlParts.lang) { if (page.language !== urlParts.lang) {
// switch language // switch language
@ -207,7 +209,7 @@ class Server {
let html = await page.content(); let html = await page.content();
return html; return html;
} else { } else {
logger.err(`failed to render ${path} for ${action}: ` + (e instanceof Error ? e.message : `${e}`)); logger.err(`failed to render ${reqUrl} for ${action}: ` + (e instanceof Error ? e.message : `${e}`));
page.repairRequested = true; page.repairRequested = true;
} }
} }
@ -228,7 +230,9 @@ class Server {
// don't bother unless the route is definitely renderable // don't bother unless the route is definitely renderable
if (rawPath.includes('/preview/') && matchedRoute.render) { if (rawPath.includes('/preview/') && matchedRoute.render) {
img = await this.cluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'screenshot' }); img = await this.cluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'screenshot', reqUrl: req.url });
} else {
logger.info('rendering not enabled for page "' + req.url + '"');
} }
if (!img) { if (!img) {
@ -258,10 +262,17 @@ class Server {
res.status(404).send(); res.status(404).send();
return; return;
} else { } else {
logger.info('proxying resource "' + req.url + '"');
if (this.secureHost) { if (this.secureHost) {
https.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res)); https.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => {
res.writeHead(got.statusCode, got.headers);
return got.pipe(res);
});
} else { } else {
http.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => got.pipe(res)); http.get(config.SERVER.HOST + rawPath, { headers: { 'user-agent': 'mempoolunfurl' }}, (got) => {
res.writeHead(got.statusCode, got.headers);
return got.pipe(res);
});
} }
return; return;
} }
@ -270,9 +281,10 @@ class Server {
let result = ''; let result = '';
try { try {
if (unfurl) { if (unfurl) {
logger.info('unfurling "' + req.url + '"');
result = await this.renderUnfurlMeta(rawPath); result = await this.renderUnfurlMeta(rawPath);
} else { } else {
result = await this.renderSEOPage(rawPath); result = await this.renderSEOPage(rawPath, req.url);
} }
if (result && result.length) { if (result && result.length) {
if (result === '404') { if (result === '404') {
@ -326,8 +338,8 @@ class Server {
</html>`; </html>`;
} }
async renderSEOPage(rawPath: string): Promise<string> { async renderSEOPage(rawPath: string, reqUrl: string): Promise<string> {
let html = await this.ssrCluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'ssr' }); let html = await this.ssrCluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'ssr', reqUrl });
// remove javascript to prevent double hydration // remove javascript to prevent double hydration
if (html && html.length) { if (html && html.length) {
html = html.replaceAll(/<script.*<\/script>/g, ""); html = html.replaceAll(/<script.*<\/script>/g, "");