Merge branch 'master' into knorrium/backend_unit_tests
This commit is contained in:
wiz
GitHub
commit
f855173b9b
34
.github/workflows/on-tag.yml
vendored
34
.github/workflows/on-tag.yml
vendored
@ -1,7 +1,7 @@
|
||||
name: Docker build on tag
|
||||
env:
|
||||
DOCKER_CLI_EXPERIMENTAL: enabled
|
||||
TAG_FMT: '^refs/tags/(((.?[0-9]+){3,4}))$'
|
||||
TAG_FMT: "^refs/tags/(((.?[0-9]+){3,4}))$"
|
||||
DOCKER_BUILDKIT: 0
|
||||
COMPOSE_DOCKER_CLI_BUILD: 0
|
||||
|
||||
@ -21,9 +21,39 @@ jobs:
|
||||
service:
|
||||
- frontend
|
||||
- backend
|
||||
runs-on: ubuntu-18.04
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 120
|
||||
name: Build and push to DockerHub
|
||||
steps:
|
||||
# Workaround based on JonasAlfredsson/docker-on-tmpfs@v1.0.1
|
||||
- name: Replace the current swap file
|
||||
shell: bash
|
||||
run: |
|
||||
sudo swapoff /mnt/swapfile
|
||||
sudo rm -v /mnt/swapfile
|
||||
sudo fallocate -l 10G /mnt/swapfile
|
||||
sudo chmod 600 /mnt/swapfile
|
||||
sudo mkswap /mnt/swapfile
|
||||
sudo swapon /mnt/swapfile
|
||||
|
||||
- name: Show current memory and swap status
|
||||
shell: bash
|
||||
run: |
|
||||
sudo free -h
|
||||
echo
|
||||
sudo swapon --show
|
||||
|
||||
- name: Mount a tmpfs over /var/lib/docker
|
||||
shell: bash
|
||||
run: |
|
||||
if [ ! -d "/var/lib/docker" ]; then
|
||||
echo "Directory '/var/lib/docker' not found"
|
||||
exit 1
|
||||
fi
|
||||
sudo mount -t tmpfs -o size=10G tmpfs /var/lib/docker
|
||||
sudo systemctl restart docker
|
||||
sudo df -h | grep docker
|
||||
|
||||
- name: Set env variables
|
||||
run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
|
||||
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
import { CpfpInfo, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
|
||||
import config from '../config';
|
||||
import { NodeSocket } from '../repositories/NodesSocketsRepository';
|
||||
import { isIP } from 'net';
|
||||
export class Common {
|
||||
static nativeAssetId = config.MEMPOOL.NETWORK === 'liquidtestnet' ?
|
||||
'144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49'
|
||||
@ -221,4 +223,35 @@ export class Common {
|
||||
const d = new Date((date || 0) * 1000);
|
||||
return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
|
||||
}
|
||||
|
||||
static formatSocket(publicKey: string, socket: {network: string, addr: string}): NodeSocket {
|
||||
let network: string | null = null;
|
||||
|
||||
if (config.LIGHTNING.BACKEND === 'cln') {
|
||||
network = socket.network;
|
||||
} else if (config.LIGHTNING.BACKEND === 'lnd') {
|
||||
if (socket.addr.indexOf('onion') !== -1) {
|
||||
if (socket.addr.split('.')[0].length >= 56) {
|
||||
network = 'torv3';
|
||||
} else {
|
||||
network = 'torv2';
|
||||
}
|
||||
} else if (socket.addr.indexOf('i2p') !== -1) {
|
||||
network = 'i2p';
|
||||
} else {
|
||||
const ipv = isIP(socket.addr.split(':')[0]);
|
||||
if (ipv === 4) {
|
||||
network = 'ipv4';
|
||||
} else if (ipv === 6) {
|
||||
network = 'ipv6';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
publicKey: publicKey,
|
||||
network: network,
|
||||
addr: socket.addr,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@ -4,7 +4,7 @@ import logger from '../logger';
|
||||
import { Common } from './common';
|
||||
|
||||
class DatabaseMigration {
|
||||
private static currentVersion = 36;
|
||||
private static currentVersion = 37;
|
||||
private queryTimeout = 120000;
|
||||
private statisticsAddedIndexed = false;
|
||||
private uniqueLogs: string[] = [];
|
||||
@ -324,6 +324,10 @@ class DatabaseMigration {
|
||||
if (databaseSchemaVersion < 36 && isBitcoin == true) {
|
||||
await this.$executeQuery('ALTER TABLE `nodes` ADD status TINYINT NOT NULL DEFAULT "1"');
|
||||
}
|
||||
|
||||
if (databaseSchemaVersion < 37 && isBitcoin == true) {
|
||||
await this.$executeQuery(this.getCreateLNNodesSocketsTableQuery(), await this.$checkIfTableExists('nodes_sockets'));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@ -737,7 +741,7 @@ class DatabaseMigration {
|
||||
names text DEFAULT NULL,
|
||||
UNIQUE KEY id (id,type),
|
||||
KEY id_2 (id)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
|
||||
}
|
||||
|
||||
private getCreateBlocksPricesTableQuery(): string {
|
||||
@ -749,6 +753,16 @@ class DatabaseMigration {
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
|
||||
}
|
||||
|
||||
private getCreateLNNodesSocketsTableQuery(): string {
|
||||
return `CREATE TABLE IF NOT EXISTS nodes_sockets (
|
||||
public_key varchar(66) NOT NULL,
|
||||
socket varchar(100) NOT NULL,
|
||||
type enum('ipv4', 'ipv6', 'torv2', 'torv3', 'i2p', 'dns', 'websocket') NULL,
|
||||
UNIQUE KEY public_key_socket (public_key, socket),
|
||||
INDEX (public_key)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
|
||||
}
|
||||
|
||||
public async $truncateIndexedData(tables: string[]) {
|
||||
const allowedTables = ['blocks', 'hashrates', 'prices'];
|
||||
|
||||
|
||||
@ -17,7 +17,7 @@ export function convertNode(clNode: any): ILightningApi.Node {
|
||||
network: addr.type,
|
||||
addr: `${addr.address}:${addr.port}`
|
||||
};
|
||||
}),
|
||||
}) ?? [],
|
||||
last_update: clNode?.last_timestamp ?? 0,
|
||||
};
|
||||
}
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
import transactionUtils from '../api/transaction-utils';
|
||||
import DB from '../database';
|
||||
import logger from '../logger';
|
||||
import { BlockAudit } from '../mempool.interfaces';
|
||||
|
||||
45
backend/src/repositories/NodesSocketsRepository.ts
Normal file
45
backend/src/repositories/NodesSocketsRepository.ts
Normal file
@ -0,0 +1,45 @@
|
||||
import { ResultSetHeader } from 'mysql2';
|
||||
import DB from '../database';
|
||||
import logger from '../logger';
|
||||
|
||||
export interface NodeSocket {
|
||||
publicKey: string;
|
||||
network: string | null;
|
||||
addr: string;
|
||||
}
|
||||
|
||||
class NodesSocketsRepository {
|
||||
public async $saveSocket(socket: NodeSocket): Promise<void> {
|
||||
try {
|
||||
await DB.query(`
|
||||
INSERT INTO nodes_sockets(public_key, socket, type)
|
||||
VALUE (?, ?, ?)
|
||||
`, [socket.publicKey, socket.addr, socket.network]);
|
||||
} catch (e: any) {
|
||||
if (e.errno !== 1062) { // ER_DUP_ENTRY - Not an issue, just ignore this
|
||||
logger.err(`Cannot save node socket (${[socket.publicKey, socket.addr, socket.network]}) into db. Reason: ` + (e instanceof Error ? e.message : e));
|
||||
// We don't throw, not a critical issue if we miss some nodes sockets
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public async $deleteUnusedSockets(publicKey: string, addresses: string[]): Promise<number> {
|
||||
if (addresses.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
try {
|
||||
const query = `
|
||||
DELETE FROM nodes_sockets
|
||||
WHERE public_key = ?
|
||||
AND socket NOT IN (${addresses.map(id => `"${id}"`).join(',')})
|
||||
`;
|
||||
const [result] = await DB.query<ResultSetHeader>(query, [publicKey]);
|
||||
return result.affectedRows;
|
||||
} catch (e) {
|
||||
logger.err(`Cannot delete unused sockets for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e));
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new NodesSocketsRepository();
|
||||
@ -10,6 +10,8 @@ import lightningApi from '../../api/lightning/lightning-api-factory';
|
||||
import nodesApi from '../../api/explorer/nodes.api';
|
||||
import { ResultSetHeader } from 'mysql2';
|
||||
import fundingTxFetcher from './sync-tasks/funding-tx-fetcher';
|
||||
import NodesSocketsRepository from '../../repositories/NodesSocketsRepository';
|
||||
import { Common } from '../../api/common';
|
||||
|
||||
class NetworkSyncService {
|
||||
loggerTimer = 0;
|
||||
@ -58,6 +60,7 @@ class NetworkSyncService {
|
||||
private async $updateNodesList(nodes: ILightningApi.Node[]): Promise<void> {
|
||||
let progress = 0;
|
||||
|
||||
let deletedSockets = 0;
|
||||
const graphNodesPubkeys: string[] = [];
|
||||
for (const node of nodes) {
|
||||
await nodesApi.$saveNode(node);
|
||||
@ -69,8 +72,15 @@ class NetworkSyncService {
|
||||
logger.info(`Updating node ${progress}/${nodes.length}`);
|
||||
this.loggerTimer = new Date().getTime() / 1000;
|
||||
}
|
||||
|
||||
const addresses: string[] = [];
|
||||
for (const socket of node.addresses) {
|
||||
await NodesSocketsRepository.$saveSocket(Common.formatSocket(node.pub_key, socket));
|
||||
addresses.push(socket.addr);
|
||||
}
|
||||
deletedSockets += await NodesSocketsRepository.$deleteUnusedSockets(node.pub_key, addresses);
|
||||
}
|
||||
logger.info(`${progress} nodes updated`);
|
||||
logger.info(`${progress} nodes updated. ${deletedSockets} sockets deleted`);
|
||||
|
||||
// If a channel if not present in the graph, mark it as inactive
|
||||
nodesApi.$setNodesInactive(graphNodesPubkeys);
|
||||
|
||||
@ -44,7 +44,7 @@
|
||||
<div class="w-100 d-block d-md-none"></div>
|
||||
<div class="col-md qrcode-col">
|
||||
<div class="qr-wrapper">
|
||||
<app-qrcode [data]="address.address" [size]="370"></app-qrcode>
|
||||
<app-qrcode [data]="address.address" [size]="448"></app-qrcode>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
h1 {
|
||||
font-size: 42px;
|
||||
font-size: 52px;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
@ -11,23 +11,26 @@ h1 {
|
||||
}
|
||||
|
||||
.qrcode-col {
|
||||
width: 420px;
|
||||
min-width: 420px;
|
||||
width: 468px;
|
||||
min-width: 468px;
|
||||
flex-grow: 0;
|
||||
flex-shrink: 0;
|
||||
text-align: center;
|
||||
padding: 0;
|
||||
margin-left: 2px;
|
||||
margin-right: 15px;
|
||||
}
|
||||
|
||||
.table {
|
||||
font-size: 24px;
|
||||
font-size: 32px;
|
||||
|
||||
::ng-deep .symbol {
|
||||
font-size: 18px;
|
||||
font-size: 24px;
|
||||
}
|
||||
}
|
||||
|
||||
.address-link {
|
||||
font-size: 20px;
|
||||
font-size: 24px;
|
||||
margin-bottom: 0.5em;
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
@ -35,7 +38,7 @@ h1 {
|
||||
.truncated-address {
|
||||
text-overflow: ellipsis;
|
||||
overflow: hidden;
|
||||
max-width: calc(505px - 4em);
|
||||
max-width: calc(640px - 4em);
|
||||
display: inline-block;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
@ -44,7 +44,6 @@ export class AddressPreviewComponent implements OnInit, OnDestroy {
|
||||
) { }
|
||||
|
||||
ngOnInit() {
|
||||
this.openGraphService.setPreviewLoading();
|
||||
this.stateService.networkChanged$.subscribe((network) => this.network = network);
|
||||
|
||||
this.addressLoadingStatus$ = this.route.paramMap
|
||||
@ -56,6 +55,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy {
|
||||
this.mainSubscription = this.route.paramMap
|
||||
.pipe(
|
||||
switchMap((params: ParamMap) => {
|
||||
this.openGraphService.waitFor('address-data');
|
||||
this.error = undefined;
|
||||
this.isLoadingAddress = true;
|
||||
this.loadedConfirmedTxCount = 0;
|
||||
@ -73,6 +73,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy {
|
||||
this.isLoadingAddress = false;
|
||||
this.error = err;
|
||||
console.log(err);
|
||||
this.openGraphService.fail('address-data');
|
||||
return of(null);
|
||||
})
|
||||
);
|
||||
@ -90,7 +91,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy {
|
||||
this.address = address;
|
||||
this.updateChainStats();
|
||||
this.isLoadingAddress = false;
|
||||
this.openGraphService.setPreviewReady();
|
||||
this.openGraphService.waitOver('address-data');
|
||||
})
|
||||
)
|
||||
.subscribe(() => {},
|
||||
@ -98,6 +99,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy {
|
||||
console.log(error);
|
||||
this.error = error;
|
||||
this.isLoadingAddress = false;
|
||||
this.openGraphService.fail('address-data');
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
@ -18,6 +18,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy {
|
||||
@Input() orientation = 'left';
|
||||
@Input() flip = true;
|
||||
@Output() txClickEvent = new EventEmitter<TransactionStripped>();
|
||||
@Output() readyEvent = new EventEmitter();
|
||||
|
||||
@ViewChild('blockCanvas')
|
||||
canvas: ElementRef<HTMLCanvasElement>;
|
||||
@ -37,6 +38,8 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy {
|
||||
selectedTx: TxView | void;
|
||||
tooltipPosition: Position;
|
||||
|
||||
readyNextFrame = false;
|
||||
|
||||
constructor(
|
||||
readonly ngZone: NgZone,
|
||||
readonly elRef: ElementRef,
|
||||
@ -78,6 +81,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy {
|
||||
setup(transactions: TransactionStripped[]): void {
|
||||
if (this.scene) {
|
||||
this.scene.setup(transactions);
|
||||
this.readyNextFrame = true;
|
||||
this.start();
|
||||
}
|
||||
}
|
||||
@ -258,6 +262,11 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy {
|
||||
this.gl.drawArrays(this.gl.TRIANGLES, 0, pointArray.length / TxSprite.vertexSize);
|
||||
}
|
||||
}
|
||||
|
||||
if (this.readyNextFrame) {
|
||||
this.readyNextFrame = false;
|
||||
this.readyEvent.emit();
|
||||
}
|
||||
}
|
||||
|
||||
/* LOOP */
|
||||
|
||||
@ -30,44 +30,42 @@
|
||||
<td i18n="block.weight">Weight</td>
|
||||
<td [innerHTML]="'‎' + (block?.weight | wuBytes: 2)"></td>
|
||||
</tr>
|
||||
<ng-template [ngIf]="webGlEnabled">
|
||||
<tr *ngIf="block?.extras?.medianFee != undefined">
|
||||
<td class="td-width" i18n="block.median-fee">Median fee</td>
|
||||
<td>~{{ block?.extras?.medianFee | number:'1.0-0' }} <span class="symbol" i18n="shared.sat-vbyte|sat/vB">sat/vB</span></td>
|
||||
</tr>
|
||||
<ng-template [ngIf]="fees !== undefined">
|
||||
<tr>
|
||||
<td i18n="block.total-fees|Total fees in a block">Total fees</td>
|
||||
<td *ngIf="network !== 'liquid' && network !== 'liquidtestnet'; else liquidTotalFees">
|
||||
<app-amount [satoshis]="block?.extras.totalFees" digitsInfo="1.2-3" [noFiat]="true"></app-amount>
|
||||
<tr *ngIf="block?.extras?.medianFee != undefined">
|
||||
<td class="td-width" i18n="block.median-fee">Median fee</td>
|
||||
<td>~{{ block?.extras?.medianFee | number:'1.0-0' }} <span class="symbol" i18n="shared.sat-vbyte|sat/vB">sat/vB</span></td>
|
||||
</tr>
|
||||
<ng-template [ngIf]="fees !== undefined">
|
||||
<tr>
|
||||
<td i18n="block.total-fees|Total fees in a block">Total fees</td>
|
||||
<td *ngIf="network !== 'liquid' && network !== 'liquidtestnet'; else liquidTotalFees">
|
||||
<app-amount [satoshis]="block?.extras.totalFees" digitsInfo="1.2-3" [noFiat]="true"></app-amount>
|
||||
</td>
|
||||
<ng-template #liquidTotalFees>
|
||||
<td>
|
||||
<app-amount [satoshis]="fees * 100000000" digitsInfo="1.2-2" [noFiat]="true"></app-amount>
|
||||
</td>
|
||||
<ng-template #liquidTotalFees>
|
||||
<td>
|
||||
<app-amount [satoshis]="fees * 100000000" digitsInfo="1.2-2" [noFiat]="true"></app-amount>
|
||||
</td>
|
||||
</ng-template>
|
||||
</tr>
|
||||
</ng-template>
|
||||
<tr *ngIf="network !== 'liquid' && network !== 'liquidtestnet'">
|
||||
<td i18n="block.miner">Miner</td>
|
||||
<td *ngIf="stateService.env.MINING_DASHBOARD">
|
||||
<a [attr.data-cy]="'block-details-miner-badge'" placement="bottom" [routerLink]="['/mining/pool' | relativeUrl, block?.extras.pool.slug]" class="badge"
|
||||
[class]="block?.extras.pool.name === 'Unknown' ? 'badge-secondary' : 'badge-primary'">
|
||||
{{ block?.extras.pool.name }}
|
||||
</a>
|
||||
</td>
|
||||
<td *ngIf="!stateService.env.MINING_DASHBOARD && stateService.env.BASE_MODULE === 'mempool'">
|
||||
<span [attr.data-cy]="'block-details-miner-badge'" placement="bottom" class="badge"
|
||||
[class]="block?.extras.pool.name === 'Unknown' ? 'badge-secondary' : 'badge-primary'">
|
||||
{{ block?.extras.pool.name }}
|
||||
</span>
|
||||
</td>
|
||||
</ng-template>
|
||||
</tr>
|
||||
</ng-template>
|
||||
<tr *ngIf="network !== 'liquid' && network !== 'liquidtestnet'">
|
||||
<td i18n="block.miner">Miner</td>
|
||||
<td *ngIf="stateService.env.MINING_DASHBOARD">
|
||||
<a [attr.data-cy]="'block-details-miner-badge'" placement="bottom" [routerLink]="['/mining/pool' | relativeUrl, block?.extras.pool.slug]" class="badge"
|
||||
[class]="block?.extras.pool.name === 'Unknown' ? 'badge-secondary' : 'badge-primary'">
|
||||
{{ block?.extras.pool.name }}
|
||||
</a>
|
||||
</td>
|
||||
<td *ngIf="!stateService.env.MINING_DASHBOARD && stateService.env.BASE_MODULE === 'mempool'">
|
||||
<span [attr.data-cy]="'block-details-miner-badge'" placement="bottom" class="badge"
|
||||
[class]="block?.extras.pool.name === 'Unknown' ? 'badge-secondary' : 'badge-primary'">
|
||||
{{ block?.extras.pool.name }}
|
||||
</span>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="col-sm chart-container" *ngIf="webGlEnabled">
|
||||
<div class="col-sm chart-container">
|
||||
<app-block-overview-graph
|
||||
#blockGraph
|
||||
[isLoading]="false"
|
||||
@ -75,7 +73,7 @@
|
||||
[blockLimit]="stateService.blockVSize"
|
||||
[orientation]="'top'"
|
||||
[flip]="false"
|
||||
(txClickEvent)="onTxClick($event)"
|
||||
(readyEvent)="onGraphReady()"
|
||||
></app-block-overview-graph>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -1,23 +1,25 @@
|
||||
.block-title {
|
||||
margin-bottom: 0.75em;
|
||||
font-size: 42px;
|
||||
margin-bottom: 48px;
|
||||
font-size: 52px;
|
||||
|
||||
::ng-deep .next-previous-blocks {
|
||||
font-size: 42px;
|
||||
font-size: 52px;
|
||||
}
|
||||
}
|
||||
|
||||
.table {
|
||||
font-size: 24px;
|
||||
font-size: 32px;
|
||||
}
|
||||
|
||||
.chart-container {
|
||||
flex-grow: 0;
|
||||
flex-shrink: 0;
|
||||
width: 420px;
|
||||
min-width: 420px;
|
||||
width: 470px;
|
||||
min-width: 470px;
|
||||
padding: 0;
|
||||
margin-right: 15px;
|
||||
}
|
||||
|
||||
::ng-deep .symbol {
|
||||
font-size: 18px;
|
||||
font-size: 24px;
|
||||
}
|
||||
|
||||
@ -1,11 +1,168 @@
|
||||
import { Component } from '@angular/core';
|
||||
import { BlockComponent } from './block.component';
|
||||
import { Component, OnInit, OnDestroy, ViewChild, ElementRef } from '@angular/core';
|
||||
import { ActivatedRoute, ParamMap } from '@angular/router';
|
||||
import { ElectrsApiService } from '../../services/electrs-api.service';
|
||||
import { switchMap, tap, throttleTime, catchError, shareReplay, startWith, pairwise, filter } from 'rxjs/operators';
|
||||
import { of, Subscription, asyncScheduler } from 'rxjs';
|
||||
import { StateService } from '../../services/state.service';
|
||||
import { SeoService } from 'src/app/services/seo.service';
|
||||
import { OpenGraphService } from 'src/app/services/opengraph.service';
|
||||
import { BlockExtended, TransactionStripped } from 'src/app/interfaces/node-api.interface';
|
||||
import { ApiService } from 'src/app/services/api.service';
|
||||
import { BlockOverviewGraphComponent } from 'src/app/components/block-overview-graph/block-overview-graph.component';
|
||||
|
||||
@Component({
|
||||
selector: 'app-block-preview',
|
||||
templateUrl: './block-preview.component.html',
|
||||
styleUrls: ['./block.component.scss', './block-preview.component.scss']
|
||||
styleUrls: ['./block-preview.component.scss']
|
||||
})
|
||||
export class BlockPreviewComponent extends BlockComponent {
|
||||
export class BlockPreviewComponent implements OnInit, OnDestroy {
|
||||
network = '';
|
||||
block: BlockExtended;
|
||||
blockHeight: number;
|
||||
blockHash: string;
|
||||
isLoadingBlock = true;
|
||||
strippedTransactions: TransactionStripped[];
|
||||
overviewTransitionDirection: string;
|
||||
isLoadingOverview = true;
|
||||
error: any;
|
||||
blockSubsidy: number;
|
||||
fees: number;
|
||||
overviewError: any = null;
|
||||
|
||||
overviewSubscription: Subscription;
|
||||
networkChangedSubscription: Subscription;
|
||||
|
||||
@ViewChild('blockGraph') blockGraph: BlockOverviewGraphComponent;
|
||||
|
||||
constructor(
|
||||
private route: ActivatedRoute,
|
||||
private electrsApiService: ElectrsApiService,
|
||||
public stateService: StateService,
|
||||
private seoService: SeoService,
|
||||
private openGraphService: OpenGraphService,
|
||||
private apiService: ApiService
|
||||
) { }
|
||||
|
||||
ngOnInit() {
|
||||
this.network = this.stateService.network;
|
||||
|
||||
const block$ = this.route.paramMap.pipe(
|
||||
switchMap((params: ParamMap) => {
|
||||
this.openGraphService.waitFor('block-viz');
|
||||
this.openGraphService.waitFor('block-data');
|
||||
|
||||
const blockHash: string = params.get('id') || '';
|
||||
this.block = undefined;
|
||||
this.error = undefined;
|
||||
this.overviewError = undefined;
|
||||
this.fees = undefined;
|
||||
|
||||
let isBlockHeight = false;
|
||||
if (/^[0-9]+$/.test(blockHash)) {
|
||||
isBlockHeight = true;
|
||||
} else {
|
||||
this.blockHash = blockHash;
|
||||
}
|
||||
|
||||
this.isLoadingBlock = true;
|
||||
this.isLoadingOverview = true;
|
||||
|
||||
if (isBlockHeight) {
|
||||
return this.electrsApiService.getBlockHashFromHeight$(parseInt(blockHash, 10))
|
||||
.pipe(
|
||||
switchMap((hash) => {
|
||||
if (hash) {
|
||||
this.blockHash = hash;
|
||||
return this.apiService.getBlock$(hash);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}),
|
||||
catchError((err) => {
|
||||
this.error = err;
|
||||
this.openGraphService.fail('block-data');
|
||||
this.openGraphService.fail('block-viz');
|
||||
return of(null);
|
||||
}),
|
||||
);
|
||||
}
|
||||
return this.apiService.getBlock$(blockHash);
|
||||
}),
|
||||
filter((block: BlockExtended | void) => block != null),
|
||||
tap((block: BlockExtended) => {
|
||||
this.block = block;
|
||||
this.blockHeight = block.height;
|
||||
|
||||
this.seoService.setTitle($localize`:@@block.component.browser-title:Block ${block.height}:BLOCK_HEIGHT:: ${block.id}:BLOCK_ID:`);
|
||||
this.isLoadingBlock = false;
|
||||
this.setBlockSubsidy();
|
||||
if (block?.extras?.reward !== undefined) {
|
||||
this.fees = block.extras.reward / 100000000 - this.blockSubsidy;
|
||||
}
|
||||
this.stateService.markBlock$.next({ blockHeight: this.blockHeight });
|
||||
this.isLoadingOverview = true;
|
||||
this.overviewError = null;
|
||||
|
||||
this.openGraphService.waitOver('block-data');
|
||||
}),
|
||||
throttleTime(50, asyncScheduler, { leading: true, trailing: true }),
|
||||
shareReplay(1)
|
||||
);
|
||||
|
||||
this.overviewSubscription = block$.pipe(
|
||||
startWith(null),
|
||||
pairwise(),
|
||||
switchMap(([prevBlock, block]) => this.apiService.getStrippedBlockTransactions$(block.id)
|
||||
.pipe(
|
||||
catchError((err) => {
|
||||
this.overviewError = err;
|
||||
this.openGraphService.fail('block-viz');
|
||||
return of([]);
|
||||
}),
|
||||
switchMap((transactions) => {
|
||||
return of({ transactions, direction: 'down' });
|
||||
})
|
||||
)
|
||||
),
|
||||
)
|
||||
.subscribe(({transactions, direction}: {transactions: TransactionStripped[], direction: string}) => {
|
||||
this.strippedTransactions = transactions;
|
||||
this.isLoadingOverview = false;
|
||||
if (this.blockGraph) {
|
||||
this.blockGraph.destroy();
|
||||
this.blockGraph.setup(this.strippedTransactions);
|
||||
}
|
||||
},
|
||||
(error) => {
|
||||
this.error = error;
|
||||
this.isLoadingOverview = false;
|
||||
this.openGraphService.fail('block-viz');
|
||||
this.openGraphService.fail('block-data');
|
||||
if (this.blockGraph) {
|
||||
this.blockGraph.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
this.networkChangedSubscription = this.stateService.networkChanged$
|
||||
.subscribe((network) => this.network = network);
|
||||
}
|
||||
|
||||
ngOnDestroy() {
|
||||
if (this.overviewSubscription) {
|
||||
this.overviewSubscription.unsubscribe();
|
||||
}
|
||||
if (this.networkChangedSubscription) {
|
||||
this.networkChangedSubscription.unsubscribe();
|
||||
}
|
||||
}
|
||||
|
||||
// TODO - Refactor this.fees/this.reward for liquid because it is not
|
||||
// used anymore on Bitcoin networks (we use block.extras directly)
|
||||
setBlockSubsidy() {
|
||||
this.blockSubsidy = 0;
|
||||
}
|
||||
|
||||
onGraphReady(): void {
|
||||
this.openGraphService.waitOver('block-viz');
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,21 +1,20 @@
|
||||
<ng-container *ngIf="{ val: network$ | async } as network">
|
||||
<div class="preview-wrapper">
|
||||
<router-outlet></router-outlet>
|
||||
|
||||
<footer>
|
||||
<span class="footer-brand" style="position: relative;">
|
||||
<img *ngIf="!officialMempoolSpace" src="/resources/mempool-logo.png" height="35" width="140" class="logo">
|
||||
<app-svg-images *ngIf="officialMempoolSpace" name="officialMempoolSpace" style="width: 140px; height: 35px" width="500" height="126" viewBox="0 0 500 126"></app-svg-images>
|
||||
<header>
|
||||
<span class="header-brand" style="position: relative;">
|
||||
<img *ngIf="!officialMempoolSpace" src="/resources/mempool-logo.png" height="50" width="200" class="logo">
|
||||
<app-svg-images *ngIf="officialMempoolSpace" name="officialMempoolSpace" style="width: 200px; height: 50px" width="500" height="126" viewBox="0 0 500 126"></app-svg-images>
|
||||
</span>
|
||||
|
||||
<div [ngSwitch]="network.val">
|
||||
<span *ngSwitchCase="'signet'" class="network signet"><img src="/resources/signet-logo.png" style="width: 30px;" class="signet mr-1" alt="logo"> Signet</span>
|
||||
<span *ngSwitchCase="'testnet'" class="network testnet"><img src="/resources/testnet-logo.png" style="width: 30px;" class="mr-1" alt="testnet logo"> Testnet</span>
|
||||
<span *ngSwitchCase="'bisq'" class="network bisq"><img src="/resources/bisq-logo.png" style="width: 30px;" class="mr-1" alt="bisq logo"> Bisq</span>
|
||||
<span *ngSwitchCase="'liquid'" class="network liquid"><img src="/resources/liquid-logo.png" style="width: 30px;" class="mr-1" alt="liquid mainnet logo"> Liquid</span>
|
||||
<span *ngSwitchCase="'liquidtestnet'" class="network liquidtestnet"><img src="/resources/liquidtestnet-logo.png" style="width: 30px;" class="mr-1" alt="liquid testnet logo"> Liquid Testnet</span>
|
||||
<span *ngSwitchDefault class="network mainnet"><img src="/resources/bitcoin-logo.png" style="width: 30px;" class="mainnet mr-1" alt="bitcoin logo"> Mainnet</span>
|
||||
<span *ngSwitchCase="'signet'" class="network signet"><img src="/resources/signet-logo.png" style="width: 45px;" class="signet mr-1" alt="logo"> Signet</span>
|
||||
<span *ngSwitchCase="'testnet'" class="network testnet"><img src="/resources/testnet-logo.png" style="width: 45px;" class="mr-1" alt="testnet logo"> Testnet</span>
|
||||
<span *ngSwitchCase="'bisq'" class="network bisq"><img src="/resources/bisq-logo.png" style="width: 45px;" class="mr-1" alt="bisq logo"> Bisq</span>
|
||||
<span *ngSwitchCase="'liquid'" class="network liquid"><img src="/resources/liquid-logo.png" style="width: 45px;" class="mr-1" alt="liquid mainnet logo"> Liquid</span>
|
||||
<span *ngSwitchCase="'liquidtestnet'" class="network liquidtestnet"><img src="/resources/liquidtestnet-logo.png" style="width: 45px;" class="mr-1" alt="liquid testnet logo"> Liquid Testnet</span>
|
||||
<span *ngSwitchDefault class="network mainnet"><img src="/resources/bitcoin-logo.png" style="width: 45px;" class="mainnet mr-1" alt="bitcoin logo"> Mainnet</span>
|
||||
</div>
|
||||
</footer>
|
||||
</header>
|
||||
<router-outlet></router-outlet>
|
||||
</div>
|
||||
</ng-container>
|
||||
|
||||
@ -2,28 +2,28 @@
|
||||
position: relative;
|
||||
display: block;
|
||||
margin: auto;
|
||||
max-width: 1024px;
|
||||
max-height: 512px;
|
||||
padding-bottom: 64px;
|
||||
max-width: 1200px;
|
||||
max-height: 600px;
|
||||
padding-top: 80px;
|
||||
|
||||
footer {
|
||||
header {
|
||||
position: absolute;
|
||||
left: 0;
|
||||
right: 0;
|
||||
bottom: 0;
|
||||
top: 0;
|
||||
z-index: 100;
|
||||
min-height: 64px;
|
||||
padding: 0rem 2rem;
|
||||
min-height: 80px;
|
||||
padding: 0rem 3rem;
|
||||
display: flex;
|
||||
flex-direction: row;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
background: #11131f;
|
||||
text-align: start;
|
||||
font-size: 1.2em;
|
||||
font-size: 1.8em;
|
||||
}
|
||||
|
||||
.footer-brand {
|
||||
.header-brand {
|
||||
width: 60%;
|
||||
}
|
||||
|
||||
|
||||
@ -12,6 +12,7 @@ import { StateService } from '../../services/state.service';
|
||||
import { chartColors, poolsColor } from 'src/app/app.constants';
|
||||
import { RelativeUrlPipe } from 'src/app/shared/pipes/relative-url/relative-url.pipe';
|
||||
import { download } from 'src/app/shared/graphs.utils';
|
||||
import { isMobile } from 'src/app/shared/common.utils';
|
||||
|
||||
@Component({
|
||||
selector: 'app-pool-ranking',
|
||||
@ -108,21 +109,23 @@ export class PoolRankingComponent implements OnInit {
|
||||
return pool;
|
||||
}
|
||||
|
||||
isMobile() {
|
||||
return (window.innerWidth <= 767.98);
|
||||
}
|
||||
|
||||
generatePoolsChartSerieData(miningStats) {
|
||||
const poolShareThreshold = this.isMobile() ? 2 : 1; // Do not draw pools which hashrate share is lower than that
|
||||
let poolShareThreshold = 0.5;
|
||||
if (isMobile()) {
|
||||
poolShareThreshold = 2;
|
||||
} else if (this.widget) {
|
||||
poolShareThreshold = 1;
|
||||
}
|
||||
|
||||
const data: object[] = [];
|
||||
let totalShareOther = 0;
|
||||
let totalBlockOther = 0;
|
||||
let totalEstimatedHashrateOther = 0;
|
||||
|
||||
let edgeDistance: any = '20%';
|
||||
if (this.isMobile() && this.widget) {
|
||||
if (isMobile() && this.widget) {
|
||||
edgeDistance = 0;
|
||||
} else if (this.isMobile() && !this.widget || this.widget) {
|
||||
} else if (isMobile() && !this.widget || this.widget) {
|
||||
edgeDistance = 10;
|
||||
}
|
||||
|
||||
@ -138,7 +141,7 @@ export class PoolRankingComponent implements OnInit {
|
||||
color: poolsColor[pool.name.replace(/[^a-zA-Z0-9]/g, '').toLowerCase()],
|
||||
},
|
||||
value: pool.share,
|
||||
name: pool.name + ((this.isMobile() || this.widget) ? `` : ` (${pool.share}%)`),
|
||||
name: pool.name + ((isMobile() || this.widget) ? `` : ` (${pool.share}%)`),
|
||||
label: {
|
||||
overflow: 'none',
|
||||
color: '#b1b1b1',
|
||||
@ -146,7 +149,7 @@ export class PoolRankingComponent implements OnInit {
|
||||
edgeDistance: edgeDistance,
|
||||
},
|
||||
tooltip: {
|
||||
show: !this.isMobile() || !this.widget,
|
||||
show: !isMobile() || !this.widget,
|
||||
backgroundColor: 'rgba(17, 19, 31, 1)',
|
||||
borderRadius: 4,
|
||||
shadowColor: 'rgba(0, 0, 0, 0.5)',
|
||||
@ -176,7 +179,7 @@ export class PoolRankingComponent implements OnInit {
|
||||
color: 'grey',
|
||||
},
|
||||
value: totalShareOther,
|
||||
name: 'Other' + (this.isMobile() ? `` : ` (${totalShareOther.toFixed(2)}%)`),
|
||||
name: 'Other' + (isMobile() ? `` : ` (${totalShareOther.toFixed(2)}%)`),
|
||||
label: {
|
||||
overflow: 'none',
|
||||
color: '#b1b1b1',
|
||||
@ -210,7 +213,7 @@ export class PoolRankingComponent implements OnInit {
|
||||
|
||||
prepareChartOptions(miningStats) {
|
||||
let pieSize = ['20%', '80%']; // Desktop
|
||||
if (this.isMobile() && !this.widget) {
|
||||
if (isMobile() && !this.widget) {
|
||||
pieSize = ['15%', '60%'];
|
||||
}
|
||||
|
||||
@ -226,7 +229,7 @@ export class PoolRankingComponent implements OnInit {
|
||||
series: [
|
||||
{
|
||||
zlevel: 0,
|
||||
minShowLabelAngle: 3.6,
|
||||
minShowLabelAngle: 1.8,
|
||||
name: 'Mining pool',
|
||||
type: 'pie',
|
||||
radius: pieSize,
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { Injectable } from '@angular/core';
|
||||
import { Injectable, NgZone } from '@angular/core';
|
||||
import { Meta } from '@angular/platform-browser';
|
||||
import { Router, ActivatedRoute, NavigationEnd } from '@angular/router';
|
||||
import { filter, map, switchMap } from 'rxjs/operators';
|
||||
@ -12,8 +12,11 @@ import { LanguageService } from './language.service';
|
||||
export class OpenGraphService {
|
||||
network = '';
|
||||
defaultImageUrl = '';
|
||||
previewLoadingEvents = {};
|
||||
previewLoadingCount = 0;
|
||||
|
||||
constructor(
|
||||
private ngZone: NgZone,
|
||||
private metaService: Meta,
|
||||
private stateService: StateService,
|
||||
private LanguageService: LanguageService,
|
||||
@ -39,6 +42,11 @@ export class OpenGraphService {
|
||||
this.clearOgImage();
|
||||
}
|
||||
});
|
||||
|
||||
// expose routing method to global scope, so we can access it from the unfurler
|
||||
window['ogService'] = {
|
||||
loadPage: (path) => { return this.loadPage(path) }
|
||||
};
|
||||
}
|
||||
|
||||
setOgImage() {
|
||||
@ -47,8 +55,8 @@ export class OpenGraphService {
|
||||
this.metaService.updateTag({ property: 'og:image', content: ogImageUrl });
|
||||
this.metaService.updateTag({ property: 'twitter:image:src', content: ogImageUrl });
|
||||
this.metaService.updateTag({ property: 'og:image:type', content: 'image/png' });
|
||||
this.metaService.updateTag({ property: 'og:image:width', content: '1024' });
|
||||
this.metaService.updateTag({ property: 'og:image:height', content: '512' });
|
||||
this.metaService.updateTag({ property: 'og:image:width', content: '1200' });
|
||||
this.metaService.updateTag({ property: 'og:image:height', content: '600' });
|
||||
}
|
||||
|
||||
clearOgImage() {
|
||||
@ -59,13 +67,53 @@ export class OpenGraphService {
|
||||
this.metaService.updateTag({ property: 'og:image:height', content: '500' });
|
||||
}
|
||||
|
||||
/// signal that the unfurler should wait for a 'ready' signal before taking a screenshot
|
||||
setPreviewLoading() {
|
||||
this.metaService.updateTag({ property: 'og:loading', content: 'loading'});
|
||||
/// register an event that needs to resolve before we can take a screenshot
|
||||
waitFor(event) {
|
||||
if (!this.previewLoadingEvents[event]) {
|
||||
this.previewLoadingEvents[event] = 1;
|
||||
this.previewLoadingCount++;
|
||||
} else {
|
||||
this.previewLoadingEvents[event]++;
|
||||
}
|
||||
this.metaService.updateTag({ property: 'og:preview:loading', content: 'loading'});
|
||||
}
|
||||
|
||||
// signal to the unfurler that the page is ready for a screenshot
|
||||
setPreviewReady() {
|
||||
this.metaService.updateTag({ property: 'og:ready', content: 'ready'});
|
||||
// mark an event as resolved
|
||||
// if all registered events have resolved, signal we are ready for a screenshot
|
||||
waitOver(event) {
|
||||
if (this.previewLoadingEvents[event]) {
|
||||
this.previewLoadingEvents[event]--;
|
||||
if (this.previewLoadingEvents[event] === 0) {
|
||||
delete this.previewLoadingEvents[event]
|
||||
this.previewLoadingCount--;
|
||||
}
|
||||
}
|
||||
if (this.previewLoadingCount === 0) {
|
||||
this.metaService.updateTag({ property: 'og:preview:ready', content: 'ready'});
|
||||
}
|
||||
}
|
||||
|
||||
fail(event) {
|
||||
if (this.previewLoadingEvents[event]) {
|
||||
this.metaService.updateTag({ property: 'og:preview:fail', content: 'fail'});
|
||||
}
|
||||
}
|
||||
|
||||
resetLoading() {
|
||||
this.previewLoadingEvents = {};
|
||||
this.previewLoadingCount = 0;
|
||||
this.metaService.removeTag("property='og:preview:loading'");
|
||||
this.metaService.removeTag("property='og:preview:ready'");
|
||||
this.metaService.removeTag("property='og:preview:fail'");
|
||||
this.metaService.removeTag("property='og:meta:ready'");
|
||||
}
|
||||
|
||||
loadPage(path) {
|
||||
if (path !== this.router.url) {
|
||||
this.resetLoading();
|
||||
this.ngZone.run(() => {
|
||||
this.router.navigateByUrl(path);
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -21,12 +21,14 @@ export class SeoService {
|
||||
this.titleService.setTitle(newTitle + ' - ' + this.getTitle());
|
||||
this.metaService.updateTag({ property: 'og:title', content: newTitle});
|
||||
this.metaService.updateTag({ property: 'twitter:title', content: newTitle});
|
||||
this.metaService.updateTag({ property: 'og:meta:ready', content: 'ready'});
|
||||
}
|
||||
|
||||
resetTitle(): void {
|
||||
this.titleService.setTitle(this.getTitle());
|
||||
this.metaService.updateTag({ property: 'og:title', content: this.getTitle()});
|
||||
this.metaService.updateTag({ property: 'twitter:title', content: this.getTitle()});
|
||||
this.metaService.updateTag({ property: 'og:meta:ready', content: 'ready'});
|
||||
}
|
||||
|
||||
setEnterpriseTitle(title: string) {
|
||||
|
||||
@ -88,8 +88,8 @@ body {
|
||||
}
|
||||
|
||||
.preview-box {
|
||||
min-height: 512px;
|
||||
padding: 2rem 3rem;
|
||||
min-height: 520px;
|
||||
padding: 1.5rem 3rem;
|
||||
}
|
||||
|
||||
@media (max-width: 767.98px) {
|
||||
|
||||
@ -5,10 +5,13 @@
|
||||
},
|
||||
"MEMPOOL": {
|
||||
"HTTP_HOST": "http://localhost",
|
||||
"HTTP_PORT": 4200
|
||||
"HTTP_PORT": 4200,
|
||||
"NETWORK": "bitcoin" // "bitcoin" | "liquid" | "bisq" (optional - defaults to "bitcoin")
|
||||
},
|
||||
"PUPPETEER": {
|
||||
"CLUSTER_SIZE": 2,
|
||||
"EXEC_PATH": "/usr/local/bin/chrome" // optional
|
||||
"EXEC_PATH": "/usr/local/bin/chrome", // optional
|
||||
"MAX_PAGE_AGE": 86400, // maximum lifetime of a page session (in seconds)
|
||||
"RENDER_TIMEOUT": 3000, // timeout for preview image rendering (in ms) (optional)
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "mempool-unfurl",
|
||||
"version": "0.0.1",
|
||||
"version": "0.0.2",
|
||||
"description": "Renderer for mempool open graph link preview images",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
|
||||
@ -1,11 +1,11 @@
|
||||
{
|
||||
"headless": true,
|
||||
"defaultViewport": {
|
||||
"width": 1024,
|
||||
"height": 512
|
||||
"width": 1200,
|
||||
"height": 600
|
||||
},
|
||||
"args": [
|
||||
"--window-size=1024,512",
|
||||
"--window-size=1200,600",
|
||||
"--autoplay-policy=user-gesture-required",
|
||||
"--disable-background-networking",
|
||||
"--disable-background-timer-throttling",
|
||||
|
||||
159
unfurler/src/concurrency/ReusablePage.ts
Normal file
159
unfurler/src/concurrency/ReusablePage.ts
Normal file
@ -0,0 +1,159 @@
|
||||
import * as puppeteer from 'puppeteer';
|
||||
import ConcurrencyImplementation from 'puppeteer-cluster/dist/concurrency/ConcurrencyImplementation';
|
||||
import { timeoutExecute } from 'puppeteer-cluster/dist/util';
|
||||
|
||||
import config from '../config';
|
||||
const mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : '');
|
||||
|
||||
const BROWSER_TIMEOUT = 8000;
|
||||
// maximum lifetime of a single page session
|
||||
const maxAgeMs = (config.PUPPETEER.MAX_PAGE_AGE || (24 * 60 * 60)) * 1000;
|
||||
const maxConcurrency = config.PUPPETEER.CLUSTER_SIZE;
|
||||
|
||||
interface RepairablePage extends puppeteer.Page {
|
||||
repairRequested?: boolean;
|
||||
language?: string | null;
|
||||
createdAt?: number;
|
||||
free?: boolean;
|
||||
index?: number;
|
||||
}
|
||||
|
||||
interface ResourceData {
|
||||
page: RepairablePage;
|
||||
}
|
||||
|
||||
export default class ReusablePage extends ConcurrencyImplementation {
|
||||
|
||||
protected browser: puppeteer.Browser | null = null;
|
||||
protected pages: RepairablePage[] = [];
|
||||
private repairing: boolean = false;
|
||||
private repairRequested: boolean = false;
|
||||
private openInstances: number = 0;
|
||||
private waitingForRepairResolvers: (() => void)[] = [];
|
||||
|
||||
public constructor(options: puppeteer.LaunchOptions, puppeteer: any) {
|
||||
super(options, puppeteer);
|
||||
}
|
||||
|
||||
private async repair() {
|
||||
if (this.openInstances !== 0 || this.repairing) {
|
||||
// already repairing or there are still pages open? wait for start/finish
|
||||
await new Promise<void>(resolve => this.waitingForRepairResolvers.push(resolve));
|
||||
return;
|
||||
}
|
||||
|
||||
this.repairing = true;
|
||||
console.log('Starting repair');
|
||||
|
||||
try {
|
||||
// will probably fail, but just in case the repair was not necessary
|
||||
await (<puppeteer.Browser>this.browser).close();
|
||||
} catch (e) {
|
||||
console.log('Unable to close browser.');
|
||||
}
|
||||
|
||||
try {
|
||||
await this.init();
|
||||
} catch (err) {
|
||||
throw new Error('Unable to restart chrome.');
|
||||
}
|
||||
this.repairRequested = false;
|
||||
this.repairing = false;
|
||||
this.waitingForRepairResolvers.forEach(resolve => resolve());
|
||||
this.waitingForRepairResolvers = [];
|
||||
}
|
||||
|
||||
public async init() {
|
||||
this.browser = await this.puppeteer.launch(this.options);
|
||||
const promises = []
|
||||
for (let i = 0; i < maxConcurrency; i++) {
|
||||
const newPage = await this.initPage();
|
||||
newPage.index = this.pages.length;
|
||||
console.log('initialized page ', newPage.index);
|
||||
this.pages.push(newPage);
|
||||
}
|
||||
}
|
||||
|
||||
public async close() {
|
||||
await (this.browser as puppeteer.Browser).close();
|
||||
}
|
||||
|
||||
protected async initPage(): Promise<RepairablePage> {
|
||||
const page = await (this.browser as puppeteer.Browser).newPage() as RepairablePage;
|
||||
page.language = null;
|
||||
page.createdAt = Date.now();
|
||||
const defaultUrl = mempoolHost + '/preview/block/1';
|
||||
page.on('pageerror', (err) => {
|
||||
page.repairRequested = true;
|
||||
});
|
||||
await page.goto(defaultUrl, { waitUntil: "load" });
|
||||
page.free = true;
|
||||
return page
|
||||
}
|
||||
|
||||
protected async createResources(): Promise<ResourceData> {
|
||||
const page = this.pages.find(p => p.free);
|
||||
if (!page) {
|
||||
console.log('no free pages!')
|
||||
throw new Error('no pages available');
|
||||
} else {
|
||||
page.free = false;
|
||||
return { page };
|
||||
}
|
||||
}
|
||||
|
||||
protected async repairPage(page) {
|
||||
// create a new page
|
||||
const newPage = await this.initPage();
|
||||
newPage.free = true;
|
||||
// replace the old page
|
||||
newPage.index = page.index;
|
||||
this.pages.splice(page.index, 1, newPage);
|
||||
// clean up the old page
|
||||
try {
|
||||
await page.goto('about:blank', {timeout: 200}); // prevents memory leak (maybe?)
|
||||
} catch (e) {
|
||||
console.log('unexpected page repair error');
|
||||
}
|
||||
await page.close();
|
||||
return newPage;
|
||||
}
|
||||
|
||||
public async workerInstance() {
|
||||
let resources: ResourceData;
|
||||
|
||||
return {
|
||||
jobInstance: async () => {
|
||||
await timeoutExecute(BROWSER_TIMEOUT, (async () => {
|
||||
resources = await this.createResources();
|
||||
})());
|
||||
this.openInstances += 1;
|
||||
|
||||
return {
|
||||
resources,
|
||||
|
||||
close: async () => {
|
||||
this.openInstances -= 1; // decrement first in case of error
|
||||
if (resources?.page != null) {
|
||||
if (resources.page.repairRequested || (Date.now() - (resources.page.createdAt || 0) > maxAgeMs)) {
|
||||
resources.page = await this.repairPage(resources.page);
|
||||
} else {
|
||||
resources.page.free = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (this.repairRequested) {
|
||||
await this.repair();
|
||||
}
|
||||
},
|
||||
};
|
||||
},
|
||||
|
||||
close: async () => {},
|
||||
|
||||
repair: async () => {
|
||||
await this.repairPage(resources.page);
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
@ -8,10 +8,13 @@ interface IConfig {
|
||||
MEMPOOL: {
|
||||
HTTP_HOST: string;
|
||||
HTTP_PORT: number;
|
||||
NETWORK?: string;
|
||||
};
|
||||
PUPPETEER: {
|
||||
CLUSTER_SIZE: number;
|
||||
EXEC_PATH?: string;
|
||||
MAX_PAGE_AGE?: number;
|
||||
RENDER_TIMEOUT?: number;
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -3,6 +3,8 @@ import { Application, Request, Response, NextFunction } from 'express';
|
||||
import * as http from 'http';
|
||||
import config from './config';
|
||||
import { Cluster } from 'puppeteer-cluster';
|
||||
import ReusablePage from './concurrency/ReusablePage';
|
||||
import { parseLanguageUrl } from './language/lang';
|
||||
const puppeteerConfig = require('../puppeteer.config.json');
|
||||
|
||||
if (config.PUPPETEER.EXEC_PATH) {
|
||||
@ -14,10 +16,14 @@ class Server {
|
||||
private app: Application;
|
||||
cluster?: Cluster;
|
||||
mempoolHost: string;
|
||||
network: string;
|
||||
defaultImageUrl: string;
|
||||
|
||||
constructor() {
|
||||
this.app = express();
|
||||
this.mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : '');
|
||||
this.network = config.MEMPOOL.NETWORK || 'bitcoin';
|
||||
this.defaultImageUrl = this.getDefaultImageUrl();
|
||||
this.startServer();
|
||||
}
|
||||
|
||||
@ -32,7 +38,7 @@ class Server {
|
||||
;
|
||||
|
||||
this.cluster = await Cluster.launch({
|
||||
concurrency: Cluster.CONCURRENCY_CONTEXT,
|
||||
concurrency: ReusablePage,
|
||||
maxConcurrency: config.PUPPETEER.CLUSTER_SIZE,
|
||||
puppeteerOptions: puppeteerConfig,
|
||||
});
|
||||
@ -47,63 +53,75 @@ class Server {
|
||||
});
|
||||
}
|
||||
|
||||
async stopServer() {
|
||||
if (this.cluster) {
|
||||
await this.cluster.idle();
|
||||
await this.cluster.close();
|
||||
}
|
||||
if (this.server) {
|
||||
await this.server.close();
|
||||
}
|
||||
}
|
||||
|
||||
setUpRoutes() {
|
||||
this.app.get('/render*', async (req, res) => { return this.renderPreview(req, res) })
|
||||
this.app.get('*', (req, res) => { return this.renderHTML(req, res) })
|
||||
}
|
||||
|
||||
async clusterTask({ page, data: { url, action } }) {
|
||||
await page.goto(url, { waitUntil: "networkidle0" });
|
||||
switch (action) {
|
||||
case 'screenshot': {
|
||||
await page.evaluate(async () => {
|
||||
// wait for all images to finish loading
|
||||
const imgs = Array.from(document.querySelectorAll("img"));
|
||||
await Promise.all([
|
||||
document.fonts.ready,
|
||||
...imgs.map((img) => {
|
||||
if (img.complete) {
|
||||
if (img.naturalHeight !== 0) return;
|
||||
throw new Error("Image failed to load");
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
img.addEventListener("load", resolve);
|
||||
img.addEventListener("error", reject);
|
||||
});
|
||||
}),
|
||||
]);
|
||||
});
|
||||
const waitForReady = await page.$('meta[property="og:loading"]');
|
||||
const alreadyReady = await page.$('meta[property="og:ready"]');
|
||||
if (waitForReady != null && alreadyReady == null) {
|
||||
try {
|
||||
await page.waitForSelector('meta[property="og:ready]"', { timeout: 10000 });
|
||||
} catch (e) {
|
||||
// probably timed out
|
||||
async clusterTask({ page, data: { url, path, action } }) {
|
||||
try {
|
||||
const urlParts = parseLanguageUrl(path);
|
||||
if (page.language !== urlParts.lang) {
|
||||
// switch language
|
||||
page.language = urlParts.lang;
|
||||
const localizedUrl = urlParts.lang ? `${this.mempoolHost}/${urlParts.lang}${urlParts.path}` : `${this.mempoolHost}${urlParts.path}` ;
|
||||
await page.goto(localizedUrl, { waitUntil: "load" });
|
||||
} else {
|
||||
const loaded = await page.evaluate(async (path) => {
|
||||
if (window['ogService']) {
|
||||
window['ogService'].loadPage(path);
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}, urlParts.path);
|
||||
if (!loaded) {
|
||||
throw new Error('failed to access open graph service');
|
||||
}
|
||||
return page.screenshot();
|
||||
} break;
|
||||
default: {
|
||||
try {
|
||||
await page.waitForSelector('meta[property="og:title"]', { timeout: 10000 })
|
||||
const tag = await page.$('meta[property="og:title"]');
|
||||
} catch (e) {
|
||||
// probably timed out
|
||||
}
|
||||
return page.content();
|
||||
}
|
||||
|
||||
const waitForReady = await page.$('meta[property="og:preview:loading"]');
|
||||
let success = true;
|
||||
if (waitForReady != null) {
|
||||
success = await Promise.race([
|
||||
page.waitForSelector('meta[property="og:preview:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => true),
|
||||
page.waitForSelector('meta[property="og:preview:fail"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => false)
|
||||
])
|
||||
}
|
||||
if (success) {
|
||||
const screenshot = await page.screenshot();
|
||||
return screenshot;
|
||||
} else {
|
||||
console.log(`failed to render page preview for ${action} due to client-side error. probably requested an invalid ID`);
|
||||
page.repairRequested = true;
|
||||
}
|
||||
} catch (e) {
|
||||
console.log(`failed to render page for ${action}`, e instanceof Error ? e.message : e);
|
||||
page.repairRequested = true;
|
||||
}
|
||||
}
|
||||
|
||||
async renderPreview(req, res) {
|
||||
try {
|
||||
// strip default language code for compatibility
|
||||
const path = req.params[0].replace('/en/', '/');
|
||||
const img = await this.cluster?.execute({ url: this.mempoolHost + path, action: 'screenshot' });
|
||||
const path = req.params[0]
|
||||
const img = await this.cluster?.execute({ url: this.mempoolHost + path, path: path, action: 'screenshot' });
|
||||
|
||||
res.contentType('image/png');
|
||||
res.send(img);
|
||||
if (!img) {
|
||||
res.status(500).send('failed to render page preview');
|
||||
} else {
|
||||
res.contentType('image/png');
|
||||
res.send(img);
|
||||
}
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
res.status(500).send(e instanceof Error ? e.message : e);
|
||||
@ -112,22 +130,89 @@ class Server {
|
||||
|
||||
async renderHTML(req, res) {
|
||||
// drop requests for static files
|
||||
const path = req.params[0];
|
||||
const match = path.match(/\.[\w]+$/);
|
||||
const rawPath = req.params[0];
|
||||
const match = rawPath.match(/\.[\w]+$/);
|
||||
if (match?.length && match[0] !== '.html') {
|
||||
res.status(404).send();
|
||||
return
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
let html = await this.cluster?.execute({ url: this.mempoolHost + req.params[0], action: 'html' });
|
||||
let previewSupported = true;
|
||||
let mode = 'mainnet'
|
||||
let ogImageUrl = this.defaultImageUrl;
|
||||
let ogTitle;
|
||||
const { lang, path } = parseLanguageUrl(rawPath);
|
||||
const parts = path.slice(1).split('/');
|
||||
|
||||
res.send(html)
|
||||
} catch (e) {
|
||||
console.log(e);
|
||||
res.status(500).send(e instanceof Error ? e.message : e);
|
||||
// handle network mode modifiers
|
||||
if (['testnet', 'signet'].includes(parts[0])) {
|
||||
mode = parts.shift();
|
||||
}
|
||||
|
||||
// handle supported preview routes
|
||||
if (parts[0] === 'block') {
|
||||
ogTitle = `Block: ${parts[1]}`;
|
||||
} else if (parts[0] === 'address') {
|
||||
ogTitle = `Address: ${parts[1]}`;
|
||||
} else {
|
||||
previewSupported = false;
|
||||
}
|
||||
|
||||
if (previewSupported) {
|
||||
ogImageUrl = `${config.SERVER.HOST}/render/${lang || 'en'}/preview${path}`;
|
||||
ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${mode !== 'mainnet' ? capitalize(mode) + ' ' : ''}${ogTitle}`;
|
||||
} else {
|
||||
ogTitle = 'The Mempool Open Source Project™';
|
||||
}
|
||||
|
||||
res.send(`
|
||||
<!doctype html>
|
||||
<html lang="en-US" dir="ltr">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>${ogTitle}</title>
|
||||
<meta name="description" content="The Mempool Open Source Project™ - our self-hosted explorer for the ${capitalize(this.network)} community."/>
|
||||
<meta property="og:image" content="${ogImageUrl}"/>
|
||||
<meta property="og:image:type" content="image/png"/>
|
||||
<meta property="og:image:width" content="${previewSupported ? 1200 : 1000}"/>
|
||||
<meta property="og:image:height" content="${previewSupported ? 600 : 500}"/>
|
||||
<meta property="og:title" content="${ogTitle}">
|
||||
<meta property="twitter:card" content="summary_large_image">
|
||||
<meta property="twitter:site" content="@mempool">
|
||||
<meta property="twitter:creator" content="@mempool">
|
||||
<meta property="twitter:title" content="${ogTitle}">
|
||||
<meta property="twitter:description" content="Our self-hosted mempool explorer for the ${capitalize(this.network)} community."/>
|
||||
<meta property="twitter:image:src" content="${ogImageUrl}"/>
|
||||
<meta property="twitter:domain" content="mempool.space">
|
||||
<body></body>
|
||||
</html>
|
||||
`);
|
||||
}
|
||||
|
||||
getDefaultImageUrl() {
|
||||
switch (this.network) {
|
||||
case 'liquid':
|
||||
return this.mempoolHost + '/resources/liquid/liquid-network-preview.png';
|
||||
case 'bisq':
|
||||
return this.mempoolHost + '/resources/bisq/bisq-markets-preview.png';
|
||||
default:
|
||||
return this.mempoolHost + '/resources/mempool-space-preview.png';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const server = new Server();
|
||||
|
||||
process.on('SIGTERM', async () => {
|
||||
console.info('Shutting down Mempool Unfurl Server');
|
||||
await server.stopServer();
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
function capitalize(str) {
|
||||
if (str && str.length) {
|
||||
return str[0].toUpperCase() + str.slice(1);
|
||||
} else {
|
||||
return str;
|
||||
}
|
||||
}
|
||||
|
||||
79
unfurler/src/language/lang.ts
Normal file
79
unfurler/src/language/lang.ts
Normal file
@ -0,0 +1,79 @@
|
||||
export interface Language {
|
||||
code: string;
|
||||
name: string;
|
||||
}
|
||||
|
||||
const languageList: Language[] = [
|
||||
{ code: 'ar', name: 'العربية' }, // Arabic
|
||||
{ code: 'bg', name: 'Български' }, // Bulgarian
|
||||
{ code: 'bs', name: 'Bosanski' }, // Bosnian
|
||||
{ code: 'ca', name: 'Català' }, // Catalan
|
||||
{ code: 'cs', name: 'Čeština' }, // Czech
|
||||
{ code: 'da', name: 'Dansk' }, // Danish
|
||||
{ code: 'de', name: 'Deutsch' }, // German
|
||||
{ code: 'et', name: 'Eesti' }, // Estonian
|
||||
{ code: 'el', name: 'Ελληνικά' }, // Greek
|
||||
{ code: 'en', name: 'English' }, // English
|
||||
{ code: 'es', name: 'Español' }, // Spanish
|
||||
{ code: 'eo', name: 'Esperanto' }, // Esperanto
|
||||
{ code: 'eu', name: 'Euskara' }, // Basque
|
||||
{ code: 'fa', name: 'فارسی' }, // Persian
|
||||
{ code: 'fr', name: 'Français' }, // French
|
||||
{ code: 'gl', name: 'Galego' }, // Galician
|
||||
{ code: 'ko', name: '한국어' }, // Korean
|
||||
{ code: 'hr', name: 'Hrvatski' }, // Croatian
|
||||
{ code: 'id', name: 'Bahasa Indonesia' },// Indonesian
|
||||
{ code: 'hi', name: 'हिन्दी' }, // Hindi
|
||||
{ code: 'it', name: 'Italiano' }, // Italian
|
||||
{ code: 'he', name: 'עברית' }, // Hebrew
|
||||
{ code: 'ka', name: 'ქართული' }, // Georgian
|
||||
{ code: 'lv', name: 'Latviešu' }, // Latvian
|
||||
{ code: 'lt', name: 'Lietuvių' }, // Lithuanian
|
||||
{ code: 'hu', name: 'Magyar' }, // Hungarian
|
||||
{ code: 'mk', name: 'Македонски' }, // Macedonian
|
||||
{ code: 'ms', name: 'Bahasa Melayu' }, // Malay
|
||||
{ code: 'nl', name: 'Nederlands' }, // Dutch
|
||||
{ code: 'ja', name: '日本語' }, // Japanese
|
||||
{ code: 'nb', name: 'Norsk' }, // Norwegian Bokmål
|
||||
{ code: 'nn', name: 'Norsk Nynorsk' }, // Norwegian Nynorsk
|
||||
{ code: 'pl', name: 'Polski' }, // Polish
|
||||
{ code: 'pt', name: 'Português' }, // Portuguese
|
||||
{ code: 'pt-BR', name: 'Português (Brazil)' }, // Portuguese (Brazil)
|
||||
{ code: 'ro', name: 'Română' }, // Romanian
|
||||
{ code: 'ru', name: 'Русский' }, // Russian
|
||||
{ code: 'sk', name: 'Slovenčina' }, // Slovak
|
||||
{ code: 'sl', name: 'Slovenščina' }, // Slovenian
|
||||
{ code: 'sr', name: 'Српски / srpski' }, // Serbian
|
||||
{ code: 'sh', name: 'Srpskohrvatski / српскохрватски' },// Serbo-Croatian
|
||||
{ code: 'fi', name: 'Suomi' }, // Finnish
|
||||
{ code: 'sv', name: 'Svenska' }, // Swedish
|
||||
{ code: 'th', name: 'ไทย' }, // Thai
|
||||
{ code: 'tr', name: 'Türkçe' }, // Turkish
|
||||
{ code: 'uk', name: 'Українська' }, // Ukrainian
|
||||
{ code: 'vi', name: 'Tiếng Việt' }, // Vietnamese
|
||||
{ code: 'zh', name: '中文' }, // Chinese
|
||||
];
|
||||
|
||||
const languageDict = {};
|
||||
languageList.forEach(lang => {
|
||||
languageDict[lang.code] = lang
|
||||
});
|
||||
export const languages = languageDict;
|
||||
|
||||
// expects path to start with a leading '/'
|
||||
export function parseLanguageUrl(path) {
|
||||
const parts = path.split('/');
|
||||
let lang;
|
||||
let rest;
|
||||
if (languages[parts[1]]) {
|
||||
lang = parts[1];
|
||||
rest = '/' + parts.slice(2).join('/');
|
||||
} else {
|
||||
lang = null;
|
||||
rest = path;
|
||||
}
|
||||
if (lang === 'en') {
|
||||
lang = null;
|
||||
}
|
||||
return { lang, path: rest };
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user