Merge branch 'master' into knorrium/fix_docker_startup_vars

.github/workflows/ci.yml

@@ -42,8 +42,10 @@ jobs:
         run: npm run lint
         working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
 
-      #  - name: Test
+      - name: Unit Tests
-      #    run: npm run test
+        if: ${{ matrix.flavor == 'dev'}}
+        run: npm run test
+        working-directory: ${{ matrix.node }}/${{ matrix.flavor }}/backend
 
       - name: Build
         run: npm run build

.github/workflows/on-tag.yml

@@ -1,7 +1,7 @@
 name: Docker build on tag
 env:
   DOCKER_CLI_EXPERIMENTAL: enabled
-  TAG_FMT: '^refs/tags/(((.?[0-9]+){3,4}))$'
+  TAG_FMT: "^refs/tags/(((.?[0-9]+){3,4}))$"
   DOCKER_BUILDKIT: 0
   COMPOSE_DOCKER_CLI_BUILD: 0
 
@@ -21,9 +21,39 @@ jobs:
         service:
           - frontend
           - backend
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
     name: Build and push to DockerHub
     steps:
+      # Workaround based on JonasAlfredsson/docker-on-tmpfs@v1.0.1
+      - name: Replace the current swap file
+        shell: bash
+        run: |
+          sudo swapoff /mnt/swapfile
+          sudo rm -v /mnt/swapfile
+          sudo fallocate -l 10G /mnt/swapfile
+          sudo chmod 600 /mnt/swapfile
+          sudo mkswap /mnt/swapfile
+          sudo swapon /mnt/swapfile
+
+      - name: Show current memory and swap status
+        shell: bash
+        run: |
+          sudo free -h
+          echo
+          sudo swapon --show
+
+      - name: Mount a tmpfs over /var/lib/docker
+        shell: bash
+        run: |
+          if [ ! -d "/var/lib/docker" ]; then
+            echo "Directory '/var/lib/docker' not found"
+            exit 1
+          fi
+          sudo mount -t tmpfs -o size=10G tmpfs /var/lib/docker
+          sudo systemctl restart docker
+          sudo df -h | grep docker
+
       - name: Set env variables
         run: echo "TAG=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
 

backend/.gitignore

@@ -1,7 +1,8 @@
 # See http://help.github.com/ignore-files/ for more about ignoring files.
 
 # production config and external assets
-
+!mempool-config.template.json
+!mempool-config.sample.json
 mempool-config.json
 pools.json
 icons.json

backend/jest.config.ts

@@ -0,0 +1,17 @@
+import type { Config } from "@jest/types"
+
+const config: Config.InitialOptions = {
+  preset: "ts-jest",
+  testEnvironment: "node",
+  verbose: true,
+  automock: false,
+  collectCoverage: true,
+  collectCoverageFrom: ["./src/**/**.ts"],
+  coverageProvider: "v8",
+  coverageThreshold: {
+    global: {
+      lines: 1
+    }
+  }
+}
+export default config;

backend/package.json

@@ -21,16 +21,17 @@
   ],
   "main": "index.ts",
   "scripts": {
-    "tsc": "./node_modules/typescript/bin/tsc",
+    "tsc": "./node_modules/typescript/bin/tsc -p tsconfig.build.json",
     "build": "npm run tsc",
     "start": "node --max-old-space-size=2048 dist/index.js",
     "start-production": "node --max-old-space-size=4096 dist/index.js",
-    "test": "echo \"Error: no test specified\" && exit 1",
+    "test": "./node_modules/.bin/jest --coverage",
     "lint": "./node_modules/.bin/eslint . --ext .ts",
     "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",
     "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""
   },
   "dependencies": {
+    "@babel/core": "^7.18.6",
     "@mempool/electrum-client": "^1.1.7",
     "@types/node": "^16.11.41",
     "axios": "~0.27.2",
@@ -46,14 +47,20 @@
     "ws": "~8.8.0"
   },
   "devDependencies": {
+    "@babel/core": "^7.18.6",
+    "@babel/code-frame": "^7.18.6",
     "@types/compression": "^1.7.2",
     "@types/crypto-js": "^4.1.1",
     "@types/express": "^4.17.13",
+    "@types/jest": "^28.1.4",
     "@types/ws": "~8.5.3",
     "@typescript-eslint/eslint-plugin": "^5.30.5",
     "@typescript-eslint/parser": "^5.30.5",
     "eslint": "^8.19.0",
     "eslint-config-prettier": "^8.5.0",
-    "prettier": "^2.7.1"
+    "jest": "^28.1.2",
+    "prettier": "^2.7.1",
+    "ts-jest": "^28.0.5",
+    "ts-node": "^10.8.2"
   }
 }

backend/src/__fixtures__/mempool-config.template.json

@@ -0,0 +1,92 @@
+{
+  "MEMPOOL": {
+    "NETWORK": "__MEMPOOL_NETWORK__",
+    "BACKEND": "__MEMPOOL_BACKEND__",
+    "BLOCKS_SUMMARIES_INDEXING": true,
+    "HTTP_PORT": 1,
+    "SPAWN_CLUSTER_PROCS": 2,
+    "API_URL_PREFIX": "__MEMPOOL_API_URL_PREFIX__",
+    "AUTOMATIC_BLOCK_REINDEXING": true,
+    "POLL_RATE_MS": 3,
+    "CACHE_DIR": "__MEMPOOL_CACHE_DIR__",
+    "CLEAR_PROTECTION_MINUTES": 4,
+    "RECOMMENDED_FEE_PERCENTILE": 5,
+    "BLOCK_WEIGHT_UNITS": 6,
+    "INITIAL_BLOCKS_AMOUNT": 7,
+    "MEMPOOL_BLOCKS_AMOUNT": 8,
+    "PRICE_FEED_UPDATE_INTERVAL": 9,
+    "USE_SECOND_NODE_FOR_MINFEE": 10,
+    "EXTERNAL_ASSETS": 11,
+    "EXTERNAL_MAX_RETRY": 12,
+    "EXTERNAL_RETRY_INTERVAL": 13,
+    "USER_AGENT": "__MEMPOOL_USER_AGENT__",
+    "STDOUT_LOG_MIN_PRIORITY": "__MEMPOOL_STDOUT_LOG_MIN_PRIORITY__",
+    "INDEXING_BLOCKS_AMOUNT": 14,
+    "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__",
+    "POOLS_JSON_URL": "__POOLS_JSON_URL__"
+  },
+  "CORE_RPC": {
+    "HOST": "__CORE_RPC_HOST__",
+    "PORT": 15,
+    "USERNAME": "__CORE_RPC_USERNAME__",
+    "PASSWORD": "__CORE_RPC_PASSWORD__"
+  },
+  "ELECTRUM": {
+    "HOST": "__ELECTRUM_HOST__",
+    "PORT": 16,
+    "TLS_ENABLED": true
+  },
+  "ESPLORA": {
+    "REST_API_URL": "__ESPLORA_REST_API_URL__"
+  },
+  "SECOND_CORE_RPC": {
+    "HOST": "__SECOND_CORE_RPC_HOST__",
+    "PORT": 17,
+    "USERNAME": "__SECOND_CORE_RPC_USERNAME__",
+    "PASSWORD": "__SECOND_CORE_RPC_PASSWORD__"
+  },
+  "DATABASE": {
+    "ENABLED": false,
+    "HOST": "__DATABASE_HOST__",
+    "SOCKET": "__DATABASE_SOCKET__",
+    "PORT": 18,
+    "DATABASE": "__DATABASE_DATABASE__",
+    "USERNAME": "__DATABASE_USERNAME__",
+    "PASSWORD": "__DATABASE_PASSWORD__"
+  },
+  "SYSLOG": {
+    "ENABLED": false,
+    "HOST": "__SYSLOG_HOST__",
+    "PORT": 19,
+    "MIN_PRIORITY": "__SYSLOG_MIN_PRIORITY__",
+    "FACILITY": "__SYSLOG_FACILITY__"
+  },
+  "STATISTICS": {
+    "ENABLED": false,
+    "TX_PER_SECOND_SAMPLE_PERIOD": 20
+  },
+  "BISQ": {
+    "ENABLED": true,
+    "DATA_PATH": "__BISQ_DATA_PATH__"
+  },
+  "SOCKS5PROXY": {
+    "ENABLED": true,
+    "USE_ONION": true,
+    "HOST": "__SOCKS5PROXY_HOST__",
+    "PORT": "__SOCKS5PROXY_PORT__",
+    "USERNAME": "__SOCKS5PROXY_USERNAME__",
+    "PASSWORD": "__SOCKS5PROXY_PASSWORD__"
+  },
+  "PRICE_DATA_SERVER": {
+    "TOR_URL": "__PRICE_DATA_SERVER_TOR_URL__",
+    "CLEARNET_URL": "__PRICE_DATA_SERVER_CLEARNET_URL__"
+  },
+  "EXTERNAL_DATA_SERVER": {
+    "MEMPOOL_API": "__EXTERNAL_DATA_SERVER_MEMPOOL_API__",
+    "MEMPOOL_ONION": "__EXTERNAL_DATA_SERVER_MEMPOOL_ONION__",
+    "LIQUID_API": "__EXTERNAL_DATA_SERVER_LIQUID_API__",
+    "LIQUID_ONION": "__EXTERNAL_DATA_SERVER_LIQUID_ONION__",
+    "BISQ_URL": "__EXTERNAL_DATA_SERVER_BISQ_URL__",
+    "BISQ_ONION": "__EXTERNAL_DATA_SERVER_BISQ_ONION__"
+  }
+}

backend/src/__tests__/config.test.ts

@@ -0,0 +1,140 @@
+import * as fs from 'fs';
+
+describe('Mempool Backend Config', () => {
+  beforeEach(() => {
+    jest.resetAllMocks();
+    jest.resetModules();
+  });
+
+  test('should return defaults when no file is present', () => {
+    jest.isolateModules(() => {
+      jest.mock('../../mempool-config.json', () => ({}), { virtual: true });
+
+      const config = jest.requireActual('../config').default;
+
+      expect(config.MEMPOOL).toStrictEqual({
+        NETWORK: 'mainnet',
+        BACKEND: 'none',
+        BLOCKS_SUMMARIES_INDEXING: false,
+        HTTP_PORT: 8999,
+        SPAWN_CLUSTER_PROCS: 0,
+        API_URL_PREFIX: '/api/v1/',
+        AUTOMATIC_BLOCK_REINDEXING: false,
+        POLL_RATE_MS: 2000,
+        CACHE_DIR: './cache',
+        CLEAR_PROTECTION_MINUTES: 20,
+        RECOMMENDED_FEE_PERCENTILE: 50,
+        BLOCK_WEIGHT_UNITS: 4000000,
+        INITIAL_BLOCKS_AMOUNT: 8,
+        MEMPOOL_BLOCKS_AMOUNT: 8,
+        INDEXING_BLOCKS_AMOUNT: 11000,
+        PRICE_FEED_UPDATE_INTERVAL: 600,
+        USE_SECOND_NODE_FOR_MINFEE: false,
+        EXTERNAL_ASSETS: [],
+        EXTERNAL_MAX_RETRY: 1,
+        EXTERNAL_RETRY_INTERVAL: 0,
+        USER_AGENT: 'mempool',
+        STDOUT_LOG_MIN_PRIORITY: 'debug',
+        POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master',
+        POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json'
+      });
+
+      expect(config.ELECTRUM).toStrictEqual({ HOST: '127.0.0.1', PORT: 3306, TLS_ENABLED: true });
+
+      expect(config.ESPLORA).toStrictEqual({ REST_API_URL: 'http://127.0.0.1:3000' });
+
+      expect(config.CORE_RPC).toStrictEqual({
+        HOST: '127.0.0.1',
+        PORT: 8332,
+        USERNAME: 'mempool',
+        PASSWORD: 'mempool'
+      });
+
+      expect(config.SECOND_CORE_RPC).toStrictEqual({
+        HOST: '127.0.0.1',
+        PORT: 8332,
+        USERNAME: 'mempool',
+        PASSWORD: 'mempool'
+      });
+
+      expect(config.DATABASE).toStrictEqual({
+        ENABLED: true,
+        HOST: '127.0.0.1',
+        SOCKET: '',
+        PORT: 3306,
+        DATABASE: 'mempool',
+        USERNAME: 'mempool',
+        PASSWORD: 'mempool'
+      });
+
+      expect(config.SYSLOG).toStrictEqual({
+        ENABLED: true,
+        HOST: '127.0.0.1',
+        PORT: 514,
+        MIN_PRIORITY: 'info',
+        FACILITY: 'local7'
+      });
+
+      expect(config.STATISTICS).toStrictEqual({ ENABLED: true, TX_PER_SECOND_SAMPLE_PERIOD: 150 });
+
+      expect(config.BISQ).toStrictEqual({ ENABLED: false, DATA_PATH: '/bisq/statsnode-data/btc_mainnet/db' });
+
+      expect(config.SOCKS5PROXY).toStrictEqual({
+        ENABLED: false,
+        USE_ONION: true,
+        HOST: '127.0.0.1',
+        PORT: 9050,
+        USERNAME: '',
+        PASSWORD: ''
+      });
+
+      expect(config.PRICE_DATA_SERVER).toStrictEqual({
+        TOR_URL: 'http://wizpriceje6q5tdrxkyiazsgu7irquiqjy2dptezqhrtu7l2qelqktid.onion/getAllMarketPrices',
+        CLEARNET_URL: 'https://price.bisq.wiz.biz/getAllMarketPrices'
+      });
+
+      expect(config.EXTERNAL_DATA_SERVER).toStrictEqual({
+        MEMPOOL_API: 'https://mempool.space/api/v1',
+        MEMPOOL_ONION: 'http://mempoolhqx4isw62xs7abwphsq7ldayuidyx2v2oethdhhj6mlo2r6ad.onion/api/v1',
+        LIQUID_API: 'https://liquid.network/api/v1',
+        LIQUID_ONION: 'http://liquidmom47f6s3m53ebfxn47p76a6tlnxib3wp6deux7wuzotdr6cyd.onion/api/v1',
+        BISQ_URL: 'https://bisq.markets/api',
+        BISQ_ONION: 'http://bisqmktse2cabavbr2xjq7xw3h6g5ottemo5rolfcwt6aly6tp5fdryd.onion/api'
+      });
+    });
+  });
+
+  test('should override the default values with the passed values', () => {
+    jest.isolateModules(() => {
+      const fixture = JSON.parse(fs.readFileSync(`${__dirname}/../__fixtures__/mempool-config.template.json`, 'utf8'));
+      jest.mock('../../mempool-config.json', () => (fixture), { virtual: true });
+
+      const config = jest.requireActual('../config').default;
+
+      expect(config.MEMPOOL).toStrictEqual(fixture.MEMPOOL);
+
+      expect(config.ELECTRUM).toStrictEqual(fixture.ELECTRUM);
+
+      expect(config.ESPLORA).toStrictEqual(fixture.ESPLORA);
+
+      expect(config.CORE_RPC).toStrictEqual(fixture.CORE_RPC);
+
+      expect(config.SECOND_CORE_RPC).toStrictEqual(fixture.SECOND_CORE_RPC);
+
+      expect(config.DATABASE).toStrictEqual(fixture.DATABASE);
+
+      expect(config.SYSLOG).toStrictEqual(fixture.SYSLOG);
+
+      expect(config.STATISTICS).toStrictEqual(fixture.STATISTICS);
+
+      expect(config.BISQ).toStrictEqual(fixture.BISQ);
+
+      expect(config.SOCKS5PROXY).toStrictEqual(fixture.SOCKS5PROXY);
+
+      expect(config.PRICE_DATA_SERVER).toStrictEqual(fixture.PRICE_DATA_SERVER);
+
+      expect(config.EXTERNAL_DATA_SERVER).toStrictEqual(fixture.EXTERNAL_DATA_SERVER);
+    });
+  });
+
+});

backend/src/api/common.ts

@@ -1,5 +1,7 @@
 import { CpfpInfo, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
 import config from '../config';
+import { NodeSocket } from '../repositories/NodesSocketsRepository';
+import { isIP } from 'net';
 export class Common {
   static nativeAssetId = config.MEMPOOL.NETWORK === 'liquidtestnet' ?
     '144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49'
@@ -221,4 +223,35 @@ export class Common {
     const d = new Date((date || 0) * 1000);
     return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
   }
+
+  static formatSocket(publicKey: string, socket: {network: string, addr: string}): NodeSocket {
+    let network: string | null = null;
+
+    if (config.LIGHTNING.BACKEND === 'cln') {
+      network = socket.network;
+    } else if (config.LIGHTNING.BACKEND === 'lnd') {
+      if (socket.addr.indexOf('onion') !== -1) {
+        if (socket.addr.split('.')[0].length >= 56) {
+          network = 'torv3';
+        } else {
+          network = 'torv2';
+        }
+      } else if (socket.addr.indexOf('i2p') !== -1) {
+        network = 'i2p';
+      } else {
+        const ipv = isIP(socket.addr.split(':')[0]);
+        if (ipv === 4) {
+          network = 'ipv4';
+        } else if (ipv === 6) {
+          network = 'ipv6';
+        }
+      }
+    }
+
+    return {
+      publicKey: publicKey,
+      network: network,
+      addr: socket.addr,
+    };
+  }
 }

backend/src/api/database-migration.ts

@@ -4,7 +4,7 @@ import logger from '../logger';
 import { Common } from './common';
 
 class DatabaseMigration {
-  private static currentVersion = 36;
+  private static currentVersion = 37;
   private queryTimeout = 120000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -324,6 +324,10 @@ class DatabaseMigration {
     if (databaseSchemaVersion < 36 && isBitcoin == true) {
       await this.$executeQuery('ALTER TABLE `nodes` ADD status TINYINT NOT NULL DEFAULT "1"');
     }
+
+    if (databaseSchemaVersion < 37 && isBitcoin == true) {
+      await this.$executeQuery(this.getCreateLNNodesSocketsTableQuery(), await this.$checkIfTableExists('nodes_sockets'));
+    }
   }
 
   /**
@@ -737,7 +741,7 @@ class DatabaseMigration {
       names text DEFAULT NULL,
       UNIQUE KEY id (id,type),
       KEY id_2 (id)
-    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
   private getCreateBlocksPricesTableQuery(): string {
@@ -749,6 +753,16 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getCreateLNNodesSocketsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS nodes_sockets (
+      public_key varchar(66) NOT NULL,
+      socket varchar(100) NOT NULL,
+      type enum('ipv4', 'ipv6', 'torv2', 'torv3', 'i2p', 'dns', 'websocket') NULL,
+      UNIQUE KEY public_key_socket (public_key, socket),
+      INDEX (public_key)
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
   public async $truncateIndexedData(tables: string[]) {
     const allowedTables = ['blocks', 'hashrates', 'prices'];
 

backend/src/api/lightning/clightning/clightning-convert.ts

@@ -17,7 +17,7 @@ export function convertNode(clNode: any): ILightningApi.Node {
         network: addr.type,
         addr: `${addr.address}:${addr.port}`
       };
-    }),
+    }) ?? [],
     last_update: clNode?.last_timestamp ?? 0,
   };
 }

backend/src/repositories/BlocksAuditsRepository.ts

@@ -1,4 +1,3 @@
-import transactionUtils from '../api/transaction-utils';
 import DB from '../database';
 import logger from '../logger';
 import { BlockAudit } from '../mempool.interfaces';

backend/src/repositories/NodesSocketsRepository.ts

@@ -0,0 +1,45 @@
+import { ResultSetHeader } from 'mysql2';
+import DB from '../database';
+import logger from '../logger';
+
+export interface NodeSocket {
+  publicKey: string;
+  network: string | null;
+  addr: string;
+}
+
+class NodesSocketsRepository {
+  public async $saveSocket(socket: NodeSocket): Promise<void> {
+    try {
+      await DB.query(`
+        INSERT INTO nodes_sockets(public_key, socket, type)
+        VALUE (?, ?, ?)
+      `, [socket.publicKey, socket.addr, socket.network]);
+    } catch (e: any) {
+      if (e.errno !== 1062) { // ER_DUP_ENTRY - Not an issue, just ignore this
+        logger.err(`Cannot save node socket (${[socket.publicKey, socket.addr, socket.network]}) into db. Reason: ` + (e instanceof Error ? e.message : e));
+        // We don't throw, not a critical issue if we miss some nodes sockets
+      }
+    }
+   }
+
+   public async $deleteUnusedSockets(publicKey: string, addresses: string[]): Promise<number> {
+    if (addresses.length === 0) {
+      return 0;
+    }
+    try {
+      const query = `
+        DELETE FROM nodes_sockets
+        WHERE public_key = ?
+        AND socket NOT IN (${addresses.map(id => `"${id}"`).join(',')})
+      `;
+      const [result] = await DB.query<ResultSetHeader>(query, [publicKey]);
+      return result.affectedRows;
+    } catch (e) {
+      logger.err(`Cannot delete unused sockets for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e));
+      return 0;
+    }
+   }
+}
+
+export default new NodesSocketsRepository();

backend/src/tasks/lightning/network-sync.service.ts

@@ -10,6 +10,8 @@ import lightningApi from '../../api/lightning/lightning-api-factory';
 import nodesApi from '../../api/explorer/nodes.api';
 import { ResultSetHeader } from 'mysql2';
 import fundingTxFetcher from './sync-tasks/funding-tx-fetcher';
+import NodesSocketsRepository from '../../repositories/NodesSocketsRepository';
+import { Common } from '../../api/common';
 
 class NetworkSyncService {
   loggerTimer = 0;
@@ -58,6 +60,7 @@ class NetworkSyncService {
   private async $updateNodesList(nodes: ILightningApi.Node[]): Promise<void> {
     let progress = 0;
 
+    let deletedSockets = 0;
     const graphNodesPubkeys: string[] = [];
     for (const node of nodes) {
       await nodesApi.$saveNode(node);
@@ -69,8 +72,15 @@ class NetworkSyncService {
         logger.info(`Updating node ${progress}/${nodes.length}`);
         this.loggerTimer = new Date().getTime() / 1000;
       }
+
+      const addresses: string[] = [];
+      for (const socket of node.addresses) {
+        await NodesSocketsRepository.$saveSocket(Common.formatSocket(node.pub_key, socket));
+        addresses.push(socket.addr);
+      }
+      deletedSockets += await NodesSocketsRepository.$deleteUnusedSockets(node.pub_key, addresses);
     }
-    logger.info(`${progress} nodes updated`);
+    logger.info(`${progress} nodes updated. ${deletedSockets} sockets deleted`);
 
     // If a channel if not present in the graph, mark it as inactive
     nodesApi.$setNodesInactive(graphNodesPubkeys);

backend/tsconfig.build.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig",
+  "exclude": ["**/*.test.*", "**/__mocks__/*", "**/__tests__/*"],
+  "compilerOptions": {
+    "types": ["node"]
+  },
+
+}

backend/tsconfig.json

@@ -1,8 +1,8 @@
 {
   "compilerOptions": {
-    "types": ["node"],
     "module": "commonjs",
     "target": "esnext",
+    "types": ["node", "jest"],
     "lib": ["es2019", "dom"],
     "strict": true,
     "noImplicitAny": false,

frontend/src/app/components/address/address-preview.component.html

@@ -44,7 +44,7 @@
     <div class="w-100 d-block d-md-none"></div>
     <div class="col-md qrcode-col">
       <div class="qr-wrapper">
-        <app-qrcode [data]="address.address" [size]="370"></app-qrcode>
+        <app-qrcode [data]="address.address" [size]="448"></app-qrcode>
       </div>
     </div>
   </div>

frontend/src/app/components/address/address-preview.component.scss

@@ -1,5 +1,5 @@
 h1 {
-  font-size: 42px;
+  font-size: 52px;
   margin: 0;
 }
 
@@ -11,23 +11,26 @@ h1 {
 }
 
 .qrcode-col {
-  width: 420px;
+  width: 468px;
-  min-width: 420px;
+  min-width: 468px;
   flex-grow: 0;
   flex-shrink: 0;
   text-align: center;
+  padding: 0;
+  margin-left: 2px;
+  margin-right: 15px;
 }
 
 .table {
-  font-size: 24px;
+  font-size: 32px;
 
   ::ng-deep .symbol {
-    font-size: 18px;
+    font-size: 24px;
   }
 }
 
 .address-link {
-  font-size: 20px;
+  font-size: 24px;
   margin-bottom: 0.5em;
   display: flex;
   flex-direction: row;
@@ -35,7 +38,7 @@ h1 {
   .truncated-address {
     text-overflow: ellipsis;
     overflow: hidden;
-    max-width: calc(505px - 4em);
+    max-width: calc(640px - 4em);
     display: inline-block;
     white-space: nowrap;
   }

frontend/src/app/components/address/address-preview.component.ts

@@ -44,7 +44,6 @@ export class AddressPreviewComponent implements OnInit, OnDestroy {
   ) { }
 
   ngOnInit() {
-    this.openGraphService.setPreviewLoading();
     this.stateService.networkChanged$.subscribe((network) => this.network = network);
 
     this.addressLoadingStatus$ = this.route.paramMap
@@ -56,6 +55,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy {
     this.mainSubscription = this.route.paramMap
       .pipe(
         switchMap((params: ParamMap) => {
+          this.openGraphService.waitFor('address-data');
           this.error = undefined;
           this.isLoadingAddress = true;
           this.loadedConfirmedTxCount = 0;
@@ -73,6 +73,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy {
                 this.isLoadingAddress = false;
                 this.error = err;
                 console.log(err);
+                this.openGraphService.fail('address-data');
                 return of(null);
               })
             );
@@ -90,7 +91,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy {
           this.address = address;
           this.updateChainStats();
           this.isLoadingAddress = false;
-          this.openGraphService.setPreviewReady();
+          this.openGraphService.waitOver('address-data');
         })
       )
       .subscribe(() => {},
@@ -98,6 +99,7 @@ export class AddressPreviewComponent implements OnInit, OnDestroy {
           console.log(error);
           this.error = error;
           this.isLoadingAddress = false;
+          this.openGraphService.fail('address-data');
         }
       );
   }

frontend/src/app/components/block-overview-graph/block-overview-graph.component.ts

@@ -18,6 +18,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy {
   @Input() orientation = 'left';
   @Input() flip = true;
   @Output() txClickEvent = new EventEmitter<TransactionStripped>();
+  @Output() readyEvent = new EventEmitter();
 
   @ViewChild('blockCanvas')
   canvas: ElementRef<HTMLCanvasElement>;
@@ -37,6 +38,8 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy {
   selectedTx: TxView | void;
   tooltipPosition: Position;
 
+  readyNextFrame = false;
+
   constructor(
     readonly ngZone: NgZone,
     readonly elRef: ElementRef,
@@ -78,6 +81,7 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy {
   setup(transactions: TransactionStripped[]): void {
     if (this.scene) {
       this.scene.setup(transactions);
+      this.readyNextFrame = true;
       this.start();
     }
   }
@@ -258,6 +262,11 @@ export class BlockOverviewGraphComponent implements AfterViewInit, OnDestroy {
           this.gl.drawArrays(this.gl.TRIANGLES, 0, pointArray.length / TxSprite.vertexSize);
         }
       }
+
+      if (this.readyNextFrame) {
+        this.readyNextFrame = false;
+        this.readyEvent.emit();
+      }
     }
 
     /* LOOP */

frontend/src/app/components/block/block-preview.component.html

@@ -30,44 +30,42 @@
             <td i18n="block.weight">Weight</td>
             <td [innerHTML]="'&lrm;' + (block?.weight | wuBytes: 2)"></td>
           </tr>
-          <ng-template [ngIf]="webGlEnabled">
+          <tr *ngIf="block?.extras?.medianFee != undefined">
-            <tr *ngIf="block?.extras?.medianFee != undefined">
+            <td class="td-width" i18n="block.median-fee">Median fee</td>
-              <td class="td-width" i18n="block.median-fee">Median fee</td>
+            <td>~{{ block?.extras?.medianFee | number:'1.0-0' }} <span class="symbol" i18n="shared.sat-vbyte|sat/vB">sat/vB</span></td>
-              <td>~{{ block?.extras?.medianFee | number:'1.0-0' }} <span class="symbol" i18n="shared.sat-vbyte|sat/vB">sat/vB</span></td>
+          </tr>
-            </tr>
+          <ng-template [ngIf]="fees !== undefined">
-            <ng-template [ngIf]="fees !== undefined">
+            <tr>
-              <tr>
+              <td i18n="block.total-fees|Total fees in a block">Total fees</td>
-                <td i18n="block.total-fees|Total fees in a block">Total fees</td>
+              <td *ngIf="network !== 'liquid' && network !== 'liquidtestnet'; else liquidTotalFees">
-                <td *ngIf="network !== 'liquid' && network !== 'liquidtestnet'; else liquidTotalFees">
+                <app-amount [satoshis]="block?.extras.totalFees" digitsInfo="1.2-3" [noFiat]="true"></app-amount>
-                  <app-amount [satoshis]="block?.extras.totalFees" digitsInfo="1.2-3" [noFiat]="true"></app-amount>
+              </td>
+              <ng-template #liquidTotalFees>
+                <td>
+                  <app-amount [satoshis]="fees * 100000000" digitsInfo="1.2-2" [noFiat]="true"></app-amount>
                 </td>
-                <ng-template #liquidTotalFees>
+              </ng-template>
-                  <td>
-                    <app-amount [satoshis]="fees * 100000000" digitsInfo="1.2-2" [noFiat]="true"></app-amount>
-                  </td>
-                </ng-template>
-              </tr>
-            </ng-template>
-            <tr *ngIf="network !== 'liquid' && network !== 'liquidtestnet'">
-              <td i18n="block.miner">Miner</td>
-              <td *ngIf="stateService.env.MINING_DASHBOARD">
-                <a [attr.data-cy]="'block-details-miner-badge'" placement="bottom" [routerLink]="['/mining/pool' | relativeUrl, block?.extras.pool.slug]" class="badge"
-                  [class]="block?.extras.pool.name === 'Unknown' ? 'badge-secondary' : 'badge-primary'">
-                  {{ block?.extras.pool.name }}
-                </a>
-              </td>
-              <td *ngIf="!stateService.env.MINING_DASHBOARD && stateService.env.BASE_MODULE === 'mempool'">
-                <span [attr.data-cy]="'block-details-miner-badge'" placement="bottom" class="badge"
-                  [class]="block?.extras.pool.name === 'Unknown' ? 'badge-secondary' : 'badge-primary'">
-                  {{ block?.extras.pool.name }}
-              </span>
-              </td>
             </tr>
           </ng-template>
+          <tr *ngIf="network !== 'liquid' && network !== 'liquidtestnet'">
+            <td i18n="block.miner">Miner</td>
+            <td *ngIf="stateService.env.MINING_DASHBOARD">
+              <a [attr.data-cy]="'block-details-miner-badge'" placement="bottom" [routerLink]="['/mining/pool' | relativeUrl, block?.extras.pool.slug]" class="badge"
+                [class]="block?.extras.pool.name === 'Unknown' ? 'badge-secondary' : 'badge-primary'">
+                {{ block?.extras.pool.name }}
+              </a>
+            </td>
+            <td *ngIf="!stateService.env.MINING_DASHBOARD && stateService.env.BASE_MODULE === 'mempool'">
+              <span [attr.data-cy]="'block-details-miner-badge'" placement="bottom" class="badge"
+                [class]="block?.extras.pool.name === 'Unknown' ? 'badge-secondary' : 'badge-primary'">
+                {{ block?.extras.pool.name }}
+            </span>
+            </td>
+          </tr>
         </tbody>
       </table>
     </div>
-    <div class="col-sm chart-container" *ngIf="webGlEnabled">
+    <div class="col-sm chart-container">
       <app-block-overview-graph
         #blockGraph
         [isLoading]="false"
@@ -75,7 +73,7 @@
         [blockLimit]="stateService.blockVSize"
         [orientation]="'top'"
         [flip]="false"
-        (txClickEvent)="onTxClick($event)"
+        (readyEvent)="onGraphReady()"
       ></app-block-overview-graph>
     </div>
   </div>

frontend/src/app/components/block/block-preview.component.scss

@@ -1,23 +1,25 @@
 .block-title {
-  margin-bottom: 0.75em;
+  margin-bottom: 48px;
-  font-size: 42px;
+  font-size: 52px;
 
   ::ng-deep .next-previous-blocks {
-    font-size: 42px;
+    font-size: 52px;
   }
 }
 
 .table {
-  font-size: 24px;
+  font-size: 32px;
 }
 
 .chart-container {
   flex-grow: 0;
   flex-shrink: 0;
-  width: 420px;
+  width: 470px;
-  min-width: 420px;
+  min-width: 470px;
+  padding: 0;
+  margin-right: 15px;
 }
 
 ::ng-deep .symbol {
-  font-size: 18px;
+  font-size: 24px;
 }

frontend/src/app/components/block/block-preview.component.ts

@@ -1,11 +1,168 @@
-import { Component } from '@angular/core';
+import { Component, OnInit, OnDestroy, ViewChild, ElementRef } from '@angular/core';
-import { BlockComponent } from './block.component';
+import { ActivatedRoute, ParamMap } from '@angular/router';
+import { ElectrsApiService } from '../../services/electrs-api.service';
+import { switchMap, tap, throttleTime, catchError, shareReplay, startWith, pairwise, filter } from 'rxjs/operators';
+import { of, Subscription, asyncScheduler } from 'rxjs';
+import { StateService } from '../../services/state.service';
+import { SeoService } from 'src/app/services/seo.service';
+import { OpenGraphService } from 'src/app/services/opengraph.service';
+import { BlockExtended, TransactionStripped } from 'src/app/interfaces/node-api.interface';
+import { ApiService } from 'src/app/services/api.service';
+import { BlockOverviewGraphComponent } from 'src/app/components/block-overview-graph/block-overview-graph.component';
 
 @Component({
   selector: 'app-block-preview',
   templateUrl: './block-preview.component.html',
-  styleUrls: ['./block.component.scss', './block-preview.component.scss']
+  styleUrls: ['./block-preview.component.scss']
 })
-export class BlockPreviewComponent extends BlockComponent {
+export class BlockPreviewComponent implements OnInit, OnDestroy {
+  network = '';
+  block: BlockExtended;
+  blockHeight: number;
+  blockHash: string;
+  isLoadingBlock = true;
+  strippedTransactions: TransactionStripped[];
+  overviewTransitionDirection: string;
+  isLoadingOverview = true;
+  error: any;
+  blockSubsidy: number;
+  fees: number;
+  overviewError: any = null;
 
+  overviewSubscription: Subscription;
+  networkChangedSubscription: Subscription;
+
+  @ViewChild('blockGraph') blockGraph: BlockOverviewGraphComponent;
+
+  constructor(
+    private route: ActivatedRoute,
+    private electrsApiService: ElectrsApiService,
+    public stateService: StateService,
+    private seoService: SeoService,
+    private openGraphService: OpenGraphService,
+    private apiService: ApiService
+  ) { }
+
+  ngOnInit() {
+    this.network = this.stateService.network;
+
+    const block$ = this.route.paramMap.pipe(
+      switchMap((params: ParamMap) => {
+        this.openGraphService.waitFor('block-viz');
+        this.openGraphService.waitFor('block-data');
+
+        const blockHash: string = params.get('id') || '';
+        this.block = undefined;
+        this.error = undefined;
+        this.overviewError = undefined;
+        this.fees = undefined;
+
+        let isBlockHeight = false;
+        if (/^[0-9]+$/.test(blockHash)) {
+          isBlockHeight = true;
+        } else {
+          this.blockHash = blockHash;
+        }
+
+        this.isLoadingBlock = true;
+        this.isLoadingOverview = true;
+
+        if (isBlockHeight) {
+          return this.electrsApiService.getBlockHashFromHeight$(parseInt(blockHash, 10))
+            .pipe(
+              switchMap((hash) => {
+                if (hash) {
+                  this.blockHash = hash;
+                  return this.apiService.getBlock$(hash);
+                } else {
+                  return null;
+                }
+              }),
+              catchError((err) => {
+                this.error = err;
+                this.openGraphService.fail('block-data');
+                this.openGraphService.fail('block-viz');
+                return of(null);
+              }),
+            );
+        }
+        return this.apiService.getBlock$(blockHash);
+      }),
+      filter((block: BlockExtended | void) => block != null),
+      tap((block: BlockExtended) => {
+        this.block = block;
+        this.blockHeight = block.height;
+
+        this.seoService.setTitle($localize`:@@block.component.browser-title:Block ${block.height}:BLOCK_HEIGHT:: ${block.id}:BLOCK_ID:`);
+        this.isLoadingBlock = false;
+        this.setBlockSubsidy();
+        if (block?.extras?.reward !== undefined) {
+          this.fees = block.extras.reward / 100000000 - this.blockSubsidy;
+        }
+        this.stateService.markBlock$.next({ blockHeight: this.blockHeight });
+        this.isLoadingOverview = true;
+        this.overviewError = null;
+
+        this.openGraphService.waitOver('block-data');
+      }),
+      throttleTime(50, asyncScheduler, { leading: true, trailing: true }),
+      shareReplay(1)
+    );
+
+    this.overviewSubscription = block$.pipe(
+      startWith(null),
+      pairwise(),
+      switchMap(([prevBlock, block]) => this.apiService.getStrippedBlockTransactions$(block.id)
+        .pipe(
+          catchError((err) => {
+            this.overviewError = err;
+            this.openGraphService.fail('block-viz');
+            return of([]);
+          }),
+          switchMap((transactions) => {
+            return of({ transactions, direction: 'down' });
+          })
+        )
+      ),
+    )
+    .subscribe(({transactions, direction}: {transactions: TransactionStripped[], direction: string}) => {
+      this.strippedTransactions = transactions;
+      this.isLoadingOverview = false;
+      if (this.blockGraph) {
+        this.blockGraph.destroy();
+        this.blockGraph.setup(this.strippedTransactions);
+      }
+    },
+    (error) => {
+      this.error = error;
+      this.isLoadingOverview = false;
+      this.openGraphService.fail('block-viz');
+      this.openGraphService.fail('block-data');
+      if (this.blockGraph) {
+        this.blockGraph.destroy();
+      }
+    });
+
+    this.networkChangedSubscription = this.stateService.networkChanged$
+      .subscribe((network) => this.network = network);
+  }
+
+  ngOnDestroy() {
+    if (this.overviewSubscription) {
+      this.overviewSubscription.unsubscribe();
+    }
+    if (this.networkChangedSubscription) {
+      this.networkChangedSubscription.unsubscribe();
+    }
+  }
+
+  // TODO - Refactor this.fees/this.reward for liquid because it is not
+  // used anymore on Bitcoin networks (we use block.extras directly)
+  setBlockSubsidy() {
+    this.blockSubsidy = 0;
+  }
+
+  onGraphReady(): void {
+    this.openGraphService.waitOver('block-viz');
+  }
 }

frontend/src/app/components/master-page-preview/master-page-preview.component.html

@@ -1,21 +1,20 @@
 <ng-container *ngIf="{ val: network$ | async } as network">
 <div class="preview-wrapper">
-  <router-outlet></router-outlet>
+  <header>
-
+    <span class="header-brand" style="position: relative;">
-  <footer>
+      <img *ngIf="!officialMempoolSpace" src="/resources/mempool-logo.png" height="50" width="200" class="logo">
-    <span class="footer-brand" style="position: relative;">
+      <app-svg-images *ngIf="officialMempoolSpace" name="officialMempoolSpace" style="width: 200px; height: 50px" width="500" height="126" viewBox="0 0 500 126"></app-svg-images>
-      <img *ngIf="!officialMempoolSpace" src="/resources/mempool-logo.png" height="35" width="140" class="logo">
-      <app-svg-images *ngIf="officialMempoolSpace" name="officialMempoolSpace" style="width: 140px; height: 35px" width="500" height="126" viewBox="0 0 500 126"></app-svg-images>
     </span>
 
     <div [ngSwitch]="network.val">
-      <span *ngSwitchCase="'signet'" class="network signet"><img src="/resources/signet-logo.png" style="width: 30px;" class="signet mr-1" alt="logo"> Signet</span>
+      <span *ngSwitchCase="'signet'" class="network signet"><img src="/resources/signet-logo.png" style="width: 45px;" class="signet mr-1" alt="logo"> Signet</span>
-      <span *ngSwitchCase="'testnet'" class="network testnet"><img src="/resources/testnet-logo.png" style="width: 30px;" class="mr-1" alt="testnet logo"> Testnet</span>
+      <span *ngSwitchCase="'testnet'" class="network testnet"><img src="/resources/testnet-logo.png" style="width: 45px;" class="mr-1" alt="testnet logo"> Testnet</span>
-      <span *ngSwitchCase="'bisq'" class="network bisq"><img src="/resources/bisq-logo.png" style="width: 30px;" class="mr-1" alt="bisq logo"> Bisq</span>
+      <span *ngSwitchCase="'bisq'" class="network bisq"><img src="/resources/bisq-logo.png" style="width: 45px;" class="mr-1" alt="bisq logo"> Bisq</span>
-      <span *ngSwitchCase="'liquid'" class="network liquid"><img src="/resources/liquid-logo.png" style="width: 30px;" class="mr-1" alt="liquid mainnet logo"> Liquid</span>
+      <span *ngSwitchCase="'liquid'" class="network liquid"><img src="/resources/liquid-logo.png" style="width: 45px;" class="mr-1" alt="liquid mainnet logo"> Liquid</span>
-      <span *ngSwitchCase="'liquidtestnet'" class="network liquidtestnet"><img src="/resources/liquidtestnet-logo.png" style="width: 30px;" class="mr-1" alt="liquid testnet logo"> Liquid Testnet</span>
+      <span *ngSwitchCase="'liquidtestnet'" class="network liquidtestnet"><img src="/resources/liquidtestnet-logo.png" style="width: 45px;" class="mr-1" alt="liquid testnet logo"> Liquid Testnet</span>
-      <span *ngSwitchDefault class="network mainnet"><img src="/resources/bitcoin-logo.png" style="width: 30px;" class="mainnet mr-1" alt="bitcoin logo"> Mainnet</span>
+      <span *ngSwitchDefault class="network mainnet"><img src="/resources/bitcoin-logo.png" style="width: 45px;" class="mainnet mr-1" alt="bitcoin logo"> Mainnet</span>
     </div>
-  </footer>
+  </header>
+  <router-outlet></router-outlet>
 </div>
 </ng-container>

frontend/src/app/components/master-page-preview/master-page-preview.component.scss

@@ -2,28 +2,28 @@
   position: relative;
   display: block;
   margin: auto;
-  max-width: 1024px;
+  max-width: 1200px;
-  max-height: 512px;
+  max-height: 600px;
-  padding-bottom: 64px;
+  padding-top: 80px;
 
-  footer {
+  header {
     position: absolute;
     left: 0;
     right: 0;
-    bottom: 0;
+    top: 0;
     z-index: 100;
-    min-height: 64px;
+    min-height: 80px;
-    padding: 0rem 2rem;
+    padding: 0rem 3rem;
     display: flex;
     flex-direction: row;
     justify-content: space-between;
     align-items: center;
     background: #11131f;
     text-align: start;
-    font-size: 1.2em;
+    font-size: 1.8em;
   }
 
-  .footer-brand {
+  .header-brand {
     width: 60%;
   }
 

frontend/src/app/components/pool-ranking/pool-ranking.component.ts

@@ -12,6 +12,7 @@ import { StateService } from '../../services/state.service';
 import { chartColors, poolsColor } from 'src/app/app.constants';
 import { RelativeUrlPipe } from 'src/app/shared/pipes/relative-url/relative-url.pipe';
 import { download } from 'src/app/shared/graphs.utils';
+import { isMobile } from 'src/app/shared/common.utils';
 
 @Component({
   selector: 'app-pool-ranking',
@@ -108,21 +109,23 @@ export class PoolRankingComponent implements OnInit {
     return pool;
   }
 
-  isMobile() {
-    return (window.innerWidth <= 767.98);
-  }
-
   generatePoolsChartSerieData(miningStats) {
-    const poolShareThreshold = this.isMobile() ? 2 : 1; // Do not draw pools which hashrate share is lower than that
+    let poolShareThreshold = 0.5;
+    if (isMobile()) {
+      poolShareThreshold = 2;
+    } else if (this.widget) {
+      poolShareThreshold = 1;
+    }
+    
     const data: object[] = [];
     let totalShareOther = 0;
     let totalBlockOther = 0;
     let totalEstimatedHashrateOther = 0;
 
     let edgeDistance: any = '20%';
-    if (this.isMobile() && this.widget) {
+    if (isMobile() && this.widget) {
       edgeDistance = 0;
-    } else if (this.isMobile() && !this.widget || this.widget) {
+    } else if (isMobile() && !this.widget || this.widget) {
       edgeDistance = 10;
     }
 
@@ -138,7 +141,7 @@ export class PoolRankingComponent implements OnInit {
           color: poolsColor[pool.name.replace(/[^a-zA-Z0-9]/g, '').toLowerCase()],
         },
         value: pool.share,
-        name: pool.name + ((this.isMobile() || this.widget) ? `` : ` (${pool.share}%)`),
+        name: pool.name + ((isMobile() || this.widget) ? `` : ` (${pool.share}%)`),
         label: {
           overflow: 'none',
           color: '#b1b1b1',
@@ -146,7 +149,7 @@ export class PoolRankingComponent implements OnInit {
           edgeDistance: edgeDistance,
         },
         tooltip: {
-          show: !this.isMobile() || !this.widget,
+          show: !isMobile() || !this.widget,
           backgroundColor: 'rgba(17, 19, 31, 1)',
           borderRadius: 4,
           shadowColor: 'rgba(0, 0, 0, 0.5)',
@@ -176,7 +179,7 @@ export class PoolRankingComponent implements OnInit {
         color: 'grey',
       },
       value: totalShareOther,
-      name: 'Other' + (this.isMobile() ? `` : ` (${totalShareOther.toFixed(2)}%)`),
+      name: 'Other' + (isMobile() ? `` : ` (${totalShareOther.toFixed(2)}%)`),
       label: {
         overflow: 'none',
         color: '#b1b1b1',
@@ -210,7 +213,7 @@ export class PoolRankingComponent implements OnInit {
 
   prepareChartOptions(miningStats) {
     let pieSize = ['20%', '80%']; // Desktop
-    if (this.isMobile() && !this.widget) {
+    if (isMobile() && !this.widget) {
       pieSize = ['15%', '60%'];
     }
 
@@ -226,7 +229,7 @@ export class PoolRankingComponent implements OnInit {
       series: [
         {
           zlevel: 0,
-          minShowLabelAngle: 3.6,
+          minShowLabelAngle: 1.8,
           name: 'Mining pool',
           type: 'pie',
           radius: pieSize,

frontend/src/app/services/opengraph.service.ts

@@ -1,4 +1,4 @@
-import { Injectable } from '@angular/core';
+import { Injectable, NgZone } from '@angular/core';
 import { Meta } from '@angular/platform-browser';
 import { Router, ActivatedRoute, NavigationEnd } from '@angular/router';
 import { filter, map, switchMap } from 'rxjs/operators';
@@ -12,8 +12,11 @@ import { LanguageService } from './language.service';
 export class OpenGraphService {
   network = '';
   defaultImageUrl = '';
+  previewLoadingEvents = {};
+  previewLoadingCount = 0;
 
   constructor(
+    private ngZone: NgZone,
     private metaService: Meta,
     private stateService: StateService,
     private LanguageService: LanguageService,
@@ -39,6 +42,11 @@ export class OpenGraphService {
         this.clearOgImage();
       }
     });
+
+    // expose routing method to global scope, so we can access it from the unfurler
+    window['ogService'] = {
+      loadPage: (path) => { return this.loadPage(path) }
+    };
   }
 
   setOgImage() {
@@ -47,8 +55,8 @@ export class OpenGraphService {
     this.metaService.updateTag({ property: 'og:image', content: ogImageUrl });
     this.metaService.updateTag({ property: 'twitter:image:src', content: ogImageUrl });
     this.metaService.updateTag({ property: 'og:image:type', content: 'image/png' });
-    this.metaService.updateTag({ property: 'og:image:width', content: '1024' });
+    this.metaService.updateTag({ property: 'og:image:width', content: '1200' });
-    this.metaService.updateTag({ property: 'og:image:height', content: '512' });
+    this.metaService.updateTag({ property: 'og:image:height', content: '600' });
   }
 
   clearOgImage() {
@@ -59,13 +67,53 @@ export class OpenGraphService {
     this.metaService.updateTag({ property: 'og:image:height', content: '500' });
   }
 
-  /// signal that the unfurler should wait for a 'ready' signal before taking a screenshot
+  /// register an event that needs to resolve before we can take a screenshot
-  setPreviewLoading() {
+  waitFor(event) {
-    this.metaService.updateTag({ property: 'og:loading', content: 'loading'});
+    if (!this.previewLoadingEvents[event]) {
+      this.previewLoadingEvents[event] = 1;
+      this.previewLoadingCount++;
+    } else {
+      this.previewLoadingEvents[event]++;
+    }
+    this.metaService.updateTag({ property: 'og:preview:loading', content: 'loading'});
   }
 
-  // signal to the unfurler that the page is ready for a screenshot
+  // mark an event as resolved
-  setPreviewReady() {
+  // if all registered events have resolved, signal we are ready for a screenshot
-    this.metaService.updateTag({ property: 'og:ready', content: 'ready'});
+  waitOver(event) {
+    if (this.previewLoadingEvents[event]) {
+      this.previewLoadingEvents[event]--;
+      if (this.previewLoadingEvents[event] === 0) {
+        delete this.previewLoadingEvents[event]
+        this.previewLoadingCount--;
+      }
+    }
+    if (this.previewLoadingCount === 0) {
+      this.metaService.updateTag({ property: 'og:preview:ready', content: 'ready'});
+    }
+  }
+
+  fail(event) {
+    if (this.previewLoadingEvents[event]) {
+      this.metaService.updateTag({ property: 'og:preview:fail', content: 'fail'});
+    }
+  }
+
+  resetLoading() {
+    this.previewLoadingEvents = {};
+    this.previewLoadingCount = 0;
+    this.metaService.removeTag("property='og:preview:loading'");
+    this.metaService.removeTag("property='og:preview:ready'");
+    this.metaService.removeTag("property='og:preview:fail'");
+    this.metaService.removeTag("property='og:meta:ready'");
+  }
+
+  loadPage(path) {
+    if (path !== this.router.url) {
+      this.resetLoading();
+      this.ngZone.run(() => {
+        this.router.navigateByUrl(path);
+      })
+    }
   }
 }

frontend/src/app/services/seo.service.ts

@@ -21,12 +21,14 @@ export class SeoService {
     this.titleService.setTitle(newTitle + ' - ' + this.getTitle());
     this.metaService.updateTag({ property: 'og:title', content: newTitle});
     this.metaService.updateTag({ property: 'twitter:title', content: newTitle});
+    this.metaService.updateTag({ property: 'og:meta:ready', content: 'ready'});
   }
 
   resetTitle(): void {
     this.titleService.setTitle(this.getTitle());
     this.metaService.updateTag({ property: 'og:title', content: this.getTitle()});
     this.metaService.updateTag({ property: 'twitter:title', content: this.getTitle()});
+    this.metaService.updateTag({ property: 'og:meta:ready', content: 'ready'});
   }
 
   setEnterpriseTitle(title: string) {

frontend/src/styles.scss

@@ -88,8 +88,8 @@ body {
 }
 
 .preview-box {
-  min-height: 512px;
+  min-height: 520px;
-  padding: 2rem 3rem;
+  padding: 1.5rem 3rem;
 }
 
 @media (max-width: 767.98px) {

unfurler/config.sample.json

@@ -5,10 +5,13 @@
   },
   "MEMPOOL": {
     "HTTP_HOST": "http://localhost",
-    "HTTP_PORT": 4200
+    "HTTP_PORT": 4200,
+    "NETWORK": "bitcoin" // "bitcoin" | "liquid" | "bisq" (optional - defaults to "bitcoin")
   },
   "PUPPETEER": {
     "CLUSTER_SIZE": 2,
-    "EXEC_PATH": "/usr/local/bin/chrome" // optional
+    "EXEC_PATH": "/usr/local/bin/chrome", // optional
+    "MAX_PAGE_AGE": 86400, // maximum lifetime of a page session (in seconds)
+    "RENDER_TIMEOUT": 3000, // timeout for preview image rendering (in ms) (optional)
   }
 }

unfurler/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mempool-unfurl",
-  "version": "0.0.1",
+  "version": "0.0.2",
   "description": "Renderer for mempool open graph link preview images",
   "repository": {
     "type": "git",

unfurler/puppeteer.config.json

@@ -1,11 +1,11 @@
 {
   "headless": true,
   "defaultViewport": {
-    "width": 1024,
+    "width": 1200,
-    "height": 512
+    "height": 600
   },
   "args": [
-    "--window-size=1024,512",
+    "--window-size=1200,600",
     "--autoplay-policy=user-gesture-required",
     "--disable-background-networking",
     "--disable-background-timer-throttling",

unfurler/src/concurrency/ReusablePage.ts

@@ -0,0 +1,159 @@
+import * as puppeteer from 'puppeteer';
+import ConcurrencyImplementation from 'puppeteer-cluster/dist/concurrency/ConcurrencyImplementation';
+import { timeoutExecute } from 'puppeteer-cluster/dist/util';
+
+import config from '../config';
+const mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : '');
+
+const BROWSER_TIMEOUT = 8000;
+// maximum lifetime of a single page session
+const maxAgeMs = (config.PUPPETEER.MAX_PAGE_AGE || (24 * 60 * 60)) * 1000;
+const maxConcurrency = config.PUPPETEER.CLUSTER_SIZE;
+
+interface RepairablePage extends puppeteer.Page {
+  repairRequested?: boolean;
+  language?: string | null;
+  createdAt?: number;
+  free?: boolean;
+  index?: number;
+}
+
+interface ResourceData {
+  page: RepairablePage;
+}
+
+export default class ReusablePage extends ConcurrencyImplementation {
+
+  protected browser: puppeteer.Browser | null = null;
+  protected pages: RepairablePage[] = [];
+  private repairing: boolean = false;
+  private repairRequested: boolean = false;
+  private openInstances: number = 0;
+  private waitingForRepairResolvers: (() => void)[] = [];
+
+  public constructor(options: puppeteer.LaunchOptions, puppeteer: any) {
+    super(options, puppeteer);
+  }
+
+  private async repair() {
+    if (this.openInstances !== 0 || this.repairing) {
+      // already repairing or there are still pages open? wait for start/finish
+      await new Promise<void>(resolve => this.waitingForRepairResolvers.push(resolve));
+      return;
+    }
+
+    this.repairing = true;
+    console.log('Starting repair');
+
+    try {
+      // will probably fail, but just in case the repair was not necessary
+      await (<puppeteer.Browser>this.browser).close();
+    } catch (e) {
+      console.log('Unable to close browser.');
+    }
+
+    try {
+      await this.init();
+    } catch (err) {
+      throw new Error('Unable to restart chrome.');
+    }
+    this.repairRequested = false;
+    this.repairing = false;
+    this.waitingForRepairResolvers.forEach(resolve => resolve());
+    this.waitingForRepairResolvers = [];
+  }
+
+  public async init() {
+    this.browser = await this.puppeteer.launch(this.options);
+    const promises = []
+    for (let i = 0; i < maxConcurrency; i++) {
+      const newPage = await this.initPage();
+      newPage.index = this.pages.length;
+      console.log('initialized page ', newPage.index);
+      this.pages.push(newPage);
+    }
+  }
+
+  public async close() {
+    await (this.browser as puppeteer.Browser).close();
+  }
+
+  protected async initPage(): Promise<RepairablePage> {
+    const page = await (this.browser as puppeteer.Browser).newPage() as RepairablePage;
+    page.language = null;
+    page.createdAt = Date.now();
+    const defaultUrl = mempoolHost + '/preview/block/1';
+    page.on('pageerror', (err) => {
+      page.repairRequested = true;
+    });
+    await page.goto(defaultUrl, { waitUntil: "load" });
+    page.free = true;
+    return page
+  }
+
+  protected async createResources(): Promise<ResourceData> {
+    const page = this.pages.find(p => p.free);
+    if (!page) {
+      console.log('no free pages!')
+      throw new Error('no pages available');
+    } else {
+      page.free = false;
+      return { page };
+    }
+  }
+
+  protected async repairPage(page) {
+    // create a new page
+    const newPage = await this.initPage();
+    newPage.free = true;
+    // replace the old page
+    newPage.index = page.index;
+    this.pages.splice(page.index, 1, newPage);
+    // clean up the old page
+    try {
+      await page.goto('about:blank', {timeout: 200}); // prevents memory leak (maybe?)
+    } catch (e) {
+      console.log('unexpected page repair error');
+    }
+    await page.close();
+    return newPage;
+  }
+
+  public async workerInstance() {
+    let resources: ResourceData;
+
+    return {
+      jobInstance: async () => {
+        await timeoutExecute(BROWSER_TIMEOUT, (async () => {
+          resources = await this.createResources();
+        })());
+        this.openInstances += 1;
+
+        return {
+          resources,
+
+          close: async () => {
+            this.openInstances -= 1; // decrement first in case of error
+            if (resources?.page != null) {
+              if (resources.page.repairRequested || (Date.now() - (resources.page.createdAt || 0) > maxAgeMs)) {
+                resources.page = await this.repairPage(resources.page);
+              } else {
+                resources.page.free = true;
+              }
+            }
+
+            if (this.repairRequested) {
+              await this.repair();
+            }
+          },
+        };
+      },
+
+      close: async () => {},
+
+      repair: async () => {
+        await this.repairPage(resources.page);
+      },
+    };
+  }
+}

unfurler/src/config.ts

@@ -8,10 +8,13 @@ interface IConfig {
   MEMPOOL: {
     HTTP_HOST: string;
     HTTP_PORT: number;
+    NETWORK?: string;
   };
   PUPPETEER: {
     CLUSTER_SIZE: number;
     EXEC_PATH?: string;
+    MAX_PAGE_AGE?: number;
+    RENDER_TIMEOUT?: number;
   };
 }
 

unfurler/src/index.ts

@@ -3,6 +3,8 @@ import { Application, Request, Response, NextFunction } from 'express';
 import * as http from 'http';
 import config from './config';
 import { Cluster } from 'puppeteer-cluster';
+import ReusablePage from './concurrency/ReusablePage';
+import { parseLanguageUrl } from './language/lang';
 const puppeteerConfig = require('../puppeteer.config.json');
 
 if (config.PUPPETEER.EXEC_PATH) {
@@ -14,10 +16,14 @@ class Server {
   private app: Application;
   cluster?: Cluster;
   mempoolHost: string;
+  network: string;
+  defaultImageUrl: string;
 
   constructor() {
     this.app = express();
     this.mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : '');
+    this.network = config.MEMPOOL.NETWORK || 'bitcoin';
+    this.defaultImageUrl = this.getDefaultImageUrl();
     this.startServer();
   }
 
@@ -32,7 +38,7 @@ class Server {
       ;
 
     this.cluster = await Cluster.launch({
-        concurrency: Cluster.CONCURRENCY_CONTEXT,
+        concurrency: ReusablePage,
         maxConcurrency: config.PUPPETEER.CLUSTER_SIZE,
         puppeteerOptions: puppeteerConfig,
     });
@@ -47,63 +53,75 @@ class Server {
     });
   }
 
+  async stopServer() {
+    if (this.cluster) {
+      await this.cluster.idle();
+      await this.cluster.close();
+    }
+    if (this.server) {
+      await this.server.close();
+    }
+  }
+
   setUpRoutes() {
     this.app.get('/render*', async (req, res) => { return this.renderPreview(req, res) })
     this.app.get('*', (req, res) => { return this.renderHTML(req, res) })
   }
 
-  async clusterTask({ page, data: { url, action } }) {
+  async clusterTask({ page, data: { url, path, action } }) {
-    await page.goto(url, { waitUntil: "networkidle0" });
+    try {
-    switch (action) {
+      const urlParts = parseLanguageUrl(path);
-      case 'screenshot': {
+      if (page.language !== urlParts.lang) {
-        await page.evaluate(async () => {
+        // switch language
-          // wait for all images to finish loading
+        page.language = urlParts.lang;
-          const imgs = Array.from(document.querySelectorAll("img"));
+        const localizedUrl = urlParts.lang ? `${this.mempoolHost}/${urlParts.lang}${urlParts.path}` : `${this.mempoolHost}${urlParts.path}` ;
-          await Promise.all([
+        await page.goto(localizedUrl, { waitUntil: "load" });
-            document.fonts.ready,
+      } else {
-            ...imgs.map((img) => {
+        const loaded = await page.evaluate(async (path) => {
-              if (img.complete) {
+          if (window['ogService']) {
-                if (img.naturalHeight !== 0) return;
+            window['ogService'].loadPage(path);
-                throw new Error("Image failed to load");
+            return true;
-              }
+          } else {
-              return new Promise((resolve, reject) => {
+            return false;
-                img.addEventListener("load", resolve);
-                img.addEventListener("error", reject);
-              });
-            }),
-          ]);
-        });
-        const waitForReady = await page.$('meta[property="og:loading"]');
-        const alreadyReady = await page.$('meta[property="og:ready"]');
-        if (waitForReady != null && alreadyReady == null) {
-          try {
-            await page.waitForSelector('meta[property="og:ready]"', { timeout: 10000 });
-          } catch (e) {
-            // probably timed out
           }
+        }, urlParts.path);
+        if (!loaded) {
+          throw new Error('failed to access open graph service');
         }
-        return page.screenshot();
-      } break;
-      default: {
-        try {
-          await page.waitForSelector('meta[property="og:title"]', { timeout: 10000 })
-          const tag = await page.$('meta[property="og:title"]');
-        } catch (e) {
-          // probably timed out
-        }
-        return page.content();
       }
+
+      const waitForReady = await page.$('meta[property="og:preview:loading"]');
+      let success = true;
+      if (waitForReady != null) {
+        success = await Promise.race([
+          page.waitForSelector('meta[property="og:preview:ready"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => true),
+          page.waitForSelector('meta[property="og:preview:fail"]', { timeout: config.PUPPETEER.RENDER_TIMEOUT || 3000 }).then(() => false)
+        ])
+      }
+      if (success) {
+        const screenshot = await page.screenshot();
+        return screenshot;
+      } else {
+        console.log(`failed to render page preview for ${action} due to client-side error. probably requested an invalid ID`);
+        page.repairRequested = true;
+      }
+    } catch (e) {
+      console.log(`failed to render page for ${action}`, e instanceof Error ? e.message : e);
+      page.repairRequested = true;
     }
   }
 
   async renderPreview(req, res) {
     try {
-      // strip default language code for compatibility
+      const path = req.params[0]
-      const path = req.params[0].replace('/en/', '/');
+      const img = await this.cluster?.execute({ url: this.mempoolHost + path, path: path, action: 'screenshot' });
-      const img = await this.cluster?.execute({ url: this.mempoolHost + path, action: 'screenshot' });
 
-      res.contentType('image/png');
+      if (!img) {
-      res.send(img);
+        res.status(500).send('failed to render page preview');
+      } else {
+        res.contentType('image/png');
+        res.send(img);
+      }
     } catch (e) {
       console.log(e);
       res.status(500).send(e instanceof Error ? e.message : e);
@@ -112,22 +130,89 @@ class Server {
 
   async renderHTML(req, res) {
     // drop requests for static files
-    const path = req.params[0];
+    const rawPath = req.params[0];
-    const match = path.match(/\.[\w]+$/);
+    const match = rawPath.match(/\.[\w]+$/);
     if (match?.length && match[0] !== '.html') {
       res.status(404).send();
-      return
+      return;
     }
 
-    try {
+    let previewSupported = true;
-      let html = await this.cluster?.execute({ url: this.mempoolHost + req.params[0], action: 'html' });
+    let mode = 'mainnet'
+    let ogImageUrl = this.defaultImageUrl;
+    let ogTitle;
+    const { lang, path } = parseLanguageUrl(rawPath);
+    const parts = path.slice(1).split('/');
 
-      res.send(html)
+    // handle network mode modifiers
-    } catch (e) {
+    if (['testnet', 'signet'].includes(parts[0])) {
-      console.log(e);
+      mode = parts.shift();
-      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+
+    // handle supported preview routes
+    if (parts[0] === 'block') {
+      ogTitle = `Block: ${parts[1]}`;
+    } else if (parts[0] === 'address') {
+      ogTitle = `Address: ${parts[1]}`;
+    } else {
+      previewSupported = false;
+    }
+
+    if (previewSupported) {
+      ogImageUrl = `${config.SERVER.HOST}/render/${lang || 'en'}/preview${path}`;
+      ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${mode !== 'mainnet' ? capitalize(mode) + ' ' : ''}${ogTitle}`;
+    } else {
+      ogTitle = 'The Mempool Open Source Project™';
+    }
+
+    res.send(`
+      <!doctype html>
+      <html lang="en-US" dir="ltr">
+      <head>
+        <meta charset="utf-8">
+        <title>${ogTitle}</title>
+        <meta name="description" content="The Mempool Open Source Project™ - our self-hosted explorer for the ${capitalize(this.network)} community."/>
+        <meta property="og:image" content="${ogImageUrl}"/>
+        <meta property="og:image:type" content="image/png"/>
+        <meta property="og:image:width" content="${previewSupported ? 1200 : 1000}"/>
+        <meta property="og:image:height" content="${previewSupported ? 600 : 500}"/>
+        <meta property="og:title" content="${ogTitle}">
+        <meta property="twitter:card" content="summary_large_image">
+        <meta property="twitter:site" content="@mempool">
+        <meta property="twitter:creator" content="@mempool">
+        <meta property="twitter:title" content="${ogTitle}">
+        <meta property="twitter:description" content="Our self-hosted mempool explorer for the ${capitalize(this.network)} community."/>
+        <meta property="twitter:image:src" content="${ogImageUrl}"/>
+        <meta property="twitter:domain" content="mempool.space">
+      <body></body>
+      </html>
+    `);
+  }
+
+  getDefaultImageUrl() {
+    switch (this.network) {
+      case 'liquid':
+        return this.mempoolHost + '/resources/liquid/liquid-network-preview.png';
+      case 'bisq':
+        return this.mempoolHost + '/resources/bisq/bisq-markets-preview.png';
+      default:
+        return this.mempoolHost + '/resources/mempool-space-preview.png';
     }
   }
 }
 
 const server = new Server();
+
+process.on('SIGTERM', async () => {
+  console.info('Shutting down Mempool Unfurl Server');
+  await server.stopServer();
+  process.exit(0);
+});
+
+function capitalize(str) {
+  if (str && str.length) {
+    return str[0].toUpperCase() + str.slice(1);
+  } else {
+    return str;
+  }
+}

unfurler/src/language/lang.ts

@@ -0,0 +1,79 @@
+export interface Language {
+  code: string;
+  name: string;
+}
+
+const languageList: Language[] = [
+  { code: 'ar', name: 'العربية' },         // Arabic
+  { code: 'bg', name: 'Български' },       // Bulgarian
+  { code: 'bs', name: 'Bosanski' },        // Bosnian
+  { code: 'ca', name: 'Català' },          // Catalan
+  { code: 'cs', name: 'Čeština' },         // Czech
+  { code: 'da', name: 'Dansk' },           // Danish
+  { code: 'de', name: 'Deutsch' },         // German
+  { code: 'et', name: 'Eesti' },           // Estonian
+  { code: 'el', name: 'Ελληνικά' },        // Greek
+  { code: 'en', name: 'English' },         // English
+  { code: 'es', name: 'Español' },         // Spanish
+  { code: 'eo', name: 'Esperanto' },       // Esperanto
+  { code: 'eu', name: 'Euskara' },         // Basque
+  { code: 'fa', name: 'فارسی' },           // Persian
+  { code: 'fr', name: 'Français' },        // French
+  { code: 'gl', name: 'Galego' },          // Galician
+  { code: 'ko', name: '한국어' },          // Korean
+  { code: 'hr', name: 'Hrvatski' },        // Croatian
+  { code: 'id', name: 'Bahasa Indonesia' },// Indonesian
+  { code: 'hi', name: 'हिन्दी' },             // Hindi
+  { code: 'it', name: 'Italiano' },        // Italian
+  { code: 'he', name: 'עברית' },           // Hebrew
+  { code: 'ka', name: 'ქართული' },         // Georgian
+  { code: 'lv', name: 'Latviešu' },        // Latvian
+  { code: 'lt', name: 'Lietuvių' },        // Lithuanian
+  { code: 'hu', name: 'Magyar' },          // Hungarian
+  { code: 'mk', name: 'Македонски' },      // Macedonian
+  { code: 'ms', name: 'Bahasa Melayu' },   // Malay
+  { code: 'nl', name: 'Nederlands' },      // Dutch
+  { code: 'ja', name: '日本語' },          // Japanese
+  { code: 'nb', name: 'Norsk' },           // Norwegian Bokmål
+  { code: 'nn', name: 'Norsk Nynorsk' },   // Norwegian Nynorsk
+  { code: 'pl', name: 'Polski' },          // Polish
+  { code: 'pt', name: 'Português' },       // Portuguese
+  { code: 'pt-BR', name: 'Português (Brazil)' }, // Portuguese (Brazil)
+  { code: 'ro', name: 'Română' },          // Romanian
+  { code: 'ru', name: 'Русский' },         // Russian
+  { code: 'sk', name: 'Slovenčina' },      // Slovak
+  { code: 'sl', name: 'Slovenščina' },     // Slovenian
+  { code: 'sr', name: 'Српски / srpski' }, // Serbian
+  { code: 'sh', name: 'Srpskohrvatski / српскохрватски' },// Serbo-Croatian
+  { code: 'fi', name: 'Suomi' },           // Finnish
+  { code: 'sv', name: 'Svenska' },         // Swedish
+  { code: 'th', name: 'ไทย' },             // Thai
+  { code: 'tr', name: 'Türkçe' },          // Turkish
+  { code: 'uk', name: 'Українська' },      // Ukrainian
+  { code: 'vi', name: 'Tiếng Việt' },      // Vietnamese
+  { code: 'zh', name: '中文' },            // Chinese
+];
+
+const languageDict = {};
+languageList.forEach(lang => {
+  languageDict[lang.code] = lang
+});
+export const languages = languageDict;
+
+// expects path to start with a leading '/'
+export function parseLanguageUrl(path) {
+  const parts = path.split('/');
+  let lang;
+  let rest;
+  if (languages[parts[1]]) {
+    lang = parts[1];
+    rest = '/' + parts.slice(2).join('/');
+  } else {
+    lang = null;
+    rest = path;
+  }
+  if (lang === 'en') {
+    lang = null;
+  }
+  return { lang, path: rest };
+}