Merge branch 'master' into nymkappa/bugfix/index-blocks-prices-often

backend/.eslintrc

@@ -15,10 +15,11 @@
     "@typescript-eslint/ban-types": 1,
     "@typescript-eslint/no-empty-function": 1,
     "@typescript-eslint/no-explicit-any": 1,
-    "@typescript-eslint/no-inferrable-types": 1,
+    "@typescript-eslint/no-inferrable-types": 0,
     "@typescript-eslint/no-namespace": 1,
     "@typescript-eslint/no-this-alias": 1,
     "@typescript-eslint/no-var-requires": 1,
+    "@typescript-eslint/explicit-function-return-type": 1,
     "no-console": 1,
     "no-constant-condition": 1,
     "no-dupe-else-if": 1,
@@ -28,6 +29,8 @@
     "no-useless-catch": 1,
     "no-var": 1,
     "prefer-const": 1,
-    "prefer-rest-params": 1
+    "prefer-rest-params": 1,
+    "quotes": [1, "single", { "allowTemplateLiterals": true }],
+    "semi": 1
   }
 }

backend/mempool-config.sample.json

@@ -66,7 +66,8 @@
   "MAXMIND": {
     "ENABLED": false,
     "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb",
-    "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb"
+    "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb",
+    "GEOIP2_ISP": "/usr/local/share/GeoIP/GeoIP2-ISP.mmdb"
   },
   "BISQ": {
     "ENABLED": false,
@@ -78,8 +79,8 @@
   },
   "LND": {
     "TLS_CERT_PATH": "tls.cert",
-    "MACAROON_PATH": "admin.macaroon",
+    "MACAROON_PATH": "readonly.macaroon",
-    "SOCKET": "localhost:10009"
+    "REST_API_URL": "https://localhost:8080"
   },
   "SOCKS5PROXY": {
     "ENABLED": false,

backend/package.json

@@ -16,7 +16,8 @@
     "mempool",
     "blockchain",
     "explorer",
-    "liquid"
+    "liquid",
+    "lightning"
   ],
   "main": "index.ts",
   "scripts": {
@@ -34,10 +35,9 @@
     "@types/node": "^16.11.41",
     "axios": "~0.27.2",
     "bitcoinjs-lib": "6.0.1",
-    "bolt07": "^1.8.1",
     "crypto-js": "^4.0.0",
     "express": "^4.18.0",
-    "lightning": "^5.16.3",
+    "fast-xml-parser": "^4.0.9",
     "maxmind": "^4.3.6",
     "mysql2": "2.3.3",
     "node-worker-threads-pool": "^1.5.1",

backend/src/api/bitcoin/bitcoin-api-abstract-factory.ts

@@ -9,6 +9,7 @@ export interface AbstractBitcoinApi {
   $getBlockHash(height: number): Promise<string>;
   $getBlockHeader(hash: string): Promise<string>;
   $getBlock(hash: string): Promise<IEsploraApi.Block>;
+  $getRawBlock(hash: string): Promise<string>;
   $getAddress(address: string): Promise<IEsploraApi.Address>;
   $getAddressTransactions(address: string, lastSeenTxId: string): Promise<IEsploraApi.Transaction[]>;
   $getAddressPrefix(prefix: string): string[];

backend/src/api/bitcoin/bitcoin-api.ts

@@ -77,7 +77,8 @@ class BitcoinApi implements AbstractBitcoinApi {
   }
 
   $getRawBlock(hash: string): Promise<string> {
-    return this.bitcoindClient.getBlock(hash, 0);
+    return this.bitcoindClient.getBlock(hash, 0)
+      .then((raw: string) => Buffer.from(raw, "hex"));
   }
 
   $getBlockHash(height: number): Promise<string> {

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -103,9 +103,10 @@ class BitcoinRoutes {
           .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/header', this.getBlockHeader)
           .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/height', this.getBlockTipHeight)
           .get(config.MEMPOOL.API_URL_PREFIX + 'blocks/tip/hash', this.getBlockTipHash)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/raw', this.getRawBlock)
+          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txids', this.getTxIdsForBlock)
           .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs', this.getBlockTransactions)
           .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txs/:index', this.getBlockTransactions)
-          .get(config.MEMPOOL.API_URL_PREFIX + 'block/:hash/txids', this.getTxIdsForBlock)
           .get(config.MEMPOOL.API_URL_PREFIX + 'block-height/:height', this.getBlockHeight)
           .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address', this.getAddress)
           .get(config.MEMPOOL.API_URL_PREFIX + 'address/:address/txs', this.getAddressTransactions)
@@ -470,6 +471,16 @@ class BitcoinRoutes {
     }
   }
 
+  private async getRawBlock(req: Request, res: Response) {
+    try {
+      const result = await bitcoinApi.$getRawBlock(req.params.hash);
+      res.setHeader('content-type', 'application/octet-stream');
+      res.send(result);
+    } catch (e) {
+      res.status(500).send(e instanceof Error ? e.message : e);
+    }
+  }
+
   private async getTxIdsForBlock(req: Request, res: Response) {
     try {
       const result = await bitcoinApi.$getTxIdsForBlock(req.params.hash);

backend/src/api/bitcoin/esplora-api.ts

@@ -50,6 +50,11 @@ class ElectrsApi implements AbstractBitcoinApi {
       .then((response) => response.data);
   }
 
+  $getRawBlock(hash: string): Promise<string> {
+    return axios.get<string>(config.ESPLORA.REST_API_URL + '/block/' + hash + "/raw", this.axiosConfig)
+      .then((response) => response.data);
+  }
+
   $getAddress(address: string): Promise<IEsploraApi.Address> {
     throw new Error('Method getAddress not implemented.');
   }

backend/src/api/common.ts

@@ -1,5 +1,6 @@
 import { CpfpInfo, TransactionExtended, TransactionStripped } from '../mempool.interfaces';
 import config from '../config';
+import { convertChannelId } from './lightning/clightning/clightning-convert';
 export class Common {
   static nativeAssetId = config.MEMPOOL.NETWORK === 'liquidtestnet' ?
     '144c654344aa716d6f3abcc1ca90e5641e4e2a7f633bc09fe3baf64585819a49'
@@ -184,4 +185,37 @@ export class Common {
       config.MEMPOOL.BLOCKS_SUMMARIES_INDEXING === true
     );
   }
+
+  static setDateMidnight(date: Date): void {
+    date.setUTCHours(0);
+    date.setUTCMinutes(0);
+    date.setUTCSeconds(0);
+    date.setUTCMilliseconds(0);
+  }
+
+  static channelShortIdToIntegerId(id: string): string {
+    if (config.LIGHTNING.BACKEND === 'lnd') {
+      return id;
+    }
+    return convertChannelId(id);
+  }
+
+  /** Decodes a channel id returned by lnd as uint64 to a short channel id */
+  static channelIntegerIdToShortId(id: string): string {
+    if (config.LIGHTNING.BACKEND === 'cln') {
+      return id;
+    }
+
+    const n = BigInt(id);
+    return [
+      n >> 40n, // nth block
+      (n >> 16n) & 0xffffffn, // nth tx of the block
+      n & 0xffffn // nth output of the tx
+    ].join('x');
+  }
+
+  static utcDateToMysql(date?: number): string {
+    const d = new Date((date || 0) * 1000);
+    return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
+  }
 }

backend/src/api/database-migration.ts

@@ -4,13 +4,13 @@ import logger from '../logger';
 import { Common } from './common';
 
 class DatabaseMigration {
-  private static currentVersion = 33;
+  private static currentVersion = 36;
   private queryTimeout = 120000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
 
-  private blocksTruncatedMessage = `'blocks' table has been truncated. Re-indexing from scratch.`;
+  private blocksTruncatedMessage = `'blocks' table has been truncated.`;
-  private hashratesTruncatedMessage = `'hashrates' table has been truncated. Re-indexing from scratch.`;
+  private hashratesTruncatedMessage = `'hashrates' table has been truncated.`;
 
   /**
    * Avoid printing multiple time the same message
@@ -256,7 +256,9 @@ class DatabaseMigration {
     }
 
     if (databaseSchemaVersion < 26 && isBitcoin === true) {
-      this.uniqueLog(logger.notice, `'lightning_stats' table has been truncated. Will re-generate historical data from scratch.`);
+      if (config.LIGHTNING.ENABLED) {
+        this.uniqueLog(logger.notice, `'lightning_stats' table has been truncated.`);
+      }
       await this.$executeQuery(`TRUNCATE lightning_stats`);
       await this.$executeQuery('ALTER TABLE `lightning_stats` ADD tor_nodes int(11) NOT NULL DEFAULT "0"');
       await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_nodes int(11) NOT NULL DEFAULT "0"');
@@ -273,6 +275,9 @@ class DatabaseMigration {
     }
     
     if (databaseSchemaVersion < 28 && isBitcoin === true) {
+      if (config.LIGHTNING.ENABLED) {
+        this.uniqueLog(logger.notice, `'lightning_stats' and 'node_stats' tables have been truncated.`);
+      }
       await this.$executeQuery(`TRUNCATE lightning_stats`);
       await this.$executeQuery(`TRUNCATE node_stats`);
       await this.$executeQuery(`ALTER TABLE lightning_stats MODIFY added DATE`);
@@ -306,6 +311,19 @@ class DatabaseMigration {
     if (databaseSchemaVersion < 33 && isBitcoin == true) {
       await this.$executeQuery('ALTER TABLE `geo_names` CHANGE `type` `type` enum("city","country","division","continent","as_organization", "country_iso_code") NOT NULL');
     }
+
+    if (databaseSchemaVersion < 34 && isBitcoin == true) {
+      await this.$executeQuery('ALTER TABLE `lightning_stats` ADD clearnet_tor_nodes int(11) NOT NULL DEFAULT "0"');
+    }
+
+    if (databaseSchemaVersion < 35 && isBitcoin == true) {
+      await this.$executeQuery('DELETE from `lightning_stats` WHERE added > "2021-09-19"');
+      await this.$executeQuery('ALTER TABLE `lightning_stats` ADD CONSTRAINT added_unique UNIQUE (added);');
+    }
+
+    if (databaseSchemaVersion < 36 && isBitcoin == true) {
+      await this.$executeQuery('ALTER TABLE `nodes` ADD status TINYINT NOT NULL DEFAULT "1"');
+    }
   }
 
   /**

backend/src/api/explorer/channels.api.ts

@@ -1,5 +1,9 @@
 import logger from '../../logger';
 import DB from '../../database';
+import nodesApi from './nodes.api';
+import { ResultSetHeader } from 'mysql2';
+import { ILightningApi } from '../lightning/lightning-api.interface';
+import { Common } from '../common';
 
 class ChannelsApi {
   public async $getAllChannels(): Promise<any[]> {
@@ -13,9 +17,10 @@ class ChannelsApi {
     }
   }
 
-  public async $getAllChannelsGeo(): Promise<any[]> {
+  public async $getAllChannelsGeo(publicKey?: string): Promise<any[]> {
     try {
-      const query = `SELECT nodes_1.public_key as node1_public_key, nodes_1.alias AS node1_alias,
+      const params: string[] = [];
+      let query = `SELECT nodes_1.public_key as node1_public_key, nodes_1.alias AS node1_alias,
         nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude,
         nodes_2.public_key as node2_public_key, nodes_2.alias AS node2_alias,
         nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude,
@@ -26,7 +31,14 @@ class ChannelsApi {
       WHERE nodes_1.latitude IS NOT NULL AND nodes_1.longitude IS NOT NULL
         AND nodes_2.latitude IS NOT NULL AND nodes_2.longitude IS NOT NULL
       `;
-      const [rows]: any = await DB.query(query);
+
+      if (publicKey !== undefined) {
+        query += ' AND (nodes_1.public_key = ? OR nodes_2.public_key = ?)';
+        params.push(publicKey);
+        params.push(publicKey);
+      }
+
+      const [rows]: any = await DB.query(query, params);
       return rows.map((row) => [
         row.node1_public_key, row.node1_alias, row.node1_longitude, row.node1_latitude,
         row.node2_public_key, row.node2_alias, row.node2_longitude, row.node2_latitude,
@@ -173,15 +185,57 @@ class ChannelsApi {
 
   public async $getChannelsForNode(public_key: string, index: number, length: number, status: string): Promise<any[]> {
     try {
-      // Default active and inactive channels
+      let channelStatusFilter;
-      let statusQuery = '< 2';
+      if (status === 'open') {
-      // Closed channels only
+        channelStatusFilter = '< 2';
-      if (status === 'closed') {
+      } else if (status === 'closed') {
-        statusQuery = '= 2';
+        channelStatusFilter = '= 2';
       }
-      const query = `SELECT n1.alias AS alias_left, n2.alias AS alias_right, channels.*, ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key WHERE (ns1.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node1_public_key) AND ns2.id = (SELECT MAX(id) FROM node_stats WHERE public_key = channels.node2_public_key)) AND (node1_public_key = ? OR node2_public_key = ?) AND status ${statusQuery} ORDER BY channels.capacity DESC LIMIT ?, ?`;
+
-      const [rows]: any = await DB.query(query, [public_key, public_key, index, length]);
+      // Channels originating from node
-      const channels = rows.map((row) => this.convertChannel(row));
+      let query = `
+        SELECT node2.alias, node2.public_key, channels.status, channels.node1_fee_rate,
+          channels.capacity, channels.short_id, channels.id
+        FROM channels
+        JOIN nodes AS node2 ON node2.public_key = channels.node2_public_key
+        WHERE node1_public_key = ? AND channels.status ${channelStatusFilter}
+      `;
+      const [channelsFromNode]: any = await DB.query(query, [public_key, index, length]);
+
+      // Channels incoming to node
+      query = `
+        SELECT node1.alias, node1.public_key, channels.status, channels.node2_fee_rate,
+          channels.capacity, channels.short_id, channels.id
+        FROM channels
+        JOIN nodes AS node1 ON node1.public_key = channels.node1_public_key
+        WHERE node2_public_key = ? AND channels.status ${channelStatusFilter}
+      `;
+      const [channelsToNode]: any = await DB.query(query, [public_key, index, length]);
+
+      let allChannels = channelsFromNode.concat(channelsToNode);
+      allChannels.sort((a, b) => {
+        return b.capacity - a.capacity;
+      });
+      allChannels = allChannels.slice(index, index + length);
+
+      const channels: any[] = []
+      for (const row of allChannels) {
+        const activeChannelsStats: any = await nodesApi.$getActiveChannelsStats(row.public_key);
+        channels.push({
+          status: row.status,
+          capacity: row.capacity ?? 0,
+          short_id: row.short_id,
+          id: row.id,
+          fee_rate: row.node1_fee_rate ?? row.node2_fee_rate ?? 0,
+          node: {
+            alias: row.alias.length > 0 ? row.alias : row.public_key.slice(0, 20),
+            public_key: row.public_key,
+            channels: activeChannelsStats.active_channel_count ?? 0,
+            capacity: activeChannelsStats.capacity ?? 0,
+          }
+        });
+      }
+
       return channels;
     } catch (e) {
       logger.err('$getChannelsForNode error: ' + (e instanceof Error ? e.message : e));
@@ -197,7 +251,12 @@ class ChannelsApi {
       if (status === 'closed') {
         statusQuery = '= 2';
       }
-      const query = `SELECT COUNT(*) AS count FROM channels WHERE (node1_public_key = ? OR node2_public_key = ?) AND status ${statusQuery}`;
+      const query = `
+        SELECT COUNT(*) AS count
+        FROM channels
+        WHERE (node1_public_key = ? OR node2_public_key = ?)
+        AND status ${statusQuery}
+      `;
       const [rows]: any = await DB.query(query, [public_key, public_key]);
       return rows[0]['count'];
     } catch (e) {
@@ -246,6 +305,135 @@ class ChannelsApi {
       },
     };
   }
+
+  /**
+   * Save or update a channel present in the graph
+   */
+  public async $saveChannel(channel: ILightningApi.Channel): Promise<void> {
+    const [ txid, vout ] = channel.chan_point.split(':');
+
+    const policy1: Partial<ILightningApi.RoutingPolicy> = channel.node1_policy || {};
+    const policy2: Partial<ILightningApi.RoutingPolicy> = channel.node2_policy || {};
+
+    const query = `INSERT INTO channels
+      (
+        id,
+        short_id,
+        capacity,
+        transaction_id,
+        transaction_vout,
+        updated_at,
+        status,
+        node1_public_key,
+        node1_base_fee_mtokens,
+        node1_cltv_delta,
+        node1_fee_rate,
+        node1_is_disabled,
+        node1_max_htlc_mtokens,
+        node1_min_htlc_mtokens,
+        node1_updated_at,
+        node2_public_key,
+        node2_base_fee_mtokens,
+        node2_cltv_delta,
+        node2_fee_rate,
+        node2_is_disabled,
+        node2_max_htlc_mtokens,
+        node2_min_htlc_mtokens,
+        node2_updated_at
+      )
+      VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON DUPLICATE KEY UPDATE
+        capacity = ?,
+        updated_at = ?,
+        status = 1,
+        node1_public_key = ?,
+        node1_base_fee_mtokens = ?,
+        node1_cltv_delta = ?,
+        node1_fee_rate = ?,
+        node1_is_disabled = ?,
+        node1_max_htlc_mtokens = ?,
+        node1_min_htlc_mtokens = ?,
+        node1_updated_at = ?,
+        node2_public_key = ?,
+        node2_base_fee_mtokens = ?,
+        node2_cltv_delta = ?,
+        node2_fee_rate = ?,
+        node2_is_disabled = ?,
+        node2_max_htlc_mtokens = ?,
+        node2_min_htlc_mtokens = ?,
+        node2_updated_at = ?
+      ;`;
+
+    await DB.query(query, [
+      Common.channelShortIdToIntegerId(channel.channel_id),
+      Common.channelIntegerIdToShortId(channel.channel_id),
+      channel.capacity,
+      txid,
+      vout,
+      Common.utcDateToMysql(channel.last_update),
+      channel.node1_pub,
+      policy1.fee_base_msat,
+      policy1.time_lock_delta,
+      policy1.fee_rate_milli_msat,
+      policy1.disabled,
+      policy1.max_htlc_msat,
+      policy1.min_htlc,
+      Common.utcDateToMysql(policy1.last_update),
+      channel.node2_pub,
+      policy2.fee_base_msat,
+      policy2.time_lock_delta,
+      policy2.fee_rate_milli_msat,
+      policy2.disabled,
+      policy2.max_htlc_msat,
+      policy2.min_htlc,
+      Common.utcDateToMysql(policy2.last_update),
+      channel.capacity,
+      Common.utcDateToMysql(channel.last_update),
+      channel.node1_pub,
+      policy1.fee_base_msat,
+      policy1.time_lock_delta,
+      policy1.fee_rate_milli_msat,
+      policy1.disabled,
+      policy1.max_htlc_msat,
+      policy1.min_htlc,
+      Common.utcDateToMysql(policy1.last_update),
+      channel.node2_pub,
+      policy2.fee_base_msat,
+      policy2.time_lock_delta,
+      policy2.fee_rate_milli_msat,
+      policy2.disabled,
+      policy2.max_htlc_msat,
+      policy2.min_htlc,
+      Common.utcDateToMysql(policy2.last_update)
+    ]);
+  }
+
+  /**
+   * Set all channels not in `graphChannelsIds` as inactive (status = 0)
+   */
+  public async $setChannelsInactive(graphChannelsIds: string[]): Promise<void> {
+    if (graphChannelsIds.length === 0) {
+      return;
+    }
+
+    try {
+      const result = await DB.query<ResultSetHeader>(`
+        UPDATE channels
+        SET status = 0
+        WHERE short_id NOT IN (
+          ${graphChannelsIds.map(id => `"${id}"`).join(',')}
+        )
+        AND status != 2
+      `);
+      if (result[0].changedRows ?? 0 > 0) {
+        logger.info(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`);
+      } else {
+        logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not in the graph`);
+      }
+    } catch (e) {
+      logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
 }
 
 export default new ChannelsApi();

backend/src/api/explorer/channels.routes.ts

@@ -11,7 +11,8 @@ class ChannelsRoutes {
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels/search/:search', this.$searchChannelsById)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels/:short_id', this.$getChannel)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels', this.$getChannelsForNode)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels-geo', this.$getChannelsGeo)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels-geo', this.$getAllChannelsGeo)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/channels-geo/:publicKey', this.$getAllChannelsGeo)
     ;
   }
 
@@ -45,9 +46,11 @@ class ChannelsRoutes {
       }
       const index = parseInt(typeof req.query.index === 'string' ? req.query.index : '0', 10) || 0;
       const status: string = typeof req.query.status === 'string' ? req.query.status : '';
-      const length = 25;
+      const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, 10, status);
-      const channels = await channelsApi.$getChannelsForNode(req.query.public_key, index, length, status);
       const channelsCount = await channelsApi.$getChannelsCountForNode(req.query.public_key, status);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.header('X-Total-Count', channelsCount.toString());
       res.json(channels);
     } catch (e) {
@@ -94,9 +97,9 @@ class ChannelsRoutes {
     }
   }
 
-  private async $getChannelsGeo(req: Request, res: Response) {
+  private async $getAllChannelsGeo(req: Request, res: Response) {
     try {
-      const channels = await channelsApi.$getAllChannelsGeo();
+      const channels = await channelsApi.$getAllChannelsGeo(req.params?.publicKey);
       res.json(channels);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);

backend/src/api/explorer/nodes.api.ts

@@ -1,43 +1,90 @@
 import logger from '../../logger';
 import DB from '../../database';
+import { ResultSetHeader } from 'mysql2';
+import { ILightningApi } from '../lightning/lightning-api.interface';
 
 class NodesApi {
   public async $getNode(public_key: string): Promise<any> {
     try {
-      const query = `
+      // General info
-        SELECT nodes.*, geo_names_as.names as as_organization, geo_names_city.names as city,
+      let query = `
-        geo_names_country.names as country, geo_names_subdivision.names as subdivision,
+        SELECT public_key, alias, UNIX_TIMESTAMP(first_seen) AS first_seen,
-          (SELECT Count(*)
+        UNIX_TIMESTAMP(updated_at) AS updated_at, color, sockets as sockets,
-          FROM channels
+        as_number, city_id, country_id, subdivision_id, longitude, latitude,
-          WHERE channels.status < 2 AND ( channels.node1_public_key = ? OR channels.node2_public_key = ? )) AS channel_count,
+        geo_names_iso.names as iso_code, geo_names_as.names as as_organization, geo_names_city.names as city,
-          (SELECT Sum(capacity)
+        geo_names_country.names as country, geo_names_subdivision.names as subdivision
-          FROM channels
-          WHERE channels.status < 2 AND ( channels.node1_public_key = ? OR channels.node2_public_key = ? )) AS capacity,
-          (SELECT Avg(capacity)
-          FROM channels
-          WHERE status < 2 AND ( node1_public_key = ? OR node2_public_key = ? )) AS channels_capacity_avg
         FROM nodes
         LEFT JOIN geo_names geo_names_as on geo_names_as.id = as_number
         LEFT JOIN geo_names geo_names_city on geo_names_city.id = city_id
         LEFT JOIN geo_names geo_names_subdivision on geo_names_subdivision.id = subdivision_id
         LEFT JOIN geo_names geo_names_country on geo_names_country.id = country_id
+        LEFT JOIN geo_names geo_names_iso ON geo_names_iso.id = nodes.country_id AND geo_names_iso.type = 'country_iso_code'
         WHERE public_key = ?
       `;
-      const [rows]: any = await DB.query(query, [public_key, public_key, public_key, public_key, public_key, public_key, public_key]);
+      let [rows]: any[] = await DB.query(query, [public_key]);
-      if (rows.length > 0) {
+      if (rows.length === 0) {
-        rows[0].as_organization = JSON.parse(rows[0].as_organization);
+        throw new Error(`This node does not exist, or our node is not seeing it yet`);
-        rows[0].subdivision = JSON.parse(rows[0].subdivision);
-        rows[0].city = JSON.parse(rows[0].city);
-        rows[0].country = JSON.parse(rows[0].country);
-        return rows[0];
       }
-      return null;
+
+      const node = rows[0];
+      node.as_organization = JSON.parse(node.as_organization);
+      node.subdivision = JSON.parse(node.subdivision);
+      node.city = JSON.parse(node.city);
+      node.country = JSON.parse(node.country);
+
+      // Active channels and capacity
+      const activeChannelsStats: any = await this.$getActiveChannelsStats(public_key);
+      node.active_channel_count = activeChannelsStats.active_channel_count ?? 0;
+      node.capacity = activeChannelsStats.capacity ?? 0;
+
+      // Opened channels count
+      query = `
+        SELECT count(short_id) as opened_channel_count
+        FROM channels
+        WHERE status != 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+      `;
+      [rows] = await DB.query(query, [public_key, public_key]);
+      node.opened_channel_count = 0;
+      if (rows.length > 0) {
+        node.opened_channel_count = rows[0].opened_channel_count;
+      }
+
+      // Closed channels count
+      query = `
+        SELECT count(short_id) as closed_channel_count
+        FROM channels
+        WHERE status = 2 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+      `;
+      [rows] = await DB.query(query, [public_key, public_key]);
+      node.closed_channel_count = 0;
+      if (rows.length > 0) {
+        node.closed_channel_count = rows[0].closed_channel_count;
+      }
+
+      return node;
     } catch (e) {
-      logger.err('$getNode error: ' + (e instanceof Error ? e.message : e));
+      logger.err(`Cannot get node information for ${public_key}. Reason: ${(e instanceof Error ? e.message : e)}`);
       throw e;
     }
   }
 
+  public async $getActiveChannelsStats(node_public_key: string): Promise<unknown> {
+    const query = `
+      SELECT count(short_id) as active_channel_count, sum(capacity) as capacity
+      FROM channels
+      WHERE status = 1 AND (channels.node1_public_key = ? OR channels.node2_public_key = ?)
+    `;
+    const [rows]: any[] = await DB.query(query, [node_public_key, node_public_key]);
+    if (rows.length > 0) {
+      return {
+        active_channel_count: rows[0].active_channel_count,
+        capacity: rows[0].capacity
+      };
+    } else {
+      return null;
+    }
+  }
+
   public async $getAllNodes(): Promise<any> {
     try {
       const query = `SELECT * FROM nodes`;
@@ -51,7 +98,12 @@ class NodesApi {
 
   public async $getNodeStats(public_key: string): Promise<any> {
     try {
-      const query = `SELECT UNIX_TIMESTAMP(added) AS added, capacity, channels FROM node_stats WHERE public_key = ? ORDER BY added DESC`;
+      const query = `
+        SELECT UNIX_TIMESTAMP(added) AS added, capacity, channels
+        FROM node_stats
+        WHERE public_key = ?
+        ORDER BY added DESC
+      `;
       const [rows]: any = await DB.query(query, [public_key]);
       return rows;
     } catch (e) {
@@ -62,8 +114,19 @@ class NodesApi {
 
   public async $getTopCapacityNodes(): Promise<any> {
     try {
-      const query = `SELECT nodes.*, node_stats.capacity, node_stats.channels FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key ORDER BY node_stats.added DESC, node_stats.capacity DESC LIMIT 10`;
+      let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats');
-      const [rows]: any = await DB.query(query);
+      const latestDate = rows[0].maxAdded;
+
+      const query = `
+        SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, node_stats.capacity, node_stats.channels
+        FROM node_stats
+        JOIN nodes ON nodes.public_key = node_stats.public_key
+        WHERE added = FROM_UNIXTIME(${latestDate})
+        ORDER BY capacity DESC
+        LIMIT 10;
+      `;
+      [rows] = await DB.query(query);
+
       return rows;
     } catch (e) {
       logger.err('$getTopCapacityNodes error: ' + (e instanceof Error ? e.message : e));
@@ -73,8 +136,19 @@ class NodesApi {
 
   public async $getTopChannelsNodes(): Promise<any> {
     try {
-      const query = `SELECT nodes.*, node_stats.capacity, node_stats.channels FROM nodes LEFT JOIN node_stats ON node_stats.public_key = nodes.public_key ORDER BY node_stats.added DESC, node_stats.channels DESC LIMIT 10`;
+      let [rows]: any[] = await DB.query('SELECT UNIX_TIMESTAMP(MAX(added)) as maxAdded FROM node_stats');
-      const [rows]: any = await DB.query(query);
+      const latestDate = rows[0].maxAdded;
+
+      const query = `
+        SELECT nodes.public_key, IF(nodes.alias = '', SUBSTRING(nodes.public_key, 1, 20), alias) as alias, node_stats.capacity, node_stats.channels
+        FROM node_stats
+        JOIN nodes ON nodes.public_key = node_stats.public_key
+        WHERE added = FROM_UNIXTIME(${latestDate})
+        ORDER BY channels DESC
+        LIMIT 10;
+      `;
+      [rows] = await DB.query(query);
+
       return rows;
     } catch (e) {
       logger.err('$getTopChannelsNodes error: ' + (e instanceof Error ? e.message : e));
@@ -94,29 +168,59 @@ class NodesApi {
     }
   }
 
-  public async $getNodesISP() {
+  public async $getNodesISP(groupBy: string, showTor: boolean) {
     try {
-      let query = `SELECT nodes.as_number as ispId, geo_names.names as names, COUNT(DISTINCT nodes.public_key) as nodesCount, SUM(capacity) as capacity
+      const orderBy = groupBy === 'capacity' ? `CAST(SUM(capacity) as INT)` : `COUNT(DISTINCT nodes.public_key)`;
+      
+      // Clearnet
+      let query = `SELECT GROUP_CONCAT(DISTINCT(nodes.as_number)) as ispId, geo_names.names as names,
+          COUNT(DISTINCT nodes.public_key) as nodesCount, CAST(SUM(capacity) as INT) as capacity
         FROM nodes
         JOIN geo_names ON geo_names.id = nodes.as_number
         JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key
-        GROUP BY as_number
+        GROUP BY geo_names.names
-        ORDER BY COUNT(DISTINCT nodes.public_key) DESC
+        ORDER BY ${orderBy} DESC
       `;      
       const [nodesCountPerAS]: any = await DB.query(query);
 
-      query = `SELECT COUNT(*) as total FROM nodes WHERE as_number IS NOT NULL`;
+      let total = 0;
-      const [nodesWithAS]: any = await DB.query(query);
-
       const nodesPerAs: any[] = [];
+
+      for (const asGroup of nodesCountPerAS) {
+        if (groupBy === 'capacity') {
+          total += asGroup.capacity;
+        } else {
+          total += asGroup.nodesCount;
+        }
+      }
+
+      // Tor
+      if (showTor) {
+        query = `SELECT COUNT(DISTINCT nodes.public_key) as nodesCount, CAST(SUM(capacity) as INT) as capacity
+          FROM nodes
+          JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key
+          ORDER BY ${orderBy} DESC
+        `;      
+        const [nodesCountTor]: any = await DB.query(query);
+
+        total += groupBy === 'capacity' ? nodesCountTor[0].capacity : nodesCountTor[0].nodesCount;
+        nodesPerAs.push({
+          ispId: null,
+          name: 'Tor',
+          count: nodesCountTor[0].nodesCount,
+          share: Math.floor((groupBy === 'capacity' ? nodesCountTor[0].capacity : nodesCountTor[0].nodesCount) / total * 10000) / 100,
+          capacity: nodesCountTor[0].capacity,
+        });
+      }
+
       for (const as of nodesCountPerAS) {
         nodesPerAs.push({
           ispId: as.ispId,
           name: JSON.parse(as.names),
           count: as.nodesCount,
-          share: Math.floor(as.nodesCount / nodesWithAS[0].total * 10000) / 100,
+          share: Math.floor((groupBy === 'capacity' ? as.capacity : as.nodesCount) / total * 10000) / 100,
           capacity: as.capacity,
-        })
+        });
       }
 
       return nodesPerAs;
@@ -129,8 +233,8 @@ class NodesApi {
   public async $getNodesPerCountry(countryId: string) {
     try {
       const query = `
-        SELECT node_stats.public_key, node_stats.capacity, node_stats.channels, nodes.alias,
+      SELECT nodes.public_key, CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, CAST(COALESCE(node_stats.channels, 0) as INT) as channels,
-          UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
+      nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
           geo_names_city.names as city
         FROM node_stats
         JOIN (
@@ -138,7 +242,7 @@ class NodesApi {
           FROM node_stats
           GROUP BY public_key
         ) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added
-        JOIN nodes ON nodes.public_key = node_stats.public_key
+        RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key
         JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
         LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
         WHERE geo_names_country.id = ?
@@ -159,8 +263,8 @@ class NodesApi {
   public async $getNodesPerISP(ISPId: string) {
     try {
       const query = `
-        SELECT node_stats.public_key, node_stats.capacity, node_stats.channels, nodes.alias,
+        SELECT nodes.public_key, CAST(COALESCE(node_stats.capacity, 0) as INT) as capacity, CAST(COALESCE(node_stats.channels, 0) as INT) as channels,
-          UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
+          nodes.alias, UNIX_TIMESTAMP(nodes.first_seen) as first_seen, UNIX_TIMESTAMP(nodes.updated_at) as updated_at,
           geo_names_city.names as city, geo_names_country.names as country
         FROM node_stats
         JOIN (
@@ -168,14 +272,14 @@ class NodesApi {
           FROM node_stats
           GROUP BY public_key
         ) as b ON b.public_key = node_stats.public_key AND b.last_added = node_stats.added
-        JOIN nodes ON nodes.public_key = node_stats.public_key
+        RIGHT JOIN nodes ON nodes.public_key = node_stats.public_key
         JOIN geo_names geo_names_country ON geo_names_country.id = nodes.country_id AND geo_names_country.type = 'country'
         LEFT JOIN geo_names geo_names_city ON geo_names_city.id = nodes.city_id AND geo_names_city.type = 'city'
-        WHERE nodes.as_number = ?
+        WHERE nodes.as_number IN (?)
         ORDER BY capacity DESC
       `;
 
-      const [rows]: any = await DB.query(query, [ISPId]);
+      const [rows]: any = await DB.query(query, [ISPId.split(',')]);
       for (let i = 0; i < rows.length; ++i) {
         rows[i].country = JSON.parse(rows[i].country);
         rows[i].city = JSON.parse(rows[i].city);
@@ -219,6 +323,66 @@ class NodesApi {
       throw e;
     }
   }
+
+  /**
+   * Save or update a node present in the graph
+   */
+  public async $saveNode(node: ILightningApi.Node): Promise<void> {
+    try {
+      const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? '';
+      const query = `INSERT INTO nodes(
+          public_key,
+          first_seen,
+          updated_at,
+          alias,
+          color,
+          sockets,
+          status
+        )
+        VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?, 1)
+        ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, color = ?, sockets = ?, status = 1`;
+
+      await DB.query(query, [
+        node.pub_key,
+        node.last_update,
+        node.alias,
+        node.color,
+        sockets,
+        node.last_update,
+        node.alias,
+        node.color,
+        sockets,
+      ]);
+    } catch (e) {
+      logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  /**
+   * Set all nodes not in `nodesPubkeys` as inactive (status = 0)
+   */
+   public async $setNodesInactive(graphNodesPubkeys: string[]): Promise<void> {
+    if (graphNodesPubkeys.length === 0) {
+      return;
+    }
+
+    try {
+      const result = await DB.query<ResultSetHeader>(`
+        UPDATE nodes
+        SET status = 0
+        WHERE public_key NOT IN (
+          ${graphNodesPubkeys.map(pubkey => `"${pubkey}"`).join(',')}
+        )
+      `);
+      if (result[0].changedRows ?? 0 > 0) {
+        logger.info(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`);
+      } else {
+        logger.debug(`Marked ${result[0].changedRows} nodes as inactive because they are not in the graph`);
+      }
+    } catch (e) {
+      logger.err('$setNodesInactive() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
 }
 
 export default new NodesApi();

backend/src/api/explorer/nodes.routes.ts

@@ -9,10 +9,10 @@ class NodesRoutes {
   public initRoutes(app: Application) {
     app
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/country/:country', this.$getNodesPerCountry)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp/:isp', this.$getNodesPerISP)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/search/:search', this.$searchNode)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/top', this.$getTopNodes)
-      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp', this.$getNodesISP)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp-ranking', this.$getISPRanking)
+      .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/isp/:isp', this.$getNodesPerISP)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/countries', this.$getNodesCountries)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key/statistics', this.$getHistoricalNodeStats)
       .get(config.MEMPOOL.API_URL_PREFIX + 'lightning/nodes/:public_key', this.$getNode)
@@ -35,6 +35,9 @@ class NodesRoutes {
         res.status(404).send('Node not found');
         return;
       }
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(node);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
@@ -44,6 +47,9 @@ class NodesRoutes {
   private async $getHistoricalNodeStats(req: Request, res: Response) {
     try {
       const statistics = await nodesApi.$getNodeStats(req.params.public_key);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(statistics);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
@@ -63,9 +69,18 @@ class NodesRoutes {
     }
   }
 
-  private async $getNodesISP(req: Request, res: Response) {
+  private async $getISPRanking(req: Request, res: Response): Promise<void> {
     try {
-      const nodesPerAs = await nodesApi.$getNodesISP();
+      const groupBy = req.query.groupBy as string;
+      const showTor = req.query.showTor as string === 'true' ? true : false;
+
+      if (!['capacity', 'node-count'].includes(groupBy)) {
+        res.status(400).send(`groupBy must be one of 'capacity' or 'node-count'`);
+        return;
+      }
+
+      const nodesPerAs = await nodesApi.$getNodesISP(groupBy, showTor);
+
       res.header('Pragma', 'public');
       res.header('Cache-control', 'public');
       res.setHeader('Expires', new Date(Date.now() + 1000 * 600).toUTCString());

backend/src/api/explorer/statistics.api.ts

@@ -6,14 +6,14 @@ class StatisticsApi {
   public async $getStatistics(interval: string | null = null): Promise<any> {
     interval = Common.getSqlInterval(interval);
 
-    let query = `SELECT UNIX_TIMESTAMP(added) AS added, channel_count, node_count, total_capacity, tor_nodes, clearnet_nodes, unannounced_nodes
+    let query = `SELECT UNIX_TIMESTAMP(added) AS added, channel_count, total_capacity, tor_nodes, clearnet_nodes, unannounced_nodes
       FROM lightning_stats`;
 
     if (interval) {
       query += ` WHERE added BETWEEN DATE_SUB(NOW(), INTERVAL ${interval}) AND NOW()`;
     }
 
-    query += ` ORDER BY id DESC`;
+    query += ` ORDER BY added DESC`;
 
     try {
       const [rows]: any = await DB.query(query);
@@ -26,8 +26,8 @@ class StatisticsApi {
 
   public async $getLatestStatistics(): Promise<any> {
     try {
-      const [rows]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY id DESC LIMIT 1`);
+      const [rows]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1`);
-      const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY id DESC LIMIT 1 OFFSET 7`);
+      const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1 OFFSET 7`);
       return {
         latest: rows[0],
         previous: rows2[0],

backend/src/api/lightning/clightning/clightning-client.ts

@@ -0,0 +1,272 @@
+// Imported from https://github.com/shesek/lightning-client-js
+
+'use strict';
+
+const methods = [
+  'addgossip',
+  'autocleaninvoice',
+  'check',
+  'checkmessage',
+  'close',
+  'connect',
+  'createinvoice',
+  'createinvoicerequest',
+  'createoffer',
+  'createonion',
+  'decode',
+  'decodepay',
+  'delexpiredinvoice',
+  'delinvoice',
+  'delpay',
+  'dev-listaddrs',
+  'dev-rescan-outputs',
+  'disableoffer',
+  'disconnect',
+  'estimatefees',
+  'feerates',
+  'fetchinvoice',
+  'fundchannel',
+  'fundchannel_cancel',
+  'fundchannel_complete',
+  'fundchannel_start',
+  'fundpsbt',
+  'getchaininfo',
+  'getinfo',
+  'getlog',
+  'getrawblockbyheight',
+  'getroute',
+  'getsharedsecret',
+  'getutxout',
+  'help',
+  'invoice',
+  'keysend',
+  'legacypay',
+  'listchannels',
+  'listconfigs',
+  'listforwards',
+  'listfunds',
+  'listinvoices',
+  'listnodes',
+  'listoffers',
+  'listpays',
+  'listpeers',
+  'listsendpays',
+  'listtransactions',
+  'multifundchannel',
+  'multiwithdraw',
+  'newaddr',
+  'notifications',
+  'offer',
+  'offerout',
+  'openchannel_abort',
+  'openchannel_bump',
+  'openchannel_init',
+  'openchannel_signed',
+  'openchannel_update',
+  'pay',
+  'payersign',
+  'paystatus',
+  'ping',
+  'plugin',
+  'reserveinputs',
+  'sendinvoice',
+  'sendonion',
+  'sendonionmessage',
+  'sendpay',
+  'sendpsbt',
+  'sendrawtransaction',
+  'setchannelfee',
+  'signmessage',
+  'signpsbt',
+  'stop',
+  'txdiscard',
+  'txprepare',
+  'txsend',
+  'unreserveinputs',
+  'utxopsbt',
+  'waitanyinvoice',
+  'waitblockheight',
+  'waitinvoice',
+  'waitsendpay',
+  'withdraw'
+];
+
+
+import EventEmitter from 'events';
+import { existsSync, statSync } from 'fs';
+import { createConnection, Socket } from 'net';
+import { homedir } from 'os';
+import path from 'path';
+import { createInterface, Interface } from 'readline';
+import logger from '../../../logger';
+import { AbstractLightningApi } from '../lightning-api-abstract-factory';
+import { ILightningApi } from '../lightning-api.interface';
+import { convertAndmergeBidirectionalChannels, convertNode } from './clightning-convert';
+
+class LightningError extends Error {
+  type: string = 'lightning';
+  message: string = 'lightning-client error';
+
+  constructor(error) {
+    super();
+    this.type = error.type;
+    this.message = error.message;
+  }
+}
+
+const defaultRpcPath = path.join(homedir(), '.lightning')
+  , fStat = (...p) => statSync(path.join(...p))
+  , fExists = (...p) => existsSync(path.join(...p))
+
+export default class CLightningClient extends EventEmitter implements AbstractLightningApi {
+  private rpcPath: string;
+  private reconnectWait: number;
+  private reconnectTimeout;
+  private reqcount: number;
+  private client: Socket;
+  private rl: Interface;
+  private clientConnectionPromise: Promise<unknown>;
+
+  constructor(rpcPath = defaultRpcPath) {
+    if (!path.isAbsolute(rpcPath)) {
+      throw new Error('The rpcPath must be an absolute path');
+    }
+
+    if (!fExists(rpcPath) || !fStat(rpcPath).isSocket()) {
+      // network directory provided, use the lightning-rpc within in
+      if (fExists(rpcPath, 'lightning-rpc')) {
+        rpcPath = path.join(rpcPath, 'lightning-rpc');
+      }
+
+      // main data directory provided, default to using the bitcoin mainnet subdirectory
+      // to be removed in v0.2.0
+      else if (fExists(rpcPath, 'bitcoin', 'lightning-rpc')) {
+        logger.warn(`[CLightningClient] ${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`)
+        logger.warn(`[CLightningClient] specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`)
+        rpcPath = path.join(rpcPath, 'bitcoin', 'lightning-rpc')
+      }
+    }
+
+    logger.debug(`[CLightningClient] Connecting to ${rpcPath}`);
+
+    super();
+    this.rpcPath = rpcPath;
+    this.reconnectWait = 0.5;
+    this.reconnectTimeout = null;
+    this.reqcount = 0;
+
+    const _self = this;
+
+    this.client = createConnection(rpcPath).on(
+      'error', () => {
+        _self.increaseWaitTime();
+        _self.reconnect();
+      }
+    );
+    this.rl = createInterface({ input: this.client }).on(
+      'error', () => {
+        _self.increaseWaitTime();
+        _self.reconnect();
+      }
+    );
+
+    this.clientConnectionPromise = new Promise<void>(resolve => {
+      _self.client.on('connect', () => {
+        logger.info(`[CLightningClient] Lightning client connected`);
+        _self.reconnectWait = 1;
+        resolve();
+      });
+
+      _self.client.on('end', () => {
+        logger.err('[CLightningClient] Lightning client connection closed, reconnecting');
+        _self.increaseWaitTime();
+        _self.reconnect();
+      });
+
+      _self.client.on('error', error => {
+        logger.err(`[CLightningClient] Lightning client connection error: ${error}`);
+        _self.increaseWaitTime();
+        _self.reconnect();
+      });
+    });
+
+    this.rl.on('line', line => {
+      line = line.trim();
+      if (!line) {
+        return;
+      }
+      const data = JSON.parse(line);
+      // logger.debug(`[CLightningClient] #${data.id} <-- ${JSON.stringify(data.error || data.result)}`);
+      _self.emit('res:' + data.id, data);
+    });
+  }
+
+  increaseWaitTime(): void {
+    if (this.reconnectWait >= 16) {
+      this.reconnectWait = 16;
+    } else {
+      this.reconnectWait *= 2;
+    }
+  }
+
+  reconnect(): void {
+    const _self = this;
+
+    if (this.reconnectTimeout) {
+      return;
+    }
+
+    this.reconnectTimeout = setTimeout(() => {
+      logger.debug('[CLightningClient] Trying to reconnect...');
+
+      _self.client.connect(_self.rpcPath);
+      _self.reconnectTimeout = null;
+    }, this.reconnectWait * 1000);
+  }
+
+  call(method, args = []): Promise<any> {
+    const _self = this;
+
+    const callInt = ++this.reqcount;
+    const sendObj = {
+      jsonrpc: '2.0',
+      method,
+      params: args,
+      id: '' + callInt
+    };
+
+    // logger.debug(`[CLightningClient] #${callInt} --> ${method} ${args}`);
+
+    // Wait for the client to connect
+    return this.clientConnectionPromise
+      .then(() => new Promise((resolve, reject) => {
+        // Wait for a response
+        this.once('res:' + callInt, res => res.error == null
+          ? resolve(res.result)
+          : reject(new LightningError(res.error))
+        );
+
+        // Send the command
+        _self.client.write(JSON.stringify(sendObj));
+      }));
+  }
+
+  async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
+    const listnodes: any[] = await this.call('listnodes');
+    const listchannels: any[] = await this.call('listchannels');
+    const channelsList = await convertAndmergeBidirectionalChannels(listchannels['channels']);
+
+    return {
+      nodes: listnodes['nodes'].map(node => convertNode(node)),
+      edges: channelsList,
+    };
+  }
+}
+
+const protify = s => s.replace(/-([a-z])/g, m => m[1].toUpperCase());
+
+methods.forEach(k => {
+  CLightningClient.prototype[protify(k)] = function (...args: any) {
+    return this.call(k, args);
+  };
+});

backend/src/api/lightning/clightning/clightning-convert.ts

@@ -0,0 +1,138 @@
+import { ILightningApi } from '../lightning-api.interface';
+import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher';
+import logger from '../../../logger';
+
+/**
+ * Convert a clightning "listnode" entry to a lnd node entry
+ */
+export function convertNode(clNode: any): ILightningApi.Node {
+  return {
+    alias: clNode.alias ?? '',
+    color: `#${clNode.color ?? ''}`,
+    features: [], // TODO parse and return clNode.feature
+    pub_key: clNode.nodeid,
+    addresses: clNode.addresses?.map((addr) => {
+      return {
+        network: addr.type,
+        addr: `${addr.address}:${addr.port}`
+      };
+    }),
+    last_update: clNode?.last_timestamp ?? 0,
+  };
+}
+
+/**
+ * Convert clightning "listchannels" response to lnd "describegraph.edges" format
+ */
+export async function convertAndmergeBidirectionalChannels(clChannels: any[]): Promise<ILightningApi.Channel[]> {
+  logger.info('Converting clightning nodes and channels to lnd graph format');
+
+  let loggerTimer = new Date().getTime() / 1000;
+  let channelProcessed = 0;
+
+  const consolidatedChannelList: ILightningApi.Channel[] = [];
+  const clChannelsDict = {};
+  const clChannelsDictCount = {};
+
+  for (const clChannel of clChannels) {
+    if (!clChannelsDict[clChannel.short_channel_id]) {
+      clChannelsDict[clChannel.short_channel_id] = clChannel;
+      clChannelsDictCount[clChannel.short_channel_id] = 1;
+    } else {
+      consolidatedChannelList.push(
+        await buildFullChannel(clChannel, clChannelsDict[clChannel.short_channel_id])
+      );
+      delete clChannelsDict[clChannel.short_channel_id];
+      clChannelsDictCount[clChannel.short_channel_id]++;
+    }
+
+    const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+    if (elapsedSeconds > 10) {
+      logger.info(`Building complete channels from clightning output. Channels processed: ${channelProcessed + 1} of ${clChannels.length}`);
+      loggerTimer = new Date().getTime() / 1000;
+    }
+
+    ++channelProcessed;
+  }
+
+  channelProcessed = 0;
+  const keys = Object.keys(clChannelsDict);
+  for (const short_channel_id of keys) {
+    consolidatedChannelList.push(await buildIncompleteChannel(clChannelsDict[short_channel_id]));
+
+    const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+    if (elapsedSeconds > 10) {
+      logger.info(`Building partial channels from clightning output. Channels processed: ${channelProcessed + 1} of ${keys.length}`);
+      loggerTimer = new Date().getTime() / 1000;
+    }
+  }
+
+  return consolidatedChannelList;
+}
+
+export function convertChannelId(channelId): string {
+  if (channelId.indexOf('/') !== -1) {
+    channelId = channelId.slice(0, -2);
+  }
+  const s = channelId.split('x').map(part => BigInt(part));
+  return ((s[0] << 40n) | (s[1] << 16n) | s[2]).toString();
+}
+
+/**
+ * Convert two clightning "getchannels" entries into a full a lnd "describegraph.edges" format
+ * In this case, clightning knows the channel policy for both nodes
+ */
+async function buildFullChannel(clChannelA: any, clChannelB: any): Promise<ILightningApi.Channel> {
+  const lastUpdate = Math.max(clChannelA.last_update ?? 0, clChannelB.last_update ?? 0);
+
+  const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannelA.short_channel_id);
+  const parts = clChannelA.short_channel_id.split('x');
+  const outputIdx = parts[2];
+
+  return {
+    channel_id: clChannelA.short_channel_id,
+    capacity: clChannelA.satoshis,
+    last_update: lastUpdate,
+    node1_policy: convertPolicy(clChannelA),
+    node2_policy: convertPolicy(clChannelB),
+    chan_point: `${tx.txid}:${outputIdx}`,
+    node1_pub: clChannelA.source,
+    node2_pub: clChannelB.source,
+  };
+}
+
+/**
+ * Convert one clightning "getchannels" entry into a full a lnd "describegraph.edges" format
+ * In this case, clightning knows the channel policy of only one node
+ */
+async function buildIncompleteChannel(clChannel: any): Promise<ILightningApi.Channel> {
+  const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannel.short_channel_id);
+  const parts = clChannel.short_channel_id.split('x');
+  const outputIdx = parts[2];
+
+  return {
+    channel_id: clChannel.short_channel_id,
+    capacity: clChannel.satoshis,
+    last_update: clChannel.last_update ?? 0,
+    node1_policy: convertPolicy(clChannel),
+    node2_policy: null,
+    chan_point: `${tx.txid}:${outputIdx}`,
+    node1_pub: clChannel.source,
+    node2_pub: clChannel.destination,
+  };
+}
+
+/**
+ * Convert a clightning "listnode" response to a lnd channel policy format
+ */
+function convertPolicy(clChannel: any): ILightningApi.RoutingPolicy {
+  return {
+    time_lock_delta: 0, // TODO
+    min_htlc: clChannel.htlc_minimum_msat.slice(0, -4),
+    max_htlc_msat: clChannel.htlc_maximum_msat.slice(0, -4),
+    fee_base_msat: clChannel.base_fee_millisatoshi,
+    fee_rate_milli_msat: clChannel.fee_per_millionth,
+    disabled: !clChannel.active,
+    last_update: clChannel.last_update ?? 0,
+  };
+}

backend/src/api/lightning/lightning-api-abstract-factory.ts

@@ -1,7 +1,5 @@
 import { ILightningApi } from './lightning-api.interface';
 
 export interface AbstractLightningApi {
-  $getNetworkInfo(): Promise<ILightningApi.NetworkInfo>;
   $getNetworkGraph(): Promise<ILightningApi.NetworkGraph>;
-  $getInfo(): Promise<ILightningApi.Info>;
 }

backend/src/api/lightning/lightning-api-factory.ts

@@ -1,9 +1,12 @@
 import config from '../../config';
+import CLightningClient from './clightning/clightning-client';
 import { AbstractLightningApi } from './lightning-api-abstract-factory';
 import LndApi from './lnd/lnd-api';
 
 function lightningApiFactory(): AbstractLightningApi {
-  switch (config.LIGHTNING.BACKEND) {
+  switch (config.LIGHTNING.ENABLED === true && config.LIGHTNING.BACKEND) {
+    case 'cln':
+      return new CLightningClient(config.CLIGHTNING.SOCKET);
     case 'lnd':
     default:
       return new LndApi();

backend/src/api/lightning/lightning-api.interface.ts

@@ -1,71 +1,85 @@
 export namespace ILightningApi {
   export interface NetworkInfo {
-    average_channel_size: number;
+    graph_diameter: number;
-    channel_count: number;
+    avg_out_degree: number;
-    max_channel_size: number;
+    max_out_degree: number;
-    median_channel_size: number;
+    num_nodes: number;
-    min_channel_size: number;
+    num_channels: number;
-    node_count: number;
+    total_network_capacity: string;
-    not_recently_updated_policy_count: number;
+    avg_channel_size: number;
-    total_capacity: number;
+    min_channel_size: string;
+    max_channel_size: string;
+    median_channel_size_sat: string;
+    num_zombie_chans: string;
   }
 
   export interface NetworkGraph {
-    channels: Channel[];
     nodes: Node[];
+    edges: Channel[];
   }
 
   export interface Channel {
-    id: string;
+    channel_id: string;
-    capacity: number;
+    chan_point: string;
-    policies: Policy[];
+    last_update: number;
-    transaction_id: string;
+    node1_pub: string;
-    transaction_vout: number;
+    node2_pub: string;
-    updated_at?: string;
+    capacity: string;
+    node1_policy: RoutingPolicy | null;
+    node2_policy: RoutingPolicy | null;
   }
 
-  interface Policy {
+  export interface RoutingPolicy {
-    public_key: string;
+    time_lock_delta: number;
-    base_fee_mtokens?: string;
+    min_htlc: string;
-    cltv_delta?: number;
+    fee_base_msat: string;
-    fee_rate?: number;
+    fee_rate_milli_msat: string;
-    is_disabled?: boolean;
+    disabled: boolean;
-    max_htlc_mtokens?: string;
+    max_htlc_msat: string;
-    min_htlc_mtokens?: string;
+    last_update: number;
-    updated_at?: string;
   }
 
   export interface Node {
+    last_update: number;
+    pub_key: string;
     alias: string;
+    addresses: {
+      network: string;
+      addr: string;
+    }[];
     color: string;
-    features: Feature[];
+    features: { [key: number]: Feature };
-    public_key: string;
-    sockets: string[];
-    updated_at?: string;
   }
 
   export interface Info {
-    chains: string[];
+    identity_pubkey: string;
-    color: string;
-    active_channels_count: number;
     alias: string;
-    current_block_hash: string;
+    num_pending_channels: number;
-    current_block_height: number;
+    num_active_channels: number;
-    features: Feature[];
+    num_peers: number;
-    is_synced_to_chain: boolean;
+    block_height: number;
-    is_synced_to_graph: boolean;
+    block_hash: string;
-    latest_block_at: string;
+    synced_to_chain: boolean;
-    peers_count: number;
+    testnet: boolean;
-    pending_channels_count: number;
+    uris: string[];
-    public_key: string;
+    best_header_timestamp: string;
-    uris: any[];
     version: string;
+    num_inactive_channels: number;
+    chains: {
+      chain: string;
+      network: string;
+    }[];
+    color: string;
+    synced_to_graph: boolean;
+    features: { [key: number]: Feature };
+    commit_hash: string;
+    /** Available on LND since v0.15.0-beta */
+    require_htlc_interceptor?: boolean;
   }
 
   export interface Feature {
-    bit: number;
+    name: string;
-    is_known: boolean;
     is_required: boolean;
-    type?: string;
+    is_known: boolean;
   }
 }

backend/src/api/lightning/lnd/lnd-api.ts

@@ -1,44 +1,40 @@
+import axios, { AxiosRequestConfig } from 'axios';
+import { Agent } from 'https';
+import * as fs from 'fs';
 import { AbstractLightningApi } from '../lightning-api-abstract-factory';
 import { ILightningApi } from '../lightning-api.interface';
-import * as fs from 'fs';
-import { authenticatedLndGrpc, getWalletInfo, getNetworkGraph, getNetworkInfo } from 'lightning';
 import config from '../../../config';
-import logger from '../../../logger';
 
 class LndApi implements AbstractLightningApi {
-  private lnd: any;
+  axiosConfig: AxiosRequestConfig = {};
+
   constructor() {
-    if (!config.LIGHTNING.ENABLED) {
+    if (config.LIGHTNING.ENABLED) {
-      return;
+      this.axiosConfig = {
-    }
+        headers: {
-    try {
+          'Grpc-Metadata-macaroon': fs.readFileSync(config.LND.MACAROON_PATH).toString('hex')
-      const tls = fs.readFileSync(config.LND.TLS_CERT_PATH).toString('base64');
+        },
-      const macaroon = fs.readFileSync(config.LND.MACAROON_PATH).toString('base64');
+        httpsAgent: new Agent({
-
+          ca: fs.readFileSync(config.LND.TLS_CERT_PATH)
-      const { lnd } = authenticatedLndGrpc({
+        }),
-        cert: tls,
+        timeout: 10000
-        macaroon: macaroon,
+      };
-        socket: config.LND.SOCKET,
-      });
-
-      this.lnd = lnd;
-    } catch (e) {
-      logger.err('Could not initiate the LND service handler: ' + (e instanceof Error ? e.message : e));
-      process.exit(1);
     }
   }
 
   async $getNetworkInfo(): Promise<ILightningApi.NetworkInfo> {
-    return await getNetworkInfo({ lnd: this.lnd });
+    return axios.get<ILightningApi.NetworkInfo>(config.LND.REST_API_URL + '/v1/graph/info', this.axiosConfig)
+      .then((response) => response.data);
   }
 
   async $getInfo(): Promise<ILightningApi.Info> {
-    // @ts-ignore
+    return axios.get<ILightningApi.Info>(config.LND.REST_API_URL + '/v1/getinfo', this.axiosConfig)
-    return await getWalletInfo({ lnd: this.lnd });
+      .then((response) => response.data);
   }
 
   async $getNetworkGraph(): Promise<ILightningApi.NetworkGraph> {
-    return await getNetworkGraph({ lnd: this.lnd });
+    return axios.get<ILightningApi.NetworkGraph>(config.LND.REST_API_URL + '/v1/graph', this.axiosConfig)
+      .then((response) => response.data);
   }
 }
 

backend/src/config.ts

@@ -31,10 +31,16 @@ interface IConfig {
   LIGHTNING: {
     ENABLED: boolean;
     BACKEND: 'lnd' | 'cln' | 'ldk';
+    TOPOLOGY_FOLDER: string;
+    STATS_REFRESH_INTERVAL: number;
+    GRAPH_REFRESH_INTERVAL: number;
   };
   LND: {
     TLS_CERT_PATH: string;
     MACAROON_PATH: string;
+    REST_API_URL: string;
+  };
+  CLIGHTNING: {
     SOCKET: string;
   };
   ELECTRUM: {
@@ -102,6 +108,7 @@ interface IConfig {
     ENABLED: boolean;
     GEOLITE2_CITY: string;
     GEOLITE2_ASN: string;
+    GEOIP2_ISP: string;
   },
 }
 
@@ -176,12 +183,18 @@ const defaults: IConfig = {
   },
   'LIGHTNING': {
     'ENABLED': false,
-    'BACKEND': 'lnd'
+    'BACKEND': 'lnd',
+    'TOPOLOGY_FOLDER': '',
+    'STATS_REFRESH_INTERVAL': 600,
+    'GRAPH_REFRESH_INTERVAL': 600,
   },
   'LND': {
     'TLS_CERT_PATH': '',
     'MACAROON_PATH': '',
-    'SOCKET': 'localhost:10009',
+    'REST_API_URL': 'https://localhost:8080',
+  },
+  'CLIGHTNING': {
+    'SOCKET': '',
   },
   'SOCKS5PROXY': {
     'ENABLED': false,
@@ -206,7 +219,8 @@ const defaults: IConfig = {
   "MAXMIND": {
     'ENABLED': false,
     "GEOLITE2_CITY": "/usr/local/share/GeoIP/GeoLite2-City.mmdb",
-    "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb"
+    "GEOLITE2_ASN": "/usr/local/share/GeoIP/GeoLite2-ASN.mmdb",
+    "GEOIP2_ISP": "/usr/local/share/GeoIP/GeoIP2-ISP.mmdb"
   },
 };
 
@@ -222,6 +236,7 @@ class Config implements IConfig {
   BISQ: IConfig['BISQ'];
   LIGHTNING: IConfig['LIGHTNING'];
   LND: IConfig['LND'];
+  CLIGHTNING: IConfig['CLIGHTNING'];
   SOCKS5PROXY: IConfig['SOCKS5PROXY'];
   PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER'];
   EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER'];
@@ -240,6 +255,7 @@ class Config implements IConfig {
     this.BISQ = configs.BISQ;
     this.LIGHTNING = configs.LIGHTNING;
     this.LND = configs.LND;
+    this.CLIGHTNING = configs.CLIGHTNING;
     this.SOCKS5PROXY = configs.SOCKS5PROXY;
     this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER;
     this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER;

backend/src/database.ts

@@ -1,7 +1,7 @@
 import config from './config';
 import { createPool, Pool, PoolConnection } from 'mysql2/promise';
 import logger from './logger';
-import { PoolOptions } from 'mysql2/typings/mysql';
+import { FieldPacket, OkPacket, PoolOptions, ResultSetHeader, RowDataPacket } from 'mysql2/typings/mysql';
 
  class DB {
   constructor() {
@@ -28,7 +28,9 @@ import { PoolOptions } from 'mysql2/typings/mysql';
     }
   }
 
-  public async query(query, params?) {
+  public async query<T extends RowDataPacket[][] | RowDataPacket[] | OkPacket |
+    OkPacket[] | ResultSetHeader>(query, params?): Promise<[T, FieldPacket[]]>
+  {
     this.checkDBFlag();
     const pool = await this.getPool();
     return pool.query(query, params);

backend/src/index.ts

@@ -28,12 +28,13 @@ import nodesRoutes from './api/explorer/nodes.routes';
 import channelsRoutes from './api/explorer/channels.routes';
 import generalLightningRoutes from './api/explorer/general.routes';
 import lightningStatsUpdater from './tasks/lightning/stats-updater.service';
-import nodeSyncService from './tasks/lightning/node-sync.service';
+import networkSyncService from './tasks/lightning/network-sync.service';
-import statisticsRoutes from "./api/statistics/statistics.routes";
+import statisticsRoutes from './api/statistics/statistics.routes';
-import miningRoutes from "./api/mining/mining-routes";
+import miningRoutes from './api/mining/mining-routes';
-import bisqRoutes from "./api/bisq/bisq.routes";
+import bisqRoutes from './api/bisq/bisq.routes';
-import liquidRoutes from "./api/liquid/liquid.routes";
+import liquidRoutes from './api/liquid/liquid.routes';
-import bitcoinRoutes from "./api/bitcoin/bitcoin.routes";
+import bitcoinRoutes from './api/bitcoin/bitcoin.routes';
+import fundingTxFetcher from "./tasks/lightning/sync-tasks/funding-tx-fetcher";
 
 class Server {
   private wss: WebSocket.Server | undefined;
@@ -136,8 +137,7 @@ class Server {
     }
 
     if (config.LIGHTNING.ENABLED) {
-      nodeSyncService.$startService()
+      this.$runLightningBackend();
-        .then(() => lightningStatsUpdater.$startService());
     }
 
     this.server.listen(config.MEMPOOL.HTTP_PORT, () => {
@@ -183,6 +183,18 @@ class Server {
     }
   }
 
+  async $runLightningBackend() {
+    try {
+      await fundingTxFetcher.$init();
+      await networkSyncService.$startService();
+      await lightningStatsUpdater.$startService();
+    } catch(e) {
+      logger.err(`Lightning backend crashed. Restarting in 1 minute. Reason: ${(e instanceof Error ? e.message : e)}`);
+      await Common.sleep$(1000 * 60);
+      this.$runLightningBackend();
+    };
+}
+
   setUpWebsocketHandling() {
     if (this.wss) {
       websocketHandler.setWebsocketServer(this.wss);

backend/src/tasks/lightning/network-sync.service.ts

@@ -1,51 +1,43 @@
-import { chanNumber } from 'bolt07';
 import DB from '../../database';
 import logger from '../../logger';
 import channelsApi from '../../api/explorer/channels.api';
-import bitcoinClient from '../../api/bitcoin/bitcoin-client';
 import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory';
 import config from '../../config';
 import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface';
-import lightningApi from '../../api/lightning/lightning-api-factory';
 import { ILightningApi } from '../../api/lightning/lightning-api.interface';
 import { $lookupNodeLocation } from './sync-tasks/node-locations';
+import lightningApi from '../../api/lightning/lightning-api-factory';
+import nodesApi from '../../api/explorer/nodes.api';
+import { ResultSetHeader } from 'mysql2';
+import fundingTxFetcher from './sync-tasks/funding-tx-fetcher';
+
+class NetworkSyncService {
+  loggerTimer = 0;
 
-class NodeSyncService {
   constructor() {}
 
-  public async $startService() {
+  public async $startService(): Promise<void> {
-    logger.info('Starting node sync service');
+    logger.info('Starting lightning network sync service');
 
-    await this.$runUpdater();
+    this.loggerTimer = new Date().getTime() / 1000;
 
-    setInterval(async () => {
+    await this.$runTasks();
-      await this.$runUpdater();
-    }, 1000 * 60 * 60);
   }
 
-  private async $runUpdater() {
+  private async $runTasks(): Promise<void> {
     try {
-      logger.info(`Updating nodes and channels...`);
+      logger.info(`Updating nodes and channels`);
 
       const networkGraph = await lightningApi.$getNetworkGraph();
-
+      if (networkGraph.nodes.length === 0 || networkGraph.edges.length === 0) {
-      for (const node of networkGraph.nodes) {
+        logger.info(`LN Network graph is empty, retrying in 10 seconds`);
-        await this.$saveNode(node);
+        setTimeout(() => { this.$runTasks(); }, 10000);
-      }
+        return;
-      logger.info(`Nodes updated.`);
-
-      if (config.MAXMIND.ENABLED) {
-        await $lookupNodeLocation();
       }
 
-      await this.$setChannelsInactive();
+      await this.$updateNodesList(networkGraph.nodes);
-
+      await this.$updateChannelsList(networkGraph.edges);
-      for (const channel of networkGraph.channels) {
+      await this.$deactivateChannelsWithoutActiveNodes();
-        await this.$saveChannel(channel);
-      }
-      logger.info(`Channels updated.`);
-
-      await this.$findInactiveNodesAndChannels();
       await this.$lookUpCreationDateFromChain();
       await this.$updateNodeFirstSeen();
       await this.$scanForClosedChannels();
@@ -54,70 +46,183 @@ class NodeSyncService {
       }
 
     } catch (e) {
-      logger.err('$updateNodes() error: ' + (e instanceof Error ? e.message : e));
+      logger.err('$runTasks() error: ' + (e instanceof Error ? e.message : e));
     }
+
+    setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.GRAPH_REFRESH_INTERVAL);
+  }
+
+  /**
+   * Update the `nodes` table to reflect the current network graph state
+   */
+  private async $updateNodesList(nodes: ILightningApi.Node[]): Promise<void> {
+    let progress = 0;
+
+    const graphNodesPubkeys: string[] = [];
+    for (const node of nodes) {
+      await nodesApi.$saveNode(node);
+      graphNodesPubkeys.push(node.pub_key);
+      ++progress;
+
+      const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+      if (elapsedSeconds > 10) {
+        logger.info(`Updating node ${progress}/${nodes.length}`);
+        this.loggerTimer = new Date().getTime() / 1000;
+      }
+    }
+    logger.info(`${progress} nodes updated`);
+
+    // If a channel if not present in the graph, mark it as inactive
+    nodesApi.$setNodesInactive(graphNodesPubkeys);
+
+    if (config.MAXMIND.ENABLED) {
+      $lookupNodeLocation();
+    }
+  }
+
+  /**
+   * Update the `channels` table to reflect the current network graph state
+   */
+  private async $updateChannelsList(channels: ILightningApi.Channel[]): Promise<void> {
+    try {
+      let progress = 0;
+
+      const graphChannelsIds: string[] = [];
+      for (const channel of channels) {
+        await channelsApi.$saveChannel(channel);
+        graphChannelsIds.push(channel.channel_id);
+        ++progress;
+
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating channel ${progress}/${channels.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+        }
+      }
+
+      logger.info(`${progress} channels updated`);
+
+      // If a channel if not present in the graph, mark it as inactive
+      channelsApi.$setChannelsInactive(graphChannelsIds);
+    } catch (e) {
+      logger.err(`Cannot update channel list. Reason: ${(e instanceof Error ? e.message : e)}`);
+    }
+
+    setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.STATS_REFRESH_INTERVAL);
   }
 
   // This method look up the creation date of the earliest channel of the node
   // and update the node to that date in order to get the earliest first seen date
-  private async $updateNodeFirstSeen() {
+  private async $updateNodeFirstSeen(): Promise<void> {
+    let progress = 0;
+    let updated = 0;
+
     try {
-      const [nodes]: any[] = await DB.query(`SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node1_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created1, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node2_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created2 FROM nodes`);
+      const [nodes]: any[] = await DB.query(`
+        SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen,
+        (
+          SELECT MIN(UNIX_TIMESTAMP(created))
+          FROM channels
+          WHERE channels.node1_public_key = nodes.public_key
+        ) AS created1,
+        (
+          SELECT MIN(UNIX_TIMESTAMP(created))
+          FROM channels
+          WHERE channels.node2_public_key = nodes.public_key
+        ) AS created2
+        FROM nodes
+      `);
+
       for (const node of nodes) {
-        let lowest = 0;
+        const lowest = Math.min(
-        if (node.created1) {
+          node.created1 ?? Number.MAX_SAFE_INTEGER,
-          if (node.created2 && node.created2 < node.created1) {
+          node.created2 ?? Number.MAX_SAFE_INTEGER,
-            lowest = node.created2;
+          node.first_seen ?? Number.MAX_SAFE_INTEGER
-          } else {
+        );
-            lowest = node.created1;
+        if (lowest < node.first_seen) {
-          }
-        } else if (node.created2) {
-          lowest = node.created2;
-        }
-        if (lowest && lowest < node.first_seen) {
           const query = `UPDATE nodes SET first_seen = FROM_UNIXTIME(?) WHERE public_key = ?`;
           const params = [lowest, node.public_key];
           await DB.query(query, params);
         }
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating node first seen date ${progress}/${nodes.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+          ++updated;
         }
-      logger.info(`Node first seen dates scan complete.`);
+      }
+      logger.info(`Updated ${updated} node first seen dates`);
     } catch (e) {
       logger.err('$updateNodeFirstSeen() error: ' + (e instanceof Error ? e.message : e));
     }
   }
 
-  private async $lookUpCreationDateFromChain() {
+  private async $lookUpCreationDateFromChain(): Promise<void> {
-    logger.info(`Running channel creation date lookup...`);
+    let progress = 0;
+
+    logger.info(`Running channel creation date lookup`);
     try {
       const channels = await channelsApi.$getChannelsWithoutCreatedDate();
       for (const channel of channels) {
-        const transaction = await bitcoinClient.getRawTransaction(channel.transaction_id, 1);
+        const transaction = await fundingTxFetcher.$fetchChannelOpenTx(channel.short_id);
-        await DB.query(`UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`, [transaction.blocktime, channel.id]);
+        await DB.query(`
+          UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`,
+          [transaction.timestamp, channel.id]
+        );
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating channel creation date ${progress}/${channels.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
         }
-      logger.info(`Channel creation dates scan complete.`);
+      }
+      logger.info(`Updated ${channels.length} channels' creation date`);
     } catch (e) {
-      logger.err('$setCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e));
+      logger.err('$lookUpCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e));
     }
   }
 
-  // Looking for channels whos nodes are inactive
+  /**
-  private async $findInactiveNodesAndChannels(): Promise<void> {
+   * If a channel does not have any active node linked to it, then also
-    logger.info(`Running inactive channels scan...`);
+   * mark that channel as inactive
+   */
+  private async $deactivateChannelsWithoutActiveNodes(): Promise<void> {
+    logger.info(`Find channels which nodes are offline`);
 
     try {
-      // @ts-ignore
+      const result = await DB.query<ResultSetHeader>(`
-      const [channels]: [ILightningApi.Channel[]] = await DB.query(`SELECT channels.id FROM channels WHERE channels.status = 1 AND ((SELECT COUNT(*) FROM nodes WHERE nodes.public_key = channels.node1_public_key) = 0 OR (SELECT COUNT(*) FROM nodes WHERE nodes.public_key = channels.node2_public_key) = 0)`);
+        UPDATE channels
+        SET status = 0
+        WHERE channels.status = 1
+        AND (
+          (
+            SELECT COUNT(*)
+            FROM nodes
+            WHERE nodes.public_key = channels.node1_public_key
+            AND nodes.status = 1
+          ) = 0
+        OR (
+            SELECT COUNT(*)
+            FROM nodes
+            WHERE nodes.public_key = channels.node2_public_key
+            AND nodes.status = 1
+          ) = 0)
+        `);
 
-      for (const channel of channels) {
+      if (result[0].changedRows ?? 0 > 0) {
-        await this.$updateChannelStatus(channel.id, 0);
+        logger.info(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`);
+      } else {
+        logger.debug(`Marked ${result[0].changedRows} channels as inactive because they are not linked to any active node`);
       }
-      logger.info(`Inactive channels scan complete.`);
     } catch (e) {
-      logger.err('$findInactiveNodesAndChannels() error: ' + (e instanceof Error ? e.message : e));
+      logger.err('$deactivateChannelsWithoutActiveNodes() error: ' + (e instanceof Error ? e.message : e));
     }
   }
 
   private async $scanForClosedChannels(): Promise<void> {
+    let progress = 0;
+
     try {
       logger.info(`Starting closed channels scan...`);
       const channels = await channelsApi.$getChannelsByStatus(0);
@@ -131,6 +236,13 @@ class NodeSyncService {
             await DB.query(`UPDATE channels SET closing_transaction_id = ? WHERE id = ?`, [spendingTx.txid, channel.id]);
           }
         }
+
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Checking if channel has been closed ${progress}/${channels.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+        }
       }
       logger.info(`Closed channels scan complete.`);
     } catch (e) {
@@ -148,6 +260,9 @@ class NodeSyncService {
     if (!config.ESPLORA.REST_API_URL) {
       return;
     }
+
+    let progress = 0;
+
     try {
       logger.info(`Started running closed channel forensics...`);
       const channels = await channelsApi.$getClosedChannelsWithoutReason();
@@ -193,6 +308,13 @@ class NodeSyncService {
           logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.');
           await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]);
         }
+
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating channel closed channel forensics ${progress}/${channels.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+        }
       }
       logger.info(`Closed channels forensics scan complete.`);
     } catch (e) {
@@ -247,157 +369,6 @@ class NodeSyncService {
       }
       return 1;
   }
-
-  private async $saveChannel(channel: ILightningApi.Channel): Promise<void> {
-    const fromChannel = chanNumber({ channel: channel.id }).number;
-
-    try {
-      const query = `INSERT INTO channels
-        (
-          id,
-          short_id,
-          capacity,
-          transaction_id,
-          transaction_vout,
-          updated_at,
-          status,
-          node1_public_key,
-          node1_base_fee_mtokens,
-          node1_cltv_delta,
-          node1_fee_rate,
-          node1_is_disabled,
-          node1_max_htlc_mtokens,
-          node1_min_htlc_mtokens,
-          node1_updated_at,
-          node2_public_key,
-          node2_base_fee_mtokens,
-          node2_cltv_delta,
-          node2_fee_rate,
-          node2_is_disabled,
-          node2_max_htlc_mtokens,
-          node2_min_htlc_mtokens,
-          node2_updated_at
-        )
-        VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-        ON DUPLICATE KEY UPDATE
-          capacity = ?,
-          updated_at = ?,
-          status = 1,
-          node1_public_key = ?,
-          node1_base_fee_mtokens = ?,
-          node1_cltv_delta = ?,
-          node1_fee_rate = ?,
-          node1_is_disabled = ?,
-          node1_max_htlc_mtokens = ?,
-          node1_min_htlc_mtokens = ?,
-          node1_updated_at = ?,
-          node2_public_key = ?,
-          node2_base_fee_mtokens = ?,
-          node2_cltv_delta = ?,
-          node2_fee_rate = ?,
-          node2_is_disabled = ?,
-          node2_max_htlc_mtokens = ?,
-          node2_min_htlc_mtokens = ?,
-          node2_updated_at = ?
-        ;`;
-
-      await DB.query(query, [
-        fromChannel,
-        channel.id,
-        channel.capacity,
-        channel.transaction_id,
-        channel.transaction_vout,
-        channel.updated_at ? this.utcDateToMysql(channel.updated_at) : 0,
-        channel.policies[0].public_key,
-        channel.policies[0].base_fee_mtokens,
-        channel.policies[0].cltv_delta,
-        channel.policies[0].fee_rate,
-        channel.policies[0].is_disabled,
-        channel.policies[0].max_htlc_mtokens,
-        channel.policies[0].min_htlc_mtokens,
-        channel.policies[0].updated_at ? this.utcDateToMysql(channel.policies[0].updated_at) : 0,
-        channel.policies[1].public_key,
-        channel.policies[1].base_fee_mtokens,
-        channel.policies[1].cltv_delta,
-        channel.policies[1].fee_rate,
-        channel.policies[1].is_disabled,
-        channel.policies[1].max_htlc_mtokens,
-        channel.policies[1].min_htlc_mtokens,
-        channel.policies[1].updated_at ? this.utcDateToMysql(channel.policies[1].updated_at) : 0,
-        channel.capacity,
-        channel.updated_at ? this.utcDateToMysql(channel.updated_at) : 0,
-        channel.policies[0].public_key,
-        channel.policies[0].base_fee_mtokens,
-        channel.policies[0].cltv_delta,
-        channel.policies[0].fee_rate,
-        channel.policies[0].is_disabled,
-        channel.policies[0].max_htlc_mtokens,
-        channel.policies[0].min_htlc_mtokens,
-        channel.policies[0].updated_at ? this.utcDateToMysql(channel.policies[0].updated_at) : 0,
-        channel.policies[1].public_key,
-        channel.policies[1].base_fee_mtokens,
-        channel.policies[1].cltv_delta,
-        channel.policies[1].fee_rate,
-        channel.policies[1].is_disabled,
-        channel.policies[1].max_htlc_mtokens,
-        channel.policies[1].min_htlc_mtokens,
-        channel.policies[1].updated_at ? this.utcDateToMysql(channel.policies[1].updated_at) : 0,
-      ]);
-    } catch (e) {
-      logger.err('$saveChannel() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $updateChannelStatus(channelShortId: string, status: number): Promise<void> {
-    try {
-      await DB.query(`UPDATE channels SET status = ? WHERE id = ?`, [status, channelShortId]);
-    } catch (e) {
-      logger.err('$updateChannelStatus() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $setChannelsInactive(): Promise<void> {
-    try {
-      await DB.query(`UPDATE channels SET status = 0 WHERE status = 1`);
-    } catch (e) {
-      logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $saveNode(node: ILightningApi.Node): Promise<void> {
-    try {
-      const updatedAt = node.updated_at ? this.utcDateToMysql(node.updated_at) : '0000-00-00 00:00:00';
-      const sockets = node.sockets.join(',');
-      const query = `INSERT INTO nodes(
-          public_key,
-          first_seen,
-          updated_at,
-          alias,
-          color,
-          sockets
-        )
-        VALUES (?, NOW(), ?, ?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = ?, alias = ?, color = ?, sockets = ?;`;
-
-      await DB.query(query, [
-        node.public_key,
-        updatedAt,
-        node.alias,
-        node.color,
-        sockets,
-        updatedAt,
-        node.alias,
-        node.color,
-        sockets,
-      ]);
-    } catch (e) {
-      logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private utcDateToMysql(dateString: string): string {
-    const d = new Date(Date.parse(dateString));
-    return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0];
-  }
 }
 
-export default new NodeSyncService();
+export default new NetworkSyncService();

backend/src/tasks/lightning/stats-updater.service.ts

@@ -1,335 +1,33 @@
-
-import DB from '../../database';
 import logger from '../../logger';
 import lightningApi from '../../api/lightning/lightning-api-factory';
-import channelsApi from '../../api/explorer/channels.api';
+import LightningStatsImporter from './sync-tasks/stats-importer';
-import * as net from 'net';
+import config from '../../config';
+import { Common } from '../../api/common';
 
 class LightningStatsUpdater {
-  hardCodedStartTime = '2018-01-12';
+  public async $startService(): Promise<void> {
-
-  public async $startService() {
     logger.info('Starting Lightning Stats service');
-    let isInSync = false;
-    let error: any;
-    try {
-      error = null;
-      isInSync = await this.$lightningIsSynced();
-    } catch (e) {
-      error = e;
-    }
-    if (!isInSync) {
-      if (error) {
-        logger.warn('Was not able to fetch Lightning Node status: ' + (error instanceof Error ? error.message : error) + '. Retrying in 1 minute...');
-      } else {
-        logger.notice('The Lightning graph is not yet in sync. Retrying in 1 minute...');
-      }
-      setTimeout(() => this.$startService(), 60 * 1000);
-      return;
-    }
 
-    await this.$populateHistoricalStatistics();
+    await this.$runTasks();
-    await this.$populateHistoricalNodeStatistics();
+    LightningStatsImporter.$run();
-
-    setTimeout(() => {
-      this.$runTasks();
-    }, this.timeUntilMidnight());
-  }
-
-  private timeUntilMidnight(): number {
-    const date = new Date();
-    this.setDateMidnight(date);
-    date.setUTCHours(24);
-    return date.getTime() - new Date().getTime();
-  }
-
-  private setDateMidnight(date: Date): void {
-    date.setUTCHours(0);
-    date.setUTCMinutes(0);
-    date.setUTCSeconds(0);
-    date.setUTCMilliseconds(0);
-  }
-
-  private async $lightningIsSynced(): Promise<boolean> {
-    const nodeInfo = await lightningApi.$getInfo();
-    return nodeInfo.is_synced_to_chain && nodeInfo.is_synced_to_graph;
   }
 
   private async $runTasks(): Promise<void> {
-    await this.$logLightningStatsDaily();
+    await this.$logStatsDaily();
-    await this.$logNodeStatsDaily();
 
-    setTimeout(() => {
+    setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.STATS_REFRESH_INTERVAL);
-      this.$runTasks();
-    }, this.timeUntilMidnight());
   }
 
-  private async $logLightningStatsDaily() {
+  /**
-    try {
+   * Update the latest entry for each node every config.LIGHTNING.STATS_REFRESH_INTERVAL seconds
-      logger.info(`Running lightning daily stats log...`);  
+   */
-
+  private async $logStatsDaily(): Promise<void> {
+    const date = new Date();
+    Common.setDateMidnight(date);
     const networkGraph = await lightningApi.$getNetworkGraph();
-      let total_capacity = 0;
+    LightningStatsImporter.computeNetworkStats(date.getTime() / 1000, networkGraph);
-      for (const channel of networkGraph.channels) {
-        if (channel.capacity) {
-          total_capacity += channel.capacity;
-        }
-      }
     
-      let clearnetNodes = 0;
+    logger.info(`Updated latest network stats`);
-      let torNodes = 0;
-      let unannouncedNodes = 0;
-      for (const node of networkGraph.nodes) {
-        let isUnnanounced = true;
-        for (const socket of node.sockets) {
-          const hasOnion = socket.indexOf('.onion') !== -1;
-          if (hasOnion) {
-            torNodes++;
-            isUnnanounced = false;
-          }
-          const hasClearnet = [4, 6].includes(net.isIP(socket.split(':')[0]));
-          if (hasClearnet) {
-            clearnetNodes++;
-            isUnnanounced = false;
-          }
-        }
-        if (isUnnanounced) {
-          unannouncedNodes++;
-        }
-      }
-
-      const channelStats = await channelsApi.$getChannelsStats();
-
-      const query = `INSERT INTO lightning_stats(
-          added,
-          channel_count,
-          node_count,
-          total_capacity,
-          tor_nodes,
-          clearnet_nodes,
-          unannounced_nodes,
-          avg_capacity,
-          avg_fee_rate,
-          avg_base_fee_mtokens,
-          med_capacity,
-          med_fee_rate,
-          med_base_fee_mtokens
-        )
-        VALUES (NOW() - INTERVAL 1 DAY, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
-
-      await DB.query(query, [
-        networkGraph.channels.length,
-        networkGraph.nodes.length,
-        total_capacity,
-        torNodes,
-        clearnetNodes,
-        unannouncedNodes,
-        channelStats.avgCapacity,
-        channelStats.avgFeeRate,
-        channelStats.avgBaseFee,
-        channelStats.medianCapacity,
-        channelStats.medianFeeRate,
-        channelStats.medianBaseFee,
-      ]);
-      logger.info(`Lightning daily stats done.`);
-    } catch (e) {
-      logger.err('$logLightningStatsDaily() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $logNodeStatsDaily() {
-    try {
-      logger.info(`Running daily node stats update...`);
-
-      const query = `SELECT nodes.public_key, c1.channels_count_left, c2.channels_count_right, c1.channels_capacity_left, c2.channels_capacity_right FROM nodes LEFT JOIN (SELECT node1_public_key, COUNT(id) AS channels_count_left, SUM(capacity) AS channels_capacity_left FROM channels WHERE channels.status < 2 GROUP BY node1_public_key) c1 ON c1.node1_public_key = nodes.public_key LEFT JOIN (SELECT node2_public_key, COUNT(id) AS channels_count_right, SUM(capacity) AS channels_capacity_right FROM channels WHERE channels.status < 2 GROUP BY node2_public_key) c2 ON c2.node2_public_key = nodes.public_key`;
-      const [nodes]: any = await DB.query(query);
-
-      for (const node of nodes) {
-        await DB.query(
-          `INSERT INTO node_stats(public_key, added, capacity, channels) VALUES (?, NOW() - INTERVAL 1 DAY, ?, ?)`,
-          [node.public_key, (parseInt(node.channels_capacity_left || 0, 10)) + (parseInt(node.channels_capacity_right || 0, 10)),
-            node.channels_count_left + node.channels_count_right]);
-      }
-      logger.info('Daily node stats has updated.');
-    } catch (e) {
-      logger.err('$logNodeStatsDaily() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  // We only run this on first launch
-  private async $populateHistoricalStatistics() {
-    try {
-      const [rows]: any = await DB.query(`SELECT COUNT(*) FROM lightning_stats`);
-      // Only run if table is empty
-      if (rows[0]['COUNT(*)'] > 0) {
-        return;
-      }
-      logger.info(`Running historical stats population...`);
-
-      const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels ORDER BY created ASC`);
-      const [nodes]: any = await DB.query(`SELECT first_seen, sockets FROM nodes ORDER BY first_seen ASC`);
-
-      const date: Date = new Date(this.hardCodedStartTime);
-      const currentDate = new Date();
-      this.setDateMidnight(currentDate);
-
-      while (date < currentDate) {
-        let totalCapacity = 0;
-        let channelsCount = 0;
-
-        for (const channel of channels) {
-          if (new Date(channel.created) > date) {
-            break;
-          }
-          if (channel.closing_date === null || new Date(channel.closing_date) > date) {
-            totalCapacity += channel.capacity;
-            channelsCount++;
-          }
-        }
-
-        let nodeCount = 0;
-        let clearnetNodes = 0;
-        let torNodes = 0;
-        let unannouncedNodes = 0;
-
-        for (const node of nodes) {
-          if (new Date(node.first_seen) > date) {
-            break;
-          }
-          nodeCount++;
-
-          const sockets = node.sockets.split(',');
-          let isUnnanounced = true;
-          for (const socket of sockets) {
-            const hasOnion = socket.indexOf('.onion') !== -1;
-            if (hasOnion) {
-              torNodes++;
-              isUnnanounced = false;
-            }
-            const hasClearnet = [4, 6].includes(net.isIP(socket.substring(0, socket.lastIndexOf(':'))));
-            if (hasClearnet) {
-              clearnetNodes++;
-              isUnnanounced = false;
-            }
-          }
-          if (isUnnanounced) {
-            unannouncedNodes++;
-          }
-        }
-
-        const query = `INSERT INTO lightning_stats(
-          added,
-          channel_count,
-          node_count,
-          total_capacity,
-          tor_nodes,
-          clearnet_nodes,
-          unannounced_nodes,
-          avg_capacity,
-          avg_fee_rate,
-          avg_base_fee_mtokens,
-          med_capacity,
-          med_fee_rate,
-          med_base_fee_mtokens
-        )
-        VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`;
-
-        const rowTimestamp = date.getTime() / 1000; // Save timestamp for the row insertion down below
-
-        date.setUTCDate(date.getUTCDate() + 1);
-
-        // Last iteration, save channels stats
-        const channelStats = (date >= currentDate ? await channelsApi.$getChannelsStats() : undefined);
-
-        await DB.query(query, [
-          rowTimestamp,
-          channelsCount,
-          nodeCount,
-          totalCapacity,
-          torNodes,
-          clearnetNodes,
-          unannouncedNodes,
-          channelStats?.avgCapacity ?? 0,
-          channelStats?.avgFeeRate ?? 0,
-          channelStats?.avgBaseFee ?? 0,
-          channelStats?.medianCapacity ?? 0,
-          channelStats?.medianFeeRate ?? 0,
-          channelStats?.medianBaseFee ?? 0,
-          ]);
-      }
-
-      logger.info('Historical stats populated.');
-    } catch (e) {
-      logger.err('$populateHistoricalData() error: ' + (e instanceof Error ? e.message : e));
-    }
-  }
-
-  private async $populateHistoricalNodeStatistics() {
-    try {
-      const [rows]: any = await DB.query(`SELECT COUNT(*) FROM node_stats`);
-      // Only run if table is empty
-      if (rows[0]['COUNT(*)'] > 0) {
-        return;
-      }
-      logger.info(`Running historical node stats population...`);
-
-      const [nodes]: any = await DB.query(`SELECT public_key, first_seen, alias FROM nodes ORDER BY first_seen ASC`);
-
-      for (const node of nodes) {
-        const [channels]: any = await DB.query(`SELECT capacity, created, closing_date FROM channels WHERE node1_public_key = ? OR node2_public_key = ? ORDER BY created ASC`, [node.public_key, node.public_key]);
-        
-        const date: Date = new Date(this.hardCodedStartTime);
-        const currentDate = new Date();
-        this.setDateMidnight(currentDate);
-
-        let lastTotalCapacity = 0;
-        let lastChannelsCount = 0;
-
-        while (date < currentDate) {
-          let totalCapacity = 0;
-          let channelsCount = 0;
-          for (const channel of channels) {
-            if (new Date(channel.created) > date) {
-              break;
-            }
-            if (channel.closing_date !== null && new Date(channel.closing_date) < date) {
-              date.setUTCDate(date.getUTCDate() + 1);
-              continue;
-            }
-            totalCapacity += channel.capacity;
-            channelsCount++;
-          }
-
-          if (lastTotalCapacity === totalCapacity && lastChannelsCount === channelsCount) {
-            date.setUTCDate(date.getUTCDate() + 1);
-            continue;
-          }
-
-          lastTotalCapacity = totalCapacity;
-          lastChannelsCount = channelsCount;
-  
-          const query = `INSERT INTO node_stats(
-            public_key,
-            added,
-            capacity,
-            channels
-          )
-          VALUES (?, FROM_UNIXTIME(?), ?, ?)`;
-
-          await DB.query(query, [
-            node.public_key,
-            date.getTime() / 1000,
-            totalCapacity,
-            channelsCount,
-          ]);
-          date.setUTCDate(date.getUTCDate() + 1);
-        }
-        logger.debug('Updated node_stats for: ' + node.alias);
-      }
-      logger.info('Historical stats populated.');
-    } catch (e) {
-      logger.err('$populateHistoricalNodeData() error: ' + (e instanceof Error ? e.message : e));
-    }
   }
 }
 

backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts

@@ -0,0 +1,118 @@
+import { existsSync, promises } from 'fs';
+import bitcoinClient from '../../../api/bitcoin/bitcoin-client';
+import { Common } from '../../../api/common';
+import config from '../../../config';
+import logger from '../../../logger';
+
+const fsPromises = promises;
+
+const BLOCKS_CACHE_MAX_SIZE = 100;  
+const CACHE_FILE_NAME = config.MEMPOOL.CACHE_DIR + '/ln-funding-txs-cache.json';
+
+class FundingTxFetcher {
+  private running = false;
+  private blocksCache = {};
+  private channelNewlyProcessed = 0;
+  public fundingTxCache = {};
+
+  async $init(): Promise<void> {
+    // Load funding tx disk cache
+    if (Object.keys(this.fundingTxCache).length === 0 && existsSync(CACHE_FILE_NAME)) {
+      try {
+        this.fundingTxCache = JSON.parse(await fsPromises.readFile(CACHE_FILE_NAME, 'utf-8'));
+      } catch (e) {
+        logger.err(`Unable to parse channels funding txs disk cache. Starting from scratch`);
+        this.fundingTxCache = {};
+      }
+      logger.debug(`Imported ${Object.keys(this.fundingTxCache).length} funding tx amount from the disk cache`);
+    }
+  }
+
+  async $fetchChannelsFundingTxs(channelIds: string[]): Promise<void> {
+    if (this.running) {
+      return;
+    }
+    this.running = true;
+    
+    const globalTimer = new Date().getTime() / 1000;
+    let cacheTimer = new Date().getTime() / 1000;
+    let loggerTimer = new Date().getTime() / 1000;
+    let channelProcessed = 0;
+    this.channelNewlyProcessed = 0;
+    for (const channelId of channelIds) {
+      await this.$fetchChannelOpenTx(channelId);
+      ++channelProcessed;
+
+      let elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+      if (elapsedSeconds > 10) {
+        elapsedSeconds = Math.round((new Date().getTime() / 1000) - globalTimer);
+        logger.info(`Indexing channels funding tx ${channelProcessed + 1} of ${channelIds.length} ` +
+          `(${Math.floor(channelProcessed / channelIds.length * 10000) / 100}%) | ` +
+          `elapsed: ${elapsedSeconds} seconds`
+        );
+        loggerTimer = new Date().getTime() / 1000;
+      }
+
+      elapsedSeconds = Math.round((new Date().getTime() / 1000) - cacheTimer);
+      if (elapsedSeconds > 60) {
+        logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`);
+        fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache));
+        cacheTimer = new Date().getTime() / 1000;
+      }
+    }
+
+    if (this.channelNewlyProcessed > 0) {
+      logger.info(`Indexed ${this.channelNewlyProcessed} additional channels funding tx`);
+      logger.debug(`Saving ${Object.keys(this.fundingTxCache).length} funding txs cache into disk`);
+      fsPromises.writeFile(CACHE_FILE_NAME, JSON.stringify(this.fundingTxCache));
+    }
+
+    this.running = false;
+  }
+  
+  public async $fetchChannelOpenTx(channelId: string): Promise<{timestamp: number, txid: string, value: number}> {
+    if (channelId.indexOf('x') === -1) {
+      channelId = Common.channelIntegerIdToShortId(channelId);
+    }
+
+    if (this.fundingTxCache[channelId]) {
+      return this.fundingTxCache[channelId];
+    }
+
+    const parts = channelId.split('x');
+    const blockHeight = parts[0];
+    const txIdx = parts[1];
+    const outputIdx = parts[2];
+
+    let block = this.blocksCache[blockHeight];
+    // Fetch it from core
+    if (!block) {
+      const blockHash = await bitcoinClient.getBlockHash(parseInt(blockHeight, 10));
+      block = await bitcoinClient.getBlock(blockHash, 1);
+    }
+    this.blocksCache[block.height] = block;
+
+    const blocksCacheHashes = Object.keys(this.blocksCache).sort((a, b) => parseInt(b) - parseInt(a)).reverse();
+    if (blocksCacheHashes.length > BLOCKS_CACHE_MAX_SIZE) {
+      for (let i = 0; i < 10; ++i) {
+        delete this.blocksCache[blocksCacheHashes[i]];
+      }
+    }
+
+    const txid = block.tx[txIdx];
+    const rawTx = await bitcoinClient.getRawTransaction(txid);
+    const tx = await bitcoinClient.decodeRawTransaction(rawTx);
+
+    this.fundingTxCache[channelId] = {
+      timestamp: block.time,
+      txid: txid,
+      value: tx.vout[outputIdx].value,
+    };
+
+    ++this.channelNewlyProcessed;
+
+    return this.fundingTxCache[channelId];
+  }
+}
+
+export default new FundingTxFetcher;

backend/src/tasks/lightning/sync-tasks/node-locations.ts

@@ -1,28 +1,55 @@
 import * as net from 'net';
-import maxmind, { CityResponse, AsnResponse } from 'maxmind';
+import maxmind, { CityResponse, AsnResponse, IspResponse } from 'maxmind';
 import nodesApi from '../../../api/explorer/nodes.api';
 import config from '../../../config';
 import DB from '../../../database';
 import logger from '../../../logger';
 
 export async function $lookupNodeLocation(): Promise<void> {
-  logger.info(`Running node location updater using Maxmind...`);
+  let loggerTimer = new Date().getTime() / 1000;
+  let progress = 0;
+
+  logger.info(`Running node location updater using Maxmind`);
   try {
     const nodes = await nodesApi.$getAllNodes();
     const lookupCity = await maxmind.open<CityResponse>(config.MAXMIND.GEOLITE2_CITY);
     const lookupAsn = await maxmind.open<AsnResponse>(config.MAXMIND.GEOLITE2_ASN);
+    const lookupIsp = await maxmind.open<IspResponse>(config.MAXMIND.GEOIP2_ISP);
 
     for (const node of nodes) {
       const sockets: string[] = node.sockets.split(',');
       for (const socket of sockets) {
         const ip = socket.substring(0, socket.lastIndexOf(':')).replace('[', '').replace(']', '');
         const hasClearnet = [4, 6].includes(net.isIP(ip));
+
         if (hasClearnet && ip !== '127.0.1.1' && ip !== '127.0.0.1') {
           const city = lookupCity.get(ip);
           const asn = lookupAsn.get(ip);
-          if (city && asn) {
+          const isp = lookupIsp.get(ip);
-            const query = `UPDATE nodes SET as_number = ?, city_id = ?, country_id = ?, subdivision_id = ?, longitude = ?, latitude = ?, accuracy_radius = ? WHERE public_key = ?`;
+
-            const params = [asn.autonomous_system_number, city.city?.geoname_id, city.country?.geoname_id, city.subdivisions ? city.subdivisions[0].geoname_id : null, city.location?.longitude, city.location?.latitude, city.location?.accuracy_radius, node.public_key];
+          if (city && (asn || isp)) {
+            const query = `
+              UPDATE nodes SET 
+                as_number = ?, 
+                city_id = ?, 
+                country_id = ?, 
+                subdivision_id = ?, 
+                longitude = ?, 
+                latitude = ?, 
+                accuracy_radius = ?
+              WHERE public_key = ?
+            `;
+
+            const params = [
+              isp?.autonomous_system_number ?? asn?.autonomous_system_number,
+              city.city?.geoname_id,
+              city.country?.geoname_id,
+              city.subdivisions ? city.subdivisions[0].geoname_id : null,
+              city.location?.longitude,
+              city.location?.latitude,
+              city.location?.accuracy_radius,
+              node.public_key
+            ];
             await DB.query(query, params);
 
             // Store Continent
@@ -61,16 +88,23 @@ export async function $lookupNodeLocation(): Promise<void> {
             }
 
             // Store AS name
-            if (asn.autonomous_system_organization) {
+            if (isp?.autonomous_system_organization ?? asn?.autonomous_system_organization) {
               await DB.query(
                 `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'as_organization', ?)`,
-                [asn.autonomous_system_number, JSON.stringify(asn.autonomous_system_organization)]);
+                [isp?.autonomous_system_number ?? asn?.autonomous_system_number, JSON.stringify(isp?.isp ?? asn?.autonomous_system_organization)]);
+            }
+          }
+
+          ++progress;
+          const elapsedSeconds = Math.round((new Date().getTime() / 1000) - loggerTimer);
+          if (elapsedSeconds > 10) {
+            logger.info(`Updating node location data ${progress}/${nodes.length}`);
+            loggerTimer = new Date().getTime() / 1000;
           }
         }
       }
     }
-    }
+    logger.info(`${progress} nodes location data updated`);
-    logger.info(`Node location data updated.`);
   } catch (e) {
     logger.err('$lookupNodeLocation() error: ' + (e instanceof Error ? e.message : e));
   }

backend/src/tasks/lightning/sync-tasks/stats-importer.ts

@@ -0,0 +1,411 @@
+import DB from '../../../database';
+import { promises } from 'fs';
+import { XMLParser } from 'fast-xml-parser';
+import logger from '../../../logger';
+import fundingTxFetcher from './funding-tx-fetcher';
+import config from '../../../config';
+
+const fsPromises = promises;
+
+interface Node {
+  id: string;
+  timestamp: number;
+  features: string;
+  rgb_color: string;
+  alias: string;
+  addresses: unknown[];
+  out_degree: number;
+  in_degree: number;
+}
+
+interface Channel {
+  channel_id: string;
+  node1_pub: string;
+  node2_pub: string;
+  timestamp: number;
+  features: string;
+  fee_base_msat: number;
+  fee_rate_milli_msat: number;
+  htlc_minimim_msat: number;
+  cltv_expiry_delta: number;
+  htlc_maximum_msat: number;
+}
+
+class LightningStatsImporter {
+  topologiesFolder = config.LIGHTNING.TOPOLOGY_FOLDER;
+  parser = new XMLParser();
+
+  async $run(): Promise<void> {
+    logger.info(`Importing historical lightning stats`);
+
+    const [channels]: any[] = await DB.query('SELECT short_id from channels;');
+    logger.info('Caching funding txs for currently existing channels');
+    await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
+
+    await this.$importHistoricalLightningStats();
+  }
+
+  /**
+   * Generate LN network stats for one day
+   */
+  public async computeNetworkStats(timestamp: number, networkGraph): Promise<unknown> {
+    // Node counts and network shares
+    let clearnetNodes = 0;
+    let torNodes = 0;
+    let clearnetTorNodes = 0;
+    let unannouncedNodes = 0;
+
+    for (const node of networkGraph.nodes) {
+      let hasOnion = false;
+      let hasClearnet = false;
+      let isUnnanounced = true;
+
+      for (const socket of (node.addresses ?? [])) {
+        hasOnion = hasOnion || ['torv2', 'torv3'].includes(socket.network);
+        hasClearnet = hasClearnet || ['ipv4', 'ipv6'].includes(socket.network);
+      }
+      if (hasOnion && hasClearnet) {
+        clearnetTorNodes++;
+        isUnnanounced = false;
+      } else if (hasOnion) {
+        torNodes++;
+        isUnnanounced = false;
+      } else if (hasClearnet) {
+        clearnetNodes++;
+        isUnnanounced = false;
+      }
+      if (isUnnanounced) {
+        unannouncedNodes++;
+      }
+    }
+
+    // Channels and node historical stats
+    const nodeStats = {};
+    let capacity = 0;
+    let avgFeeRate = 0;
+    let avgBaseFee = 0;
+    const capacities: number[] = [];
+    const feeRates: number[] = [];
+    const baseFees: number[] = [];
+    const alreadyCountedChannels = {};
+    
+    for (const channel of networkGraph.edges) {
+      let short_id = channel.channel_id;
+      if (short_id.indexOf('/') !== -1) {
+        short_id = short_id.slice(0, -2);
+      }
+
+      const tx = await fundingTxFetcher.$fetchChannelOpenTx(short_id);
+      if (!tx) {
+        logger.err(`Unable to fetch funding tx for channel ${short_id}. Capacity and creation date is unknown. Skipping channel.`);
+        continue;
+      }
+
+      if (!nodeStats[channel.node1_pub]) {
+        nodeStats[channel.node1_pub] = {
+          capacity: 0,
+          channels: 0,
+        };
+      }
+      if (!nodeStats[channel.node2_pub]) {
+        nodeStats[channel.node2_pub] = {
+          capacity: 0,
+          channels: 0,
+        };
+      }
+      
+      if (!alreadyCountedChannels[short_id]) {
+        capacity += Math.round(tx.value * 100000000);
+        capacities.push(Math.round(tx.value * 100000000));
+        alreadyCountedChannels[short_id] = true;
+
+        nodeStats[channel.node1_pub].capacity += Math.round(tx.value * 100000000);
+        nodeStats[channel.node1_pub].channels++;
+        nodeStats[channel.node2_pub].capacity += Math.round(tx.value * 100000000);
+        nodeStats[channel.node2_pub].channels++;
+      }
+
+      if (channel.node1_policy !== undefined) { // Coming from the node
+        for (const policy of [channel.node1_policy, channel.node2_policy]) {
+          if (policy && policy.fee_rate_milli_msat < 5000) {
+            avgFeeRate += parseInt(policy.fee_rate_milli_msat, 10);
+            feeRates.push(parseInt(policy.fee_rate_milli_msat, 10));
+          }  
+          if (policy && policy.fee_base_msat < 5000) {
+            avgBaseFee += parseInt(policy.fee_base_msat, 10);
+            baseFees.push(parseInt(policy.fee_base_msat, 10));
+          }
+        }
+      } else { // Coming from the historical import
+        if (channel.fee_rate_milli_msat < 5000) {
+          avgFeeRate += parseInt(channel.fee_rate_milli_msat, 10);
+          feeRates.push(parseInt(channel.fee_rate_milli_msat), 10);
+        }  
+        if (channel.fee_base_msat < 5000) {
+          avgBaseFee += parseInt(channel.fee_base_msat, 10);
+          baseFees.push(parseInt(channel.fee_base_msat), 10);
+        }
+      }
+    }
+
+    avgFeeRate /= Math.max(networkGraph.edges.length, 1);
+    avgBaseFee /= Math.max(networkGraph.edges.length, 1);
+    const medCapacity = capacities.sort((a, b) => b - a)[Math.round(capacities.length / 2 - 1)];
+    const medFeeRate = feeRates.sort((a, b) => b - a)[Math.round(feeRates.length / 2 - 1)];
+    const medBaseFee = baseFees.sort((a, b) => b - a)[Math.round(baseFees.length / 2 - 1)];
+    const avgCapacity = Math.round(capacity / Math.max(capacities.length, 1));
+
+    let query = `INSERT INTO lightning_stats(
+        added,
+        channel_count,
+        node_count,
+        total_capacity,
+        tor_nodes,
+        clearnet_nodes,
+        unannounced_nodes,
+        clearnet_tor_nodes,
+        avg_capacity,
+        avg_fee_rate,
+        avg_base_fee_mtokens,
+        med_capacity,
+        med_fee_rate,
+        med_base_fee_mtokens
+      )
+      VALUES (FROM_UNIXTIME(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON DUPLICATE KEY UPDATE
+      added = FROM_UNIXTIME(?),
+      channel_count = ?,
+      node_count = ?,
+      total_capacity = ?,
+      tor_nodes = ?,
+      clearnet_nodes = ?,
+      unannounced_nodes = ?,
+      clearnet_tor_nodes = ?,
+      avg_capacity = ?,
+      avg_fee_rate = ?,
+      avg_base_fee_mtokens = ?,
+      med_capacity = ?,
+      med_fee_rate = ?,
+      med_base_fee_mtokens = ?
+    `;
+
+    await DB.query(query, [
+      timestamp,
+      capacities.length,
+      networkGraph.nodes.length,
+      capacity,
+      torNodes,
+      clearnetNodes,
+      unannouncedNodes,
+      clearnetTorNodes,
+      avgCapacity,
+      avgFeeRate,
+      avgBaseFee,
+      medCapacity,
+      medFeeRate,
+      medBaseFee,
+      timestamp,
+      capacities.length,
+      networkGraph.nodes.length,
+      capacity,
+      torNodes,
+      clearnetNodes,
+      unannouncedNodes,
+      clearnetTorNodes,
+      avgCapacity,
+      avgFeeRate,
+      avgBaseFee,
+      medCapacity,
+      medFeeRate,
+      medBaseFee,
+    ]);
+
+    for (const public_key of Object.keys(nodeStats)) {
+      query = `INSERT INTO node_stats(
+          public_key,
+          added,
+          capacity,
+          channels
+        )
+        VALUES (?, FROM_UNIXTIME(?), ?, ?)
+        ON DUPLICATE KEY UPDATE
+        added = FROM_UNIXTIME(?),
+        capacity = ?,
+        channels = ?
+      `;
+
+      await DB.query(query, [
+        public_key,
+        timestamp,
+        nodeStats[public_key].capacity,
+        nodeStats[public_key].channels,
+        timestamp,
+        nodeStats[public_key].capacity,
+        nodeStats[public_key].channels,
+      ]);
+    }
+
+    return {
+      added: timestamp,
+      node_count: networkGraph.nodes.length
+    };
+  }
+
+  /**
+   * Import topology files LN historical data into the database
+   */
+  async $importHistoricalLightningStats(): Promise<void> {
+    let latestNodeCount = 1;
+
+    const fileList = await fsPromises.readdir(this.topologiesFolder);
+    // Insert history from the most recent to the oldest
+    // This also put the .json cached files first
+    fileList.sort().reverse();
+
+    const [rows]: any[] = await DB.query(`
+      SELECT UNIX_TIMESTAMP(added) AS added, node_count
+      FROM lightning_stats
+      ORDER BY added DESC
+    `);
+    const existingStatsTimestamps = {};
+    for (const row of rows) {
+      existingStatsTimestamps[row.added] = row;
+    }
+
+    // For logging purpose
+    let processed = 10;
+    let totalProcessed = -1;
+
+    for (const filename of fileList) {
+      processed++;
+      totalProcessed++;
+
+      const timestamp = parseInt(filename.split('_')[1], 10);
+
+      // Stats exist already, don't calculate/insert them
+      if (existingStatsTimestamps[timestamp] !== undefined) {
+        latestNodeCount = existingStatsTimestamps[timestamp].node_count;
+        continue;
+      }
+
+      logger.debug(`Reading ${this.topologiesFolder}/${filename}`);
+      const fileContent = await fsPromises.readFile(`${this.topologiesFolder}/${filename}`, 'utf8');
+
+      let graph;
+      if (filename.indexOf('.json') !== -1) {
+        try {
+          graph = JSON.parse(fileContent);
+        } catch (e) {
+          logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`);
+          continue;
+        }
+      } else {
+        graph = this.parseFile(fileContent);
+        if (!graph) {
+          logger.debug(`Invalid topology file ${this.topologiesFolder}/${filename}, cannot parse the content`);
+          continue;
+        }
+        await fsPromises.writeFile(`${this.topologiesFolder}/${filename}.json`, JSON.stringify(graph));
+      }
+
+      if (timestamp > 1556316000) {
+        // "No, the reason most likely is just that I started collection in 2019,
+        // so what I had before that is just the survivors from before, which weren't that many"
+        const diffRatio = graph.nodes.length / latestNodeCount;
+        if (diffRatio < 0.9) {
+          // Ignore drop of more than 90% of the node count as it's probably a missing data point
+          logger.debug(`Nodes count diff ratio threshold reached, ignore the data for this day ${graph.nodes.length} nodes vs ${latestNodeCount}`);
+          continue;
+        }
+      }
+      latestNodeCount = graph.nodes.length;
+
+      const datestr = `${new Date(timestamp * 1000).toUTCString()} (${timestamp})`;
+      logger.debug(`${datestr}: Found ${graph.nodes.length} nodes and ${graph.edges.length} channels`);
+
+      if (processed > 10) {
+        logger.info(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`);
+        processed = 0;
+      } else {
+        logger.debug(`Generating LN network stats for ${datestr}. Processed ${totalProcessed}/${fileList.length} files`);
+      }
+      await fundingTxFetcher.$fetchChannelsFundingTxs(graph.edges.map(channel => channel.channel_id.slice(0, -2)));
+      const stat = await this.computeNetworkStats(timestamp, graph);
+
+      existingStatsTimestamps[timestamp] = stat;
+    }
+
+    logger.info(`Lightning network stats historical import completed`);
+  }
+
+  /**
+   * Parse the file content into XML, and return a list of nodes and channels
+   */
+  private parseFile(fileContent): any {
+    const graph = this.parser.parse(fileContent);
+    if (Object.keys(graph).length === 0) {
+      return null;
+    }
+
+    const nodes: Node[] = [];
+    const channels: Channel[] = [];
+
+    // If there is only one entry, the parser does not return an array, so we override this
+    if (!Array.isArray(graph.graphml.graph.node)) {
+      graph.graphml.graph.node = [graph.graphml.graph.node];
+    }
+    if (!Array.isArray(graph.graphml.graph.edge)) {
+      graph.graphml.graph.edge = [graph.graphml.graph.edge];
+    }
+
+    for (const node of graph.graphml.graph.node) {
+      if (!node.data) {
+        continue;
+      }
+      const addresses: unknown[] = [];
+      const sockets = node.data[5].split(',');
+      for (const socket of sockets) {
+        const parts = socket.split('://');
+        addresses.push({
+          network: parts[0],
+          addr: parts[1],
+        });
+      }
+      nodes.push({
+        id: node.data[0],
+        timestamp: node.data[1],
+        features: node.data[2],
+        rgb_color: node.data[3],
+        alias: node.data[4],
+        addresses: addresses,
+        out_degree: node.data[6],
+        in_degree: node.data[7],
+      });
+    }
+
+    for (const channel of graph.graphml.graph.edge) {
+      if (!channel.data) {
+        continue;
+      }
+      channels.push({
+        channel_id: channel.data[0],
+        node1_pub: channel.data[1],
+        node2_pub: channel.data[2],
+        timestamp: channel.data[3],
+        features: channel.data[4],
+        fee_base_msat: channel.data[5],
+        fee_rate_milli_msat: channel.data[6],
+        htlc_minimim_msat: channel.data[7],
+        cltv_expiry_delta: channel.data[8],
+        htlc_maximum_msat: channel.data[9],
+      });
+    }
+
+    return {
+      nodes: nodes,
+      edges: channels,
+    };
+  }
+}
+
+export default new LightningStatsImporter;

contributors/oleonardolima.txt

@@ -0,0 +1,3 @@
+I hereby accept the terms of the Contributor License Agreement in the CONTRIBUTING.md file of the mempool/mempool git repository as of July 25, 2022.
+
+Signed: oleonardolima

frontend/.eslintrc

@@ -14,10 +14,11 @@
     "@typescript-eslint/ban-types": 1,
     "@typescript-eslint/no-empty-function": 1,
     "@typescript-eslint/no-explicit-any": 1,
-    "@typescript-eslint/no-inferrable-types": 1,
+    "@typescript-eslint/no-inferrable-types": 0,
     "@typescript-eslint/no-namespace": 1,
     "@typescript-eslint/no-this-alias": 1,
     "@typescript-eslint/no-var-requires": 1,
+    "@typescript-eslint/explicit-function-return-type": 1,
     "no-case-declarations": 1,
     "no-console": 1,
     "no-constant-condition": 1,
@@ -29,6 +30,8 @@
     "no-useless-catch": 1,
     "no-var": 1,
     "prefer-const": 1,
-    "prefer-rest-params": 1
+    "prefer-rest-params": 1,
+    "quotes": [1, "single", { "allowTemplateLiterals": true }],
+    "semi": 1
   }
 }

frontend/src/app/app-routing.module.ts

@@ -4,8 +4,11 @@ import { StartComponent } from './components/start/start.component';
 import { TransactionComponent } from './components/transaction/transaction.component';
 import { BlockComponent } from './components/block/block.component';
 import { BlockAuditComponent } from './components/block-audit/block-audit.component';
+import { BlockPreviewComponent } from './components/block/block-preview.component';
 import { AddressComponent } from './components/address/address.component';
+import { AddressPreviewComponent } from './components/address/address-preview.component';
 import { MasterPageComponent } from './components/master-page/master-page.component';
+import { MasterPagePreviewComponent } from './components/master-page-preview/master-page-preview.component';
 import { AboutComponent } from './components/about/about.component';
 import { StatusViewComponent } from './components/status-view/status-view.component';
 import { TermsOfServiceComponent } from './components/terms-of-service/terms-of-service.component';
@@ -67,7 +70,10 @@ let routes: Routes = [
           {
             path: 'address/:id',
             children: [],
-            component: AddressComponent
+            component: AddressComponent,
+            data: {
+              ogImage: true
+            }
           },
           {
             path: 'tx',
@@ -85,7 +91,10 @@ let routes: Routes = [
               children: [
               {
                 path: ':id',
-                component: BlockComponent
+                component: BlockComponent,
+                data: {
+                  ogImage: true
+                }
               },
             ],
           },
@@ -170,7 +179,10 @@ let routes: Routes = [
           {
             path: 'address/:id',
             children: [],
-            component: AddressComponent
+            component: AddressComponent,
+            data: {
+              ogImage: true
+            }
           },
           {
             path: 'tx',
@@ -188,7 +200,10 @@ let routes: Routes = [
             children: [
               {
                 path: ':id',
-                component: BlockComponent
+                component: BlockComponent,
+                data: {
+                  ogImage: true
+                }
               },
             ],
           },
@@ -270,7 +285,10 @@ let routes: Routes = [
       {
         path: 'address/:id',
         children: [],
-        component: AddressComponent
+        component: AddressComponent,
+        data: {
+          ogImage: true
+        }
       },
       {
         path: 'tx',
@@ -288,7 +306,10 @@ let routes: Routes = [
         children: [
           {
             path: ':id',
-            component: BlockComponent
+            component: BlockComponent,
+            data: {
+              ogImage: true
+            }
           },
         ],
       },
@@ -315,6 +336,39 @@ let routes: Routes = [
       },
     ],
   },
+  {
+    path: 'preview',
+    component: MasterPagePreviewComponent,
+    children: [
+      {
+        path: 'block/:id',
+        component: BlockPreviewComponent
+      },
+      {
+        path: 'testnet/block/:id',
+        component: BlockPreviewComponent
+      },
+      {
+        path: 'signet/block/:id',
+        component: BlockPreviewComponent
+      },
+      {
+        path: 'address/:id',
+        children: [],
+        component: AddressPreviewComponent
+      },
+      {
+        path: 'testnet/address/:id',
+        children: [],
+        component: AddressPreviewComponent
+      },
+      {
+        path: 'signet/address/:id',
+        children: [],
+        component: AddressPreviewComponent
+      },
+    ],
+  },
   {
     path: 'status',
     component: StatusViewComponent
@@ -386,7 +440,10 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
             {
               path: 'address/:id',
               children: [],
-              component: AddressComponent
+              component: AddressComponent,
+              data: {
+                ogImage: true
+              }
             },
             {
               path: 'tx',
@@ -404,7 +461,10 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
               children: [
                 {
                   path: ':id',
-                  component: BlockComponent
+                  component: BlockComponent,
+                  data: {
+                    ogImage: true
+                  }
                 },
               ],
             },
@@ -490,7 +550,10 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
         {
           path: 'address/:id',
           children: [],
-          component: AddressComponent
+          component: AddressComponent,
+          data: {
+            ogImage: true
+          }
         },
         {
           path: 'tx',
@@ -508,7 +571,10 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
           children: [
             {
               path: ':id',
-              component: BlockComponent
+              component: BlockComponent,
+              data: {
+                ogImage: true
+              }
             },
           ],
         },
@@ -548,6 +614,30 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
         },
       ],
     },
+    {
+      path: 'preview',
+      component: MasterPagePreviewComponent,
+      children: [
+        {
+          path: 'block/:id',
+          component: BlockPreviewComponent
+        },
+        {
+          path: 'testnet/block/:id',
+          component: BlockPreviewComponent
+        },
+        {
+          path: 'address/:id',
+          children: [],
+          component: AddressPreviewComponent
+        },
+        {
+          path: 'testnet/address/:id',
+          children: [],
+          component: AddressPreviewComponent
+        },
+      ],
+    },
     {
       path: 'status',
       component: StatusViewComponent
@@ -576,4 +666,3 @@ if (browserWindowEnv && browserWindowEnv.BASE_MODULE === 'liquid') {
   })],
 })
 export class AppRoutingModule { }
-

frontend/src/app/app.module.ts

@@ -10,6 +10,7 @@ import { EnterpriseService } from './services/enterprise.service';
 import { WebsocketService } from './services/websocket.service';
 import { AudioService } from './services/audio.service';
 import { SeoService } from './services/seo.service';
+import { OpenGraphService } from './services/opengraph.service';
 import { SharedModule } from './shared/shared.module';
 import { StorageService } from './services/storage.service';
 import { HttpCacheInterceptor } from './services/http-cache.interceptor';
@@ -36,6 +37,7 @@ import { CapAddressPipe } from './shared/pipes/cap-address-pipe/cap-address-pipe
     WebsocketService,
     AudioService,
     SeoService,
+    OpenGraphService,
     StorageService,
     EnterpriseService,
     LanguageService,

frontend/src/app/bitcoin.utils.ts

@@ -5,13 +5,18 @@ const P2SH_P2WSH_COST  = 35 * 4; // the WU cost for the non-witness part of P2SH
 
 export function calcSegwitFeeGains(tx: Transaction) {
   // calculated in weight units
-  let realizedGains = 0;
+  let realizedSegwitGains = 0;
-  let potentialBech32Gains = 0;
+  let potentialSegwitGains = 0;
-  let potentialP2shGains = 0;
+  let potentialP2shSegwitGains = 0;
+  let potentialTaprootGains = 0;
+  let realizedTaprootGains = 0;
 
   for (const vin of tx.vin) {
     if (!vin.prevout) { continue; }
 
+    const isP2pk         = vin.prevout.scriptpubkey_type === 'p2pk';
+    // const isBareMultisig = vin.prevout.scriptpubkey_type === 'multisig'; // type will be unknown, so use the multisig helper from the address labels
+    const isBareMultisig = !!parseMultisigScript(vin.prevout.scriptpubkey_asm);
     const isP2pkh        = vin.prevout.scriptpubkey_type === 'p2pkh';
     const isP2sh         = vin.prevout.scriptpubkey_type === 'p2sh';
     const isP2wsh        = vin.prevout.scriptpubkey_type === 'v0_p2wsh';
@@ -19,52 +24,140 @@ export function calcSegwitFeeGains(tx: Transaction) {
     const isP2tr         = vin.prevout.scriptpubkey_type === 'v1_p2tr';
 
     const op = vin.scriptsig ? vin.scriptsig_asm.split(' ')[0] : null;
-    const isP2sh2Wpkh = isP2sh && !!vin.witness && op === 'OP_PUSHBYTES_22';
+    const isP2shP2Wpkh = isP2sh && !!vin.witness && op === 'OP_PUSHBYTES_22';
-    const isP2sh2Wsh = isP2sh && !!vin.witness && op === 'OP_PUSHBYTES_34';
+    const isP2shP2Wsh  = isP2sh && !!vin.witness && op === 'OP_PUSHBYTES_34';
 
     switch (true) {
-      // Native Segwit - P2WPKH/P2WSH (Bech32)
+      // Native Segwit - P2WPKH/P2WSH/P2TR
       case isP2wpkh:
       case isP2wsh:
       case isP2tr:
         // maximal gains: the scriptSig is moved entirely to the witness part
-        realizedGains += witnessSize(vin) * 3;
+        // if taproot is used savings are 42 WU higher because it produces smaller signatures and doesn't require a pubkey in the witness
+        // this number is explained above `realizedTaprootGains += 42;`
+        realizedSegwitGains += (witnessSize(vin) + (isP2tr ? 42 : 0)) * 3;
         // XXX P2WSH output creation is more expensive, should we take this into consideration?
         break;
 
       // Backward compatible Segwit - P2SH-P2WPKH
-      case isP2sh2Wpkh:
+      case isP2shP2Wpkh:
-        // the scriptSig is moved to the witness, but we have extra 21 extra non-witness bytes (48 WU)
+        // the scriptSig is moved to the witness, but we have extra 21 extra non-witness bytes (84 WU)
-        realizedGains += witnessSize(vin) * 3 - P2SH_P2WPKH_COST;
+        realizedSegwitGains += witnessSize(vin) * 3 - P2SH_P2WPKH_COST;
-        potentialBech32Gains += P2SH_P2WPKH_COST;
+        potentialSegwitGains += P2SH_P2WPKH_COST;
         break;
 
       // Backward compatible Segwit - P2SH-P2WSH
-      case isP2sh2Wsh:
+      case isP2shP2Wsh:
-        // the scriptSig is moved to the witness, but we have extra 35 extra non-witness bytes
+        // the scriptSig is moved to the witness, but we have extra 35 extra non-witness bytes (140 WU)
-        realizedGains += witnessSize(vin) * 3 - P2SH_P2WSH_COST;
+        realizedSegwitGains += witnessSize(vin) * 3 - P2SH_P2WSH_COST;
-        potentialBech32Gains += P2SH_P2WSH_COST;
+        potentialSegwitGains += P2SH_P2WSH_COST;
         break;
 
-      // Non-segwit P2PKH/P2SH
+      // Non-segwit P2PKH/P2SH/P2PK/bare multisig
       case isP2pkh:
       case isP2sh:
-        const fullGains = scriptSigSize(vin) * 3;
+      case isP2pk:
-        potentialBech32Gains += fullGains;
+      case isBareMultisig: {
-        potentialP2shGains += fullGains - (isP2pkh ? P2SH_P2WPKH_COST : P2SH_P2WSH_COST);
+        let fullGains = scriptSigSize(vin) * 3;
+        if (isBareMultisig) {
+          // a _bare_ multisig has the keys in the output script, but P2SH and P2WSH require them to be in the scriptSig/scriptWitness
+          fullGains -= vin.prevout.scriptpubkey.length / 2;
+        }
+        potentialSegwitGains += fullGains;
+        potentialP2shSegwitGains += fullGains - (isP2pkh ? P2SH_P2WPKH_COST : P2SH_P2WSH_COST);
         break;
+      }
+    }
 
-    // TODO: should we also consider P2PK and pay-to-bare-script (non-p2sh-wrapped) as upgradable to P2WPKH and P2WSH?
+    if (isP2tr) {
+      if (vin.witness.length === 1) {
+        // key path spend
+        // we don't know if this was a multisig or single sig (the goal of taproot :)),
+        // so calculate fee savings by comparing to the cheapest single sig input type: P2WPKH and say "saved at least ...%"
+        // the witness size of P2WPKH is 1 (stack size) + 1 (size) + 72 (low s signature) + 1 (size) + 33 (pubkey) = 108 WU
+        // the witness size of key path P2TR is 1 (stack size) + 1 (size) + 64 (signature) = 66 WU
+        realizedTaprootGains += 42;
+      } else {
+        // script path spend
+        // complex scripts with multiple spending paths can often be made around 2x to 3x smaller with the Taproot script tree
+        // because only the hash of the alternative spending path has the be in the witness data, not the entire script,
+        // but only assumptions can be made because the scripts themselves are unknown (again, the goal of taproot :))
+        // TODO maybe add some complex scripts that are specified somewhere, so that size is known, such as lightning scripts
+      }
+    } else {
+      const script = isP2shP2Wsh || isP2wsh ? vin.inner_witnessscript_asm : vin.inner_redeemscript_asm;
+      let replacementSize: number;
+      if (
+        // single sig
+        isP2pk || isP2pkh || isP2wpkh || isP2shP2Wpkh ||
+        // multisig
+        isBareMultisig || parseMultisigScript(script)
+      ) {
+        // the scriptSig and scriptWitness can all be replaced by a 66 witness WU with taproot
+        replacementSize = 66;
+      } else if (script) {
+        // not single sig, not multisig: the complex scripts
+        // rough calculations on spending paths
+        // every OP_IF and OP_NOTIF indicates an _extra_ spending path, so add 1
+        const spendingPaths = script.split(' ').filter(op => /^(OP_IF|OP_NOTIF)$/g.test(op)).length + 1;
+        // now assume the script could have been split in ${spendingPaths} equal tapleaves
+        replacementSize = script.length / 2 / spendingPaths +
+        // but account for the leaf and branch hashes and internal key in the control block
+          32 * Math.log2((spendingPaths - 1) || 1) + 33;
+      }
+      potentialTaprootGains += witnessSize(vin) + scriptSigSize(vin) * 4 - replacementSize;
     }
   }
 
   // returned as percentage of the total tx weight
-  return { realizedGains: realizedGains / (tx.weight + realizedGains) // percent of the pre-segwit tx size
+  return {
-         , potentialBech32Gains: potentialBech32Gains / tx.weight
+    realizedSegwitGains: realizedSegwitGains / (tx.weight + realizedSegwitGains), // percent of the pre-segwit tx size
-         , potentialP2shGains: potentialP2shGains / tx.weight
+    potentialSegwitGains: potentialSegwitGains / tx.weight,
+    potentialP2shSegwitGains: potentialP2shSegwitGains / tx.weight,
+    potentialTaprootGains: potentialTaprootGains / tx.weight,
+    realizedTaprootGains: realizedTaprootGains / (tx.weight + realizedTaprootGains)
   };
 }
 
+/** extracts m and n from a multisig script (asm), returns nothing if it is not a multisig script */
+export function parseMultisigScript(script: string): void | { m: number, n: number } {
+  if (!script) {
+    return;
+  }
+  const ops = script.split(' ');
+  if (ops.length < 3 || ops.pop() !== 'OP_CHECKMULTISIG') {
+    return;
+  }
+  const opN = ops.pop();
+  if (!opN.startsWith('OP_PUSHNUM_')) {
+    return;
+  }
+  const n = parseInt(opN.match(/[0-9]+/)[0], 10);
+  if (ops.length < n * 2 + 1) {
+    return;
+  }
+  // pop n public keys
+  for (let i = 0; i < n; i++) {
+    if (!/^0((2|3)\w{64}|4\w{128})$/.test(ops.pop())) {
+      return;
+    }
+    if (!/^OP_PUSHBYTES_(33|65)$/.test(ops.pop())) {
+      return;
+    }
+  }
+  const opM = ops.pop();
+  if (!opM.startsWith('OP_PUSHNUM_')) {
+    return;
+  }
+  const m = parseInt(opM.match(/[0-9]+/)[0], 10);
+
+  if (ops.length) {
+    return;
+  }
+
+  return { m, n };
+}
+
 // https://github.com/shesek/move-decimal-point
 export function moveDec(num: number, n: number) {
   let frac, int, neg, ref;
@@ -101,12 +194,12 @@ export function moveDec(num: number, n: number) {
   return neg + (int || '0') + (frac.length ? '.' + frac : '');
 }
 
-function zeros(n) {
+function zeros(n: number) {
   return new Array(n + 1).join('0');
 }
 
 // Formats a number for display. Treats the number as a string to avoid rounding errors.
-export const formatNumber = (s, precision = null) => {
+export const formatNumber = (s: number | string, precision: number | null = null) => {
   let [ whole, dec ] = s.toString().split('.');
 
   // divide numbers into groups of three separated with a thin space (U+202F, "NARROW NO-BREAK SPACE"),
@@ -128,31 +221,31 @@ export const formatNumber = (s, precision = null) => {
 };
 
 // Utilities for segwitFeeGains
-const witnessSize = (vin: Vin) => vin.witness.reduce((S, w) => S + (w.length / 2), 0);
+const witnessSize = (vin: Vin) => vin.witness ? vin.witness.reduce((S, w) => S + (w.length / 2), 0) : 0;
 const scriptSigSize = (vin: Vin) => vin.scriptsig ? vin.scriptsig.length / 2 : 0;
 
 // Power of ten wrapper
-export function selectPowerOfTen(val: number) {
+export function selectPowerOfTen(val: number): { divider: number, unit: string } {
   const powerOfTen = {
     exa: Math.pow(10, 18),
     peta: Math.pow(10, 15),
-    terra: Math.pow(10, 12),
+    tera: Math.pow(10, 12),
     giga: Math.pow(10, 9),
     mega: Math.pow(10, 6),
     kilo: Math.pow(10, 3),
   };
 
-  let selectedPowerOfTen;
+  let selectedPowerOfTen: { divider: number, unit: string };
   if (val < powerOfTen.kilo) {
     selectedPowerOfTen = { divider: 1, unit: '' }; // no scaling
   } else if (val < powerOfTen.mega) {
     selectedPowerOfTen = { divider: powerOfTen.kilo, unit: 'k' };
   } else if (val < powerOfTen.giga) {
     selectedPowerOfTen = { divider: powerOfTen.mega, unit: 'M' };
-  } else if (val < powerOfTen.terra) {
+  } else if (val < powerOfTen.tera) {
     selectedPowerOfTen = { divider: powerOfTen.giga, unit: 'G' };
   } else if (val < powerOfTen.peta) {
-    selectedPowerOfTen = { divider: powerOfTen.terra, unit: 'T' };
+    selectedPowerOfTen = { divider: powerOfTen.tera, unit: 'T' };
   } else if (val < powerOfTen.exa) {
     selectedPowerOfTen = { divider: powerOfTen.peta, unit: 'P' };
   } else {

frontend/src/app/components/address-labels/address-labels.component.ts

@@ -1,6 +1,7 @@
 import { Component, ChangeDetectionStrategy, Input, OnChanges } from '@angular/core';
 import { Vin, Vout } from '../../interfaces/electrs.interface';
 import { StateService } from 'src/app/services/state.service';
+import { parseMultisigScript } from 'src/app/bitcoin.utils';
 
 @Component({
   selector: 'app-address-labels',
@@ -98,41 +99,11 @@ export class AddressLabelsComponent implements OnChanges {
   }
 
   detectMultisig(script: string) {
-    if (!script) {
+    const ms = parseMultisigScript(script);
-      return;
-    }
-    const ops = script.split(' ');
-    if (ops.length < 3 || ops.pop() !== 'OP_CHECKMULTISIG') {
-      return;
-    }
-    const opN = ops.pop();
-    if (!opN.startsWith('OP_PUSHNUM_')) {
-      return;
-    }
-    const n = parseInt(opN.match(/[0-9]+/)[0], 10);
-    if (ops.length < n * 2 + 1) {
-      return;
-    }
-    // pop n public keys
-    for (let i = 0; i < n; i++) {
-      if (!/^0((2|3)\w{64}|4\w{128})$/.test(ops.pop())) {
-        return;
-      }
-      if (!/^OP_PUSHBYTES_(33|65)$/.test(ops.pop())) {
-        return;
-      }
-    }
-    const opM = ops.pop();
-    if (!opM.startsWith('OP_PUSHNUM_')) {
-      return;
-    }
-    const m = parseInt(opM.match(/[0-9]+/)[0], 10);
 
-    if (ops.length) {
+    if (ms) {
-      return;
+      this.label = $localize`:@@address-label.multisig:Multisig ${ms.m}:multisigM: of ${ms.n}:multisigN:`;
     }
-
-    this.label = $localize`:@@address-label.multisig:Multisig ${m}:multisigM: of ${n}:multisigN:`;
   }
 
   handleVout() {

frontend/src/app/components/address/address-preview.component.html

@@ -0,0 +1,55 @@
+<div class="box preview-box" *ngIf="address && !error">
+  <div class="row">
+    <div class="col-md">
+      <div class="title-address">
+        <h1 i18n="shared.address">Address</h1>
+      </div>
+      <a [routerLink]="['/address/' | relativeUrl, addressString]" class="address-link" >
+        <span class="truncated-address">{{addressString.slice(0,-4)}}</span><span class="last-four">{{addressString.slice(-4)}}</span>
+      </a>
+      <table class="table table-borderless table-striped">
+        <tbody>
+          <tr *ngIf="addressInfo && addressInfo.unconfidential">
+            <td i18n="address.unconfidential">Unconfidential</td>
+            <td><a [routerLink]="['/address/' | relativeUrl, addressInfo.unconfidential]">
+              <span class="d-inline d-lg-none">{{ addressInfo.unconfidential | shortenString : 14 }}</span>
+              <span class="d-none d-lg-inline">{{ addressInfo.unconfidential }}</span>
+            </a> <app-clipboard [text]="addressInfo.unconfidential"></app-clipboard></td>
+          </tr>
+          <ng-template [ngIf]="!address.electrum">
+            <tr>
+              <td i18n="address.total-received">Total received</td>
+              <td *ngIf="address.chain_stats.funded_txo_sum !== undefined; else confidentialTd"><app-amount [satoshis]="received" [noFiat]="true"></app-amount></td>
+            </tr>
+            <tr>
+              <td i18n="address.total-sent">Total sent</td>
+              <td *ngIf="address.chain_stats.spent_txo_sum !== undefined; else confidentialTd"><app-amount [satoshis]="sent" [noFiat]="true"></app-amount></td>
+            </tr>
+          </ng-template>
+          <tr>
+            <td i18n="address.balance">Balance</td>
+            <td *ngIf="address.chain_stats.funded_txo_sum !== undefined; else confidentialTd"><app-amount [satoshis]="received - sent" [noFiat]="true"></app-amount></td>
+          </tr>
+          <tr>
+            <td i18n="address.transactions">Transactions</td>
+            <td>{{ txCount | number }}</td>
+          </tr>
+          <tr>
+            <td i18n="address.unspent_txos">Unspent TXOs</td>
+            <td>{{ totalUnspent | number }}</td>
+          </tr>
+        </tbody>
+      </table>
+    </div>
+    <div class="w-100 d-block d-md-none"></div>
+    <div class="col-md qrcode-col">
+      <div class="qr-wrapper">
+        <app-qrcode [data]="address.address" [size]="370"></app-qrcode>
+      </div>
+    </div>
+  </div>
+</div>
+
+<ng-template #confidentialTd>
+  <td i18n="shared.confidential">Confidential</td>
+</ng-template>

frontend/src/app/components/address/address-preview.component.scss

@@ -0,0 +1,46 @@
+h1 {
+  font-size: 42px;
+  margin: 0;
+}
+
+.qr-wrapper {
+  background-color: #FFF;
+  padding: 10px;
+  padding-bottom: 5px;
+  display: inline-block;
+}
+
+.qrcode-col {
+  width: 420px;
+  min-width: 420px;
+  flex-grow: 0;
+  flex-shrink: 0;
+  text-align: center;
+}
+
+.table {
+  font-size: 24px;
+
+  ::ng-deep .symbol {
+    font-size: 18px;
+  }
+}
+
+.address-link {
+  font-size: 20px;
+  margin-bottom: 0.5em;
+  display: flex;
+  flex-direction: row;
+  align-items: baseline;
+  .truncated-address {
+    text-overflow: ellipsis;
+    overflow: hidden;
+    max-width: calc(505px - 4em);
+    display: inline-block;
+    white-space: nowrap;
+  }
+  .last-four {
+    display: inline-block;
+    white-space: nowrap;
+  }
+}

frontend/src/app/components/address/address-preview.component.ts

@@ -0,0 +1,116 @@
+import { Component, OnInit, OnDestroy } from '@angular/core';
+import { ActivatedRoute, ParamMap } from '@angular/router';
+import { ElectrsApiService } from '../../services/electrs-api.service';
+import { switchMap, filter, catchError, map, tap } from 'rxjs/operators';
+import { Address, Transaction } from '../../interfaces/electrs.interface';
+import { StateService } from 'src/app/services/state.service';
+import { OpenGraphService } from 'src/app/services/opengraph.service';
+import { AudioService } from 'src/app/services/audio.service';
+import { ApiService } from 'src/app/services/api.service';
+import { of, merge, Subscription, Observable } from 'rxjs';
+import { SeoService } from 'src/app/services/seo.service';
+import { AddressInformation } from 'src/app/interfaces/node-api.interface';
+
+@Component({
+  selector: 'app-address-preview',
+  templateUrl: './address-preview.component.html',
+  styleUrls: ['./address-preview.component.scss']
+})
+export class AddressPreviewComponent implements OnInit, OnDestroy {
+  network = '';
+
+  address: Address;
+  addressString: string;
+  isLoadingAddress = true;
+  error: any;
+  mainSubscription: Subscription;
+  addressLoadingStatus$: Observable<number>;
+  addressInfo: null | AddressInformation = null;
+
+  totalConfirmedTxCount = 0;
+  loadedConfirmedTxCount = 0;
+  txCount = 0;
+  received = 0;
+  sent = 0;
+  totalUnspent = 0;
+
+  constructor(
+    private route: ActivatedRoute,
+    private electrsApiService: ElectrsApiService,
+    private stateService: StateService,
+    private apiService: ApiService,
+    private seoService: SeoService,
+    private openGraphService: OpenGraphService,
+  ) { }
+
+  ngOnInit() {
+    this.openGraphService.setPreviewLoading();
+    this.stateService.networkChanged$.subscribe((network) => this.network = network);
+
+    this.addressLoadingStatus$ = this.route.paramMap
+      .pipe(
+        switchMap(() => this.stateService.loadingIndicators$),
+        map((indicators) => indicators['address-' + this.addressString] !== undefined ? indicators['address-' + this.addressString] : 0)
+      );
+
+    this.mainSubscription = this.route.paramMap
+      .pipe(
+        switchMap((params: ParamMap) => {
+          this.error = undefined;
+          this.isLoadingAddress = true;
+          this.loadedConfirmedTxCount = 0;
+          this.address = null;
+          this.addressInfo = null;
+          this.addressString = params.get('id') || '';
+          if (/^[A-Z]{2,5}1[AC-HJ-NP-Z02-9]{8,100}$/.test(this.addressString)) {
+            this.addressString = this.addressString.toLowerCase();
+          }
+          this.seoService.setTitle($localize`:@@address.component.browser-title:Address: ${this.addressString}:INTERPOLATION:`);
+
+          return this.electrsApiService.getAddress$(this.addressString)
+            .pipe(
+              catchError((err) => {
+                this.isLoadingAddress = false;
+                this.error = err;
+                console.log(err);
+                return of(null);
+              })
+            );
+        })
+      )
+      .pipe(
+        filter((address) => !!address),
+        tap((address: Address) => {
+          if ((this.stateService.network === 'liquid' || this.stateService.network === 'liquidtestnet') && /^([m-zA-HJ-NP-Z1-9]{26,35}|[a-z]{2,5}1[ac-hj-np-z02-9]{8,100}|[a-km-zA-HJ-NP-Z1-9]{80})$/.test(address.address)) {
+            this.apiService.validateAddress$(address.address)
+              .subscribe((addressInfo) => {
+                this.addressInfo = addressInfo;
+              });
+          }
+          this.address = address;
+          this.updateChainStats();
+          this.isLoadingAddress = false;
+          this.openGraphService.setPreviewReady();
+        })
+      )
+      .subscribe(() => {},
+        (error) => {
+          console.log(error);
+          this.error = error;
+          this.isLoadingAddress = false;
+        }
+      );
+  }
+
+  updateChainStats() {
+    this.received = this.address.chain_stats.funded_txo_sum + this.address.mempool_stats.funded_txo_sum;
+    this.sent = this.address.chain_stats.spent_txo_sum + this.address.mempool_stats.spent_txo_sum;
+    this.txCount = this.address.chain_stats.tx_count + this.address.mempool_stats.tx_count;
+    this.totalConfirmedTxCount = this.address.chain_stats.tx_count;
+    this.totalUnspent = this.address.chain_stats.funded_txo_count - this.address.chain_stats.spent_txo_count;
+  }
+
+  ngOnDestroy() {
+    this.mainSubscription.unsubscribe();
+  }
+}

frontend/src/app/components/app/app.component.ts

@@ -2,6 +2,7 @@ import { Location } from '@angular/common';
 import { Component, HostListener, OnInit, Inject, LOCALE_ID, HostBinding } from '@angular/core';
 import { Router, NavigationEnd } from '@angular/router';
 import { StateService } from 'src/app/services/state.service';
+import { OpenGraphService } from 'src/app/services/opengraph.service';
 import { NgbTooltipConfig } from '@ng-bootstrap/ng-bootstrap';
 
 @Component({
@@ -16,6 +17,7 @@ export class AppComponent implements OnInit {
   constructor(
     public router: Router,
     private stateService: StateService,
+    private openGraphService: OpenGraphService,
     private location: Location,
     tooltipConfig: NgbTooltipConfig,
     @Inject(LOCALE_ID) private locale: string,

frontend/src/app/components/block/block-preview.component.html

@@ -0,0 +1,82 @@
+<div class="box preview-box" *ngIf="!error">
+  <div class="row">
+    <div class="col-sm">
+      <h1 class="block-title">
+        <ng-template [ngIf]="blockHeight === 0"><ng-container i18n="@@2303359202781425764">Genesis</ng-container>
+          <span class="next-previous-blocks">
+            <a [routerLink]="['/block/' | relativeUrl, blockHash]">{{ blockHeight }}</a>
+          </span>
+        </ng-template>
+        <ng-template [ngIf]="blockHeight" i18n="shared.block-title">Block <ng-container *ngTemplateOutlet="blockTemplateContent"></ng-container></ng-template>
+        <ng-template #blockTemplateContent>
+          <span class="next-previous-blocks">
+            <a [routerLink]="['/block/' | relativeUrl, blockHash]">{{ blockHeight }}</a>
+          </span>
+        </ng-template>
+      </h1>
+      <table class="table table-borderless table-striped">
+        <tbody>
+          <!-- <tr>
+          <td class="td-width" i18n="block.hash">Hash</td>
+          <td>&lrm;<a [routerLink]="['/block/' | relativeUrl, block?.id]" title="{{ block?.id }}">{{ block?.id | shortenString : 13 }}</a></td>
+          </tr> -->
+          <tr>
+            <td i18n="block.timestamp">Timestamp</td>
+            <td>
+              {{ block?.timestamp * 1000 | date:'yyyy-MM-dd HH:mm' }}
+            </td>
+          </tr>
+          <tr>
+            <td i18n="block.weight">Weight</td>
+            <td [innerHTML]="'&lrm;' + (block?.weight | wuBytes: 2)"></td>
+          </tr>
+          <ng-template [ngIf]="webGlEnabled">
+            <tr *ngIf="block?.extras?.medianFee != undefined">
+              <td class="td-width" i18n="block.median-fee">Median fee</td>
+              <td>~{{ block?.extras?.medianFee | number:'1.0-0' }} <span class="symbol" i18n="shared.sat-vbyte|sat/vB">sat/vB</span></td>
+            </tr>
+            <ng-template [ngIf]="fees !== undefined">
+              <tr>
+                <td i18n="block.total-fees|Total fees in a block">Total fees</td>
+                <td *ngIf="network !== 'liquid' && network !== 'liquidtestnet'; else liquidTotalFees">
+                  <app-amount [satoshis]="block?.extras.totalFees" digitsInfo="1.2-3" [noFiat]="true"></app-amount>
+                </td>
+                <ng-template #liquidTotalFees>
+                  <td>
+                    <app-amount [satoshis]="fees * 100000000" digitsInfo="1.2-2" [noFiat]="true"></app-amount>
+                  </td>
+                </ng-template>
+              </tr>
+            </ng-template>
+            <tr *ngIf="network !== 'liquid' && network !== 'liquidtestnet'">
+              <td i18n="block.miner">Miner</td>
+              <td *ngIf="stateService.env.MINING_DASHBOARD">
+                <a [attr.data-cy]="'block-details-miner-badge'" placement="bottom" [routerLink]="['/mining/pool' | relativeUrl, block?.extras.pool.slug]" class="badge"
+                  [class]="block?.extras.pool.name === 'Unknown' ? 'badge-secondary' : 'badge-primary'">
+                  {{ block?.extras.pool.name }}
+                </a>
+              </td>
+              <td *ngIf="!stateService.env.MINING_DASHBOARD && stateService.env.BASE_MODULE === 'mempool'">
+                <span [attr.data-cy]="'block-details-miner-badge'" placement="bottom" class="badge"
+                  [class]="block?.extras.pool.name === 'Unknown' ? 'badge-secondary' : 'badge-primary'">
+                  {{ block?.extras.pool.name }}
+              </span>
+              </td>
+            </tr>
+          </ng-template>
+        </tbody>
+      </table>
+    </div>
+    <div class="col-sm chart-container" *ngIf="webGlEnabled">
+      <app-block-overview-graph
+        #blockGraph
+        [isLoading]="false"
+        [resolution]="75"
+        [blockLimit]="stateService.blockVSize"
+        [orientation]="'top'"
+        [flip]="false"
+        (txClickEvent)="onTxClick($event)"
+      ></app-block-overview-graph>
+    </div>
+  </div>
+</div>

frontend/src/app/components/block/block-preview.component.scss

@@ -0,0 +1,23 @@
+.block-title {
+  margin-bottom: 0.75em;
+  font-size: 42px;
+
+  ::ng-deep .next-previous-blocks {
+    font-size: 42px;
+  }
+}
+
+.table {
+  font-size: 24px;
+}
+
+.chart-container {
+  flex-grow: 0;
+  flex-shrink: 0;
+  width: 420px;
+  min-width: 420px;
+}
+
+::ng-deep .symbol {
+  font-size: 18px;
+}

frontend/src/app/components/block/block-preview.component.ts

@@ -0,0 +1,11 @@
+import { Component } from '@angular/core';
+import { BlockComponent } from './block.component';
+
+@Component({
+  selector: 'app-block-preview',
+  templateUrl: './block-preview.component.html',
+  styleUrls: ['./block.component.scss', './block-preview.component.scss']
+})
+export class BlockPreviewComponent extends BlockComponent {
+  
+}

frontend/src/app/components/graphs/graphs.component.html

@@ -37,7 +37,7 @@
       <a class="dropdown-item" routerLinkActive="active" [routerLink]="['/graphs/lightning/nodes-per-isp' | relativeUrl]"
         i18n="lightning.nodes-per-isp">Lightning nodes per ISP</a>
       <a class="dropdown-item" routerLinkActive="active" [routerLink]="['/graphs/lightning/nodes-per-country' | relativeUrl]"
-        i18n="lightning.nodes-per-isp">Lightning nodes per country</a>
+        i18n="lightning.nodes-per-country">Lightning nodes per country</a>
       <a class="dropdown-item" routerLinkActive="active" [routerLink]="['/graphs/lightning/nodes-map' | relativeUrl]"
         i18n="lightning.lightning.nodes-heatmap">Lightning nodes world map</a>
       <a class="dropdown-item" routerLinkActive="active" [routerLink]="['/graphs/lightning/nodes-channels-map' | relativeUrl]"

frontend/src/app/components/master-page-preview/master-page-preview.component.html

@@ -0,0 +1,21 @@
+<ng-container *ngIf="{ val: network$ | async } as network">
+<div class="preview-wrapper">
+  <router-outlet></router-outlet>
+
+  <footer>
+    <span class="footer-brand" style="position: relative;">
+      <img *ngIf="!officialMempoolSpace" src="/resources/mempool-logo.png" height="35" width="140" class="logo">
+      <app-svg-images *ngIf="officialMempoolSpace" name="officialMempoolSpace" style="width: 140px; height: 35px" width="500" height="126" viewBox="0 0 500 126"></app-svg-images>
+    </span>
+
+    <div [ngSwitch]="network.val">
+      <span *ngSwitchCase="'signet'" class="network signet"><img src="/resources/signet-logo.png" style="width: 30px;" class="signet mr-1" alt="logo"> Signet</span>
+      <span *ngSwitchCase="'testnet'" class="network testnet"><img src="/resources/testnet-logo.png" style="width: 30px;" class="mr-1" alt="testnet logo"> Testnet</span>
+      <span *ngSwitchCase="'bisq'" class="network bisq"><img src="/resources/bisq-logo.png" style="width: 30px;" class="mr-1" alt="bisq logo"> Bisq</span>
+      <span *ngSwitchCase="'liquid'" class="network liquid"><img src="/resources/liquid-logo.png" style="width: 30px;" class="mr-1" alt="liquid mainnet logo"> Liquid</span>
+      <span *ngSwitchCase="'liquidtestnet'" class="network liquidtestnet"><img src="/resources/liquidtestnet-logo.png" style="width: 30px;" class="mr-1" alt="liquid testnet logo"> Liquid Testnet</span>
+      <span *ngSwitchDefault class="network mainnet"><img src="/resources/bitcoin-logo.png" style="width: 30px;" class="mainnet mr-1" alt="bitcoin logo"> Mainnet</span>
+    </div>
+  </footer>
+</div>
+</ng-container>

frontend/src/app/components/master-page-preview/master-page-preview.component.scss

@@ -0,0 +1,36 @@
+.preview-wrapper {
+  position: relative;
+  display: block;
+  margin: auto;
+  max-width: 1024px;
+  max-height: 512px;
+  padding-bottom: 64px;
+
+  footer {
+    position: absolute;
+    left: 0;
+    right: 0;
+    bottom: 0;
+    z-index: 100;
+    min-height: 64px;
+    padding: 0rem 2rem;
+    display: flex;
+    flex-direction: row;
+    justify-content: space-between;
+    align-items: center;
+    background: #11131f;
+    text-align: start;
+    font-size: 1.2em;
+  }
+
+  .footer-brand {
+    width: 60%;
+  }
+
+  .network {
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-start;
+    align-items: center;
+  }
+}

frontend/src/app/components/master-page-preview/master-page-preview.component.ts

@@ -0,0 +1,25 @@
+import { Component, OnInit } from '@angular/core';
+import { StateService } from '../../services/state.service';
+import { Observable, merge, of } from 'rxjs';
+import { LanguageService } from 'src/app/services/language.service';
+
+@Component({
+  selector: 'app-master-page-preview',
+  templateUrl: './master-page-preview.component.html',
+  styleUrls: ['./master-page-preview.component.scss'],
+})
+export class MasterPagePreviewComponent implements OnInit {
+  network$: Observable<string>;
+  officialMempoolSpace = this.stateService.env.OFFICIAL_MEMPOOL_SPACE;
+  urlLanguage: string;
+
+  constructor(
+    public stateService: StateService,
+    private languageService: LanguageService,
+  ) { }
+
+  ngOnInit() {
+    this.network$ = merge(of(''), this.stateService.networkChanged$);
+    this.urlLanguage = this.languageService.getLanguageForUrl();
+  }
+}

frontend/src/app/components/master-page/master-page.component.html

@@ -1,15 +1,15 @@
 <ng-container *ngIf="{ val: network$ | async } as network">
 <header>
   <nav class="navbar navbar-expand-md navbar-dark bg-dark">
-  <a class="navbar-brand" [routerLink]="['/' | relativeUrl]">
+  <a class="navbar-brand" [ngClass]="{'dual-logos': subdomain}" [routerLink]="['/' | relativeUrl]">
   <ng-template [ngIf]="subdomain">
     <div class="subdomain_container">
       <img [src]="'/api/v1/enterprise/images/' + subdomain + '/logo'" class="subdomain_logo">
     </div>
   </ng-template>
     <ng-container *ngIf="{ val: connectionState$ | async } as connectionState">
-      <img *ngIf="!officialMempoolSpace" src="/resources/mempool-logo.png" height="35" width="140" class="logo" [ngStyle]="{'opacity': connectionState.val === 2 ? 1 : 0.5 }" alt="The Mempool Open Source Project logo">
+      <img *ngIf="!officialMempoolSpace" src="/resources/mempool-logo.png" class="mempool-logo" [ngStyle]="{'opacity': connectionState.val === 2 ? 1 : 0.5 }" alt="The Mempool Open Source Project logo">
-      <app-svg-images *ngIf="officialMempoolSpace" name="officialMempoolSpace" style="width: 140px; height: 35px" width="500" height="126" viewBox="0 0 500 126"></app-svg-images>
+      <app-svg-images *ngIf="officialMempoolSpace" name="officialMempoolSpace" viewBox="0 0 500 126"></app-svg-images>
       <div class="connection-badge">
         <div class="badge badge-warning" *ngIf="connectionState.val === 0" i18n="master-page.offline">Offline</div>
         <div class="badge badge-warning" *ngIf="connectionState.val === 1" i18n="master-page.reconnecting">Reconnecting...</div>

frontend/src/app/components/master-page/master-page.component.scss

@@ -9,12 +9,18 @@ fa-icon {
 .navbar {
   z-index: 100;
   min-height: 64px;
+  width: 100%;
 }
 
 li.nav-item {
   margin: auto 10px;
   padding-left: 10px;
   padding-right: 10px;
+  @media (max-width: 992px) {
+    margin: auto 7px;
+    padding-left: 8px;
+    padding-right: 8px;
+  }
 }
 
 @media (min-width: 992px) {
@@ -78,6 +84,14 @@ li.nav-item {
 
 .navbar-brand {
   position: relative;
+  height: 65px;
+}
+
+.navbar-brand.dual-logos {
+  justify-content: space-between;
+  @media (max-width: 767.98px) {
+    width: 100%;
+  }
 }
 
 nav {
@@ -86,7 +100,7 @@ nav {
 
 .connection-badge {
   position: absolute;
-  top: 13px;
+  top: 22px;
   width: 100%;
 }
 
@@ -150,6 +164,7 @@ nav {
   width: 140px;
   margin-right: 15px;
   text-align: center;
+  align-self: center;
 }
 
 .logo-holder {
@@ -161,3 +176,9 @@ nav {
   flex-direction: row;
   display: flex;
 }
+
+.mempool-logo, app-svg-images {
+  align-self: center;
+  width: 140px;
+  height: 35px;
+}

frontend/src/app/components/tx-features/tx-features.component.html

@@ -1,13 +1,21 @@
-<span *ngIf="segwitGains.realizedGains && !segwitGains.potentialBech32Gains; else segwitTwo" class="badge badge-success mr-1" i18n-ngbTooltip="ngbTooltip about segwit gains" ngbTooltip="This transaction saved {{ segwitGains.realizedGains * 100 | number:  '1.0-0' }}% on fees by using native SegWit-Bech32" placement="bottom" i18n="tx-features.tag.segwit|SegWit">SegWit</span>
+<span *ngIf="segwitGains.realizedSegwitGains && !segwitGains.potentialSegwitGains; else segwitTwo" class="badge badge-success mr-1" i18n-ngbTooltip="ngbTooltip about segwit gains" ngbTooltip="This transaction saved {{ segwitGains.realizedSegwitGains * 100 | number:  '1.0-0' }}% on fees by using native SegWit" placement="bottom" i18n="tx-features.tag.segwit|SegWit">SegWit</span>
 <ng-template #segwitTwo>
-  <span *ngIf="segwitGains.realizedGains && segwitGains.potentialBech32Gains else potentialP2shGains" class="badge badge-warning mr-1" i18n-ngbTooltip="ngbTooltip about double segwit gains" ngbTooltip="This transaction saved {{ segwitGains.realizedGains * 100 | number:  '1.0-0' }}% on fees by using SegWit and could save {{ segwitGains.potentialBech32Gains * 100 | number : '1.0-0' }}% more by fully upgrading to native SegWit-Bech32" placement="bottom" i18n="tx-features.tag.segwit|SegWit">SegWit</span>
+  <span *ngIf="segwitGains.realizedSegwitGains && segwitGains.potentialSegwitGains; else potentialP2shSegwitGains" class="badge badge-warning mr-1" i18n-ngbTooltip="ngbTooltip about double segwit gains" ngbTooltip="This transaction saved {{ segwitGains.realizedSegwitGains * 100 | number:  '1.0-0' }}% on fees by using SegWit and could save {{ segwitGains.potentialSegwitGains * 100 | number : '1.0-0' }}% more by fully upgrading to native SegWit" placement="bottom" i18n="tx-features.tag.segwit|SegWit">SegWit</span>
-  <ng-template #potentialP2shGains>
+  <ng-template #potentialP2shSegwitGains>
-    <span *ngIf="segwitGains.potentialP2shGains" class="badge badge-danger mr-1" i18n-ngbTooltip="ngbTooltip about missed out gains" ngbTooltip="This transaction could save {{ segwitGains.potentialBech32Gains * 100 | number : '1.0-0' }}% on fees by upgrading to native SegWit-Bech32 or {{ segwitGains.potentialP2shGains * 100 | number:  '1.0-0' }}% by upgrading to SegWit-P2SH" placement="bottom"><del i18n="tx-features.tag.segwit|SegWit">SegWit</del></span>
+    <span *ngIf="segwitGains.potentialP2shSegwitGains" class="badge badge-danger mr-1" i18n-ngbTooltip="ngbTooltip about missed out gains" ngbTooltip="This transaction could save {{ segwitGains.potentialSegwitGains * 100 | number : '1.0-0' }}% on fees by upgrading to native SegWit or {{ segwitGains.potentialP2shSegwitGains * 100 | number:  '1.0-0' }}% by upgrading to SegWit-P2SH" placement="bottom"><del i18n="tx-features.tag.segwit|SegWit">SegWit</del></span>
   </ng-template>
 </ng-template>
-<span *ngIf="isTaproot; else noTaproot" class="badge badge-success mr-1" i18n-ngbTooltip="Taproot tooltip" ngbTooltip="This transaction uses Taproot" placement="bottom" i18n="tx-features.tag.taproot">Taproot</span>
+
-<ng-template #noTaproot>
+<span *ngIf="segwitGains.realizedTaprootGains && !segwitGains.potentialTaprootGains; else notFullyTaproot" class="badge badge-success mr-1" i18n-ngbTooltip="Tooltip about privacy and fees saved with taproot" ngbTooltip="This transaction uses Taproot and thereby increased the user's privacy and saved at least {{ segwitGains.realizedTaprootGains * 100 | number: '1.0-0' }}% on fees" placement="bottom" i18n="tx-features.tag.taproot|Taproot">Taproot</span>
-  <span class="badge badge-danger mr-1" i18n-ngbTooltip="No Taproot tooltip" ngbTooltip="This transaction could save on fees and improve privacy by using Taproot" placement="bottom"><del i18n="tx-features.tag.taproot">Taproot</del></span>
+<ng-template #notFullyTaproot>
+  <span *ngIf="segwitGains.realizedTaprootGains && segwitGains.potentialTaprootGains; else noTaproot" class="badge badge-warning mr-1" i18n-ngbTooltip="Tooltip about privacy and more fees that could be saved with more taproot" ngbTooltip="This transaction uses Taproot and thereby increased the user's privacy and already saved at least {{ segwitGains.realizedTaprootGains * 100 | number: '1.0-0' }}% on fees, but could save an additional {{ segwitGains.potentialTaprootGains * 100 | number: '1.0-0' }}% by fully using Taproot" placement="bottom" i18n="tx-features.tag.taproot|Taproot">Taproot</span>
+  <ng-template #noTaproot>
+    <span *ngIf="segwitGains.potentialTaprootGains; else taprootButNoGains" class="badge badge-danger mr-1" i18n-ngbTooltip="Tooltip about privacy and fees that could be saved with taproot" ngbTooltip="This transaction could increase the user's privacy and save {{ segwitGains.potentialTaprootGains * 100 | number: '1.0-0' }}% on fees by using Taproot" placement="bottom"><del i18n="tx-features.tag.taproot|Taproot">Taproot</del></span>
+    <ng-template #taprootButNoGains>
+      <span *ngIf="isTaproot" class="badge badge-success mr-1" i18n-ngbTooltip="Tooltip about privacy with taproot" ngbTooltip="This transaction uses Taproot and thereby increased the user's privacy" placement="bottom" i18n="tx-features.tag.taproot|Taproot">Taproot</span>
+    </ng-template>
+  </ng-template>
 </ng-template>
-<span *ngIf="isRbfTransaction; else rbfDisabled" class="badge badge-success" i18n-ngbTooltip="RBF tooltip" ngbTooltip="This transaction support Replace-By-Fee (RBF) allowing fee bumping" placement="bottom" i18n="tx-features.tag.rbf|RBF">RBF</span>
+
+<span *ngIf="isRbfTransaction; else rbfDisabled" class="badge badge-success" i18n-ngbTooltip="RBF tooltip" ngbTooltip="This transaction supports Replace-By-Fee (RBF) allowing fee bumping" placement="bottom" i18n="tx-features.tag.rbf|RBF">RBF</span>
 <ng-template #rbfDisabled><span class="badge badge-danger mr-1" i18n-ngbTooltip="RBF disabled tooltip" ngbTooltip="This transaction does NOT support Replace-By-Fee (RBF) and cannot be fee bumped using this method" placement="bottom"><del i18n="tx-features.tag.rbf|RBF">RBF</del></span></ng-template>

frontend/src/app/components/tx-features/tx-features.component.ts

@@ -12,9 +12,11 @@ export class TxFeaturesComponent implements OnChanges {
   @Input() tx: Transaction;
 
   segwitGains = {
-    realizedGains: 0,
+    realizedSegwitGains: 0,
-    potentialBech32Gains: 0,
+    potentialSegwitGains: 0,
-    potentialP2shGains: 0,
+    potentialP2shSegwitGains: 0,
+    potentialTaprootGains: 0,
+    realizedTaprootGains: 0
   };
   isRbfTransaction: boolean;
   isTaproot: boolean;

frontend/src/app/lightning/channels-list/channels-list.component.html

@@ -1,34 +1,36 @@
 <div *ngIf="channels$ | async as response; else skeleton">
-  <h2 class="float-left">Channels ({{ response.totalItems }})</h2>
-
   <form [formGroup]="channelStatusForm" class="formRadioGroup float-right">
     <div class="btn-group btn-group-toggle" ngbRadioGroup name="radioBasic" formControlName="status">
       <label ngbButtonLabel class="btn-primary btn-sm">
-        <input ngbButton type="radio" [value]="'open'" fragment="open"> Open
+        <input ngbButton type="radio" [value]="'open'" fragment="open" i18n="open">Open
       </label>
       <label ngbButtonLabel class="btn-primary btn-sm">
-        <input ngbButton type="radio" [value]="'closed'" fragment="closed"> Closed
+        <input ngbButton type="radio" [value]="'closed'" fragment="closed" i18n="closed">Closed
       </label>
     </div>
   </form>
 
-  <table class="table table-borderless">
+  <table class="table table-borderless" *ngIf="response.channels.length > 0">
     <ng-container *ngTemplateOutlet="tableHeader"></ng-container>
     <tbody>
       <tr *ngFor="let channel of response.channels; let i = index;">
-        <ng-container *ngTemplateOutlet="tableTemplate; context: { $implicit: channel, node: channel.node_left.public_key === publicKey ? channel.node_right : channel.node_left }"></ng-container>
+        <ng-container *ngTemplateOutlet="tableTemplate; context: { $implicit: channel, node: channel.node }"></ng-container>
       </tr>
     </tbody>
   </table>
   
-  <ngb-pagination class="pagination-container float-right" [size]="paginationSize" [collectionSize]="response.totalItems" [rotate]="true" [pageSize]="itemsPerPage" [(page)]="page" (pageChange)="pageChange(page)" [maxSize]="paginationMaxSize" [boundaryLinks]="true" [ellipses]="false"></ngb-pagination>
+  <ngb-pagination *ngIf="response.channels.length > 0" class="pagination-container float-right" [size]="paginationSize" [collectionSize]="response.totalItems" [rotate]="true" [pageSize]="itemsPerPage" [(page)]="page" (pageChange)="pageChange(page)" [maxSize]="paginationMaxSize" [boundaryLinks]="true" [ellipses]="false"></ngb-pagination>
+
+  <table class="table table-borderless" *ngIf="response.channels.length === 0">
+    <div class="d-flex justify-content-center" i18n="lightning.empty-channels-list">No channels to display</div>
+  </table>
 </div>
   
 <ng-template #tableHeader>
   <thead>
     <th class="alias text-left" i18n="nodes.alias">Node Alias</th>
     <th class="alias text-left d-none d-md-table-cell" i18n="channels.transaction">&nbsp;</th>
-    <th class="alias text-left d-none d-md-table-cell" i18n="nodes.alias">Status</th>
+    <th class="alias text-left d-none d-md-table-cell" i18n="status">Status</th>
     <th class="channels text-left d-none d-md-table-cell" i18n="channels.rate">Fee Rate</th>
     <th class="capacity text-right d-none d-md-table-cell" i18n="nodes.capacity">Capacity</th>
     <th class="capacity text-right" i18n="channels.id">Channel ID</th>
@@ -40,31 +42,41 @@
     <div>{{ node.alias || '?' }}</div>
     <div class="second-line">
       <a [routerLink]="['/lightning/node' | relativeUrl, node.public_key]">
-        <span>{{ node.public_key | shortenString : 10 }}</span>
+        <span>{{ node.public_key | shortenString : publicKeySize }}</span>
       </a>
       <app-clipboard [text]="node.public_key" size="small"></app-clipboard>
     </div>
   </td>
   <td class="alias text-left d-none d-md-table-cell">
     <div class="second-line">{{ node.channels }} channels</div>
-    <div class="second-line"><app-amount [satoshis]="node.capacity" digitsInfo="1.2-2"></app-amount></div>
+    <div class="second-line">
+      <app-amount *ngIf="node.capacity > 100000000; else smallnode" [satoshis]="node.capacity" [digitsInfo]="'1.2-2'" [noFiat]="true"></app-amount>
+      <ng-template #smallnode>
+        {{ node.capacity | amountShortener: 1 }}
+        <span class="sats" i18n="shared.sats">sats</span>
+      </ng-template>
+    </div>
   </td>
   <td class="d-none d-md-table-cell">
-    <span class="badge rounded-pill badge-secondary" *ngIf="channel.status === 0">Inactive</span>
+    <span class="badge rounded-pill badge-secondary" *ngIf="channel.status === 0" i18n="lightning.inactive">Inactive</span>
-    <span class="badge rounded-pill badge-success" *ngIf="channel.status === 1">Active</span>
+    <span class="badge rounded-pill badge-success" *ngIf="channel.status === 1" i18n="lightning.active">Active</span>
     <ng-template [ngIf]="channel.status === 2">
-      <span class="badge rounded-pill badge-secondary" *ngIf="!channel.closing_reason; else closingReason">Closed</span>
+      <span class="badge rounded-pill badge-secondary" *ngIf="!channel.closing_reason; else closingReason" i18n="lightning.closed">Closed</span>
       <ng-template #closingReason>
         <app-closing-type [type]="channel.closing_reason"></app-closing-type>
       </ng-template>
     </ng-template>
   </td>
   <td class="capacity text-left d-none d-md-table-cell">
-    {{ node.fee_rate }} <span class="symbol">ppm ({{ node.fee_rate / 10000 | number }}%)</span>
+    {{ channel.fee_rate }} <span class="symbol">ppm ({{ channel.fee_rate / 10000 | number }}%)</span>
   </td>
   <td class="capacity text-right d-none d-md-table-cell">
-    <app-amount [satoshis]="channel.capacity" digitsInfo="1.2-2"></app-amount>
+    <app-amount *ngIf="channel.capacity > 100000000; else smallchannel" [satoshis]="channel.capacity" [digitsInfo]="'1.2-2'" [noFiat]="true"></app-amount>
-  </td>
+    <ng-template #smallchannel>
+      {{ channel.capacity | amountShortener: 1 }}
+      <span class="sats" i18n="shared.sats">sats</span>
+    </ng-template>
+</td>
   <td class="capacity text-right">
     <a [routerLink]="['/lightning/channel' | relativeUrl, channel.id]">{{ channel.short_id }}</a>
    </td>

frontend/src/app/lightning/channels-list/channels-list.component.scss

@@ -1,3 +1,9 @@
 .second-line {
   font-size: 12px;
 }
+
+.sats {
+  color: #ffffff66;
+  font-size: 12px;
+  top: 0px;
+}

frontend/src/app/lightning/channels-list/channels-list.component.ts

@@ -1,7 +1,8 @@
-import { ChangeDetectionStrategy, Component, Input, OnChanges, OnInit } from '@angular/core';
+import { ChangeDetectionStrategy, Component, EventEmitter, Input, OnChanges, OnInit, Output } from '@angular/core';
 import { FormBuilder, FormGroup } from '@angular/forms';
-import { BehaviorSubject, combineLatest, merge, Observable, of } from 'rxjs';
+import { BehaviorSubject, merge, Observable } from 'rxjs';
-import { map, startWith, switchMap } from 'rxjs/operators';
+import { map, switchMap, tap } from 'rxjs/operators';
+import { isMobile } from 'src/app/shared/common.utils';
 import { LightningApiService } from '../lightning-api.service';
 
 @Component({
@@ -12,16 +13,19 @@ import { LightningApiService } from '../lightning-api.service';
 })
 export class ChannelsListComponent implements OnInit, OnChanges {
   @Input() publicKey: string;
+  @Output() channelsStatusChangedEvent = new EventEmitter<string>();
   channels$: Observable<any>;
 
   // @ts-ignore
   paginationSize: 'sm' | 'lg' = 'md';
   paginationMaxSize = 10;
-  itemsPerPage = 25;
+  itemsPerPage = 10;
   page = 1;
   channelsPage$ = new BehaviorSubject<number>(1);
   channelStatusForm: FormGroup;
   defaultStatus = 'open';
+  status = 'open';
+  publicKeySize = 25;
 
   constructor(
     private lightningApiService: LightningApiService,
@@ -30,9 +34,12 @@ export class ChannelsListComponent implements OnInit, OnChanges {
     this.channelStatusForm = this.formBuilder.group({
       status: [this.defaultStatus],
     });
+    if (isMobile()) {
+      this.publicKeySize = 12;
+    }
   }
 
-  ngOnInit() {
+  ngOnInit(): void {
     if (document.body.clientWidth < 670) {
       this.paginationSize = 'sm';
       this.paginationMaxSize = 3;
@@ -40,24 +47,36 @@ export class ChannelsListComponent implements OnInit, OnChanges {
   }
 
   ngOnChanges(): void {
-    this.channelStatusForm.get('status').setValue(this.defaultStatus, { emitEvent: false })
+    this.channelStatusForm.get('status').setValue(this.defaultStatus, { emitEvent: false });
+    this.channelsPage$.next(1);
 
-    this.channels$ = combineLatest([
+    this.channels$ = merge(
       this.channelsPage$,
-      this.channelStatusForm.get('status').valueChanges.pipe(startWith(this.defaultStatus))
+      this.channelStatusForm.get('status').valueChanges,
-    ])
+    )
     .pipe(
-      switchMap(([page, status]) =>this.lightningApiService.getChannelsByNodeId$(this.publicKey, (page -1) * this.itemsPerPage, status)),
+      tap((val) => {
+        if (typeof val === 'string') {
+          this.status = val;
+          this.page = 1;
+        } else if (typeof val === 'number') {
+          this.page = val;
+        }
+      }),
+      switchMap(() => {
+          this.channelsStatusChangedEvent.emit(this.status);
+          return this.lightningApiService.getChannelsByNodeId$(this.publicKey, (this.page - 1) * this.itemsPerPage, this.status);
+      }),
       map((response) => {
         return {
           channels: response.body,
-          totalItems: parseInt(response.headers.get('x-total-count'), 10)
+          totalItems: parseInt(response.headers.get('x-total-count'), 10) + 1 
         };
       }),
     );
   }
 
-  pageChange(page: number) {
+  pageChange(page: number): void {
     this.channelsPage$.next(page);
   }
 

frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html

@@ -1,3 +1,5 @@
+<app-nodes-channels-map [style]="'widget'"></app-nodes-channels-map>
+
 <div class="container-xl dashboard-container">
 
   <div class="row row-cols-1 row-cols-md-2">
@@ -51,7 +53,7 @@
         <div class="card-body">
           <h5 class="card-title">Top Capacity Nodes</h5>
           <app-nodes-list [nodes$]="nodesByCapacity$"></app-nodes-list>
-          <div><a [routerLink]="['/lightning/nodes' | relativeUrl]" i18n="dashboard.view-more">View more &raquo;</a></div>
+          <!-- <div><a [routerLink]="['/lightning/nodes' | relativeUrl]" i18n="dashboard.view-more">View more &raquo;</a></div> -->
         </div>
       </div>
     </div>
@@ -61,7 +63,7 @@
         <div class="card-body">
           <h5 class="card-title">Most Connected Nodes</h5>
           <app-nodes-list [nodes$]="nodesByChannels$"></app-nodes-list>
-          <div><a [routerLink]="['/lightning/nodes' | relativeUrl]" i18n="dashboard.view-more">View more &raquo;</a></div>
+          <!-- <div><a [routerLink]="['/lightning/nodes' | relativeUrl]" i18n="dashboard.view-more">View more &raquo;</a></div> -->
         </div>
       </div>
     </div>

frontend/src/app/lightning/node-statistics-chart/node-statistics-chart.component.ts

@@ -169,9 +169,6 @@ export class NodeStatisticsChartComponent implements OnInit {
       },
       yAxis: data.channels.length === 0 ? undefined : [
         {
-          min: (value) => {
-            return value.min * 0.9;
-          },
           type: 'value',
           axisLabel: {
             color: 'rgb(110, 112, 121)',
@@ -188,9 +185,6 @@ export class NodeStatisticsChartComponent implements OnInit {
           },
         },
         {
-          min: (value) => {
-            return value.min * 0.9;
-          },
           type: 'value',
           position: 'right',
           axisLabel: {
@@ -225,15 +219,6 @@ export class NodeStatisticsChartComponent implements OnInit {
               opacity: 1,
               width: 1,
             },
-            data: [{
-              yAxis: 1,
-              label: {
-                position: 'end',
-                show: true,
-                color: '#ffffff',
-                formatter: `1 MB`
-              }
-            }],
           }
         },
         {

frontend/src/app/lightning/node/node.component.html

@@ -1,45 +1,66 @@
 <div class="container-xl" *ngIf="(node$ | async) as node">
-  <div class="title-container mb-2">
+  <div class="title-container mb-2" *ngIf="!error">
     <h1 class="mb-0">{{ node.alias }}</h1>
     <span class="tx-link">
-      <a [routerLink]="['/lightning/node' | relativeUrl, node.public_key]">{{ node.public_key | shortenString : 12 }}</a>
+      <a [routerLink]="['/lightning/node' | relativeUrl, node.public_key]">
+        {{ node.public_key | shortenString : publicKeySize }}
+      </a>
       <app-clipboard [text]="node.public_key"></app-clipboard>
     </span>
   </div>
 
   <div class="clearfix"></div>
 
-    <div class="box">
+  <div *ngIf="error" class="d-flex flex-column justify-content-around align-items-center mt-5 w-100" style="min-height: 100px">
+    <span i18n="lightning.node-not-found">No node found for public key "{{ node.public_key | shortenString : 12}}"</span>
+    <a [routerLink]="['/lightning' | relativeUrl]" i18n="lightning.back-to-lightning-dashboard">Back to the lightning dashboard</a>
+  </div>
+
+  <div class="box" *ngIf="!error">
 
     <div class="row">
       <div class="col-md">
         <table class="table table-borderless table-striped">
           <tbody>
             <tr>
-                <td i18n="address.total-received">Total capacity</td>
+              <td i18n="lightning.active-capacity">Active capacity</td>
               <td>
-                  <app-sats [satoshis]="node.capacity"></app-sats><app-fiat [value]="node.capacity" digitsInfo="1.0-0"></app-fiat>
+                <app-sats [satoshis]="node.capacity"></app-sats>
+                <app-fiat [value]="node.capacity" digitsInfo="1.0-0"></app-fiat>
               </td>
             </tr>
             <tr>
-                <td i18n="address.total-sent">Total channels</td>
+              <td i18n="lightning.active-channels">Active channels</td>
               <td>
-                  {{ node.channel_count }}
+                {{ node.active_channel_count }}
               </td>
             </tr>
             <tr>
-                <td i18n="address.total-received">Average channel size</td>
+              <td i18n="lightning.active-channels-avg">Average channel size</td>
               <td>
-                  <app-sats [satoshis]="node.channels_capacity_avg"></app-sats><app-fiat [value]="node.channels_capacity_avg" digitsInfo="1.0-0"></app-fiat>
+                <app-sats [satoshis]="node.avgCapacity"></app-sats>
+                <app-fiat [value]="node.avgCapacity" digitsInfo="1.0-0"></app-fiat>
               </td>
             </tr>
             <tr *ngIf="node.country && node.city && node.subdivision">
               <td i18n="location">Location</td>
-                <td>{{ node.city.en }}, {{ node.subdivision.en }}<br>{{ node.country.en }}</td>
+              <td>
+                <span>{{ node.city.en }}, {{ node.subdivision.en }}</span>
+                <br>
+                <a class="d-flex align-items-center" [routerLink]="['/lightning/nodes/country' | relativeUrl, node.iso_code]">
+                  <span class="link">{{ node.country.en }}</span>
+                  &nbsp;
+                  <span class="flag">{{ node.flag }}</span>
+                </a>
+              </td>
             </tr>
             <tr *ngIf="node.country && !node.city">
               <td i18n="location">Location</td>
-                <td>{{ node.country.en }}</td>
+              <td>
+                <a [routerLink]="['/lightning/nodes/country' | relativeUrl, node.iso_code]">
+                  {{ node.country.en }} {{ node.flag }}
+                </a>
+              </td>
             </tr>
           </tbody>
         </table>
@@ -51,23 +72,27 @@
             <tr>
               <td i18n="address.total-received">First seen</td>
               <td>
-                  <app-timestamp [dateString]="node.first_seen"></app-timestamp>
+                <app-timestamp [unixTime]="node.first_seen"></app-timestamp>
               </td>
             </tr>
             <tr>
               <td i18n="address.total-sent">Last update</td>
               <td>
-                  <app-timestamp [dateString]="node.updated_at"></app-timestamp>
+                <app-timestamp [unixTime]="node.updated_at"></app-timestamp>
               </td>
             </tr>
             <tr>
               <td i18n="address.balance">Color</td>
-                <td><div [ngStyle]="{'color': node.color}">{{ node.color }}</div></td>
+              <td>
+                <div [ngStyle]="{'color': node.color}">{{ node.color }}</div>
+              </td>
             </tr>
             <tr *ngIf="node.country">
               <td i18n="isp">ISP</td>
               <td>
+                <a [routerLink]="['/lightning/nodes/isp' | relativeUrl, node.as_number]">
                   {{ node.as_organization }} [ASN {{node.as_number}}]
+                </a>                
               </td>
             </tr>
           </tbody>
@@ -77,20 +102,25 @@
 
   </div>
 
-    <br>
+  <div class="input-group mt-3" *ngIf="!error && node.socketsObject.length">
-
+    <div class="d-inline-block" ngbDropdown #myDrop="ngbDropdown"
-    <div class="input-group mb-3" *ngIf="node.socketsObject.length">
+      *ngIf="node.socketsObject.length > 1; else noDropdown">
-      <div class="d-inline-block" ngbDropdown #myDrop="ngbDropdown" *ngIf="node.socketsObject.length > 1; else noDropdown">
+      <button class="btn btn-secondary dropdown-toggle" type="button" aria-expanded="false" ngbDropdownAnchor
-        <button class="btn btn-secondary dropdown-toggle" type="button" aria-expanded="false" ngbDropdownAnchor (focus)="myDrop.open()"><div class="dropdownLabel">{{ node.socketsObject[selectedSocketIndex].label }}</div></button>
+        (focus)="myDrop.open()">
+        <div class="dropdownLabel">{{ node.socketsObject[selectedSocketIndex].label }}</div>
+      </button>
       <div ngbDropdownMenu aria-labelledby="dropdownManual">
-          <button *ngFor="let socket of node.socketsObject; let i = index;" ngbDropdownItem (click)="changeSocket(i)">{{ socket.label }}</button>
+        <button *ngFor="let socket of node.socketsObject; let i = index;" ngbDropdownItem (click)="changeSocket(i)">{{
+          socket.label }}</button>
       </div>
     </div>
     <ng-template #noDropdown>
       <span class="input-group-text" id="basic-addon3">{{ node.socketsObject[selectedSocketIndex].label }}</span>
     </ng-template>
-      <input type="text" class="form-control" aria-label="Text input with dropdown button" [value]="node.socketsObject[selectedSocketIndex].socket">
+    <input type="text" class="form-control" aria-label="Text input with dropdown button"
-      <button class="btn btn-secondary ml-1" type="button" id="inputGroupFileAddon04" (mouseover)="qrCodeVisible = true" (mouseout)="qrCodeVisible = false">
+      [value]="node.socketsObject[selectedSocketIndex].socket">
+    <button class="btn btn-secondary ml-1" type="button" id="inputGroupFileAddon04" (mouseover)="qrCodeVisible = true"
+      (mouseout)="qrCodeVisible = false">
       <fa-icon [icon]="['fas', 'qrcode']" [fixedWidth]="true"></fa-icon>
       <div class="qr-wrapper" [hidden]="!qrCodeVisible">
         <app-qrcode [size]="200" [data]="node.socketsObject[selectedSocketIndex].socket"></app-qrcode>
@@ -101,13 +131,15 @@
     </button>
   </div>
 
-    <br>
+  <app-nodes-channels-map *ngIf="!error" [style]="'nodepage'" [publicKey]="node.public_key"></app-nodes-channels-map>
+  <app-node-statistics-chart [publicKey]="node.public_key" *ngIf="!error"></app-node-statistics-chart>
 
-    <app-node-statistics-chart [publicKey]="node.public_key"></app-node-statistics-chart>
+  <div class="d-flex justify-content-between" *ngIf="!error">
+    <h2>Channels ({{ channelsListStatus === 'open' ? node.opened_channel_count : node.closed_channel_count }})</h2>
+  </div>
 
-    <br>
+  <app-channels-list *ngIf="!error" [publicKey]="node.public_key"
-
+    (channelsStatusChangedEvent)="onChannelsListStatusChanged($event)"></app-channels-list>
-    <app-channels-list [publicKey]="node.public_key"></app-channels-list>
     
 </div>
 

frontend/src/app/lightning/node/node.component.scss

@@ -57,4 +57,3 @@ app-fiat {
     margin-left: 10px;
   }
 }
-

frontend/src/app/lightning/node/node.component.ts

@@ -1,9 +1,11 @@
 import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core';
 import { ActivatedRoute, ParamMap } from '@angular/router';
 import { Observable } from 'rxjs';
-import { map, switchMap } from 'rxjs/operators';
+import { catchError, map, switchMap } from 'rxjs/operators';
 import { SeoService } from 'src/app/services/seo.service';
+import { getFlagEmoji } from 'src/app/shared/graphs.utils';
 import { LightningApiService } from '../lightning-api.service';
+import { isMobile } from '../../shared/common.utils';
 
 @Component({
   selector: 'app-node',
@@ -17,17 +19,27 @@ export class NodeComponent implements OnInit {
   publicKey$: Observable<string>;
   selectedSocketIndex = 0;
   qrCodeVisible = false;
+  channelsListStatus: string;
+  error: Error;
+  publicKey: string;
+
+  publicKeySize = 99;
 
   constructor(
     private lightningApiService: LightningApiService,
     private activatedRoute: ActivatedRoute,
     private seoService: SeoService,
-  ) { }
+  ) {
+    if (isMobile()) {
+      this.publicKeySize = 12;
+    }
+  }
 
   ngOnInit(): void {
     this.node$ = this.activatedRoute.paramMap
       .pipe(
         switchMap((params: ParamMap) => {
+          this.publicKey = params.get('public_key');
           return this.lightningApiService.getNode$(params.get('public_key'));
         }),
         map((node) => {
@@ -46,14 +58,23 @@ export class NodeComponent implements OnInit {
             } else if (socket.indexOf('onion') > -1) {
               label = 'Tor';
             }
+            node.flag = getFlagEmoji(node.iso_code);
             socketsObject.push({
               label: label,
               socket: node.public_key + '@' + socket,
             });
           }
           node.socketsObject = socketsObject;
+          node.avgCapacity = node.capacity / Math.max(1, node.active_channel_count);
           return node;
         }),
+        catchError(err => {
+          this.error = err;
+          return [{
+            alias: this.publicKey,
+            public_key: this.publicKey,
+          }];
+        })
       );
   }
 
@@ -61,4 +82,7 @@ export class NodeComponent implements OnInit {
     this.selectedSocketIndex = index;
   }
 
+  onChannelsListStatusChanged(e) {
+    this.channelsListStatus = e;
+  }
 }

frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.html

@@ -1,11 +1,8 @@
-<div class="full-container">
+<div [class]="'full-container ' + style">
 
-  <div class="card-header">
+  <div *ngIf="style === 'graph'" class="card-header">
     <div class="d-flex d-md-block align-items-baseline" style="margin-bottom: -5px">
       <span i18n="lightning.nodes-channels-world-map">Lightning nodes channels world map</span>
-      <button class="btn p-0 pl-2" style="margin: 0 0 4px 0px">
-        <fa-icon [icon]="['fas', 'download']" [fixedWidth]="true" (click)="onSaveChart()"></fa-icon>
-      </button>
     </div>
     <small style="color: #ffffff66" i18n="lightning.tor-nodes-excluded">(Tor nodes excluded)</small>
   </div>

frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.scss

@@ -1,6 +1,7 @@
 .card-header {
   border-bottom: 0;
   font-size: 18px;
+
   @media (min-width: 465px) {
     font-size: 20px;
   }
@@ -11,16 +12,38 @@
   width: 100%;
   min-height: 500px;
   height: calc(100% - 150px);
+
   @media (max-width: 992px) {
     height: 100%;
     padding-bottom: 100px;
-  };
+  }
+}
+
+.full-container.nodepage {
+  margin-top: 25px;
+  margin-bottom: 25px;
+}
+
+.full-container.widget {
+  height: 250px;
+  min-height: 250px;
+}
+
+.widget {
+  width: 99vw;
+  height: 250px;
+  -webkit-mask: linear-gradient(0deg, #11131f00 5%, #11131fff 25%);
+}
+
+.widget > .chart {
+  -webkit-mask: linear-gradient(180deg, #11131f00 0%, #11131fff 20%);
+  min-height: 250px;
 }
 
 .chart {
+  min-height: 500px;
   width: 100%;
   height: 100%;
-  padding-bottom: 20px;
   padding-right: 10px;
   @media (max-width: 992px) {
     padding-bottom: 25px;

frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts

@@ -1,15 +1,13 @@
-import { ChangeDetectionStrategy, Component, NgZone, OnDestroy, OnInit } from '@angular/core';
+import { ChangeDetectionStrategy, Component, HostListener, Input, NgZone, OnDestroy, OnInit, ViewChild } from '@angular/core';
 import { SeoService } from 'src/app/services/seo.service';
 import { ApiService } from 'src/app/services/api.service';
-import { Observable, tap, zip } from 'rxjs';
+import { Observable, switchMap, tap, zip } from 'rxjs';
 import { AssetsService } from 'src/app/services/assets.service';
-import { download } from 'src/app/shared/graphs.utils';
+import { ActivatedRoute, ParamMap, Router } from '@angular/router';
-import { Router } from '@angular/router';
 import { RelativeUrlPipe } from 'src/app/shared/pipes/relative-url/relative-url.pipe';
 import { StateService } from 'src/app/services/state.service';
 import { EChartsOption, registerMap } from 'echarts';
 import 'echarts-gl';
-import { SSL_OP_SSLEAY_080_CLIENT_DH_BUG } from 'constants';
 
 @Component({
   selector: 'app-nodes-channels-map',
@@ -18,10 +16,18 @@ import { SSL_OP_SSLEAY_080_CLIENT_DH_BUG } from 'constants';
   changeDetection: ChangeDetectionStrategy.OnPush,
 })
 export class NodesChannelsMap implements OnInit, OnDestroy {
+  @Input() style: 'graph' | 'nodepage' | 'widget' = 'graph';
+  @Input() publicKey: string | undefined;
+
   observable$: Observable<any>;
   
+  center: number[] | undefined;
+  zoom: number | undefined;
+  channelWidth = 0.6;
+  channelOpacity = 0.1;
+
   chartInstance = undefined;
-  chartOptions: EChartsOption = {color: 'dark'};
+  chartOptions: EChartsOption = {};
   chartInitOptions = {
     renderer: 'canvas',
   }; 
@@ -33,38 +39,87 @@ export class NodesChannelsMap implements OnInit, OnDestroy {
     private assetsService: AssetsService,
     private router: Router,
     private zone: NgZone,
+    private activatedRoute: ActivatedRoute,
   ) {
   }
 
   ngOnDestroy(): void {}
 
   ngOnInit(): void {
-    this.seoService.setTitle($localize`Lightning nodes channels world map`);
+    this.center = this.style === 'widget' ? [0, 40] : [0, 5];
+    this.zoom = this.style === 'widget' ? 3.5 : 1.3;
 
-    this.observable$ = zip(
+    if (this.style === 'graph') {
+      this.seoService.setTitle($localize`Lightning nodes channels world map`);
+    }
+    
+    this.observable$ = this.activatedRoute.paramMap
+     .pipe(
+       switchMap((params: ParamMap) => {
+        return zip(
           this.assetsService.getWorldMapJson$,
-      this.apiService.getChannelsGeo$(),
+          this.apiService.getChannelsGeo$(params.get('public_key') ?? undefined),
+          [params.get('public_key') ?? undefined]
         ).pipe(tap((data) => {
           registerMap('world', data[0]);
 
           const channelsLoc = [];
           const nodes = [];
+          const nodesPubkeys = {};
+          let thisNodeGPS: number[] | undefined = undefined;
           for (const channel of data[1]) {
-        channelsLoc.push([[channel[2], channel[3]], [channel[6], channel[7]]]);
+            if (!thisNodeGPS && data[2] === channel[0]) {
-        nodes.push({
+              thisNodeGPS = [channel[2], channel[3]];
-          publicKey: channel[0],
+            } else if (!thisNodeGPS && data[2] === channel[4]) {
-          name: channel[1],
+              thisNodeGPS = [channel[6], channel[7]];
-          value: [channel[2], channel[3]],
+            }
-        });
+
-        nodes.push({
+            // We add a bit of noise so nodes at the same location are not all
-          publicKey: channel[4],
+            // on top of each other
-          name: channel[5],
+            let random = Math.random() * 2 * Math.PI;
-          value: [channel[6], channel[7]],
+            let random2 = Math.random() * 0.01;
-        });
+            
+            if (!nodesPubkeys[channel[0]]) {
+              nodes.push([
+                channel[2] + random2 * Math.cos(random),
+                channel[3] + random2 * Math.sin(random),
+                1,
+                channel[0],
+                channel[1]
+              ]);
+              nodesPubkeys[channel[0]] = nodes[nodes.length - 1];
+            }
+
+            random = Math.random() * 2 * Math.PI;
+            random2 = Math.random() * 0.01;
+
+            if (!nodesPubkeys[channel[4]]) {
+              nodes.push([
+                channel[6] + random2 * Math.cos(random),
+                channel[7] + random2 * Math.sin(random),
+                1,
+                channel[4],
+                channel[5]
+              ]);
+              nodesPubkeys[channel[4]] = nodes[nodes.length - 1];
+            }
+
+            const channelLoc = [];
+            channelLoc.push(nodesPubkeys[channel[0]].slice(0, 2));            
+            channelLoc.push(nodesPubkeys[channel[4]].slice(0, 2));
+            channelsLoc.push(channelLoc);
+          }
+          if (this.style === 'nodepage' && thisNodeGPS) {
+            this.center = [thisNodeGPS[0], thisNodeGPS[1]];
+            this.zoom = 10;
+            this.channelWidth = 1;
+            this.channelOpacity = 1;
           }
 
           this.prepareChartOptions(nodes, channelsLoc);
         }));
+      })
+     );
   }
 
   prepareChartOptions(nodes, channels) {
@@ -75,79 +130,87 @@ export class NodesChannelsMap implements OnInit, OnDestroy {
           color: 'grey',
           fontSize: 15
         },
-        text: $localize`No data to display yet`,
+        text: $localize`No geolocation data available`,
         left: 'center',
         top: 'center'
       };
     }
 
     this.chartOptions = {
-      geo3D: {
+      silent: this.style === 'widget',
-        map: 'world',
+      title: title ?? undefined,
-        shading: 'color',
+      tooltip: {},
+      geo: {
+        animation: false,
         silent: true,
-        postEffect: {
+        center: this.center,
-          enable: true,
+        zoom: this.zoom,
-          bloom: {
+        tooltip: {
-            intensity: 0.01,
+          show: true
-          }
-        },
-        viewControl: {
-          minDistance: 1,
-          distance: 60,
-          alpha: 89,
-          panMouseButton: 'left',
-          rotateMouseButton: 'right',
-          zoomSensivity: 0.5,
         },
+        map: 'world',
+        roam: this.style === 'widget' ? false : true,
         itemStyle: {
-          color: '#FFFFFF',
-          opacity: 0.02,
-          borderWidth: 1,
           borderColor: 'black',
+          color: '#ffffff44'
         },
-        regionHeight: 0.01,
+        scaleLimit: {
+          min: 1.3,
+          max: 100000,
+        }
       },
       series: [
         {
-          // @ts-ignore
+          large: true,
-          type: 'lines3D',
+          progressive: 200,
-          coordinateSystem: 'geo3D',
+          type: 'scatter',
-          blendMode: 'lighter',
+          data: nodes,
-          lineStyle: {
+          coordinateSystem: 'geo',
-            width: 1,
+          geoIndex: 0,
-            opacity: 0.025,
+          symbolSize: 4,
+          tooltip: {
+            backgroundColor: 'rgba(17, 19, 31, 1)',
+            borderRadius: 4,
+            shadowColor: 'rgba(0, 0, 0, 0.5)',
+            textStyle: {
+              color: '#b1b1b1',
+              align: 'left',
             },
-          data: channels
+            borderColor: '#000',
+            formatter: (value) => {
+              const data = value.data;
+              const alias = data[4].length > 0 ? data[4] : data[3].slice(0, 20);
+              return `<b style="color: white">${alias}</b>`;
+            }
+          },
+          itemStyle: {
+            color: 'white',
+            borderColor: 'black',
+            borderWidth: 2,
+            opacity: 1,
+          },
+          blendMode: 'lighter',
+          zlevel: 1,
         },
         {
-          // @ts-ignore
+          large: true,
-          type: 'scatter3D',
+          progressive: 200,
-          symbol: 'circle',
+          silent: true,
+          type: 'lines',
+          coordinateSystem: 'geo',
+          data: channels,
+          lineStyle: {
+            opacity: this.channelOpacity,
+            width: this.channelWidth,
+            curveness: 0,
+            color: '#466d9d',
+          },
           blendMode: 'lighter',
-          coordinateSystem: 'geo3D',
+          tooltip: {
-          symbolSize: 3,
+            show: false,
-          itemStyle: {
-            color: '#BBFFFF',
-            opacity: 1,
-            borderColor: '#FFFFFF00',
           },
-          data: nodes,
+          zlevel: 2,
-          emphasis: {
-            label: {
-              position: 'top',
-              // @ts-ignore
-              textStyle: {
-                color: 'white',
-                fontSize: 16,
-              },
-              formatter: function(value) {
-                return value.name;
-              },
-              show: true,
         }
-          }
-        },
       ]
     };
   }
@@ -159,31 +222,42 @@ export class NodesChannelsMap implements OnInit, OnDestroy {
 
     this.chartInstance = ec;
 
-    this.chartInstance.on('click', (e) => {
+    if (this.style === 'widget') {
-      if (e.data && e.data.publicKey) {
+      this.chartInstance.getZr().on('click', (e) => {
         this.zone.run(() => {
-          const url = new RelativeUrlPipe(this.stateService).transform(`/lightning/node/${e.data.publicKey}`);
+          const url = new RelativeUrlPipe(this.stateService).transform(`/graphs/lightning/nodes-channels-map`);
+          this.router.navigate([url]);
+        });
+      });
+    }
+      
+    this.chartInstance.on('click', (e) => {
+      if (e.data) {
+        this.zone.run(() => {
+          const url = new RelativeUrlPipe(this.stateService).transform(`/lightning/node/${e.data[3]}`);
           this.router.navigate([url]);
         });
       }
     });
+
+    this.chartInstance.on('georoam', (e) => {
+      if (!e.zoom || this.style === 'nodepage') {
+        return;
       }
 
-  onSaveChart() {
+      const speed = 0.005;
-    // @ts-ignore
+      const chartOptions = {
-    const prevBottom = this.chartOptions.grid.bottom;
+        series: this.chartOptions.series
-    const now = new Date();
+      };
-    // @ts-ignore
+
-    this.chartOptions.grid.bottom = 30;
+      chartOptions.series[1].lineStyle.opacity += e.zoom > 1 ? speed : -speed;
-    this.chartOptions.backgroundColor = '#11131f';
+      chartOptions.series[1].lineStyle.width += e.zoom > 1 ? speed : -speed;
-    this.chartInstance.setOption(this.chartOptions);
+      chartOptions.series[0].symbolSize += e.zoom > 1 ? speed * 10 : -speed * 10;
-    download(this.chartInstance.getDataURL({
+      chartOptions.series[1].lineStyle.opacity = Math.max(0.05, Math.min(0.5, chartOptions.series[1].lineStyle.opacity));
-      pixelRatio: 2,
+      chartOptions.series[1].lineStyle.width = Math.max(0.5, Math.min(1, chartOptions.series[1].lineStyle.width));
-      excludeComponents: ['dataZoom'],
+      chartOptions.series[0].symbolSize = Math.max(4, Math.min(5.5, chartOptions.series[0].symbolSize));
-    }), `lightning-nodes-heatmap-clearnet-${Math.round(now.getTime() / 1000)}.svg`);
+
-    // @ts-ignore
+      this.chartInstance.setOption(chartOptions);
-    this.chartOptions.grid.bottom = prevBottom;
+    });
-    this.chartOptions.backgroundColor = 'none';
-    this.chartInstance.setOption(this.chartOptions);
   }
 }

frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.ts

@@ -59,7 +59,7 @@ export class NodesNetworksChartComponent implements OnInit {
     let firstRun = true;
 
     if (this.widget) {
-      this.miningWindowPreference = '1y';
+      this.miningWindowPreference = '3y';
     } else {
       this.seoService.setTitle($localize`Lightning nodes per network`);
       this.miningWindowPreference = this.miningService.getDefaultTimespan('all');
@@ -83,7 +83,6 @@ export class NodesNetworksChartComponent implements OnInit {
               tap((response) => {
                 const data = response.body;
                 this.prepareChartOptions({
-                  node_count: data.map(val => [val.added * 1000, val.node_count]), 
                   tor_nodes: data.map(val => [val.added * 1000, val.tor_nodes]),
                   clearnet_nodes: data.map(val => [val.added * 1000, val.clearnet_nodes]),
                   unannounced_nodes: data.map(val => [val.added * 1000, val.unannounced_nodes]),
@@ -103,7 +102,7 @@ export class NodesNetworksChartComponent implements OnInit {
 
   prepareChartOptions(data) {
     let title: object;
-    if (data.node_count.length === 0) {
+    if (data.tor_nodes.length === 0) {
       title = {
         textStyle: {
           color: 'grey',
@@ -145,33 +144,34 @@ export class NodesNetworksChartComponent implements OnInit {
         },
         borderColor: '#000',
         formatter: (ticks) => {
+          let total = 0;
           const date = new Date(ticks[0].data[0]).toLocaleDateString(this.locale, { year: 'numeric', month: 'short', day: 'numeric' });
           let tooltip = `<b style="color: white; margin-left: 2px">${date}</b><br>`;
 
-          for (const tick of ticks) {
+          for (const tick of ticks.reverse()) {
-            if (tick.seriesIndex === 0) { // Total
+            if (tick.seriesIndex === 0) { // Tor
               tooltip += `${tick.marker} ${tick.seriesName}: ${formatNumber(tick.data[1], this.locale, '1.0-0')}`;
-            } else if (tick.seriesIndex === 1) { // Tor
+            } else if (tick.seriesIndex === 1) { // Clearnet
               tooltip += `${tick.marker} ${tick.seriesName}: ${formatNumber(tick.data[1], this.locale, '1.0-0')}`;
-            } else if (tick.seriesIndex === 2) { // Clearnet
+            } else if (tick.seriesIndex === 2) { // Unannounced
-              tooltip += `${tick.marker} ${tick.seriesName}: ${formatNumber(tick.data[1], this.locale, '1.0-0')}`;
-            } else if (tick.seriesIndex === 3) { // Unannounced
               tooltip += `${tick.marker} ${tick.seriesName}: ${formatNumber(tick.data[1], this.locale, '1.0-0')}`;
             }
             tooltip += `<br>`;
+            total += tick.data[1];
           }
+          tooltip += `<b>Total:</b> ${formatNumber(total, this.locale, '1.0-0')} nodes`;
 
           return tooltip;
         }
       },
-      xAxis: data.node_count.length === 0 ? undefined : {
+      xAxis: data.tor_nodes.length === 0 ? undefined : {
         type: 'time',
         splitNumber: (this.isMobile() || this.widget) ? 5 : 10,
         axisLabel: {
           hideOverlap: true,
         }
       },
-      legend: data.node_count.length === 0 ? undefined : {
+      legend: data.tor_nodes.length === 0 ? undefined : {
         padding: 10,
         data: [
           {
@@ -214,7 +214,7 @@ export class NodesNetworksChartComponent implements OnInit {
           'Unannounced': true,
         }
       },
-      yAxis: data.node_count.length === 0 ? undefined : [
+      yAxis: data.tor_nodes.length === 0 ? undefined : [
         {
           type: 'value',
           position: 'left',
@@ -236,45 +236,23 @@ export class NodesNetworksChartComponent implements OnInit {
           },
         }
       ],
-      series: data.node_count.length === 0 ? [] : [
+      series: data.tor_nodes.length === 0 ? [] : [
-        {
-          zlevel: 1,
-          name: $localize`Total`,
-          showSymbol: false,
-          symbol: 'none',
-          data: data.node_count,
-          type: 'line',
-          lineStyle: {
-            width: 2,
-          },
-          markLine: {
-            silent: true,
-            symbol: 'none',
-            lineStyle: {
-              type: 'solid',
-              color: '#ffffff66',
-              opacity: 1,
-              width: 1,
-            },
-          },
-          areaStyle: {
-            opacity: 0.25,
-          },
-        },
         {
           zlevel: 1,
           yAxisIndex: 0,
-          name: $localize`Tor`,
+          name: $localize`Unannounced`,
           showSymbol: false,
           symbol: 'none',
-          data: data.tor_nodes,
+          data: data.unannounced_nodes,
           type: 'line',
           lineStyle: {
             width: 2,
           },
           areaStyle: {
-            opacity: 0.25,
+            opacity: 0.5,
           },
+          stack: 'Total',
+          color: '#FDD835',
         },
         {
           zlevel: 1,
@@ -288,24 +266,28 @@ export class NodesNetworksChartComponent implements OnInit {
             width: 2,
           },
           areaStyle: {
-            opacity: 0.25,
+            opacity: 0.5,
           },
+          stack: 'Total',
+          color: '#00ACC1',
         },
         {
           zlevel: 1,
           yAxisIndex: 0,
-          name: $localize`Unannounced`,
+          name: $localize`Tor`,
           showSymbol: false,
           symbol: 'none',
-          data: data.unannounced_nodes,
+          data: data.tor_nodes,
           type: 'line',
           lineStyle: {
             width: 2,
           },
           areaStyle: {
-            opacity: 0.25,
+            opacity: 0.5,
+          },
+          stack: 'Total',
+          color: '#7D4698',
         },
-        } 
       ],
       dataZoom: this.widget ? null : [{
         type: 'inside',

frontend/src/app/lightning/nodes-per-country-chart/nodes-per-country-chart.component.html

@@ -35,11 +35,11 @@
         <tr *ngFor="let country of countries">
           <td class="text-left rank">{{ country.rank }}</td>
           <td class="text-left text-truncate name">
-            <div class="d-flex">
+            <a class="d-flex align-items-center" [routerLink]="['/lightning/nodes/country' | relativeUrl, country.iso]">
-              <span style="font-size: 20px">{{ country.flag }}</span>
+              <span class="flag">{{ country.flag }}</span>
               &nbsp;
-              <a class="mt-auto mb-auto" [routerLink]="['/lightning/nodes/country' | relativeUrl, country.iso]">{{ country.name.en }}</a>
+              <span class="link">{{ country.name.en }}</span>
-            </div>
+            </a>
           </td>
           <td class="text-right share">{{ country.share }}%</td>
           <td class="text-right nodes">{{ country.count }}</td>

frontend/src/app/lightning/nodes-per-country-chart/nodes-per-country-chart.component.scss

@@ -79,3 +79,15 @@
     max-width: 100px;
   }
 }
+
+a {
+  text-decoration: none;
+}
+
+a:hover .link {
+  text-decoration: underline;
+}
+
+.flag {
+  font-size: 20px;
+}

frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html

@@ -7,7 +7,9 @@
         <fa-icon [icon]="['fas', 'download']" [fixedWidth]="true"></fa-icon>
       </button>
     </div>
-    <small style="color: #ffffff66" i18n="lightning.tor-nodes-excluded">(Tor nodes excluded)</small>
+    <small class="d-block" style="color: #ffffff66; min-height: 25px" i18n="lightning.tor-nodes-excluded">
+      <span *ngIf="!(showTorObservable$ | async)">(Tor nodes excluded)</span>
+    </small>
   </div>
 
   <div class="container pb-lg-0 bottom-padding">
@@ -21,6 +23,11 @@
       <div class="spinner-border text-light"></div>
     </div>
 
+    <div class="d-flex toggle">
+      <app-toggle [textLeft]="'Show Tor'" [textRight]="" (toggleStatusChanged)="onTorToggleStatusChanged($event)"></app-toggle>
+      <app-toggle [textLeft]="'Nodes'" [textRight]="'Capacity'" (toggleStatusChanged)="onGroupToggleStatusChanged($event)"></app-toggle>
+    </div>
+
     <table class="table table-borderless text-center m-auto" style="max-width: 900px">
       <thead>
         <tr>
@@ -34,8 +41,9 @@
       <tbody [attr.data-cy]="'pools-table'" *ngIf="(nodesPerAsObservable$ | async) as asList">
         <tr *ngFor="let asEntry of asList">
           <td class="rank text-left pl-0">{{ asEntry.rank }}</td>
-          <td class="name text-left text-truncate"  style="max-width: 100px">
+          <td class="name text-left text-truncate">
-          <a [routerLink]="[('/lightning/nodes/isp/' + asEntry.ispId) | relativeUrl]">{{ asEntry.name }}</a>
+            <a *ngIf="asEntry.ispId" [routerLink]="[('/lightning/nodes/isp/' + asEntry.ispId) | relativeUrl]">{{ asEntry.name }}</a>
+            <span *ngIf="!asEntry.ispId">{{ asEntry.name }}</span>
           </td>
           <td class="share text-right">{{ asEntry.share }}%</td>
           <td class="nodes text-right">{{ asEntry.count }}</td>

frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss

@@ -36,16 +36,16 @@
 }
 
 .rank {
-  width: 20%;
+  width: 15%;
   @media (max-width: 576px) {
     display: none
   }
 }
 
 .name {
-  width: 20%;
+  width: 25%;
   @media (max-width: 576px) {
-    width: 80%;
+    width: 70%;
     max-width: 150px;
     padding-left: 0;
     padding-right: 0;
@@ -69,7 +69,17 @@
 .capacity {
   width: 20%;
   @media (max-width: 576px) {
-    width: 10%;
+    width: 20%;
     max-width: 100px;
   }
 }
+
+.toggle {
+  justify-content: space-between;
+  padding-top: 15px;
+  @media (min-width: 576px) {
+    padding-bottom: 15px;
+    padding-left: 105px;
+    padding-right: 105px;
+  }
+}

frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.ts

@@ -1,7 +1,7 @@
 import { ChangeDetectionStrategy, Component, OnInit, HostBinding, NgZone } from '@angular/core';
 import { Router } from '@angular/router';
 import { EChartsOption, PieSeriesOption } from 'echarts';
-import { map, Observable, share, tap } from 'rxjs';
+import { combineLatest, map, Observable, share, Subject, switchMap, tap } from 'rxjs';
 import { chartColors } from 'src/app/app.constants';
 import { ApiService } from 'src/app/services/api.service';
 import { SeoService } from 'src/app/services/seo.service';
@@ -17,19 +17,20 @@ import { RelativeUrlPipe } from 'src/app/shared/pipes/relative-url/relative-url.
   changeDetection: ChangeDetectionStrategy.OnPush,
 })
 export class NodesPerISPChartComponent implements OnInit {
-  miningWindowPreference: string;
-
   isLoading = true;
   chartOptions: EChartsOption = {};
   chartInitOptions = {
     renderer: 'svg',
   };
   timespan = '';
-  chartInstance: any = undefined;
+  chartInstance = undefined;
 
   @HostBinding('attr.dir') dir = 'ltr';
 
   nodesPerAsObservable$: Observable<any>;
+  showTorObservable$: Observable<boolean>;
+  groupBySubject = new Subject<boolean>();
+  showTorSubject = new Subject<boolean>();
 
   constructor(
     private apiService: ApiService,
@@ -44,7 +45,14 @@ export class NodesPerISPChartComponent implements OnInit {
   ngOnInit(): void {
     this.seoService.setTitle($localize`Lightning nodes per ISP`);
 
-    this.nodesPerAsObservable$ = this.apiService.getNodesPerAs()
+    this.showTorObservable$ = this.showTorSubject.asObservable();
+    this.nodesPerAsObservable$ = combineLatest([this.groupBySubject, this.showTorSubject])
+      .pipe(
+        switchMap((selectedFilters) => {
+          return this.apiService.getNodesPerAs(
+            selectedFilters[0] ? 'capacity' : 'node-count',
+            selectedFilters[1] // Show Tor nodes
+          )
             .pipe(
               tap(data => {
                 this.isLoading = false;
@@ -55,13 +63,15 @@ export class NodesPerISPChartComponent implements OnInit {
                   data[i].rank = i + 1;
                 }
                 return data.slice(0, 100);
+              })
+            );
         }),
         share()
       );
   }
 
-  generateChartSerieData(as) {
+  generateChartSerieData(as): PieSeriesOption[] {
-    const shareThreshold = this.isMobile() ? 2 : 1;
+    const shareThreshold = this.isMobile() ? 2 : 0.5;
     const data: object[] = [];
     let totalShareOther = 0;
     let totalNodeOther = 0;
@@ -78,6 +88,9 @@ export class NodesPerISPChartComponent implements OnInit {
         return;
       }
       data.push({
+        itemStyle: {
+          color: as.ispId === null ? '#7D4698' : undefined,
+        },
         value: as.share,
         name: as.name + (this.isMobile() ? `` : ` (${as.share}%)`),
         label: {
@@ -138,14 +151,14 @@ export class NodesPerISPChartComponent implements OnInit {
     return data;
   }
 
-  prepareChartOptions(as) {
+  prepareChartOptions(as): void {
     let pieSize = ['20%', '80%']; // Desktop
     if (this.isMobile()) {
       pieSize = ['15%', '60%'];
     }
 
     this.chartOptions = {
-      color: chartColors,
+      color: chartColors.slice(3),
       tooltip: {
         trigger: 'item',
         textStyle: {
@@ -155,7 +168,7 @@ export class NodesPerISPChartComponent implements OnInit {
       series: [
         {
           zlevel: 0,
-          minShowLabelAngle: 3.6,
+          minShowLabelAngle: 1.8,
           name: 'Lightning nodes',
           type: 'pie',
           radius: pieSize,
@@ -191,18 +204,18 @@ export class NodesPerISPChartComponent implements OnInit {
     };
   }
 
-  isMobile() {
+  isMobile(): boolean {
     return (window.innerWidth <= 767.98);
   }
 
-  onChartInit(ec) {
+  onChartInit(ec): void {
     if (this.chartInstance !== undefined) {
       return;
     }
     this.chartInstance = ec;
 
     this.chartInstance.on('click', (e) => {
-      if (e.data.data === 9999) { // "Other"
+      if (e.data.data === 9999 || e.data.data === null) { // "Other" or Tor
         return;
       }
       this.zone.run(() => {
@@ -212,7 +225,7 @@ export class NodesPerISPChartComponent implements OnInit {
     });
   }
 
-  onSaveChart() {
+  onSaveChart(): void {
     const now = new Date();
     this.chartOptions.backgroundColor = '#11131f';
     this.chartInstance.setOption(this.chartOptions);
@@ -224,8 +237,12 @@ export class NodesPerISPChartComponent implements OnInit {
     this.chartInstance.setOption(this.chartOptions);
   }
 
-  isEllipsisActive(e) {
+  onTorToggleStatusChanged(e): void {
-    return (e.offsetWidth < e.scrollWidth);
+    this.showTorSubject.next(e);
+  }
+
+  onGroupToggleStatusChanged(e): void {
+    this.groupBySubject.next(e);
   }
 }
 

frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.ts

@@ -58,7 +58,7 @@ export class LightningStatisticsChartComponent implements OnInit {
     let firstRun = true;
 
     if (this.widget) {
-      this.miningWindowPreference = '1y';
+      this.miningWindowPreference = '3y';
     } else {
       this.seoService.setTitle($localize`Channels and Capacity`);
       this.miningWindowPreference = this.miningService.getDefaultTimespan('all');

frontend/src/app/services/api.service.ts

@@ -255,8 +255,9 @@ export class ApiService {
     return this.httpClient.get<any[]>(this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/search', { params });
   }
 
-  getNodesPerAs(): Observable<any> {
+  getNodesPerAs(groupBy: 'capacity' | 'node-count', showTorNodes: boolean): Observable<any> {
-    return this.httpClient.get<any[]>(this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/nodes/isp');
+    return this.httpClient.get<any[]>(this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/nodes/isp-ranking'
+      + `?groupBy=${groupBy}&showTor=${showTorNodes}`);
   }
 
   getNodeForCountry$(country: string): Observable<any> {
@@ -271,7 +272,10 @@ export class ApiService {
     return this.httpClient.get<any[]>(this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/nodes/countries');
   }
 
-  getChannelsGeo$(): Observable<any> {
+  getChannelsGeo$(publicKey?: string): Observable<any> {
-    return this.httpClient.get<any[]>(this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/channels-geo');
+    return this.httpClient.get<any[]>(
+      this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/channels-geo' +
+        (publicKey !== undefined ? `/${publicKey}` : '')
+    );
   }
 }

frontend/src/app/services/enterprise.service.ts

@@ -24,7 +24,7 @@ export class EnterpriseService {
       this.subdomain = subdomain;
       this.fetchSubdomainInfo();
       this.disableSubnetworks();
-    } else {
+    } else if (document.location.hostname === 'mempool.space') {
       this.insertMatomo();
     }
   }
@@ -49,7 +49,7 @@ export class EnterpriseService {
     },
     (error) => {
       if (error.status === 404) {
-        window.location.href = 'https://mempool.space';
+        window.location.href = 'https://mempool.space' + window.location.pathname;
       }
     });
   }

frontend/src/app/services/opengraph.service.ts

@@ -0,0 +1,71 @@
+import { Injectable } from '@angular/core';
+import { Meta } from '@angular/platform-browser';
+import { Router, ActivatedRoute, NavigationEnd } from '@angular/router';
+import { filter, map, switchMap } from 'rxjs/operators';
+import { combineLatest } from 'rxjs';
+import { StateService } from './state.service';
+import { LanguageService } from './language.service';
+
+@Injectable({
+  providedIn: 'root'
+})
+export class OpenGraphService {
+  network = '';
+  defaultImageUrl = '';
+
+  constructor(
+    private metaService: Meta,
+    private stateService: StateService,
+    private LanguageService: LanguageService,
+    private router: Router,
+    private activatedRoute: ActivatedRoute,
+  ) {
+    // save og:image tag from original template
+    const initialOgImageTag = metaService.getTag("property='og:image'");
+    this.defaultImageUrl = initialOgImageTag?.content || 'https://mempool.space/resources/mempool-space-preview.png';
+    this.router.events.pipe(
+      filter(event => event instanceof NavigationEnd),
+      map(() => this.activatedRoute),
+      map(route => {
+        while (route.firstChild) route = route.firstChild;
+        return route;
+      }),
+      filter(route => route.outlet === 'primary'),
+      switchMap(route => route.data),
+    ).subscribe((data) => {
+      if (data.ogImage) {
+        this.setOgImage();
+      } else {
+        this.clearOgImage();
+      }
+    });
+  }
+
+  setOgImage() {
+    const lang = this.LanguageService.getLanguage();
+    const ogImageUrl = `${window.location.protocol}//${window.location.host}/render/${lang}/preview${this.router.url}`;
+    this.metaService.updateTag({ property: 'og:image', content: ogImageUrl });
+    this.metaService.updateTag({ property: 'twitter:image:src', content: ogImageUrl });
+    this.metaService.updateTag({ property: 'og:image:type', content: 'image/png' });
+    this.metaService.updateTag({ property: 'og:image:width', content: '1024' });
+    this.metaService.updateTag({ property: 'og:image:height', content: '512' });
+  }
+
+  clearOgImage() {
+    this.metaService.updateTag({ property: 'og:image', content: this.defaultImageUrl });
+    this.metaService.updateTag({ property: 'twitter:image:src', content: this.defaultImageUrl });
+    this.metaService.updateTag({ property: 'og:image:type', content: 'image/png' });
+    this.metaService.updateTag({ property: 'og:image:width', content: '1000' });
+    this.metaService.updateTag({ property: 'og:image:height', content: '500' });
+  }
+
+  /// signal that the unfurler should wait for a 'ready' signal before taking a screenshot
+  setPreviewLoading() {
+    this.metaService.updateTag({ property: 'og:loading', content: 'loading'});
+  }
+
+  // signal to the unfurler that the page is ready for a screenshot
+  setPreviewReady() {
+    this.metaService.updateTag({ property: 'og:ready', content: 'ready'});
+  }
+}

frontend/src/app/services/seo.service.ts

@@ -20,11 +20,13 @@ export class SeoService {
   setTitle(newTitle: string): void {
     this.titleService.setTitle(newTitle + ' - ' + this.getTitle());
     this.metaService.updateTag({ property: 'og:title', content: newTitle});
+    this.metaService.updateTag({ property: 'twitter:title', content: newTitle});
   }
 
   resetTitle(): void {
     this.titleService.setTitle(this.getTitle());
     this.metaService.updateTag({ property: 'og:title', content: this.getTitle()});
+    this.metaService.updateTag({ property: 'twitter:title', content: this.getTitle()});
   }
 
   setEnterpriseTitle(title: string) {

frontend/src/app/shared/common.utils.ts

@@ -0,0 +1,3 @@
+export function isMobile() {
+  return (window.innerWidth <= 767.98);
+}

frontend/src/app/shared/components/toggle/toggle.component.html

@@ -0,0 +1,8 @@
+<div class="d-flex align-items-center">
+  <span style="margin-bottom: 0.5rem">{{ textLeft }}</span>&nbsp;
+  <label class="switch">
+    <input type="checkbox" (change)="onToggleStatusChanged($event)">
+    <span class="slider round"></span>
+  </label>
+  &nbsp;<span style="margin-bottom: 0.5rem">{{ textRight }}</span>
+</div>

frontend/src/app/shared/components/toggle/toggle.component.scss

@@ -0,0 +1,62 @@
+/* The switch - the box around the slider */
+.switch {
+  position: relative;
+  display: inline-block;
+  width: 30px;
+  height: 17px;
+}
+
+/* Hide default HTML checkbox */
+.switch input {
+  opacity: 0;
+  width: 0;
+  height: 0;
+}
+
+/* The slider */
+.slider {
+  position: absolute;
+  cursor: pointer;
+  top: 0;
+  left: 0;
+  right: 0;
+  bottom: 0;
+  background-color: #ccc;
+  -webkit-transition: .4s;
+  transition: .4s;
+}
+
+.slider:before {
+  position: absolute;
+  content: "";
+  height: 13px;
+  width: 13px;
+  left: 2px;
+  bottom: 2px;
+  background-color: white;
+  -webkit-transition: .4s;
+  transition: .4s;
+}
+
+input:checked+.slider {
+  background-color: #2196F3;
+}
+
+input:focus+.slider {
+  box-shadow: 0 0 1px #2196F3;
+}
+
+input:checked+.slider:before {
+  -webkit-transform: translateX(13px);
+  -ms-transform: translateX(13px);
+  transform: translateX(13px);
+}
+
+/* Rounded sliders */
+.slider.round {
+  border-radius: 17px;
+}
+
+.slider.round:before {
+  border-radius: 50%;
+}

frontend/src/app/shared/components/toggle/toggle.component.ts

@@ -0,0 +1,21 @@
+import { Component, Input, Output, ChangeDetectionStrategy, EventEmitter, AfterViewInit } from '@angular/core';
+
+@Component({
+  selector: 'app-toggle',
+  templateUrl: './toggle.component.html',
+  styleUrls: ['./toggle.component.scss'],
+  changeDetection: ChangeDetectionStrategy.OnPush,
+})
+export class ToggleComponent implements AfterViewInit {
+  @Output() toggleStatusChanged = new EventEmitter<boolean>();
+  @Input() textLeft: string;
+  @Input() textRight: string;
+
+  ngAfterViewInit(): void {
+    this.toggleStatusChanged.emit(false);
+  }
+
+  onToggleStatusChanged(e): void {
+    this.toggleStatusChanged.emit(e.target.checked);
+  }
+}

frontend/src/app/shared/graphs.utils.ts

@@ -92,6 +92,9 @@ export function detectWebGL() {
 }
 
 export function getFlagEmoji(countryCode) {
+  if (!countryCode) {
+    return '';
+  }
   const codePoints = countryCode
     .toUpperCase()
     .split('')

frontend/src/app/shared/shared.module.ts

@@ -7,6 +7,7 @@ import { faFilter, faAngleDown, faAngleUp, faAngleRight, faAngleLeft, faBolt, fa
   faFileAlt, faRedoAlt, faArrowAltCircleRight, faExternalLinkAlt, faBook, faListUl, faDownload, faQrcode } from '@fortawesome/free-solid-svg-icons';
 import { InfiniteScrollModule } from 'ngx-infinite-scroll';
 import { MasterPageComponent } from '../components/master-page/master-page.component';
+import { MasterPagePreviewComponent } from '../components/master-page-preview/master-page-preview.component';
 import { BisqMasterPageComponent } from '../components/bisq-master-page/bisq-master-page.component';
 import { LiquidMasterPageComponent } from '../components/liquid-master-page/liquid-master-page.component';
 import { AboutComponent } from '../components/about/about.component';
@@ -44,10 +45,12 @@ import { StartComponent } from '../components/start/start.component';
 import { TransactionComponent } from '../components/transaction/transaction.component';
 import { TransactionsListComponent } from '../components/transactions-list/transactions-list.component';
 import { BlockComponent } from '../components/block/block.component';
+import { BlockPreviewComponent } from '../components/block/block-preview.component';
 import { BlockAuditComponent } from '../components/block-audit/block-audit.component';
 import { BlockOverviewGraphComponent } from '../components/block-overview-graph/block-overview-graph.component';
 import { BlockOverviewTooltipComponent } from '../components/block-overview-tooltip/block-overview-tooltip.component';
 import { AddressComponent } from '../components/address/address.component';
+import { AddressPreviewComponent } from '../components/address/address-preview.component';
 import { SearchFormComponent } from '../components/search-form/search-form.component';
 import { AddressLabelsComponent } from '../components/address-labels/address-labels.component';
 import { FooterComponent } from '../components/footer/footer.component';
@@ -78,6 +81,7 @@ import { ChangeComponent } from '../components/change/change.component';
 import { SatsComponent } from './components/sats/sats.component';
 import { SearchResultsComponent } from '../components/search-form/search-results/search-results.component';
 import { TimestampComponent } from './components/timestamp/timestamp.component';
+import { ToggleComponent } from './components/toggle/toggle.component';
 
 @NgModule({
   declarations: [
@@ -110,16 +114,19 @@ import { TimestampComponent } from './components/timestamp/timestamp.component';
     AmountComponent,
     AboutComponent,
     MasterPageComponent,
+    MasterPagePreviewComponent,
     BisqMasterPageComponent,
     LiquidMasterPageComponent,
     StartComponent,
     TransactionComponent,
     BlockComponent,
+    BlockPreviewComponent,
     BlockAuditComponent,
     BlockOverviewGraphComponent,
     BlockOverviewTooltipComponent,
     TransactionsListComponent,
     AddressComponent,
+    AddressPreviewComponent,
     SearchFormComponent,
     TimeSpanComponent,
     AddressLabelsComponent,
@@ -150,6 +157,7 @@ import { TimestampComponent } from './components/timestamp/timestamp.component';
     SatsComponent,
     SearchResultsComponent,
     TimestampComponent,
+    ToggleComponent,
   ],
   imports: [
     CommonModule,
@@ -215,11 +223,13 @@ import { TimestampComponent } from './components/timestamp/timestamp.component';
     StartComponent,
     TransactionComponent,
     BlockComponent,
+    BlockPreviewComponent,
     BlockAuditComponent,
     BlockOverviewGraphComponent,
     BlockOverviewTooltipComponent,
     TransactionsListComponent,
     AddressComponent,
+    AddressPreviewComponent,
     SearchFormComponent,
     TimeSpanComponent,
     AddressLabelsComponent,
@@ -250,6 +260,7 @@ import { TimestampComponent } from './components/timestamp/timestamp.component';
     SatsComponent,
     SearchResultsComponent,
     TimestampComponent,
+    ToggleComponent,
   ]
 })
 export class SharedModule {

frontend/src/locale/messages.pl.xlf

@@ -1997,6 +1997,7 @@
       </trans-unit>
       <trans-unit id="4793828002882320882" datatype="html">
         <source>At block: <x id="PH" equiv-text="data[0].data[2]"/></source>
+        <target>W bloku: <x id="PH" equiv-text="data[0].data[2]"/></target>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.ts</context>
           <context context-type="linenumber">188</context>
@@ -2020,6 +2021,7 @@
       </trans-unit>
       <trans-unit id="8918254921747459635" datatype="html">
         <source>Around block: <x id="PH" equiv-text="data[0].data[2]"/></source>
+        <target>W okolicu bloku: <x id="PH" equiv-text="data[0].data[2]"/></target>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/block-fee-rates-graph/block-fee-rates-graph.component.ts</context>
           <context context-type="linenumber">190</context>
@@ -2234,6 +2236,7 @@
       </trans-unit>
       <trans-unit id="d7d5fcf50179ad70c938491c517efb82de2c8146" datatype="html">
         <source>Block Prediction Accuracy</source>
+        <target>Dokładność prognoz bloków</target>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/block-prediction-graph/block-prediction-graph.component.html</context>
           <context context-type="linenumber">5,7</context>
@@ -2250,6 +2253,7 @@
       </trans-unit>
       <trans-unit id="829186404427796443" datatype="html">
         <source>Match rate</source>
+        <target>Częstość trafień</target>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/block-prediction-graph/block-prediction-graph.component.ts</context>
           <context context-type="linenumber">176,174</context>
@@ -2867,6 +2871,7 @@
       </trans-unit>
       <trans-unit id="49bba8e970aa3b1bec6fcff7228ef95ceb335f59" datatype="html">
         <source>Usually places your transaction in between the second and third mempool blocks</source>
+        <target>Zazwyczaj umieszcza Twoją transakcje między drugim a trzecim blokiem w mempool</target>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/fees-box/fees-box.component.html</context>
           <context context-type="linenumber">8,9</context>
@@ -2888,6 +2893,7 @@
       </trans-unit>
       <trans-unit id="eeeeabc97373285d75acf0f013f68434a6f1935b" datatype="html">
         <source>Usually places your transaction in between the first and second mempool blocks</source>
+        <target>Zazwyczaj umieszcza Twoją transakcje między pierwszym a drugim blokiem w mempool</target>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/fees-box/fees-box.component.html</context>
           <context context-type="linenumber">9,10</context>
@@ -3072,6 +3078,7 @@
       </trans-unit>
       <trans-unit id="8105839921891777281" datatype="html">
         <source>Hashrate (MA)</source>
+        <target>Prędkość haszowania (MA)</target>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/hashrate-chart/hashrate-chart.component.ts</context>
           <context context-type="linenumber">288,287</context>
@@ -3244,6 +3251,7 @@
       </trans-unit>
       <trans-unit id="2711844b4304254e88358d1761f9c732e5aefc69" datatype="html">
         <source>Pools luck (1 week)</source>
+        <target>Szczęście kolektywu (1 tydzień)</target>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/pool-ranking/pool-ranking.component.html</context>
           <context context-type="linenumber">9</context>
@@ -3252,6 +3260,7 @@
       </trans-unit>
       <trans-unit id="ea1a87734b5cc78ea8b268343497d92136855cd1" datatype="html">
         <source>Pools luck</source>
+        <target>Szczęście kolektywu</target>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/pool-ranking/pool-ranking.component.html</context>
           <context context-type="linenumber">9,11</context>
@@ -3260,6 +3269,7 @@
       </trans-unit>
       <trans-unit id="e910ea39a964514d51802d34cad96c75b14947d1" datatype="html">
         <source>The overall luck of all mining pools over the past week. A luck bigger than 100% means the average block time for the current epoch is less than 10 minutes.</source>
+        <target>Ogólne szczęście wszystkich kolektywów wydobywczych w ciągu ostatniego tygodnia. Szczęście większe niż 100% oznacza, że średni czas bloku dla danej epoki jest mniejszy niż 10 minut.</target>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/pool-ranking/pool-ranking.component.html</context>
           <context context-type="linenumber">11,15</context>
@@ -3268,6 +3278,7 @@
       </trans-unit>
       <trans-unit id="9740454c3c55ca2cfa437ff9ec07374c9b9d25b5" datatype="html">
         <source>Pools count (1w)</source>
+        <target>Liczba kolektywów (1t)</target>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/pool-ranking/pool-ranking.component.html</context>
           <context context-type="linenumber">17</context>
@@ -3276,6 +3287,7 @@
       </trans-unit>
       <trans-unit id="1107f1b39cd8474087d438971892967a331a6c7d" datatype="html">
         <source>Pools count</source>
+        <target>Liczba kolektywów</target>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/pool-ranking/pool-ranking.component.html</context>
           <context context-type="linenumber">17,19</context>
@@ -3284,6 +3296,7 @@
       </trans-unit>
       <trans-unit id="500e13dffc7300bf7e4822a6bbf29a71a55d7b75" datatype="html">
         <source>How many unique pools found at least one block over the past week.</source>
+        <target>Ile unikatowych kolektywów znalazło conajmniej jeden blok w ciągu ostatniego tygodnia.</target>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/pool-ranking/pool-ranking.component.html</context>
           <context context-type="linenumber">19,23</context>
@@ -3309,6 +3322,7 @@
       </trans-unit>
       <trans-unit id="c9e8defa185fa8e342548958bf206de97afc97a6" datatype="html">
         <source>The number of blocks found over the past week.</source>
+        <target>Liczba bloków znalezionych w ciągu ostatniego tygodnia.</target>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/components/pool-ranking/pool-ranking.component.html</context>
           <context context-type="linenumber">27,31</context>
@@ -4465,6 +4479,7 @@
       </trans-unit>
       <trans-unit id="cd2330c7e9c74256f6a91e83bccf10e2905f8556" datatype="html">
         <source>REST API service</source>
+        <target>Usługa REST API</target>
         <context-group purpose="location">
           <context context-type="sourcefile">src/app/docs/api-docs/api-docs.component.html</context>
           <context context-type="linenumber">34,35</context>

frontend/src/styles.scss

@@ -87,6 +87,11 @@ body {
   box-shadow: 0.125rem 0.125rem 0.25rem rgba(0,0,0,0.075);
 }
 
+.preview-box {
+  min-height: 512px;
+  padding: 2rem 3rem;
+}
+
 @media (max-width: 767.98px) {
   .box {
     padding: 0.75rem;

production/bitcoin.conf

@@ -18,12 +18,18 @@ whitelist=2401:b140::/32
 #uacomment=@wiz
 
 [main]
-bind=0.0.0.0:8333
-bind=[::]:8333
 rpcbind=127.0.0.1:8332
 rpcbind=[::1]:8332
-zmqpubrawblock=tcp://127.0.0.1:18332
+bind=0.0.0.0:8333
-zmqpubrawtx=tcp://127.0.0.1:18333
+bind=[::]:8333
+zmqpubrawblock=tcp://127.0.0.1:8334
+zmqpubrawtx=tcp://127.0.0.1:8335
+#addnode=[2401:b140:1::92:201]:8333
+#addnode=[2401:b140:1::92:202]:8333
+#addnode=[2401:b140:1::92:203]:8333
+#addnode=[2401:b140:1::92:204]:8333
+#addnode=[2401:b140:1::92:205]:8333
+#addnode=[2401:b140:1::92:206]:8333
 #addnode=[2401:b140:2::92:201]:8333
 #addnode=[2401:b140:2::92:202]:8333
 #addnode=[2401:b140:2::92:203]:8333
@@ -33,10 +39,18 @@ zmqpubrawtx=tcp://127.0.0.1:18333
 
 [test]
 daemon=1
-bind=0.0.0.0:18333
-bind=[::]:18333
 rpcbind=127.0.0.1:18332
 rpcbind=[::1]:18332
+bind=0.0.0.0:18333
+bind=[::]:18333
+zmqpubrawblock=tcp://127.0.0.1:18334
+zmqpubrawtx=tcp://127.0.0.1:18335
+#addnode=[2401:b140:1::92:201]:18333
+#addnode=[2401:b140:1::92:202]:18333
+#addnode=[2401:b140:1::92:203]:18333
+#addnode=[2401:b140:1::92:204]:18333
+#addnode=[2401:b140:1::92:205]:18333
+#addnode=[2401:b140:1::92:206]:18333
 #addnode=[2401:b140:2::92:201]:18333
 #addnode=[2401:b140:2::92:202]:18333
 #addnode=[2401:b140:2::92:203]:18333
@@ -46,10 +60,18 @@ rpcbind=[::1]:18332
 
 [signet]
 daemon=1
-bind=0.0.0.0:38333
-bind=[::]:38333
 rpcbind=127.0.0.1:38332
 rpcbind=[::1]:38332
+bind=0.0.0.0:38333
+bind=[::]:38333
+zmqpubrawblock=tcp://127.0.0.1:38334
+zmqpubrawtx=tcp://127.0.0.1:38335
+#addnode=[2401:b140:1::92:201]:38333
+#addnode=[2401:b140:1::92:202]:38333
+#addnode=[2401:b140:1::92:203]:38333
+#addnode=[2401:b140:1::92:204]:38333
+#addnode=[2401:b140:1::92:205]:38333
+#addnode=[2401:b140:1::92:206]:38333
 #addnode=[2401:b140:2::92:201]:38333
 #addnode=[2401:b140:2::92:202]:38333
 #addnode=[2401:b140:2::92:203]:38333

production/elements.crontab

@@ -1,6 +1,10 @@
+# start elements on reboot
 @reboot sleep 60 ; /usr/local/bin/elementsd -chain=liquidv1 >/dev/null 2>&1
 @reboot sleep 60 ; /usr/local/bin/elementsd -chain=liquidtestnet >/dev/null 2>&1
+
+# start electrs on reboot
 @reboot sleep 90 ; screen -dmS liquidv1 /elements/electrs/electrs-start-liquid
 @reboot sleep 90 ; screen -dmS liquidtestnet /elements/electrs/electrs-start-liquidtestnet
-6 * * * * cd $HOME/asset_registry_db && git pull origin master >/dev/null 2>&1
+
-6 * * * * cd $HOME/asset_registry_testnet_db && git pull origin master >/dev/null 2>&1
+# hourly asset update and electrs restart
+6 * * * * cd $HOME/asset_registry_db && git pull --quiet origin master && cd $HOME/asset_registry_testnet_db && git pull --quiet origin master && killall electrs

production/install

@@ -39,6 +39,9 @@ BITCOIN_INSTALL=ON
 BISQ_INSTALL=ON
 ELEMENTS_INSTALL=ON
 
+# install UNFURL
+UNFURL_INSTALL=ON
+
 # configure 4 network instances
 BITCOIN_MAINNET_ENABLE=ON
 BITCOIN_MAINNET_MINFEE_ENABLE=ON
@@ -49,8 +52,10 @@ ELEMENTS_LIQUID_ENABLE=ON
 ELEMENTS_LIQUIDTESTNET_ENABLE=ON
 
 # enable lightmode and disable compaction to fit on 1TB SSD drive
+BITCOIN_ELECTRS_INSTALL=ON
 BITCOIN_ELECTRS_LIGHT_MODE=ON
 BITCOIN_ELECTRS_COMPACTION=OFF
+ELEMENTS_ELECTRS_INSTALL=ON
 ELEMENTS_ELECTRS_LIGHT_MODE=ON
 ELEMENTS_ELECTRS_COMPACTION=OFF
 
@@ -177,7 +182,6 @@ case $OS in
         ROOT_USER=root
         ROOT_GROUP=wheel
         ROOT_HOME=/root
-        TOR_HOME=/var/db/tor
         TOR_CONFIGURATION=/usr/local/etc/tor/torrc
         TOR_RESOURCES=/var/db/tor
         TOR_PKG=tor
@@ -193,7 +197,6 @@ case $OS in
         ROOT_USER=root
         ROOT_GROUP=root
         ROOT_HOME=/root
-        TOR_HOME=/etc/tor
         TOR_CONFIGURATION=/etc/tor/torrc
         TOR_RESOURCES=/var/lib/tor
         TOR_PKG=tor
@@ -218,6 +221,21 @@ MYSQL_HOME=/mysql
 MYSQL_USER=mysql
 MYSQL_GROUP=mysql
 
+# mempool mysql user/password
+MEMPOOL_MAINNET_USER='mempool'
+MEMPOOL_TESTNET_USER='mempool_testnet'
+MEMPOOL_SIGNET_USER='mempool_signet'
+MEMPOOL_LIQUID_USER='mempool_liquid'
+MEMPOOL_LIQUIDTESTNET_USER='mempool_liquidtestnet'
+MEMPOOL_BISQ_USER='mempool_bisq'
+# generate random hex string
+MEMPOOL_MAINNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}')
+MEMPOOL_TESTNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}')
+MEMPOOL_SIGNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}')
+MEMPOOL_LIQUID_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}')
+MEMPOOL_LIQUIDTESTNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}')
+MEMPOOL_BISQ_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}')
+
 # mempool data folder and user/group
 MEMPOOL_HOME=/mempool
 MEMPOOL_USER=mempool
@@ -262,6 +280,14 @@ BISQ_USER=bisq
 BISQ_GROUP=bisq
 # bisq home folder, needs about 1GB
 BISQ_HOME=/bisq
+# tor HS folder
+BISQ_TOR_HS=bisq
+
+# Unfurl user/group
+UNFURL_USER=unfurl
+UNFURL_GROUP=unfurl
+# Unfurl home folder
+UNFURL_HOME=/unfurl
 
 # liquid user/group
 ELEMENTS_USER=elements
@@ -272,6 +298,8 @@ ELEMENTS_HOME=/elements
 ELECTRS_HOME=/electrs
 # elements electrs source/binaries
 ELEMENTS_ELECTRS_HOME=${ELEMENTS_HOME}/electrs
+# tor HS folder
+LIQUID_TOR_HS=liquid
 
 # minfee user/group
 MINFEE_USER=minfee
@@ -300,6 +328,13 @@ BISQ_REPO_BRANCH=master
 BISQ_LATEST_RELEASE=master
 echo -n '.'
 
+UNFURL_REPO_URL=https://github.com/mempool/mempool
+UNFURL_REPO_NAME=unfurl
+UNFURL_REPO_BRANCH=master
+#UNFURL_LATEST_RELEASE=$(curl -s https://api.github.com/repos/mempool/mempool/releases/latest|grep tag_name|head -1|cut -d '"' -f4)
+UNFURL_LATEST_RELEASE=master
+echo -n '.'
+
 ELEMENTS_REPO_URL=https://github.com/ElementsProject/elements
 ELEMENTS_REPO_NAME=elements
 ELEMENTS_REPO_BRANCH=master
@@ -336,6 +371,10 @@ DEBIAN_PKG+=(libboost-system-dev libboost-filesystem-dev libboost-chrono-dev lib
 DEBIAN_PKG+=(nodejs npm mariadb-server nginx-core python3-certbot-nginx rsync ufw)
 DEBIAN_PKG+=(geoipupdate)
 
+DEBIAN_UNFURL_PKG=()
+DEBIAN_UNFURL_PKG+=(cups chromium-bsu libatk1.0 libatk-bridge2.0 libxkbcommon-dev libxcomposite-dev)
+DEBIAN_UNFURL_PKG+=(libxdamage-dev libxrandr-dev libgbm-dev libpango1.0-dev libasound-dev)
+
 # packages needed for mempool ecosystem
 FREEBSD_PKG=()
 FREEBSD_PKG+=(zsh sudo git screen curl wget calc neovim)
@@ -653,129 +692,150 @@ ext4CreateDir()
 
 # does bitcoin exist?
 
-###########
+##########
-## dialog #
+# dialog #
-###########
+##########
-#
+
-#: ${DIALOG=dialog}
+: ${DIALOG=dialog}
-#
+
-#: ${DIALOG_OK=0}
+: ${DIALOG_OK=0}
-#: ${DIALOG_CANCEL=1}
+: ${DIALOG_CANCEL=1}
-#: ${DIALOG_HELP=2}
+: ${DIALOG_HELP=2}
-#: ${DIALOG_EXTRA=3}
+: ${DIALOG_EXTRA=3}
-#: ${DIALOG_ITEM_HELP=4}
+: ${DIALOG_ITEM_HELP=4}
-#: ${DIALOG_ESC=255}
+: ${DIALOG_ESC=255}
-#
+
-#: ${SIG_OFFNE=0}
+: ${SIG_OFFNE=0}
-#: ${SIG_HUP=1}
+: ${SIG_HUP=1}
-#: ${SIG_INT=2}
+: ${SIG_INT=2}
-#: ${SIG_QUIT=3}
+: ${SIG_QUIT=3}
-#: ${SIG_KILL=9}
+: ${SIG_KILL=9}
-#: ${SIG_TERM=15}
+: ${SIG_TERM=15}
-#
+
-#input=`tempfile 2>/dev/null` || input=/tmp/input$$
+input=`tempfile 2>/dev/null` || input=/tmp/input$$
-#output=`tempfile 2>/dev/null` || output=/tmp/test$$
+output=`tempfile 2>/dev/null` || output=/tmp/test$$
-#trap "rm -f $input $output" $SIG_OFFNE $SIG_HUP $SIG_INT $SIG_TRAP $SIG_TERM
+trap "rm -f $input $output" $SIG_OFFNE $SIG_HUP $SIG_INT $SIG_TRAP $SIG_TERM
-#
+
-#DIALOG_ERROR=254
+DIALOG_ERROR=254
-#export DIALOG_ERROR
+export DIALOG_ERROR
-#
+
-#backtitle="Mempool Fullnode Installer"
+backtitle="Mempool Fullnode Installer"
-#title="Mempool Fullnode Installer"
+title="Mempool Fullnode Installer"
-#returncode=0
+returncode=0
-#
+
-##################
+#################
-## dialog part 1 #
+# dialog part 1 #
-##################
+#################
-#
+
-#$CUT >$input <<-EOF
+$CUT >$input <<-EOF
-#Tor:Enable Tor v3 HS Onion:ON
+Tor:Enable Tor v3 HS Onion:ON
-#Certbot:Enable HTTPS using Certbot:ON
+Mainnet:Enable Bitcoin Mainnet:ON
-#Mainnet:Enable Bitcoin Mainnet:ON
+Mainnet-Minfee:Enable Bitcoin Mainnet Minfee:ON
-#Mainnet-Minfee:Enable Bitcoin Mainnet Minfee:ON
+Testnet:Enable Bitcoin Testnet:ON
-#Testnet:Enable Bitcoin Testnet:ON
+Signet:Enable Bitcoin Signet:ON
-#Liquid:Enable Elements Liquid:ON
+Liquid:Enable Elements Liquid:ON
-#Bisq:Enable Bisq:ON
+Liquidtestnet:Enable Elements Liquidtestnet:ON
-#Lightmode:Enable Electrs Lightmode to save disk space:ON
+Bisq:Enable Bisq:ON
-#Smalldisk:Disable Electrs Compaction to save disk space:ON
+Unfurl:Enable Unfurl:ON
-#Firewall:Enable Firewall:ON
+EOF
-#EOF
+
-#
+cat $input | sed -e 's/^/"/' -e 's/:/" "/g' -e 's/$/"/' >$output
-#cat $input | sed -e 's/^/"/' -e 's/:/" "/g' -e 's/$/"/' >$output
+cat $output >$input
-#cat $output >$input
+
-#
+$DIALOG --backtitle "${backtitle}" \
-#$DIALOG --backtitle "${backtitle}" \
+        --title "${title}" "$@" \
-#        --title "${title}" "$@" \
+        --checklist "Toggle the features below to configure your fullnode:\n" \
-#        --checklist "Toggle the features below to configure your fullnode:\n" \
+        20 80 10 \
-#        20 80 10 \
+        --file $input 2> $output
-#        --file $input 2> $output
+
-#
+retval=$?
-#retval=$?
+
-#
+tempfile=$output
-#tempfile=$output
+if [ $retval != $DIALOG_OK ];then
-#if [ $retval != $DIALOG_OK ];then
+    echo "Installation aborted."
-#    echo "Installation aborted."
+    exit 1
-#    exit 1
+fi
-#fi
+
-#
+if grep Tor $tempfile >/dev/null 2>&1;then
-#if grep Tor $tempfile >/dev/null 2>&1;then
+    TOR_INSTALL=ON
-#    TOR_INSTALL=ON
+else
-#else
+    TOR_INSTALL=OFF
-#    TOR_INSTALL=OFF
+fi
-#fi
+
-#
+if grep Mainnet $tempfile >/dev/null 2>&1;then
-#if grep Certbot $tempfile >/dev/null 2>&1;then
+    BITCOIN_MAINNET_ENABLE=ON
-#    CERTBOT_INSTALL=ON
+else
-#else
+    BITCOIN_MAINNET_ENABLE=OFF
-#    CERTBOT_INSTALL=OFF
+fi
-#fi
+
-#
+if grep Mainnet-Minfee $tempfile >/dev/null 2>&1;then
-#if grep Mainnet $tempfile >/dev/null 2>&1;then
+    BITCOIN_MAINNET_MINFEE_ENABLE=ON
-#    BITCOIN_MAINNET_ENABLE=ON
+else
-#else
+    BITCOIN_MAINNET_MINFEE_ENABLE=OFF
-#    BITCOIN_MAINNET_ENABLE=OFF
+fi
-#fi
+
-#
+if grep Testnet $tempfile >/dev/null 2>&1;then
-#if grep Mainnet-Minfee $tempfile >/dev/null 2>&1;then
+    BITCOIN_TESTNET_ENABLE=ON
-#    BITCOIN_MAINNET_MINFEE_ENABLE=ON
+else
-#else
+    BITCOIN_TESTNET_ENABLE=OFF
-#    BITCOIN_MAINNET_MINFEE_ENABLE=OFF
+fi
-#fi
+
-#
+if grep Signet $tempfile >/dev/null 2>&1;then
-#if grep Testnet $tempfile >/dev/null 2>&1;then
+    BITCOIN_SIGNET_ENABLE=ON
-#    BITCOIN_TESTNET_ENABLE=ON
+else
-#else
+    BITCOIN_SIGNET_ENABLE=OFF
-#    BITCOIN_TESTNET_ENABLE=OFF
+fi
-#fi
+
-#
+if [ "${BITCOIN_MAINNET_ENABLE}" = ON -o "${BITCOIN_MAINNET_MINFEE_ENABLE}" = ON -o "${BITCOIN_TESTNET_ENABLE}" = ON -o "${BITCOIN_SIGNET_ENABLE}" = ON ];then
-#if grep Liquid $tempfile >/dev/null 2>&1;then
+    BITCOIN_INSTALL=ON
-#    ELEMENTS_INSTALL=ON
+else
-#    ELEMENTS_LIQUID_ENABLE=ON
+    BITCOIN_INSTALL=OFF
-#else
+fi
-#    ELEMENTS_INSTALL=OFF
+
-#    ELEMENTS_LIQUID_ENABLE=OFF
+if grep Liquid $tempfile >/dev/null 2>&1;then
-#fi
+    ELEMENTS_LIQUID_ENABLE=ON
-#
+else
-#if grep Bisq $tempfile >/dev/null 2>&1;then
+    ELEMENTS_LIQUID_ENABLE=OFF
-#    BISQ_INSTALL=ON
+fi
-#    BISQ_MAINNET_ENABLE=ON
+
-#else
+if grep Liquidtestnet $tempfile >/dev/null 2>&1;then
-#    BISQ_INSTALL=OFF
+    ELEMENTS_LIQUIDTESTNET_ENABLE=ON
-#    BISQ_MAINNET_ENABLE=OFF
+else
-#fi
+    ELEMENTS_LIQUIDTESTNET_ENABLE=OFF
-#
+fi
-#if grep Lightmode $tempfile >/dev/null 2>&1;then
+
-#    BITCOIN_ELECTRS_LIGHT_MODE=ON
+if [ "${ELEMENTS_LIQUID_ENABLE}" = ON -o "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then
-#else
+    ELEMENTS_INSTALL=ON
-#    BITCOIN_ELECTRS_LIGHT_MODE=OFF
+else
-#fi
+    ELEMENTS_INSTALL=OFF
-#
+fi
-#if grep Smalldisk $tempfile >/dev/null 2>&1;then
+
-#    BITCOIN_ELECTRS_LIGHT_MODE=ON
+if [ "${BITCOIN_MAINNET_ENABLE}" = ON -o "${BITCOIN_TESTNET_ENABLE}" = ON -o "${BITCOIN_SIGNET_ENABLE}" = ON ];then
-#else
+    BITCOIN_ELECTRS_INSTALL=ON
-#    BITCOIN_ELECTRS_LIGHT_MODE=OFF
+else
-#fi
+    BITCOIN_ELECTRS_INSTALL=OFF
-#
+fi
+
+if [ "${ELEMENTS_LIQUID_ENABLE}" = ON -o "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then
+    ELEMENTS_ELECTRS_INSTALL=ON
+else
+    ELEMENTS_ELECTRS_INSTALL=OFF
+fi
+
+if grep Bisq $tempfile >/dev/null 2>&1;then
+    BISQ_INSTALL=ON
+    BISQ_MAINNET_ENABLE=ON
+else
+    BISQ_INSTALL=OFF
+    BISQ_MAINNET_ENABLE=OFF
+fi
+
+if grep Unfurl $tempfile >/dev/null 2>&1;then
+    UNFURL_INSTALL=ON
+else
+    UNFURL_INSTALL=OFF
+fi
+
 ##################
 ## dialog part 2 #
 ##################
@@ -923,15 +983,34 @@ if [ "${TOR_INSTALL}" = ON ];then
     osPackageInstall "${TOR_PKG}"
 
     echo "[*] Installing Tor base configuration"
-    osSudo "${ROOT_USER}" install -c -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/torrc" "${TOR_HOME}/torrc"
+    osSudo "${ROOT_USER}" install -c -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/torrc" "${TOR_CONFIGURATION}"
+    osSudo "${ROOT_USER}" sed -i.orig "s!__TOR_RESOURCES__!${TOR_RESOURCES}!" "${TOR_CONFIGURATION}"
 
-    echo "[*] Adding Tor HS configuration"
+    echo "[*] Adding Tor HS configuration for Mempool"
-    if ! grep "${MEMPOOL_TOR_HS}" /etc/tor/torrc >/dev/null 2>&1;then
+    if [ "${MEMPOOL_ENABLE}" = "ON" ];then
+        if ! grep "${MEMPOOL_TOR_HS}" "${TOR_CONFIGURATION}" >/dev/null 2>&1;then
             osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceDir ${TOR_RESOURCES}/${MEMPOOL_TOR_HS}/ >> ${TOR_CONFIGURATION}"
             osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServicePort 80 127.0.0.1:81 >> ${TOR_CONFIGURATION}"
             osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceVersion 3 >> ${TOR_CONFIGURATION}"
-    else
+        fi
-        osSudo "${ROOT_USER}" sed -i.orig "s!__TOR_RESOURCES__!${TOR_RESOURCES}!" "${TOR_CONFIGURATION}"
+    fi
+
+    echo "[*] Adding Tor HS configuration for Bisq"
+    if [ "${BISQ_ENABLE}" = "ON" ];then
+        if ! grep "${BISQ_TOR_HS}" "${TOR_CONFIGURATION}" >/dev/null 2>&1;then
+            osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceDir ${TOR_RESOURCES}/${BISQ_TOR_HS}/ >> ${TOR_CONFIGURATION}"
+            osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServicePort 80 127.0.0.1:82 >> ${TOR_CONFIGURATION}"
+            osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceVersion 3 >> ${TOR_CONFIGURATION}"
+        fi
+    fi
+
+    echo "[*] Adding Tor HS configuration for Liquid"
+    if [ "${LIQUID_ENABLE}" = "ON" ];then
+        if ! grep "${LIQUID_TOR_HS}" "${TOR_CONFIGURATION}" >/dev/null 2>&1;then
+            osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceDir ${TOR_RESOURCES}/${LIQUID_TOR_HS}/ >> ${TOR_CONFIGURATION}"
+            osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServicePort 80 127.0.0.1:83 >> ${TOR_CONFIGURATION}"
+            osSudo "${ROOT_USER}" /bin/sh -c "echo HiddenServiceVersion 3 >> ${TOR_CONFIGURATION}"
+        fi
     fi
 
     case $OS in
@@ -954,12 +1033,20 @@ if [ "${BITCOIN_INSTALL}" = ON ];then
 
     echo "[*] Creating Bitcoin user with Tor access"
     osGroupCreate "${BITCOIN_GROUP}"
+    if [ "${TOR_INSTALL}" = ON ];then
         osUserCreate "${BITCOIN_USER}" "${BITCOIN_HOME}" "${BITCOIN_GROUP}" "${TOR_GROUP}"
+    else
+        osUserCreate "${BITCOIN_USER}" "${BITCOIN_HOME}" "${BITCOIN_GROUP}"
+    fi
     osSudo "${ROOT_USER}" chsh -s `which zsh` "${BITCOIN_USER}"
 
     echo "[*] Creating Bitcoin minfee user with Tor access"
     osGroupCreate "${MINFEE_GROUP}"
+    if [ "${TOR_INSTALL}" = ON ];then
         osUserCreate "${MINFEE_USER}" "${MINFEE_HOME}" "${MINFEE_GROUP}" "${TOR_GROUP}"
+    else
+        osUserCreate "${MINFEE_USER}" "${MINFEE_HOME}" "${MINFEE_GROUP}"
+    fi
     osSudo "${ROOT_USER}" chown -R "${MINFEE_USER}:${MINFEE_GROUP}" "${MINFEE_HOME}"
     osSudo "${ROOT_USER}" chsh -s `which zsh` "${MINFEE_USER}"
     osSudo "${MINFEE_USER}" touch "${MINFEE_HOME}/.zshrc"
@@ -1007,7 +1094,11 @@ if [ "${ELEMENTS_INSTALL}" = ON ];then
 
     echo "[*] Creating Elements user with Tor access"
     osGroupCreate "${ELEMENTS_GROUP}"
+    if [ "${TOR_INSTALL}" = ON ];then
         osUserCreate "${ELEMENTS_USER}" "${ELEMENTS_HOME}" "${ELEMENTS_GROUP}" "${TOR_GROUP}"
+    else
+        osUserCreate "${ELEMENTS_USER}" "${ELEMENTS_HOME}" "${ELEMENTS_GROUP}"
+    fi
     osSudo "${ROOT_USER}" chsh -s `which zsh` "${ELEMENTS_USER}"
 
     echo "[*] Creating Elements data folder"
@@ -1045,21 +1136,29 @@ fi
 # Bitcoin -> Electrs installation #
 ###################################
 
-echo "[*] Creating Bitcoin Electrs data folder"
+if [ "${BITCOIN_ELECTRS_INSTALL}" = ON ];then
-osSudo "${ROOT_USER}" mkdir -p "${BITCOIN_ELECTRS_HOME}"
-osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${BITCOIN_ELECTRS_HOME}"
-osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_MAINNET_DATA}"
-osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_TESTNET_DATA}"
-osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_SIGNET_DATA}"
 
-echo "[*] Cloning Bitcoin Electrs repo from ${BITCOIN_ELECTRS_REPO_URL}"
+    echo "[*] Creating Bitcoin Electrs data folder"
-osSudo "${BITCOIN_USER}" git config --global advice.detachedHead false
+    osSudo "${ROOT_USER}" mkdir -p "${BITCOIN_ELECTRS_HOME}"
-osSudo "${BITCOIN_USER}" git clone --branch "${BITCOIN_ELECTRS_REPO_BRANCH}" "${BITCOIN_ELECTRS_REPO_URL}" "${BITCOIN_HOME}/${BITCOIN_ELECTRS_REPO_NAME}"
+    osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${BITCOIN_ELECTRS_HOME}"
+    if [ "${BITCOIN_MAINNET_ENABLE}" = ON ];then
+        osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_MAINNET_DATA}"
+    fi
+    if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then
+        osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_TESTNET_DATA}"
+    fi
+    if [ "${BITCOIN_SIGNET_ENABLE}" = ON ];then
+        osSudo "${ROOT_USER}" chown -R "${BITCOIN_USER}:${BITCOIN_GROUP}" "${ELECTRS_SIGNET_DATA}"
+    fi
 
-echo "[*] Checking out Electrs ${BITCOIN_ELECTRS_LATEST_RELEASE}"
+    echo "[*] Cloning Bitcoin Electrs repo from ${BITCOIN_ELECTRS_REPO_URL}"
-osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_HOME}/${BITCOIN_ELECTRS_REPO_NAME} && git checkout ${BITCOIN_ELECTRS_LATEST_RELEASE}"
+    osSudo "${BITCOIN_USER}" git config --global advice.detachedHead false
+    osSudo "${BITCOIN_USER}" git clone --branch "${BITCOIN_ELECTRS_REPO_BRANCH}" "${BITCOIN_ELECTRS_REPO_URL}" "${BITCOIN_HOME}/${BITCOIN_ELECTRS_REPO_NAME}"
 
-case $OS in
+    echo "[*] Checking out Electrs ${BITCOIN_ELECTRS_LATEST_RELEASE}"
+    osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_HOME}/${BITCOIN_ELECTRS_REPO_NAME} && git checkout ${BITCOIN_ELECTRS_LATEST_RELEASE}"
+
+    case $OS in
         FreeBSD)
            echo "[*] Installing Rust from pkg install"
         ;;
@@ -1067,12 +1166,12 @@ case $OS in
            echo "[*] Installing Rust from rustup.rs"
            osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y"
         ;;
-esac
+    esac
 
-echo "[*] Building Bitcoin Electrs release binary"
+    echo "[*] Building Bitcoin Electrs release binary"
-osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && cargo run --release --bin electrs -- --version" || true
+    osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && cargo run --release --bin electrs -- --version" || true
 
-case $OS in
+    case $OS in
         FreeBSD)
             echo "[*] Patching Bitcoin Electrs code for FreeBSD"
             osSudo "${BITCOIN_USER}" sh -c "cd \"${BITCOIN_HOME}/.cargo/registry/src/github.com-1ecc6299db9ec823/sysconf-0.3.4\" && patch -p1 < \"${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/freebsd/sysconf.patch\""
@@ -1081,51 +1180,59 @@ case $OS in
         ;;
         Debian)
         ;;
-esac
+    esac
 
-echo "[*] Building Bitcoin Electrs release binary"
+    echo "[*] Building Bitcoin Electrs release binary"
-osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && cargo run --release --bin electrs -- --version"
+    osSudo "${BITCOIN_USER}" sh -c "cd ${BITCOIN_ELECTRS_HOME} && cargo run --release --bin electrs -- --version"
+fi
 
 ##################################
 # Liquid -> Electrs installation #
 ##################################
 
-echo "[*] Creating Liquid Electrs data folder"
+if [ "${ELEMENTS_ELECTRS_INSTALL}" = ON ];then
-osSudo "${ROOT_USER}" mkdir -p "${ELEMENTS_ELECTRS_HOME}"
-osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELEMENTS_HOME}"
-osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELEMENTS_ELECTRS_HOME}"
-osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELECTRS_LIQUID_DATA}"
-osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELECTRS_LIQUIDTESTNET_DATA}"
 
-echo "[*] Cloning Liquid Electrs repo from ${ELEMENTS_ELECTRS_REPO_URL}"
+    echo "[*] Creating Liquid Electrs data folder"
-osSudo "${ELEMENTS_USER}" git config --global advice.detachedHead false
+    osSudo "${ROOT_USER}" mkdir -p "${ELEMENTS_ELECTRS_HOME}"
-osSudo "${ELEMENTS_USER}" git clone --branch "${ELEMENTS_ELECTRS_REPO_BRANCH}" "${ELEMENTS_ELECTRS_REPO_URL}" "${ELEMENTS_HOME}/${ELEMENTS_ELECTRS_REPO_NAME}"
+    osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELEMENTS_HOME}"
+    osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELEMENTS_ELECTRS_HOME}"
+    if [ "${ELEMENTS_LIQUID_ENABLE}" = ON ];then
+        osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELECTRS_LIQUID_DATA}"
+    fi
+    if [ "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then
+        osSudo "${ROOT_USER}" chown -R "${ELEMENTS_USER}:${ELEMENTS_GROUP}" "${ELECTRS_LIQUIDTESTNET_DATA}"
+    fi
     
-echo "[*] Checking out Liquid Electrs ${ELEMENTS_ELECTRS_LATEST_RELEASE}"
+    echo "[*] Cloning Liquid Electrs repo from ${ELEMENTS_ELECTRS_REPO_URL}"
-osSudo "${ELEMENTS_USER}" sh -c "cd ${ELEMENTS_HOME}/${ELEMENTS_ELECTRS_REPO_NAME} && git checkout ${ELEMENTS_ELECTRS_LATEST_RELEASE}"
+    osSudo "${ELEMENTS_USER}" git config --global advice.detachedHead false
+    osSudo "${ELEMENTS_USER}" git clone --branch "${ELEMENTS_ELECTRS_REPO_BRANCH}" "${ELEMENTS_ELECTRS_REPO_URL}" "${ELEMENTS_HOME}/${ELEMENTS_ELECTRS_REPO_NAME}"
     
-echo "[*] Cloning Liquid Asset Registry repo from ${LIQUID_ASSET_REGISTRY_DB_URL}"
+    echo "[*] Checking out Liquid Electrs ${ELEMENTS_ELECTRS_LATEST_RELEASE}"
-osSudo "${ELEMENTS_USER}" git config --global advice.detachedHead false
+    osSudo "${ELEMENTS_USER}" sh -c "cd ${ELEMENTS_HOME}/${ELEMENTS_ELECTRS_REPO_NAME} && git checkout ${ELEMENTS_ELECTRS_LATEST_RELEASE}"
-osSudo "${ELEMENTS_USER}" git clone "${LIQUID_ASSET_REGISTRY_DB_URL}" "${ELEMENTS_HOME}/${LIQUID_ASSET_REGISTRY_DB_NAME}"
     
-echo "[*] Cloning Liquid Asset Registry testnet repo from ${LIQUIDTESTNET_ASSET_REGISTRY_DB_URL}"
+    echo "[*] Cloning Liquid Asset Registry repo from ${LIQUID_ASSET_REGISTRY_DB_URL}"
-osSudo "${ELEMENTS_USER}" git config --global advice.detachedHead false
+    osSudo "${ELEMENTS_USER}" git config --global advice.detachedHead false
-osSudo "${ELEMENTS_USER}" git clone "${LIQUIDTESTNET_ASSET_REGISTRY_DB_URL}" "${ELEMENTS_HOME}/${LIQUIDTESTNET_ASSET_REGISTRY_DB_NAME}"
+    osSudo "${ELEMENTS_USER}" git clone "${LIQUID_ASSET_REGISTRY_DB_URL}" "${ELEMENTS_HOME}/${LIQUID_ASSET_REGISTRY_DB_NAME}"
     
-echo "[*] Building Liquid Electrs release binary"
+    echo "[*] Cloning Liquid Asset Registry testnet repo from ${LIQUIDTESTNET_ASSET_REGISTRY_DB_URL}"
-osSudo "${ELEMENTS_USER}" sh -c "cd ${ELEMENTS_ELECTRS_HOME} && cargo run --release --features liquid --bin electrs -- --network liquid --version" || true
+    osSudo "${ELEMENTS_USER}" git config --global advice.detachedHead false
+    osSudo "${ELEMENTS_USER}" git clone "${LIQUIDTESTNET_ASSET_REGISTRY_DB_URL}" "${ELEMENTS_HOME}/${LIQUIDTESTNET_ASSET_REGISTRY_DB_NAME}"
     
-case $OS in
+    echo "[*] Building Liquid Electrs release binary"
+    osSudo "${ELEMENTS_USER}" sh -c "cd ${ELEMENTS_ELECTRS_HOME} && cargo run --release --features liquid --bin electrs -- --network liquid --version" || true
+    
+    case $OS in
         FreeBSD)
             echo "[*] Patching Liquid Electrs code for FreeBSD"
             osSudo "${ELEMENTS_USER}" sh -c "cd \"${ELEMENTS_HOME}/.cargo/registry/src/github.com-1ecc6299db9ec823/sysconf-0.3.4\" && patch -p1 < \"${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/freebsd/sysconf.patch\""
         ;;
         Debian)
         ;;
-esac
+    esac
     
-echo "[*] Building Liquid Electrs release binary"
+    echo "[*] Building Liquid Electrs release binary"
-osSudo "${ELEMENTS_USER}" sh -c "cd ${ELEMENTS_ELECTRS_HOME} && cargo run --release --features liquid --bin electrs -- --network liquid --version" || true
+    osSudo "${ELEMENTS_USER}" sh -c "cd ${ELEMENTS_ELECTRS_HOME} && cargo run --release --features liquid --bin electrs -- --network liquid --version" || true
+fi
     
 #####################
 # Bisq installation #
@@ -1135,7 +1242,11 @@ if [ "${BISQ_INSTALL}" = ON ];then
 
     echo "[*] Creating Bisq user with Tor access"
     osGroupCreate "${BISQ_GROUP}"
+    if [ "${TOR_INSTALL}" = ON ];then
         osUserCreate "${BISQ_USER}" "${BISQ_HOME}" "${BISQ_GROUP}" "${TOR_GROUP}"
+    else
+        osUserCreate "${BISQ_USER}" "${BISQ_HOME}" "${BISQ_GROUP}"
+    fi
     osSudo "${ROOT_USER}" chsh -s `which zsh` "${BISQ_USER}"
 
     echo "[*] Creating Bisq data folder"
@@ -1204,6 +1315,50 @@ if [ "${BISQ_INSTALL}" = ON ];then
     esac
 fi
 
+#######################
+# Unfurl installation #
+#######################
+
+if [ "${UNFURL_INSTALL}" = ON ];then
+
+    echo "[*] Creating Unfurl user"
+    osGroupCreate "${UNFURL_GROUP}"
+    osUserCreate "${UNFURL_USER}" "${UNFURL_HOME}" "${UNFURL_GROUP}"
+    osSudo "${ROOT_USER}" chsh -s `which zsh` "${UNFURL_USER}"
+
+    echo "[*] Creating Unfurl folder"
+    osSudo "${ROOT_USER}" mkdir -p "${UNFURL_HOME}"
+    osSudo "${ROOT_USER}" chown -R "${UNFURL_USER}:${UNFURL_GROUP}" "${UNFURL_HOME}"
+    osSudo "${UNFURL_USER}" touch "${UNFURL_HOME}/.zshrc"
+
+    echo "[*] Insalling Unfurl source"
+    case $OS in
+
+        FreeBSD)
+            echo "[*] FIXME: Unfurl must be installed manually on FreeBSD"
+        ;;
+
+        Debian)
+            echo "[*] Installing packages for Unfurl"
+            osPackageInstall ${DEBIAN_UNFURL_PKG[@]}
+            echo "[*] Cloning Mempool (Unfurl) repo from ${UNFURL_REPO_URL}"
+            osSudo "${UNFURL_USER}" git config --global pull.rebase true
+            osSudo "${UNFURL_USER}" git config --global advice.detachedHead false
+            osSudo "${UNFURL_USER}" git clone --branch "${UNFURL_REPO_BRANCH}" "${UNFURL_REPO_URL}" "${UNFURL_HOME}/${UNFURL_REPO_NAME}"
+            osSudo "${UNFURL_USER}" ln -s unfurl/production/unfurl-build upgrade
+            osSudo "${UNFURL_USER}" ln -s unfurl/production/unfurl-kill stop
+            osSudo "${UNFURL_USER}" ln -s unfurl/production/unfurl-start start
+            echo "[*] Installing nvm.sh from GitHub"
+            osSudo "${UNFURL_USER}" sh -c 'curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | zsh'
+
+            echo "[*] Building NodeJS via nvm.sh"
+            osSudo "${UNFURL_USER}" zsh -c 'source ~/.zshrc ; nvm install v16.16.0 --shared-zlib'
+
+        ;;
+    esac
+
+fi
+
 ################################
 # Bitcoin instance for Mainnet #
 ################################
@@ -1420,7 +1575,9 @@ case $OS in
             echo "[*] Installing Electrs Signet Cronjob"
             crontab_bitcoin+="@reboot sleep 90 ; screen -dmS signet /bitcoin/electrs/electrs-start-signet\n"
         fi
+        if [ "${BITCOIN_MAINNET_ENABLE}" = ON -o "${BITCOIN_TESTNET_ENABLE}" = ON -o "${BITCOIN_SIGNET_ENABLE}" = ON ];then
             echo "${crontab_bitcoin}" | crontab -u "${BITCOIN_USER}" -
+        fi
 
         crontab_elements=()
         if [ "${ELEMENTS_LIQUID_ENABLE}" = ON ];then
@@ -1431,7 +1588,9 @@ case $OS in
             echo "[*] Installing Liquid Asset Testnet Cronjob"
             crontab_elements+="6 * * * * cd $HOME/asset_registry_testnet_db && git pull origin master >/dev/null 2>&1\n"
         fi
+        if [ "${ELEMENTS_LIQUID_ENABLE}" = ON -o "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then
             echo "${crontab_elements}" | crontab -u "${ELEMENTS_USER}" -
+        fi
     ;;
 esac
 
@@ -1444,7 +1603,7 @@ fi
 
 ##### Mempool -> Bitcoin Mainnet instance
 
-if [ "${BITCOIN_MAINNET_ENABLE}" = ON ];then
+if [ "${BITCOIN_MAINNET_ENABLE}" = ON -o "${BITCOIN_TESTNET_ENABLE}" = ON -o "${BITCOIN_SIGNET_ENABLE}" = ON ];then
     echo "[*] Creating Mempool instance for Bitcoin Mainnet"
     osSudo "${MEMPOOL_USER}" git config --global advice.detachedHead false
     osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/mainnet"
@@ -1513,32 +1672,42 @@ esac
 
 mysql << _EOF_
 create database mempool;
-grant all on mempool.* to 'mempool'@'localhost' identified by 'mempool';
+grant all on mempool.* to '${MEMPOOL_MAINNET_USER}'@'localhost' identified by '${MEMPOOL_MAINNET_PASS}';
 
 create database mempool_testnet;
-grant all on mempool_testnet.* to 'mempool_testnet'@'localhost' identified by 'mempool_testnet';
+grant all on mempool_testnet.* to '${MEMPOOL_TESTNET_USER}'@'localhost' identified by '${MEMPOOL_TESTNET_PASS}';
 
 create database mempool_signet;
-grant all on mempool_signet.* to 'mempool_signet'@'localhost' identified by 'mempool_signet';
+grant all on mempool_signet.* to '${MEMPOOL_SIGNET_USER}'@'localhost' identified by '${MEMPOOL_SIGNET_PASS}';
 
 create database mempool_liquid;
-grant all on mempool_liquid.* to 'mempool_liquid'@'localhost' identified by 'mempool_liquid';
+grant all on mempool_liquid.* to '${MEMPOOL_LIQUID_USER}'@'localhost' identified by '${MEMPOOL_LIQUID_PASS}';
 
 create database mempool_liquidtestnet;
-grant all on mempool_liquidtestnet.* to 'mempool_liquidtestnet'@'localhost' identified by 'mempool_liquidtestnet';
+grant all on mempool_liquidtestnet.* to '${MEMPOOL_LIQUIDTESTNET_USER}'@'localhost' identified by '${MEMPOOL_LIQUIDTESTNET_PASS}';
 
 create database mempool_bisq;
-grant all on mempool_bisq.* to 'mempool_bisq'@'localhost' identified by 'mempool_bisq';
+grant all on mempool_bisq.* to '${MEMPOOL_BISQ_USER}'@'localhost' identified by '${MEMPOOL_BISQ_PASS}';
+_EOF_
+
+echo "[*] save MySQL credentials"
+cat > ${MEMPOOL_HOME}/mysql_credentials << _EOF_
+declare -x MEMPOOL_MAINNET_USER="${MEMPOOL_MAINNET_USER}"
+declare -x MEMPOOL_MAINNET_PASS="${MEMPOOL_MAINNET_PASS}"
+declare -x MEMPOOL_TESTNET_USER="${MEMPOOL_TESTNET_USER}"
+declare -x MEMPOOL_TESTNET_PASS="${MEMPOOL_TESTNET_PASS}"
+declare -x MEMPOOL_SIGNET_USER="${MEMPOOL_SIGNET_USER}"
+declare -x MEMPOOL_SIGNET_PASS="${MEMPOOL_SIGNET_PASS}"
+declare -x MEMPOOL_LIQUID_USER="${MEMPOOL_LIQUID_USER}"
+declare -x MEMPOOL_LIQUID_PASS="${MEMPOOL_LIQUID_PASS}"
+declare -x MEMPOOL_LIQUIDTESTNET_USER="${MEMPOOL_LIQUIDTESTNET_USER}"
+declare -x MEMPOOL_LIQUIDTESTNET_PASS="${MEMPOOL_LIQUIDTESTNET_PASS}"
+declare -x MEMPOOL_BISQ_USER="${MEMPOOL_BISQ_USER}"
+declare -x MEMPOOL_BISQ_PASS="${MEMPOOL_BISQ_PASS}"
 _EOF_
 
 ##### nginx
 
-
-echo "[*] Read tor v3 onion hostnames"
-NGINX_MEMPOOL_ONION=$(cat "${TOR_RESOURCES}/mempool/hostname")
-NGINX_BISQ_ONION=$(cat "${TOR_RESOURCES}/bisq/hostname")
-NGINX_LIQUID_ONION=$(cat "${TOR_RESOURCES}/liquid/hostname")
-
 echo "[*] Adding Nginx configuration"
 osSudo "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/nginx/nginx.conf" "${NGINX_CONFIGURATION}"
 mkdir -p /var/cache/nginx/services /var/cache/nginx/api
@@ -1546,9 +1715,15 @@ chown ${NGINX_USER}: /var/cache/nginx/services /var/cache/nginx/api
 ln -s /mempool/mempool /etc/nginx/mempool
 osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_USER__!${NGINX_USER}!" "${NGINX_CONFIGURATION}"
 osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_ETC_FOLDER__!${NGINX_ETC_FOLDER}!" "${NGINX_CONFIGURATION}"
-osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_MEMPOOL_ONION__!${NGINX_MEMPOOL_ONION%.onion}!" "${NGINX_CONFIGURATION}"
+if [ "${TOR_INSTALL}" = ON ];then
-osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_BISQ_ONION__!${NGINX_BISQ_ONION%.onion}!" "${NGINX_CONFIGURATION}"
+echo "[*] Read tor v3 onion hostnames"
-osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_LIQUID_ONION__!${NGINX_LIQUID_ONIONi%.onion}!" "${NGINX_CONFIGURATION}"
+    NGINX_MEMPOOL_ONION=$(cat "${TOR_RESOURCES}/mempool/hostname")
+    NGINX_BISQ_ONION=$(cat "${TOR_RESOURCES}/bisq/hostname")
+    NGINX_LIQUID_ONION=$(cat "${TOR_RESOURCES}/liquid/hostname")
+    osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_MEMPOOL_ONION__!${NGINX_MEMPOOL_ONION%.onion}!" "${NGINX_CONFIGURATION}"
+    osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_BISQ_ONION__!${NGINX_BISQ_ONION%.onion}!" "${NGINX_CONFIGURATION}"
+    osSudo "${ROOT_USER}" sed -i.orig "s!__NGINX_LIQUID_ONION__!${NGINX_LIQUID_ONIONi%.onion}!" "${NGINX_CONFIGURATION}"
+fi
 echo "[*] Restarting Nginx"
 osSudo "${ROOT_USER}" service nginx restart
 
@@ -1611,7 +1786,7 @@ esac
 ##### Build Mempool
 
 echo "[*] Build Mempool"
-osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME} && ./upgrade"
+osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME} && ./upgrade" || true
 
 
 
@@ -1696,10 +1871,12 @@ case $OS in
     ;;
 
     Debian)
+        if [ "${TOR_INSTALL}" = ON ];then
             echo "This are the generated Tor addresses:"
             echo "${NGINX_MEMPOOL_ONION}"
             echo "${NGINX_BISQ_ONION}"
             echo "${NGINX_LIQUID_ONION}"
+        fi
     ;;
 esac
 

production/mempool-build-all

@@ -11,6 +11,9 @@ BITCOIN_RPC_PASS=$(grep '^rpcpassword' /bitcoin/bitcoin.conf | cut -d '=' -f2)
 ELEMENTS_RPC_USER=$(grep '^rpcuser' /elements/elements.conf | cut -d '=' -f2)
 ELEMENTS_RPC_PASS=$(grep '^rpcpassword' /elements/elements.conf | cut -d '=' -f2)
 
+# get mysql credentials
+. /mempool/mysql_credentials
+
 if [ -f "${LOCKFILE}" ];then
     echo "upgrade already running? check lockfile ${LOCKFILE}"
     exit 1
@@ -73,6 +76,18 @@ build_backend()
 	-e "s!__BITCOIN_RPC_PASS__!${BITCOIN_RPC_PASS}!" \
 	-e "s!__ELEMENTS_RPC_USER__!${ELEMENTS_RPC_USER}!" \
 	-e "s!__ELEMENTS_RPC_PASS__!${ELEMENTS_RPC_PASS}!" \
+        -e "s!__MEMPOOL_MAINNET_USER__!${MEMPOOL_MAINNET_USER}!" \
+        -e "s!__MEMPOOL_MAINNET_PASS__!${MEMPOOL_MAINNET_PASS}!" \
+        -e "s!__MEMPOOL_TESTNET_USER__!${MEMPOOL_TESTNET_USER}!" \
+        -e "s!__MEMPOOL_TESTNET_PASS__!${MEMPOOL_TESTNET_PASS}!" \
+        -e "s!__MEMPOOL_SIGNET_USER__!${MEMPOOL_SIGNET_USER}!" \
+        -e "s!__MEMPOOL_SIGNET_PASS__!${MEMPOOL_SIGNET_PASS}!" \
+        -e "s!__MEMPOOL_LIQUID_USER__!${MEMPOOL_LIQUID_USER}!" \
+        -e "s!__MEMPOOL_LIQUID_PASS__!${MEMPOOL_LIQUID_PASS}!" \
+        -e "s!__MEMPOOL_LIQUIDTESTNET_USER__!${LIQUIDTESTNET_USER}!" \
+        -e "s!__MEMPOOL_LIQUIDTESTNET_PASS__!${MEMPOOL_LIQUIDTESTNET_PASS}!" \
+        -e "s!__MEMPOOL_BISQ_USER__!${MEMPOOL_BISQ_USER}!" \
+        -e "s!__MEMPOOL_BISQ_PASS__!${MEMPOOL_BISQ_PASS}!" \
 	"mempool-config.json"
     fi
     npm install --omit=dev --omit=optional || exit 1

production/mempool-config.bisq.json

@@ -21,8 +21,8 @@
     "ENABLED": false,
     "HOST": "127.0.0.1",
     "PORT": 3306,
-    "USERNAME": "mempool_bisq",
+    "USERNAME": "__MEMPOOL_BISQ_USER__",
-    "PASSWORD": "mempool_bisq",
+    "PASSWORD": "__MEMPOOL_BISQ_PASS__",
     "DATABASE": "mempool_bisq"
   },
   "STATISTICS": {

production/mempool-config.liquid.json

@@ -28,8 +28,8 @@
     "ENABLED": true,
     "HOST": "127.0.0.1",
     "PORT": 3306,
-    "USERNAME": "mempool_liquid",
+    "USERNAME": "__MEMPOOL_LIQUID_USER__",
-    "PASSWORD": "mempool_liquid",
+    "PASSWORD": "__MEMPOOL_LIQUID_PASS__",
     "DATABASE": "mempool_liquid"
   },
   "STATISTICS": {

production/mempool-config.liquidtestnet.json

@@ -28,8 +28,8 @@
     "ENABLED": true,
     "HOST": "127.0.0.1",
     "PORT": 3306,
-    "USERNAME": "mempool_liquidtestnet",
+    "USERNAME": "__MEMPOOL_LIQUIDTESTNET_USER__",
-    "PASSWORD": "mempool_liquidtestnet",
+    "PASSWORD": "__MEMPOOL_LIQUIDTESTNET_PASS__",
     "DATABASE": "mempool_liquidtestnet"
   },
   "STATISTICS": {

production/mempool-config.mainnet.json

@@ -32,8 +32,8 @@
     "ENABLED": true,
     "HOST": "127.0.0.1",
     "PORT": 3306,
-    "USERNAME": "mempool",
+    "USERNAME": "__MEMPOOL_MAINNET_USER__",
-    "PASSWORD": "mempool",
+    "PASSWORD": "__MEMPOOL_MAINNET_PASS__",
     "DATABASE": "mempool"
   },
   "STATISTICS": {

production/mempool-config.signet.json

@@ -24,8 +24,8 @@
     "ENABLED": true,
     "HOST": "127.0.0.1",
     "PORT": 3306,
-    "USERNAME": "mempool_signet",
+    "USERNAME": "__MEMPOOL_SIGNET_USER__",
-    "PASSWORD": "mempool_signet",
+    "PASSWORD": "__MEMPOOL_SIGNET_PASS__",
     "DATABASE": "mempool_signet"
   },
   "STATISTICS": {

production/mempool-config.testnet.json

@@ -24,8 +24,8 @@
     "ENABLED": true,
     "HOST": "127.0.0.1",
     "PORT": 3306,
-    "USERNAME": "mempool_testnet",
+    "USERNAME": "__MEMPOOL_TESTNET_USER__",
-    "PASSWORD": "mempool_testnet",
+    "PASSWORD": "__MEMPOOL_TESTNET_PASS__",
     "DATABASE": "mempool_testnet"
   },
   "STATISTICS": {

production/mempool-config.unfurl.json

@@ -0,0 +1,13 @@
+{
+  "SERVER": {
+    "HOST": "https://mempool.space",
+    "HTTP_PORT": 8001
+  },
+  "MEMPOOL": {
+    "HTTP_HOST": "https://mempool.space",
+    "HTTP_PORT": 443
+  },
+  "PUPPETEER": {
+    "CLUSTER_SIZE": 8
+  }
+}

production/mempool.crontab

@@ -1,3 +1,12 @@
+# start on reboot
 @reboot sleep 10 ; $HOME/start
-37 13 * * * sleep 30 ; /mempool/mempool.space/backup >/dev/null 2>&1 &
+
+# start cache warmer on reboot
 @reboot sleep 180 ; /mempool/mempool/production/nginx-cache-warmer >/dev/null 2>&1 &
+
+# daily backup
+37 13 * * * sleep 30 ; /mempool/mempool.space/backup >/dev/null 2>&1 &
+
+# hourly liquid asset update
+6 * * * * cd $HOME/liquid/frontend && npm run sync-assets && rsync -av $HOME/liquid/frontend/dist/mempool/browser/en-US/resources/assets* $HOME/public_html/liquid/en-US/resources/ >/dev/null 2>&1
+

production/newsyslog-mempool-backend.conf

@@ -1,2 +1,2 @@
-/var/log/mempool			640  10	   *	@T00	C
+/var/log/mempool			644  10	   *	@T00	C
-/var/log/mempool.debug			640  10	   *	@T00	C
+/var/log/mempool.debug			644  10	   *	@T00	C

production/torrc

@@ -13,11 +13,3 @@ CookieAuthFileGroupReadable 1
 HiddenServiceDir __TOR_RESOURCES__/mempool
 HiddenServicePort 80 127.0.0.1:81
 HiddenServiceVersion 3
-
-HiddenServiceDir __TOR_RESOURCES__/bisq
-HiddenServicePort 80 127.0.0.1:82
-HiddenServiceVersion 3
-
-HiddenServiceDir __TOR_RESOURCES__/liquid
-HiddenServicePort 80 127.0.0.1:83
-HiddenServiceVersion 3

production/unfurl-build

@@ -0,0 +1,62 @@
+#!/usr/bin/env zsh
+PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:$HOME/bin
+HOSTNAME=$(hostname)
+LOCATION=$(hostname|cut -d . -f2)
+LOCKFILE="${HOME}/lock"
+REF=$(echo "${1:=origin/master}"|sed -e 's!:!/!')
+
+if [ -f "${LOCKFILE}" ];then
+    echo "upgrade already running? check lockfile ${LOCKFILE}"
+    exit 1
+fi
+
+# on exit, remove lockfile but preserve exit code
+trap "rv=\$?; rm -f "${LOCKFILE}"; exit \$rv" INT TERM EXIT
+
+# create lockfile
+touch "${LOCKFILE}"
+
+# notify logged in users
+echo "Upgrading unfurler to ${REF}" | wall
+
+update_repo()
+{
+    echo "[*] Upgrading unfurler to ${REF}"
+    cd "$HOME/unfurl/unfurler" || exit 1
+
+    git fetch origin || exit 1
+    for remote in origin;do
+        git remote add "${remote}" "https://github.com/${remote}/mempool" >/dev/null 2>&1
+        git fetch "${remote}" || exit 1
+    done
+
+    if [ $(git tag -l "${REF}") ];then
+        git reset --hard "tags/${REF}" || exit 1
+    elif [ $(git branch -r -l "origin/${REF}") ];then
+        git reset --hard "origin/${REF}" || exit 1
+    else
+        git reset --hard "${REF}" || exit 1
+    fi
+    export HASH=$(git rev-parse HEAD)
+}
+
+build_backend()
+{
+    echo "[*] Building backend for unfurler"
+    [ -z "${HASH}" ] && exit 1
+    cd "$HOME/unfurl/unfurler" || exit 1
+    if [ ! -e "config.json" ];then
+        cp "${HOME}/unfurl/production/mempool-config.unfurl.json" "config.json"
+    fi
+    npm install || exit 1
+    npm run build || exit 1
+}
+
+update_repo
+build_backend
+
+# notify everyone
+echo "${HOSTNAME} unfurl updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general mempool.dev
+echo "${HOSTNAME} unfurl updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general "mempool.ops.${LOCATION}"
+
+exit 0

production/unfurl-kill

@@ -0,0 +1,2 @@
+#!/usr/bin/env zsh
+killall sh node

production/unfurl-start

@@ -0,0 +1,6 @@
+#!/usr/bin/env zsh
+export NVM_DIR="$HOME/.nvm"
+source "$NVM_DIR/nvm.sh"
+
+cd "${HOME}/unfurl/unfurler/" && \
+screen -dmS "unfurl" sh -c 'while true;do npm run start-production;sleep 1;done'

unfurler/.editorconfig

@@ -0,0 +1,17 @@
+# Editor configuration, see https://editorconfig.org
+root = true
+
+[*]
+charset = utf-8
+indent_style = space
+indent_size = 2
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.ts]
+quote_type = single
+
+[*.md]
+max_line_length = off
+trim_trailing_whitespace = false
+

unfurler/.eslintignore

@@ -0,0 +1,2 @@
+node_modules
+dist

unfurler/.eslintrc

@@ -0,0 +1,33 @@
+{
+  "root": true,
+  "parser": "@typescript-eslint/parser",
+  "plugins": [
+    "@typescript-eslint"
+  ],
+  "extends": [
+    "eslint:recommended",
+    "plugin:@typescript-eslint/eslint-recommended",
+    "plugin:@typescript-eslint/recommended",
+    "prettier"
+  ],
+  "rules": {
+    "@typescript-eslint/ban-ts-comment": 1,
+    "@typescript-eslint/ban-types": 1,
+    "@typescript-eslint/no-empty-function": 1,
+    "@typescript-eslint/no-explicit-any": 1,
+    "@typescript-eslint/no-inferrable-types": 1,
+    "@typescript-eslint/no-namespace": 1,
+    "@typescript-eslint/no-this-alias": 1,
+    "@typescript-eslint/no-var-requires": 1,
+    "no-console": 1,
+    "no-constant-condition": 1,
+    "no-dupe-else-if": 1,
+    "no-empty": 1,
+    "no-prototype-builtins": 1,
+    "no-self-assign": 1,
+    "no-useless-catch": 1,
+    "no-var": 1,
+    "prefer-const": 1,
+    "prefer-rest-params": 1
+  }
+}