Merge branch 'master' into simon/lightning-node-channel-skeleton-loaders

backend/src/api/bitcoin/bitcoin.routes.ts

@@ -510,7 +510,12 @@ class BitcoinRoutes {
 
   private getDifficultyChange(req: Request, res: Response) {
     try {
-      res.json(difficultyAdjustment.getDifficultyAdjustment());
+      const da = difficultyAdjustment.getDifficultyAdjustment();
+      if (da) {
+        res.json(da);
+      } else {
+        res.status(503).send(`Service Temporarily Unavailable`);
+      }
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);
     }

backend/src/api/difficulty-adjustment.ts

@@ -81,14 +81,15 @@ export function calcDifficultyAdjustment(
 }
 
 class DifficultyAdjustmentApi {
-  constructor() { }
+  public getDifficultyAdjustment(): IDifficultyAdjustment | null {
-
-  public getDifficultyAdjustment(): IDifficultyAdjustment {
     const DATime = blocks.getLastDifficultyAdjustmentTime();
     const previousRetarget = blocks.getPreviousDifficultyRetarget();
     const blockHeight = blocks.getCurrentBlockHeight();
     const blocksCache = blocks.getBlocks();
     const latestBlock = blocksCache[blocksCache.length - 1];
+    if (!latestBlock) {
+      return null;
+    }
     const nowSeconds = Math.floor(new Date().getTime() / 1000);
 
     return calcDifficultyAdjustment(

backend/src/api/explorer/statistics.api.ts

@@ -27,7 +27,7 @@ class StatisticsApi {
   public async $getLatestStatistics(): Promise<any> {
     try {
       const [rows]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1`);
-      const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats ORDER BY added DESC LIMIT 1 OFFSET 7`);
+      const [rows2]: any = await DB.query(`SELECT * FROM lightning_stats WHERE DATE(added) = DATE(NOW() - INTERVAL 7 DAY)`);
       return {
         latest: rows[0],
         previous: rows2[0],

backend/src/tasks/lightning/sync-tasks/node-locations.ts

@@ -4,6 +4,7 @@ import nodesApi from '../../../api/explorer/nodes.api';
 import config from '../../../config';
 import DB from '../../../database';
 import logger from '../../../logger';
+import * as IPCheck from '../../../utils/ipcheck.js';
 
 export async function $lookupNodeLocation(): Promise<void> {
   let loggerTimer = new Date().getTime() / 1000;
@@ -27,6 +28,26 @@ export async function $lookupNodeLocation(): Promise<void> {
           const asn = lookupAsn.get(ip);
           const isp = lookupIsp.get(ip);
 
+          let asOverwrite: any | undefined;
+          if (asn && (IPCheck.match(ip, '170.75.160.0/20') || IPCheck.match(ip, '172.81.176.0/21'))) {
+            asOverwrite = {
+              asn: 394745,
+              name: 'Lunanode',
+            };
+          }
+          else if (asn && (IPCheck.match(ip, '50.7.0.0/16') || IPCheck.match(ip, '66.90.64.0/18'))) {
+            asOverwrite = {
+              asn: 30058,
+              name: 'FDCservers.net',
+            };
+          }
+          else if (asn && asn.autonomous_system_number === 174) {
+            asOverwrite = {
+              asn: 174,
+              name: 'Cogent Communications',
+            };
+          }
+
           if (city && (asn || isp)) {
             const query = `
               UPDATE nodes SET 
@@ -41,7 +62,7 @@ export async function $lookupNodeLocation(): Promise<void> {
             `;
 
             const params = [
-              isp?.autonomous_system_number ?? asn?.autonomous_system_number,
+              asOverwrite?.asn ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number,
               city.city?.geoname_id,
               city.country?.geoname_id,
               city.subdivisions ? city.subdivisions[0].geoname_id : null,
@@ -91,7 +112,10 @@ export async function $lookupNodeLocation(): Promise<void> {
             if (isp?.autonomous_system_organization ?? asn?.autonomous_system_organization) {
               await DB.query(
                 `INSERT IGNORE INTO geo_names (id, type, names) VALUES (?, 'as_organization', ?)`,
-                [isp?.autonomous_system_number ?? asn?.autonomous_system_number, JSON.stringify(isp?.isp ?? asn?.autonomous_system_organization)]);
+                [
+                  asOverwrite?.asn ?? isp?.autonomous_system_number ?? asn?.autonomous_system_number,
+                  JSON.stringify(asOverwrite?.name ?? isp?.isp ?? asn?.autonomous_system_organization)
+                ]);
             }
           }
 

backend/src/utils/ipcheck.js

@@ -0,0 +1,119 @@
+var net = require('net');
+
+var IPCheck = module.exports = function(input) {
+  var self = this;
+
+  if (!(self instanceof IPCheck)) {
+    return new IPCheck(input);
+  }
+
+  self.input = input;
+  self.parse();
+};
+
+IPCheck.prototype.parse = function() {
+  var self = this;
+
+  if (!self.input || typeof self.input !== 'string') return self.valid = false;
+
+  var ip;
+
+  var pos = self.input.lastIndexOf('/');
+  if (pos !== -1) {
+    ip = self.input.substring(0, pos);
+    self.mask = +self.input.substring(pos + 1);
+  } else {
+    ip = self.input;
+    self.mask = null;
+  }
+
+  self.ipv = net.isIP(ip);
+  self.valid = !!self.ipv && !isNaN(self.mask);
+
+  if (!self.valid) return;
+
+  // default mask = 32 for ipv4 and 128 for ipv6
+  if (self.mask === null) self.mask = self.ipv === 4 ? 32 : 128;
+
+  if (self.ipv === 4) {
+    // difference between ipv4 and ipv6 masks
+    self.mask += 96;
+  }
+
+  if (self.mask < 0 || self.mask > 128) {
+    self.valid = false;
+    return;
+  }
+
+  self.address = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ];
+
+  if(self.ipv === 4){
+    self.parseIPv4(ip);
+  }else{
+    self.parseIPv6(ip);
+  }
+};
+
+IPCheck.prototype.parseIPv4 = function(ip) {
+  var self = this;
+
+  // ipv4 addresses live under ::ffff:0:0
+  self.address[10] = self.address[11] = 0xff;
+
+  var octets = ip.split('.');
+  for (var i = 0; i < 4; i++) {
+    self.address[i + 12] = parseInt(octets[i], 10);
+  }
+};
+
+
+var V6_TRANSITIONAL = /:(\d+\.\d+\.\d+\.\d+)$/;
+
+IPCheck.prototype.parseIPv6 = function(ip) {
+  var self = this;
+
+  var transitionalMatch = V6_TRANSITIONAL.exec(ip);
+  if(transitionalMatch){
+    self.parseIPv4(transitionalMatch[1]);
+    return;
+  }
+
+  var bits = ip.split(':');
+  if (bits.length < 8) {
+    ip = ip.replace('::', Array(11 - bits.length).join(':'));
+    bits = ip.split(':');
+  }
+
+  var j = 0;
+  for (var i = 0; i < bits.length; i += 1) {
+    var x = bits[i] ? parseInt(bits[i], 16) : 0;
+    self.address[j++] = x >> 8;
+    self.address[j++] = x & 0xff;
+  }
+};
+
+IPCheck.prototype.match = function(cidr) {
+  var self = this;
+
+  if (!(cidr instanceof IPCheck)) cidr = new IPCheck(cidr);
+  if (!self.valid || !cidr.valid) return false;
+
+  var mask = cidr.mask;
+  var i = 0;
+
+  while (mask >= 8) {
+    if (self.address[i] !== cidr.address[i]) return false;
+
+    i++;
+    mask -= 8;
+  }
+
+  var shift = 8 - mask;
+  return (self.address[i] >>> shift) === (cidr.address[i] >>> shift);
+};
+
+
+IPCheck.match = function(ip, cidr) {
+  ip = ip instanceof IPCheck ? ip : new IPCheck(ip);
+  return ip.match(cidr);
+};

backend/tsconfig.json

@@ -13,7 +13,8 @@
       "node_modules/@types"
     ],
     "allowSyntheticDefaultImports": true,
-    "esModuleInterop": true
+    "esModuleInterop": true,
+    "allowJs": true,
   },
   "include": [
     "src/**/*.ts"

frontend/src/app/lightning/channels-statistics/channels-statistics.component.html

@@ -9,44 +9,44 @@
 <div class="fee-estimation-wrapper" *ngIf="statistics$ | async as statistics; else loadingReward">
 
   <div class="fee-estimation-container" *ngIf="mode === 'avg'">
-    <div class="item">
+    <div class="item" [class]="!statistics.previous ? 'more-padding' : ''">
       <h5 class="card-title" i18n="ln.average-capacity">Avg Capacity</h5>
       <div class="card-text">
-        <div class="fee-text">
+        <div class="fee-text" [class]="!statistics.previous ? 'no-border' : ''">
           {{ statistics.latest?.avg_capacity || 0 | number: '1.0-0' }}
           <span i18n="shared.sat-vbyte|sat/vB">sats</span>
         </div>
-        <span class="fiat">
+        <span class="fiat" *ngIf="statistics.previous">
           <app-change [current]="statistics.latest?.avg_capacity" [previous]="statistics.previous?.avg_capacity"></app-change>
         </span>
       </div>
     </div>
 
-    <div class="item">
+    <div class="item" [class]="!statistics.previous ? 'more-padding' : ''">
       <h5 class="card-title" i18n="ln.average-feerate">Avg Fee Rate</h5>
       <div class="card-text" i18n-ngbTooltip="ln.average-feerate-desc"
         ngbTooltip="The average fee rate charged by routing nodes, ignoring fee rates > 0.5% or 5000ppm"
         placement="bottom">
-        <div class="fee-text">
+        <div class="fee-text" [class]="!statistics.previous ? 'no-border' : ''">
           {{ statistics.latest?.avg_fee_rate || 0 | number: '1.0-0' }}
           <span i18n="shared.sat-vbyte|sat/vB">ppm</span>
         </div>
-        <span class="fiat">
+        <span class="fiat" *ngIf="statistics.previous">
           <app-change [current]="statistics.latest?.avg_fee_rate" [previous]="statistics.previous?.avg_fee_rate"></app-change>
         </span>
       </div>
     </div>
 
-    <div class="item">
+    <div class="item" [class]="!statistics.previous ? 'more-padding' : ''">
       <h5 class="card-title" i18n="ln.average-basefee">Avg Base Fee</h5>
       <div class="card-text" i18n-ngbTooltip="ln.average-basefee-desc"
         ngbTooltip="The average base fee charged by routing nodes, ignoring base fees > 5000ppm" placement="bottom">
         <div class="card-text">
-          <div class="fee-text">
+          <div class="fee-text" [class]="!statistics.previous ? 'no-border' : ''">
             {{ statistics.latest?.avg_base_fee_mtokens || 0 | number: '1.0-0' }}
             <span i18n="shared.sat-vbyte|sat/vB">msats</span>
           </div>
-          <span class="fiat">
+          <span class="fiat" *ngIf="statistics.previous">
             <app-change [current]="statistics.latest?.avg_base_fee_mtokens" [previous]="statistics.previous?.avg_base_fee_mtokens"></app-change>
           </span>
         </div>
@@ -55,43 +55,45 @@
   </div>
 
   <div class="fee-estimation-container" *ngIf="mode === 'med'">
-    <div class="item">
+    <div class="item" [class]="!statistics.previous ? 'more-padding' : ''">
       <h5 class="card-title" i18n="ln.median-capacity">Med Capacity</h5>
       <div class="card-text">
-        <div class="fee-text">
+        <div class="fee-text" [class]="!statistics.previous ? 'no-border' : ''">
           {{ statistics.latest?.med_capacity || 0 | number: '1.0-0' }}
           <span i18n="shared.sat-vbyte|sat/vB">sats</span>
         </div>
-        <span class="fiat">
+        <span class="fiat" *ngIf="statistics.previous">
           <app-change [current]="statistics.latest?.med_capacity" [previous]="statistics.previous?.med_capacity"></app-change>
         </span>
       </div>
     </div>
-    <div class="item">
+
+    <div class="item" [class]="!statistics.previous ? 'more-padding' : ''">
       <h5 class="card-title" i18n="ln.average-feerate">Med Fee Rate</h5>
       <div class="card-text" i18n-ngbTooltip="ln.median-feerate-desc"
-        ngbTooltip="The average fee rate charged by routing nodes, ignoring fee rates > 0.5% or 5000ppm"
+        ngbTooltip="The median fee rate charged by routing nodes, ignoring fee rates > 0.5% or 5000ppm"
         placement="bottom">
-        <div class="fee-text">
+        <div class="fee-text" [class]="!statistics.previous ? 'no-border' : ''">
           {{ statistics.latest?.med_fee_rate || 0 | number: '1.0-0' }}
           <span i18n="shared.sat-vbyte|sat/vB">ppm</span>
         </div>
-        <span class="fiat">
+        <span class="fiat" *ngIf="statistics.previous">
           <app-change [current]="statistics.latest?.med_fee_rate" [previous]="statistics.previous?.med_fee_rate"></app-change>
         </span>
       </div>
     </div>
-    <div class="item">
+
+    <div class="item" [class]="!statistics.previous ? 'more-padding' : ''">
       <h5 class="card-title" i18n="ln.median-basefee">Med Base Fee</h5>
       <div class="card-text" i18n-ngbTooltip="ln.median-basefee-desc"
         ngbTooltip="The median base fee charged by routing nodes, ignoring base fees > 5000ppm" placement="bottom">
         <div class="card-text">
-          <div class="fee-text">
+          <div class="fee-text" [class]="!statistics.previous ? 'no-border' : ''">
             {{ statistics.latest?.med_base_fee_mtokens || 0 | number: '1.0-0' }}
             <span i18n="shared.sat-vbyte|sat/vB">msats</span>
           </div>
         </div>
-        <span class="fiat">
+        <span class="fiat" *ngIf="statistics.previous">
           <app-change [current]="statistics.latest?.med_base_fee_mtokens" [previous]="statistics.previous?.med_base_fee_mtokens"></app-change>
         </span>
       </div>

frontend/src/app/lightning/channels-statistics/channels-statistics.component.scss

@@ -18,6 +18,10 @@
   }
 }
 
+.fee-estimation-wrapper {
+  min-height: 77px;
+}
+
 .fee-estimation-container {
   display: flex;
   justify-content: space-between;
@@ -31,6 +35,9 @@
     @media (min-width: 376px) {
       margin: 0 auto 0px;
     }
+    &.more-padding {
+      padding-top: 10px;
+    }  
     &:first-child{
       display: none;
       @media (min-width: 485px) {
@@ -57,6 +64,9 @@
       margin: auto;
       line-height: 1.45;
       padding: 0px 2px;
+      &.no-border {
+        border-bottom: none;
+      }
     }
     .fiat {
       display: block;

frontend/src/app/lightning/node-statistics/node-statistics.component.html

@@ -1,76 +1,64 @@
 <div class="fee-estimation-wrapper" *ngIf="statistics$ | async as statistics; else loadingReward">
   <div class="fee-estimation-container">
-    <div class="item">
+    <div class="item" [class]="!statistics.previous ? 'more-padding' : ''">
-      <h5 class="card-title" i18n="mining.average-fee">Capacity</h5>
+      <h5 class="card-title" i18n="lightning.capacity">Capacity</h5>
-      <div class="card-text" i18n-ngbTooltip="mining.average-fee" ngbTooltip="Percentage change past week"
+      <div class="card-text" i18n-ngbTooltip="mining.percentage-change-last-week" ngbTooltip="Percentage change past week"
-        placement="bottom">
+        [disableTooltip]="!statistics.previous" placement="bottom">
-        <div class="fee-text">
+        <div class="fee-text" [class]="!statistics.previous ? 'no-border' : ''">
           <app-amount [satoshis]="statistics.latest?.total_capacity" digitsInfo="1.2-2"></app-amount>
         </div>
-        <span class="fiat">
+        <span class="fiat" *ngIf="statistics.previous">
           <app-change [current]="statistics.latest?.total_capacity" [previous]="statistics.previous?.total_capacity">
           </app-change>
         </span>
       </div>
     </div>
-    <div class="item">
+    <div class="item" [class]="!statistics.previous ? 'more-padding' : ''">
-      <h5 class="card-title" i18n="mining.rewards">Nodes</h5>
+      <h5 class="card-title" i18n="lightning.nodes">Nodes</h5>
-      <div class="card-text" i18n-ngbTooltip="mining.rewards-desc" ngbTooltip="Percentage change past week"
+      <div class="card-text" i18n-ngbTooltip="mining.percentage-change-last-week" ngbTooltip="Percentage change past week"
-        placement="bottom">
+      [disableTooltip]="!statistics.previous">
-        <div class="fee-text">
+        <div class="fee-text" [class]="!statistics.previous ? 'no-border' : ''">
           {{ statistics.latest?.node_count || 0 | number }}
         </div>
-        <span class="fiat">
+        <span class="fiat" *ngIf="statistics.previous">
           <app-change [current]="statistics.latest?.node_count" [previous]="statistics.previous?.node_count"></app-change>
         </span>
       </div>
     </div>
-    <div class="item">
+    <div class="item" [class]="!statistics.previous ? 'more-padding' : ''">
-      <h5 class="card-title" i18n="mining.rewards-per-tx">Channels</h5>
+      <h5 class="card-title" i18n="lightning.channels">Channels</h5>
-      <div class="card-text" i18n-ngbTooltip="mining.rewards-per-tx-desc" ngbTooltip="Percentage change past week"
+      <div class="card-text" i18n-ngbTooltip="mining.percentage-change-last-week" ngbTooltip="Percentage change past week"
-        placement="bottom">
+      [disableTooltip]="!statistics.previous">
-        <div class="fee-text">
+        <div class="fee-text" [class]="!statistics.previous ? 'no-border' : ''">
           {{ statistics.latest?.channel_count || 0 | number }}
         </div>
-        <span class="fiat">
+        <span class="fiat" *ngIf="statistics.previous">
           <app-change [current]="statistics.latest?.channel_count" [previous]="statistics.previous?.channel_count">
           </app-change>
         </span>
       </div>
     </div>
-    <!--
-    <div class="item">
-      <h5 class="card-title" i18n="mining.average-fee">Average Channel</h5>
-      <div class="card-text" i18n-ngbTooltip="mining.average-fee"
-        ngbTooltip="Fee paid on average for each transaction in the past 144 blocks" placement="bottom">
-        <app-amount [satoshis]="statistics.latest.average_channel_size" digitsInfo="1.2-3"></app-amount>
-        <span class="fiat">
-          <app-change [current]="statistics.latest.average_channel_size" [previous]="statistics.previous.average_channel_size"></app-change>
-        </span>
-      </div>
-    </div>
-    -->
   </div>
 </div>
 
 <ng-template #loadingReward>
   <div class="fee-estimation-container loading-container">
     <div class="item">
-      <h5 class="card-title" i18n="mining.rewards">Nodes</h5>
+      <h5 class="card-title" i18n="lightning.nodes">Nodes</h5>
       <div class="card-text">
         <div class="skeleton-loader"></div>
         <div class="skeleton-loader"></div>
       </div>
     </div>
     <div class="item">
-      <h5 class="card-title" i18n="mining.rewards-per-tx">Channels</h5>
+      <h5 class="card-title" i18n="lightning.channels">Channels</h5>
       <div class="card-text">
         <div class="skeleton-loader"></div>
         <div class="skeleton-loader"></div>
       </div>
     </div>
     <div class="item">
-      <h5 class="card-title" i18n="mining.average-fee">Average Channel</h5>
+      <h5 class="card-title" i18n="lightning.average-channels">Average Channel</h5>
       <div class="card-text">
         <div class="skeleton-loader"></div>
         <div class="skeleton-loader"></div>

frontend/src/app/lightning/node-statistics/node-statistics.component.scss

@@ -18,6 +18,10 @@
   }
 }
 
+.fee-estimation-wrapper {
+  min-height: 77px;
+}
+
 .fee-estimation-container {
   display: flex;
   justify-content: space-between;
@@ -31,6 +35,9 @@
     @media (min-width: 376px) {
       margin: 0 auto 0px;
     }
+    &.more-padding {
+      padding-top: 10px;
+    }  
     &:first-child{
       display: none;
       @media (min-width: 485px) {
@@ -57,6 +64,9 @@
       margin: auto;
       line-height: 1.45;
       padding: 0px 2px;
+      &.no-border {
+        border-bottom: none;
+      }
     }
     .fiat {
       display: block;

frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts

@@ -238,7 +238,7 @@ export class NodesChannelsMap implements OnInit {
         roam: this.style === 'widget' ? false : true,
         itemStyle: {
           borderColor: 'black',
-          color: '#ffffff44'
+          color: '#272b3f'
         },
         scaleLimit: {
           min: 1.3,

frontend/src/app/lightning/nodes-channels/node-channels.component.ts

@@ -44,13 +44,13 @@ export class NodeChannels implements OnChanges {
         switchMap((response) => {
           this.isLoading = true;
           if ((response.body?.length ?? 0) <= 0) {
-            return [];
+            this.isLoading = false;
+            return [''];
           }
           return [response.body];
         }),
         tap((body: any[]) => {
-          if (body.length === 0) {
+          if (body.length === 0 || body[0].length === 0) {
-            this.isLoading = false;
             return;
           }
           const biggestCapacity = body[0].capacity;
@@ -130,10 +130,6 @@ export class NodeChannels implements OnChanges {
   }
 
   onChartInit(ec: ECharts): void {
-    if (this.chartInstance !== undefined) {
-      return;
-    }
-
     this.chartInstance = ec;
 
     this.chartInstance.on('click', (e) => {

frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html

@@ -28,7 +28,7 @@
 
   <div class="card-header" *ngIf="!widget">
     <div class="d-flex d-md-block align-items-baseline" style="margin-bottom: -5px">
-      <span i18n="lightning.top-100-isp-ln">Top 100 ISP hosting LN nodes</span>
+      <span i18n="lightning.top-100-isp-ln">Top 100 ISPs hosting LN nodes</span>
       <button class="btn p-0 pl-2" style="margin: 0 0 4px 0px" (click)="onSaveChart()">
         <fa-icon [icon]="['fas', 'download']" [fixedWidth]="true"></fa-icon>
       </button>

frontend/src/index.bisq.html

@@ -5,7 +5,7 @@
   <title>mempool - Bisq Markets</title>
   <base href="/">
 
-  <meta name="description" content="The Mempool Open Source Project™ - our self-hosted explorer for the Bisq Network.">
+  <meta name="description" content="The Mempool Open Source Project™ - Explore the full Bitcoin ecosystem.">
 
   <meta property="og:image" content="https://bisq.markets/resources/bisq/bisq-markets-preview.png" />
   <meta property="og:image:type" content="image/jpeg" />
@@ -14,7 +14,7 @@
   <meta property="twitter:site" content="https://bisq.markets/">
   <meta property="twitter:creator" content="@bisq_network">
   <meta property="twitter:title" content="The Mempool Open Source Project™">
-  <meta property="twitter:description" content="Our self-hosted markets explorer for the Bisq community.">
+  <meta property="twitter:description" content="Explore the full Bitcoin ecosystem with mempool.space™" />
   <meta property="twitter:image:src" content="https://bisq.markets/resources/bisq/bisq-markets-preview.png" />
   <meta property="twitter:domain" content="bisq.markets">
 

frontend/src/index.liquid.html

@@ -5,7 +5,7 @@
   <title>mempool - Liquid Network</title>
   <base href="/">
 
-  <meta name="description" content="The Mempool Open Source Project™ - our self-hosted explorer for the Liquid Network.">
+  <meta name="description" content="The Mempool Open Source Project™ - Explore the full Bitcoin ecosystem.">
   <meta property="og:image" content="https://liquid.network/resources/liquid/liquid-network-preview.png" />
   <meta property="og:image:type" content="image/png" />
   <meta property="og:image:width" content="1000" />
@@ -14,7 +14,7 @@
   <meta property="twitter:site" content="@mempool">
   <meta property="twitter:creator" content="@mempool">
   <meta property="twitter:title" content="The Mempool Open Source Project™">
-  <meta property="twitter:description" content="Our self-hosted network explorer for the Liquid community.">
+  <meta property="twitter:description" content="Explore the full Bitcoin ecosystem with mempool.space™" />
   <meta property="twitter:image:src" content="https://liquid.network/resources/liquid/liquid-network-preview.png" />
   <meta property="twitter:domain" content="liquid.network">
   

frontend/src/index.mempool.html

@@ -5,7 +5,7 @@
   <title>mempool - Bitcoin Explorer</title>
   <base href="/">
 
-  <meta name="description" content="The Mempool Open Source Project™ - our self-hosted explorer for the Bitcoin community." />
+  <meta name="description" content="The Mempool Open Source Project™ - Explore the full Bitcoin ecosystem." />
   <meta property="og:image" content="https://mempool.space/resources/mempool-space-preview.png" />
   <meta property="og:image:type" content="image/png" />
   <meta property="og:image:width" content="1000" />
@@ -14,7 +14,7 @@
   <meta property="twitter:site" content="@mempool">
   <meta property="twitter:creator" content="@mempool">
   <meta property="twitter:title" content="The Mempool Open Source Project™">
-  <meta property="twitter:description" content="Our self-hosted mempool explorer for the Bitcoin community." />
+  <meta property="twitter:description" content="Explore the full Bitcoin ecosystem with mempool.space™" />
   <meta property="twitter:image:src" content="https://mempool.space/resources/mempool-space-preview.png" />
   <meta property="twitter:domain" content="mempool.space">
   

production/install

@@ -48,6 +48,9 @@ BITCOIN_MAINNET_ENABLE=ON
 BITCOIN_MAINNET_MINFEE_ENABLE=ON
 BITCOIN_TESTNET_ENABLE=ON
 BITCOIN_SIGNET_ENABLE=ON
+LN_BITCOIN_MAINNET_ENABLE=ON
+LN_BITCOIN_TESTNET_ENABLE=ON
+LN_BITCOIN_SIGNET_ENABLE=ON
 BISQ_MAINNET_ENABLE=ON
 ELEMENTS_LIQUID_ENABLE=ON
 ELEMENTS_LIQUIDTESTNET_ENABLE=ON
@@ -227,6 +230,9 @@ MYSQL_GROUP=mysql
 MEMPOOL_MAINNET_USER='mempool'
 MEMPOOL_TESTNET_USER='mempool_testnet'
 MEMPOOL_SIGNET_USER='mempool_signet'
+LN_MEMPOOL_MAINNET_USER='mempool_mainnet_lightning'
+LN_MEMPOOL_TESTNET_USER='mempool_testnet_lightning'
+LN_MEMPOOL_SIGNET_USER='mempool_signet_lightning'
 MEMPOOL_LIQUID_USER='mempool_liquid'
 MEMPOOL_LIQUIDTESTNET_USER='mempool_liquidtestnet'
 MEMPOOL_BISQ_USER='mempool_bisq'
@@ -234,6 +240,9 @@ MEMPOOL_BISQ_USER='mempool_bisq'
 MEMPOOL_MAINNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}')
 MEMPOOL_TESTNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}')
 MEMPOOL_SIGNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}')
+LN_MEMPOOL_MAINNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}')
+LN_MEMPOOL_TESTNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}')
+LN_MEMPOOL_SIGNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}')
 MEMPOOL_LIQUID_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}')
 MEMPOOL_LIQUIDTESTNET_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}')
 MEMPOOL_BISQ_PASS=$(head -150 /dev/urandom | ${MD5} | awk '{print $1}')
@@ -391,6 +400,10 @@ FREEBSD_PKG+=(boost-libs autoconf automake gmake gcc libevent libtool pkgconf)
 FREEBSD_PKG+=(nginx rsync py39-certbot-nginx mariadb105-server keybase)
 FREEBSD_PKG+=(geoipupdate)
 
+FREEBSD_UNFURL_PKG=()
+FREEBSD_UNFURL_PKG+=(nvidia-driver-470-470.129.06 chromium xinit xterm twm ja-sourcehansans-otf)
+FREEBSD_UNFURL_PKG+=(zh-sourcehansans-sc-otf ko-aleefonts-ttf lohit tlwg-ttf)
+
 #############################
 ##### utility functions #####
 #############################
@@ -747,6 +760,9 @@ $CUT >$input <<-EOF
 Tor:Enable Tor v3 HS Onion:ON
 Mainnet:Enable Bitcoin Mainnet:ON
 Mainnet-Minfee:Enable Bitcoin Mainnet Minfee:ON
+LN-Mainnet:Enable Bitcoin Mainnet Lightning:ON
+LN-Testnet:Enable Bitcoin Testnet Lightning:ON
+LN-Signet:Enable Bitcoin Signet Lightning:ON
 Testnet:Enable Bitcoin Testnet:ON
 Signet:Enable Bitcoin Signet:ON
 Liquid:Enable Elements Liquid:ON
@@ -809,6 +825,24 @@ else
     BITCOIN_INSTALL=OFF
 fi
 
+if grep LN-Mainnet $tempfile >/dev/null 2>&1;then
+    LN_BITCOIN_MAINNET_ENABLE=ON
+else
+    LN_BITCOIN_MAINNET_ENABLE=OFF
+fi
+
+if grep LN-Testnet $tempfile >/dev/null 2>&1;then
+    LN_BITCOIN_TESTNET_ENABLE=ON
+else
+    LN_BITCOIN_TESTNET_ENABLE=OFF
+fi
+
+if grep LN-Signet $tempfile >/dev/null 2>&1;then
+    LN_BITCOIN_SIGNET_ENABLE=ON
+else
+    LN_BITCOIN_SIGNET_ENABLE=OFF
+fi
+
 if grep Liquid $tempfile >/dev/null 2>&1;then
     ELEMENTS_LIQUID_ENABLE=ON
 else
@@ -831,6 +865,7 @@ if grep CoreLN $tempfile >/dev/null 2>&1;then
     CLN_INSTALL=ON
 else
     CLN_INSTALL=OFF
+fi
 
 if [ "${BITCOIN_MAINNET_ENABLE}" = ON -o "${BITCOIN_TESTNET_ENABLE}" = ON -o "${BITCOIN_SIGNET_ENABLE}" = ON ];then
     BITCOIN_ELECTRS_INSTALL=ON
@@ -1279,8 +1314,11 @@ case $OS in
         echo "[*] Creating Core Lightning user"
         osGroupCreate "${CLN_GROUP}"
         osUserCreate "${CLN_USER}" "${CLN_HOME}" "${CLN_GROUP}"
+        osSudo "${ROOT_USER}" pw usermod ${MEMPOOL_USER} -G "${CLN_GROUP}"
         osSudo "${ROOT_USER}" chsh -s `which zsh` "${CLN_USER}"
         echo "export PATH=$PATH:$HOME/.local/bin" >> "${CLN_HOME}/.zshrc"
+        osSudo "${ROOT_USER}" mkdir -p "${CLN_HOME}/.lightning/{bitcoin,signet,testnet}"
+        osSudo "${ROOT_USER}" chmod 750 "${CLN_HOME}" "${CLN_HOME}/.lightning" "${CLN_HOME}/.lightning/{bitcoin,signet,testnet}"
         osSudo "${ROOT_USER}" chown -R "${CLN_USER}:${CLN_GROUP}" "${CLN_HOME}"
 
         echo "[*] Installing Core Lightning package"
@@ -1397,7 +1435,42 @@ if [ "${UNFURL_INSTALL}" = ON ];then
     case $OS in
 
         FreeBSD)
-            echo "[*] FIXME: Unfurl must be installed manually on FreeBSD"
+
+            if pciconf -lv|grep -i nvidia >/dev/null 2>&1;then
+                echo "[*] GPU detected: Installing packages for Unfurl"
+                osPackageInstall ${FREEBSD_UNFURL_PKG[@]}
+
+                echo 'allowed_users = anybody' >> /usr/local/etc/X11/Xwrapper.config
+                echo 'kld_list="nvidia"' >> /etc/rc.conf
+                echo 'nvidia_xorg_enable="YES"' >> /etc/rc.conf
+
+                echo "[*] Installing color emoji"
+                osSudo "${ROOT_USER}" curl "https://github.com/samuelngs/apple-emoji-linux/releases/download/ios-15.4/AppleColorEmoji.ttf" -o /usr/local/share/fonts/TTF/AppleColorEmoji.ttf
+                cat >> /usr/local/etc/fonts/conf.d/01-emoji.conf <<EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE fontconfig SYSTEM "fonts.dtd">
+<fontconfig>
+<match>
+<test name="family"><string>sans-serif</string></test>
+<edit name="family" mode="prepend" binding="strong">
+<string>Apple Color Emoji</string>
+</edit>
+</match>
+<match>
+<test name="family"><string>serif</string></test>
+<edit name="family" mode="prepend" binding="strong">
+<string>Apple Color Emoji</string>
+</edit>
+</match>
+<match>
+<test name="family"><string>Apple Color Emoji</string></test>
+<edit name="family" mode="prepend" binding="strong">
+<string>Apple Color Emoji</string>
+</edit>
+</match>
+</fontconfig>
+EOF
+            fi
         ;;
 
         Debian)
@@ -1671,7 +1744,16 @@ if [ "${BITCOIN_MAINNET_ENABLE}" = ON -o "${BITCOIN_TESTNET_ENABLE}" = ON -o "${
     osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/mainnet"
 
     echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Bitcoin Mainnet"
-    osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}"
+    osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/mainnet && git checkout ${MEMPOOL_LATEST_RELEASE}"
+fi
+
+if [ "${LN_BITCOIN_MAINNET_ENABLE}" = ON ];then
+    echo "[*] Creating Mempool instance for Lightning Network on Bitcoin Mainnet"
+    osSudo "${MEMPOOL_USER}" git config --global advice.detachedHead false
+    osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/mainnet-lightning"
+
+    echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Lightning Network on Bitcoin Mainnet"
+    osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/mainnet-lightning && git checkout ${MEMPOOL_LATEST_RELEASE}"
 fi
 
 if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then
@@ -1680,7 +1762,16 @@ if [ "${BITCOIN_TESTNET_ENABLE}" = ON ];then
     osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/testnet"
 
     echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Bitcoin Testnet"
-    osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}"
+    osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/testnet && git checkout ${MEMPOOL_LATEST_RELEASE}"
+fi
+
+if [ "${LN_BITCOIN_TESTNET_ENABLE}" = ON ];then
+    echo "[*] Creating Mempool instance for Lightning Network on Bitcoin Testnet"
+    osSudo "${MEMPOOL_USER}" git config --global advice.detachedHead false
+    osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/testnet-lightning"
+
+    echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Lightning Network on Bitcoin Testnet"
+    osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/testnet-lightning && git checkout ${MEMPOOL_LATEST_RELEASE}"
 fi
 
 if [ "${BITCOIN_SIGNET_ENABLE}" = ON ];then
@@ -1689,7 +1780,16 @@ if [ "${BITCOIN_SIGNET_ENABLE}" = ON ];then
     osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/signet"
 
     echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Bitcoin Signet"
-    osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}"
+    osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/signet && git checkout ${MEMPOOL_LATEST_RELEASE}"
+fi
+
+if [ "${LN_BITCOIN_SIGNET_ENABLE}" = ON ];then
+    echo "[*] Creating Mempool instance for Lightning Network on Bitcoin Signet"
+    osSudo "${MEMPOOL_USER}" git config --global advice.detachedHead false
+    osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/signet-lightning"
+
+    echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Lightning Network on Bitcoin Signet"
+    osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/signet-lightning && git checkout ${MEMPOOL_LATEST_RELEASE}"
 fi
 
 if [ "${ELEMENTS_LIQUID_ENABLE}" = ON ];then
@@ -1698,7 +1798,7 @@ if [ "${ELEMENTS_LIQUID_ENABLE}" = ON ];then
     osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/liquid"
 
     echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Liquid"
-    osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}"
+    osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/liquid && git checkout ${MEMPOOL_LATEST_RELEASE}"
 fi
 
 if [ "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then
@@ -1707,7 +1807,7 @@ if [ "${ELEMENTS_LIQUIDTESTNET_ENABLE}" = ON ];then
     osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/liquidtestnet"
 
     echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Liquid Testnet"
-    osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}"
+    osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/liquidtestnet && git checkout ${MEMPOOL_LATEST_RELEASE}"
 fi
 
 if [ "${BISQ_INSTALL}" = ON ];then
@@ -1716,7 +1816,7 @@ if [ "${BISQ_INSTALL}" = ON ];then
     osSudo "${MEMPOOL_USER}" git clone --branch "${MEMPOOL_REPO_BRANCH}" "${MEMPOOL_REPO_URL}" "${MEMPOOL_HOME}/bisq"
 
     echo "[*] Checking out Mempool ${MEMPOOL_LATEST_RELEASE} for Bisq"
-    osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME} && git checkout ${MEMPOOL_LATEST_RELEASE}"
+    osSudo "${MEMPOOL_USER}" sh -c "cd ${MEMPOOL_HOME}/bisq && git checkout ${MEMPOOL_LATEST_RELEASE}"
 fi
 
 ##### mariadb
@@ -1742,6 +1842,15 @@ grant all on mempool_testnet.* to '${MEMPOOL_TESTNET_USER}'@'localhost' identifi
 create database mempool_signet;
 grant all on mempool_signet.* to '${MEMPOOL_SIGNET_USER}'@'localhost' identified by '${MEMPOOL_SIGNET_PASS}';
 
+create database mempool_mainnet_lightning;
+grant all on mempool_mainnet_lightning.* to '${LN_MEMPOOL_MAINNET_USER}'@'%' identified by '${LN_MEMPOOL_MAINNET_PASS}';
+
+create database mempool_testnet_lightning;
+grant all on mempool_testnet_lightning.* to '${LN_MEMPOOL_TESTNET_USER}'@'%' identified by '${LN_MEMPOOL_TESTNET_PASS}';
+
+create database mempool_signet_lightning;
+grant all on mempool_signet_lightning.* to '${LN_MEMPOOL_SIGNET_USER}'@'%' identified by '${LN_MEMPOOL_SIGNET_PASS}';
+
 create database mempool_liquid;
 grant all on mempool_liquid.* to '${MEMPOOL_LIQUID_USER}'@'localhost' identified by '${MEMPOOL_LIQUID_PASS}';
 
@@ -1760,6 +1869,12 @@ declare -x MEMPOOL_TESTNET_USER="${MEMPOOL_TESTNET_USER}"
 declare -x MEMPOOL_TESTNET_PASS="${MEMPOOL_TESTNET_PASS}"
 declare -x MEMPOOL_SIGNET_USER="${MEMPOOL_SIGNET_USER}"
 declare -x MEMPOOL_SIGNET_PASS="${MEMPOOL_SIGNET_PASS}"
+declare -x LN_MEMPOOL_MAINNET_USER="${LN_MEMPOOL_MAINNET_USER}"
+declare -x LN_MEMPOOL_MAINNET_PASS="${LN_MEMPOOL_MAINNET_PASS}"
+declare -x LN_MEMPOOL_TESTNET_USER="${LN_MEMPOOL_TESTNET_USER}"
+declare -x LN_MEMPOOL_TESTNET_PASS="${LN_MEMPOOL_TESTNET_PASS}"
+declare -x LN_MEMPOOL_SIGNET_USER="${LN_MEMPOOL_SIGNET_USER}"
+declare -x LN_MEMPOOL_SIGNET_PASS="${LN_MEMPOOL_SIGNET_PASS}"
 declare -x MEMPOOL_LIQUID_USER="${MEMPOOL_LIQUID_USER}"
 declare -x MEMPOOL_LIQUID_PASS="${MEMPOOL_LIQUID_PASS}"
 declare -x MEMPOOL_LIQUIDTESTNET_USER="${MEMPOOL_LIQUIDTESTNET_USER}"
@@ -1770,6 +1885,12 @@ _EOF_
 
 ##### nginx
 
+case $OS in
+
+    FreeBSD)
+    ;;
+
+Debian)
         echo "[*] Adding Nginx configuration"
         osSudo "${ROOT_USER}" install -c -o "${ROOT_USER}" -g "${ROOT_GROUP}" -m 644 "${MEMPOOL_HOME}/${MEMPOOL_REPO_NAME}/production/nginx/nginx.conf" "${NGINX_CONFIGURATION}"
         mkdir -p /var/cache/nginx/services /var/cache/nginx/api
@@ -1788,6 +1909,8 @@ echo "[*] Read tor v3 onion hostnames"
         fi
         echo "[*] Restarting Nginx"
         osSudo "${ROOT_USER}" service nginx restart
+    ;;
+esac
 
 ##### OS systemd
 

production/mempool-build-all

@@ -98,6 +98,12 @@ build_backend()
         -e "s!__MEMPOOL_TESTNET_PASS__!${MEMPOOL_TESTNET_PASS}!" \
         -e "s!__MEMPOOL_SIGNET_USER__!${MEMPOOL_SIGNET_USER}!" \
         -e "s!__MEMPOOL_SIGNET_PASS__!${MEMPOOL_SIGNET_PASS}!" \
+        -e "s!__LN_MEMPOOL_MAINNET_USER__!${LN_MEMPOOL_MAINNET_USER}!" \
+        -e "s!__LN_MEMPOOL_MAINNET_PASS__!${LN_MEMPOOL_MAINNET_PASS}!" \
+        -e "s!__LN_MEMPOOL_TESTNET_USER__!${LN_MEMPOOL_TESTNET_USER}!" \
+        -e "s!__LN_MEMPOOL_TESTNET_PASS__!${LN_MEMPOOL_TESTNET_PASS}!" \
+        -e "s!__LN_MEMPOOL_SIGNET_USER__!${LN_MEMPOOL_SIGNET_USER}!" \
+        -e "s!__LN_MEMPOOL_SIGNET_PASS__!${LN_MEMPOOL_SIGNET_PASS}!" \
         -e "s!__MEMPOOL_LIQUID_USER__!${MEMPOOL_LIQUID_USER}!" \
         -e "s!__MEMPOOL_LIQUID_PASS__!${MEMPOOL_LIQUID_PASS}!" \
         -e "s!__MEMPOOL_LIQUIDTESTNET_USER__!${LIQUIDTESTNET_USER}!" \
@@ -145,7 +151,7 @@ for repo in $backend_repos;do
 done
 
 # build unfurlers
-for repo in mainnet liquid;do
+for repo in mainnet liquid bisq;do
     build_unfurler "${repo}"
 done
 

production/mempool-start-all

@@ -9,17 +9,19 @@ for site in mainnet mainnet-lightning testnet testnet-lightning signet signet-li
     screen -dmS "${site}" sh -c 'while true;do npm run start-production;sleep 1;done'
 done
 
-# only start unfurler if GPU present
+# only start xorg if GPU present
 if pciconf -lv|grep -i nvidia >/dev/null 2>&1;then
     export DISPLAY=:0
     screen -dmS x startx
     sleep 3
-    for site in mainnet liquid;do
+fi
+
+# start unfurlers for each frontend
+for site in mainnet liquid bisq;do
     cd "$HOME/${site}/unfurler" && \
     echo "starting mempool unfurler: ${site}" && \
     screen -dmS "unfurler-${site}" sh -c 'while true;do npm run unfurler;sleep 2;done'
 done
-fi
 
 # start nginx warm cacher
 for site in mainnet;do

production/nginx-cache-warmer

@@ -99,7 +99,7 @@ do for url in / \
 	'/api/v1/lightning/nodes/isp/39572' `# DataWeb` \
 	'/api/v1/lightning/nodes/isp/14061' `# Digital Ocean` \
 	'/api/v1/lightning/nodes/isp/24940,213230' `# Hetzner` \
-	'/api/v1/lightning/nodes/isp/174' `# LunaNode` \
+	'/api/v1/lightning/nodes/isp/394745' `# LunaNode` \
 	'/api/v1/lightning/nodes/isp/45102' `# Alibaba` \
 	'/api/v1/lightning/nodes/isp/3209' `# Vodafone Germany` \
 	'/api/v1/lightning/nodes/isp/7922' `# Comcast Cable` \

production/nginx/server-common.conf

@@ -48,6 +48,9 @@ add_header Vary Cookie;
 # for exact / requests, redirect based on $lang
 # cache redirect for 5 minutes
 location = / {
+	if ($unfurlbot) {
+		proxy_pass $mempoolSpaceUnfurler;
+	}
 	if ($lang != '') {
 		return 302 $scheme://$host/$lang/;
 	}

production/unfurler-config.bisq.json

@@ -0,0 +1,17 @@
+{
+  "SERVER": {
+    "HOST": "https://bisq.fra.mempool.space",
+    "HTTP_PORT": 8002
+  },
+  "MEMPOOL": {
+    "HTTP_HOST": "http://127.0.0.1",
+    "HTTP_PORT": 82,
+    "NETWORK": "bisq"
+  },
+  "PUPPETEER": {
+    "CLUSTER_SIZE": 8,
+    "EXEC_PATH": "/usr/local/bin/chrome",
+    "MAX_PAGE_AGE": 86400,
+    "RENDER_TIMEOUT": 3000
+  }
+}

production/unfurler-config.liquid.json

@@ -1,12 +1,12 @@
 {
   "SERVER": {
-    "HOST": "https://liquid.network",
+    "HOST": "https://liquid.fra.mempool.space",
-    "HTTP_PORT": 8002
+    "HTTP_PORT": 8003
   },
   "MEMPOOL": {
-    "HTTP_HOST": "https://liquid.network",
+    "HTTP_HOST": "http://127.0.0.1",
-    "HTTP_PORT": 443,
+    "HTTP_PORT": 83,
-    "NETWORK": "liquid"
+    "NETWORK": "bitcoin"
   },
   "PUPPETEER": {
     "CLUSTER_SIZE": 8,

production/unfurler-config.mainnet.json

@@ -1,11 +1,11 @@
 {
   "SERVER": {
-    "HOST": "https://mempool.space",
+    "HOST": "https://mempool.fra.mempool.space",
     "HTTP_PORT": 8001
   },
   "MEMPOOL": {
-    "HTTP_HOST": "https://mempool.space",
+    "HTTP_HOST": "http://127.0.0.1",
-    "HTTP_PORT": 443,
+    "HTTP_PORT": 81,
     "NETWORK": "bitcoin"
   },
   "PUPPETEER": {

unfurler/config.sample.json

@@ -9,6 +9,7 @@
     "NETWORK": "bitcoin" // "bitcoin" | "liquid" | "bisq" (optional - defaults to "bitcoin")
   },
   "PUPPETEER": {
+    "DISABLE": false, // optional, boolean, disables puppeteer and /render endpoints
     "CLUSTER_SIZE": 2,
     "EXEC_PATH": "/usr/local/bin/chrome", // optional
     "MAX_PAGE_AGE": 86400, // maximum lifetime of a page session (in seconds)

unfurler/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "mempool-unfurl",
-  "version": "0.0.1",
+  "version": "0.1.0",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "mempool-unfurl",
-      "version": "0.0.1",
+      "version": "0.1.0",
       "dependencies": {
         "@types/node": "^16.11.41",
         "express": "^4.18.0",

unfurler/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mempool-unfurl",
-  "version": "0.0.2",
+  "version": "0.1.0",
   "description": "Renderer for mempool open graph link preview images",
   "repository": {
     "type": "git",

unfurler/src/config.ts

@@ -11,6 +11,7 @@ interface IConfig {
     NETWORK?: string;
   };
   PUPPETEER: {
+    DISABLE: boolean;
     CLUSTER_SIZE: number;
     EXEC_PATH?: string;
     MAX_PAGE_AGE?: number;
@@ -28,6 +29,7 @@ const defaults: IConfig = {
     'HTTP_PORT': 4200,
   },
   'PUPPETEER': {
+    'DISABLE': false,
     'CLUSTER_SIZE': 1,
   },
 };

unfurler/src/index.ts

@@ -1,10 +1,12 @@
 import express from "express";
 import { Application, Request, Response, NextFunction } from 'express';
 import * as http from 'http';
+import * as https from 'https';
 import config from './config';
 import { Cluster } from 'puppeteer-cluster';
 import ReusablePage from './concurrency/ReusablePage';
 import { parseLanguageUrl } from './language/lang';
+import { matchRoute } from './routes';
 const puppeteerConfig = require('../puppeteer.config.json');
 
 if (config.PUPPETEER.EXEC_PATH) {
@@ -17,13 +19,13 @@ class Server {
   cluster?: Cluster;
   mempoolHost: string;
   network: string;
-  defaultImageUrl: string;
+  secureHost = true;
 
   constructor() {
     this.app = express();
     this.mempoolHost = config.MEMPOOL.HTTP_HOST + (config.MEMPOOL.HTTP_PORT ? ':' + config.MEMPOOL.HTTP_PORT : '');
+    this.secureHost = this.mempoolHost.startsWith('https');
     this.network = config.MEMPOOL.NETWORK || 'bitcoin';
-    this.defaultImageUrl = this.getDefaultImageUrl();
     this.startServer();
   }
 
@@ -37,12 +39,14 @@ class Server {
       .use(express.text())
       ;
 
+    if (!config.PUPPETEER.DISABLE) {
       this.cluster = await Cluster.launch({
           concurrency: ReusablePage,
           maxConcurrency: config.PUPPETEER.CLUSTER_SIZE,
           puppeteerOptions: puppeteerConfig,
       });
       await this.cluster?.task(async (args) => { return this.clusterTask(args) });
+    }
 
     this.setUpRoutes();
 
@@ -64,7 +68,11 @@ class Server {
   }
 
   setUpRoutes() {
+    if (!config.PUPPETEER.DISABLE) {
       this.app.get('/render*', async (req, res) => { return this.renderPreview(req, res) })
+    } else {
+      this.app.get('/render*', async (req, res) => { return this.renderDisabled(req, res) })
+    }
     this.app.get('*', (req, res) => { return this.renderHTML(req, res) })
   }
 
@@ -111,13 +119,31 @@ class Server {
     }
   }
 
+  async renderDisabled(req, res) {
+    res.status(500).send("preview rendering disabled");
+  }
+
   async renderPreview(req, res) {
     try {
-      const path = req.params[0]
+      const rawPath = req.params[0];
-      const img = await this.cluster?.execute({ url: this.mempoolHost + path, path: path, action: 'screenshot' });
+
+      let img = null;
+
+      const { lang, path } = parseLanguageUrl(rawPath);
+      const matchedRoute = matchRoute(this.network, path);
+
+      // don't bother unless the route is definitely renderable
+      if (rawPath.includes('/preview/') && matchedRoute.render) {
+        img = await this.cluster?.execute({ url: this.mempoolHost + rawPath, path: rawPath, action: 'screenshot' });
+      }
 
       if (!img) {
-        res.status(500).send('failed to render page preview');
+        // proxy fallback image from the frontend
+        if (this.secureHost) {
+          https.get(config.SERVER.HOST + matchedRoute.fallbackImg, (got) => got.pipe(res));
+        } else {
+          http.get(config.SERVER.HOST + matchedRoute.fallbackImg, (got) => got.pipe(res));
+        }
       } else {
         res.contentType('image/png');
         res.send(img);
@@ -137,50 +163,14 @@ class Server {
       return;
     }
 
-    let previewSupported = true;
-    let mode = 'mainnet'
-    let ogImageUrl = this.defaultImageUrl;
-    let ogTitle;
     const { lang, path } = parseLanguageUrl(rawPath);
-    const parts = path.slice(1).split('/');
+    const matchedRoute = matchRoute(this.network, path);
+    let ogImageUrl = config.SERVER.HOST + (matchedRoute.staticImg || matchedRoute.fallbackImg);
+    let ogTitle = 'The Mempool Open Source Project™';
 
-    // handle network mode modifiers
+    if (matchedRoute.render) {
-    if (['testnet', 'signet'].includes(parts[0])) {
-      mode = parts.shift();
-    }
-
-    // handle supported preview routes
-    switch (parts[0]) {
-      case 'block':
-        ogTitle = `Block: ${parts[1]}`;
-      break;
-      case 'address':
-        ogTitle = `Address: ${parts[1]}`;
-      break;
-      case 'tx':
-        ogTitle = `Transaction: ${parts[1]}`;
-      break;
-      case 'lightning':
-        switch (parts[1]) {
-          case 'node':
-            ogTitle = `Lightning Node: ${parts[2]}`;
-          break;
-          case 'channel':
-            ogTitle = `Lightning Channel: ${parts[2]}`;
-          break;
-          default:
-            previewSupported = false;
-        }
-      break;
-      default:
-        previewSupported = false;
-    }
-
-    if (previewSupported) {
       ogImageUrl = `${config.SERVER.HOST}/render/${lang || 'en'}/preview${path}`;
-      ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${mode !== 'mainnet' ? capitalize(mode) + ' ' : ''}${ogTitle}`;
+      ogTitle = `${this.network ? capitalize(this.network) + ' ' : ''}${matchedRoute.networkMode !== 'mainnet' ? capitalize(matchedRoute.networkMode) + ' ' : ''}${matchedRoute.title}`;
-    } else {
-      ogTitle = 'The Mempool Open Source Project™';
     }
 
     res.send(`
@@ -189,34 +179,23 @@ class Server {
       <head>
         <meta charset="utf-8">
         <title>${ogTitle}</title>
-        <meta name="description" content="The Mempool Open Source Project™ - our self-hosted explorer for the ${capitalize(this.network)} community."/>
+        <meta name="description" content="The Mempool Open Source Project™ - Explore the full Bitcoin ecosystem with mempool.space™"/>
         <meta property="og:image" content="${ogImageUrl}"/>
         <meta property="og:image:type" content="image/png"/>
-        <meta property="og:image:width" content="${previewSupported ? 1200 : 1000}"/>
+        <meta property="og:image:width" content="${matchedRoute.render ? 1200 : 1000}"/>
-        <meta property="og:image:height" content="${previewSupported ? 600 : 500}"/>
+        <meta property="og:image:height" content="${matchedRoute.render ? 600 : 500}"/>
         <meta property="og:title" content="${ogTitle}">
         <meta property="twitter:card" content="summary_large_image">
         <meta property="twitter:site" content="@mempool">
         <meta property="twitter:creator" content="@mempool">
         <meta property="twitter:title" content="${ogTitle}">
-        <meta property="twitter:description" content="Our self-hosted mempool explorer for the ${capitalize(this.network)} community."/>
+        <meta property="twitter:description" content="Explore the full Bitcoin ecosystem with mempool.space"/>
         <meta property="twitter:image:src" content="${ogImageUrl}"/>
         <meta property="twitter:domain" content="mempool.space">
       <body></body>
       </html>
     `);
   }
-
-  getDefaultImageUrl() {
-    switch (this.network) {
-      case 'liquid':
-        return this.mempoolHost + '/resources/liquid/liquid-network-preview.png';
-      case 'bisq':
-        return this.mempoolHost + '/resources/bisq/bisq-markets-preview.png';
-      default:
-        return this.mempoolHost + '/resources/mempool-space-preview.png';
-    }
-  }
 }
 
 const server = new Server();

unfurler/src/routes.ts

@@ -0,0 +1,124 @@
+interface Match {
+  render: boolean;
+  title: string;
+  fallbackImg: string;
+  staticImg?: string;
+  networkMode: string;
+}
+
+const routes = {
+  block: {
+    render: true,
+    params: 1,
+    getTitle(path) {
+      return `Block: ${path[0]}`;
+    }
+  },
+  address: {
+    render: true,
+    params: 1,
+    getTitle(path) {
+      return `Address: ${path[0]}`;
+    }
+  },
+  tx: {
+    render: true,
+    params: 1,
+    getTitle(path) {
+      return `Transaction: ${path[0]}`;
+    }
+  },
+  lightning: {
+    title: "Lightning",
+    fallbackImg: '/resources/previews/lightning.png',
+    routes: {
+      node: {
+        render: true,
+        params: 1,
+        getTitle(path) {
+          return `Lightning Node: ${path[0]}`;
+        }
+      },
+      channel: {
+        render: true,
+        params: 1,
+        getTitle(path) {
+          return `Lightning Channel: ${path[0]}`;
+        }
+      },
+    }
+  },
+  mining: {
+    title: "Mining",
+    fallbackImg: '/resources/previews/mining.png'
+  }
+};
+
+const networks = {
+  bitcoin: {
+    fallbackImg: '/resources/mempool-space-preview.png',
+    staticImg: '/resources/previews/dashboard.png',
+    routes: {
+      ...routes // all routes supported
+    }
+  },
+  liquid: {
+    fallbackImg: '/resources/liquid/liquid-network-preview.png',
+    routes: { // only block, address & tx routes supported
+      block: routes.block,
+      address: routes.address,
+      tx: routes.tx
+    }
+  },
+  bisq: {
+    fallbackImg: '/resources/bisq/bisq-markets-preview.png',
+    routes: {} // no routes supported
+  }
+};
+
+export function matchRoute(network: string, path: string): Match {
+  const match: Match = {
+    render: false,
+    title: '',
+    fallbackImg: '',
+    networkMode: 'mainnet'
+  }
+
+  const parts = path.slice(1).split('/').filter(p => p.length);
+
+  if (parts[0] === 'preview') {
+    parts.shift();
+  }
+  if (['testnet', 'signet'].includes(parts[0])) {
+    match.networkMode = parts.shift() || 'mainnet';
+  }
+
+  let route = networks[network] || networks.bitcoin;
+  match.fallbackImg = route.fallbackImg;
+
+  // traverse the route tree until we run out of route or tree, or hit a renderable match
+  while (!route.render && route.routes && parts.length && route.routes[parts[0]]) {
+    route = route.routes[parts[0]];
+    parts.shift();
+    if (route.fallbackImg) {
+      match.fallbackImg = route.fallbackImg;
+    }
+  }
+
+  // enough route parts left for title & rendering
+  if (route.render && parts.length >= route.params) {
+    match.render = true;
+  }
+  // only use set a static image for exact matches
+  if (!parts.length && route.staticImg) {
+    match.staticImg = route.staticImg;
+  }
+  // apply the title function if present
+  if (route.getTitle && typeof route.getTitle === 'function') {
+    match.title = route.getTitle(parts);
+  } else {
+    match.title = route.title;
+  }
+
+  return match;
+}