Merge branch 'master' into nymkappa/bugfix/node-page-js-error

backend/src/tasks/lightning/sync-tasks/stats-importer.ts

@@ -8,6 +8,7 @@ import { isIP } from 'net';
 import { Common } from '../../../api/common';
 import channelsApi from '../../../api/explorer/channels.api';
 import nodesApi from '../../../api/explorer/nodes.api';
+import { ResultSetHeader } from 'mysql2';
 
 const fsPromises = promises;
 
@@ -20,6 +21,7 @@ class LightningStatsImporter {
     await fundingTxFetcher.$fetchChannelsFundingTxs(channels.map(channel => channel.short_id));
 
     await this.$importHistoricalLightningStats();
+    await this.$cleanupIncorrectSnapshot();
   }
 
   /**
@@ -367,6 +369,12 @@ class LightningStatsImporter {
           continue;
         }
     
+        if (this.isIncorrectSnapshot(timestamp, graph)) {
+          logger.debug(`Ignoring ${this.topologiesFolder}/${filename}, because we defined it as an incorrect snapshot`);
+          ++totalProcessed;
+          continue;
+        }
+
         if (!logStarted) {
           logger.info(`Founds a topology file that we did not import. Importing historical lightning stats now.`);
           logStarted = true;
@@ -397,7 +405,7 @@ class LightningStatsImporter {
     }
   }
 
-  async cleanupTopology(graph) {
+  cleanupTopology(graph): ILightningApi.NetworkGraph {
     const newGraph = {
       nodes: <ILightningApi.Node[]>[],
       edges: <ILightningApi.Channel[]>[],
@@ -456,6 +464,69 @@ class LightningStatsImporter {
 
     return newGraph;
   }
+
+  private isIncorrectSnapshot(timestamp, graph): boolean {
+    if (timestamp >= 1549065600 /* 2019-02-02 */ && timestamp <= 1550620800 /* 2019-02-20 */ && graph.nodes.length < 2600) {
+        return true;
+    }
+    if (timestamp >= 1552953600 /* 2019-03-19 */ && timestamp <= 1556323200 /* 2019-05-27 */ && graph.nodes.length < 4000) {
+      return true;
+    }
+    if (timestamp >= 1557446400 /* 2019-05-10 */ && timestamp <= 1560470400 /* 2019-06-14 */ && graph.nodes.length < 4000) {
+      return true;
+    }
+    if (timestamp >= 1561680000 /* 2019-06-28 */ && timestamp <= 1563148800 /* 2019-07-15 */ && graph.nodes.length < 4000) {
+      return true;
+    }
+    if (timestamp >= 1571270400 /* 2019-11-17 */ && timestamp <= 1580601600 /* 2020-02-02 */ && graph.nodes.length < 4500) {
+      return true;
+    }
+    if (timestamp >= 1591142400 /* 2020-06-03 */ && timestamp <= 1592006400 /* 2020-06-13 */ && graph.nodes.length < 5500) {
+      return true;
+    }
+    if (timestamp >= 1632787200 /* 2021-09-28 */ && timestamp <= 1633564800 /* 2021-10-07 */ && graph.nodes.length < 13000) {
+      return true;
+    }
+    if (timestamp >= 1634256000 /* 2021-10-15 */ && timestamp <= 1645401600 /* 2022-02-21 */ && graph.nodes.length < 17000) {
+      return true;
+    }
+    if (timestamp >= 1654992000 /* 2022-06-12 */ && timestamp <= 1661472000 /* 2022-08-26 */ && graph.nodes.length < 14000) {
+      return true;
+    }
+
+    return false;
+  }
+
+  private async $cleanupIncorrectSnapshot(): Promise<void> {
+    // We do not run this one automatically because those stats are not supposed to be inserted in the first
+    // place, but I write them here to remind us we manually run those queries
+
+    // DELETE FROM lightning_stats
+    // WHERE (
+    //   UNIX_TIMESTAMP(added) >= 1549065600 AND UNIX_TIMESTAMP(added) <= 1550620800 AND node_count < 2600 OR
+    //   UNIX_TIMESTAMP(added) >= 1552953600 AND UNIX_TIMESTAMP(added) <= 1556323200 AND node_count < 4000 OR
+    //   UNIX_TIMESTAMP(added) >= 1557446400 AND UNIX_TIMESTAMP(added) <= 1560470400 AND node_count < 4000 OR
+    //   UNIX_TIMESTAMP(added) >= 1561680000 AND UNIX_TIMESTAMP(added) <= 1563148800 AND node_count < 4000 OR
+    //   UNIX_TIMESTAMP(added) >= 1571270400 AND UNIX_TIMESTAMP(added) <= 1580601600 AND node_count < 4500 OR
+    //   UNIX_TIMESTAMP(added) >= 1591142400 AND UNIX_TIMESTAMP(added) <= 1592006400 AND node_count < 5500 OR
+    //   UNIX_TIMESTAMP(added) >= 1632787200 AND UNIX_TIMESTAMP(added) <= 1633564800 AND node_count < 13000 OR
+    //   UNIX_TIMESTAMP(added) >= 1634256000 AND UNIX_TIMESTAMP(added) <= 1645401600 AND node_count < 17000 OR
+    //   UNIX_TIMESTAMP(added) >= 1654992000 AND UNIX_TIMESTAMP(added) <= 1661472000 AND node_count < 14000
+    // )
+
+    // DELETE FROM node_stats
+    // WHERE (
+    //   UNIX_TIMESTAMP(added) >= 1549065600 AND UNIX_TIMESTAMP(added) <= 1550620800 OR
+    //   UNIX_TIMESTAMP(added) >= 1552953600 AND UNIX_TIMESTAMP(added) <= 1556323200 OR
+    //   UNIX_TIMESTAMP(added) >= 1557446400 AND UNIX_TIMESTAMP(added) <= 1560470400 OR
+    //   UNIX_TIMESTAMP(added) >= 1561680000 AND UNIX_TIMESTAMP(added) <= 1563148800 OR
+    //   UNIX_TIMESTAMP(added) >= 1571270400 AND UNIX_TIMESTAMP(added) <= 1580601600 OR
+    //   UNIX_TIMESTAMP(added) >= 1591142400 AND UNIX_TIMESTAMP(added) <= 1592006400 OR
+    //   UNIX_TIMESTAMP(added) >= 1632787200 AND UNIX_TIMESTAMP(added) <= 1633564800 OR
+    //   UNIX_TIMESTAMP(added) >= 1634256000 AND UNIX_TIMESTAMP(added) <= 1645401600 OR
+    //   UNIX_TIMESTAMP(added) >= 1654992000 AND UNIX_TIMESTAMP(added) <= 1661472000 
+    // )
+  }
 }
 
 export default new LightningStatsImporter;

frontend/src/app/components/transaction/transaction-preview.component.html

@@ -2,6 +2,9 @@
 
   <div class="page-title">
     <h1 i18n="shared.transaction">Transaction</h1>
+    <a class="tx-link" [routerLink]="['/tx/' | relativeUrl, txId]">
+      <span class="truncated">{{txId.slice(0,-4)}}</span><span class="last-four">{{txId.slice(-4)}}</span>
+    </a>
     <div *ngIf="network !== 'liquid' && network !== 'liquidtestnet'" class="features">
       <app-tx-features [tx]="tx"></app-tx-features>
       <span *ngIf="cpfpInfo && cpfpInfo.bestDescendant" class="badge badge-primary mr-1">
@@ -13,104 +16,50 @@
     </div>
   </div>
 
-  <a [routerLink]="['/tx/' | relativeUrl, txId]" class="tx-link">
+  <div class="top-data row">
-    {{ txId }}
+    <span class="field col-sm-4 text-left">
-  </a>
+      <ng-template [ngIf]="isLiquid && haveBlindedOutputValues(tx)" [ngIfElse]="defaultAmount" i18n="shared.confidential">Confidential</ng-template>
+      <ng-template #defaultAmount>
+        <app-amount [satoshis]="totalValue"></app-amount>
+      </ng-template>
+    </span>
+    <span class="field col-sm-4 text-center">&lrm;{{ (tx.status.confirmed ? tx.status.block_time : transactionTime) * 1000 | date:'yyyy-MM-dd HH:mm' }}</span>
+    <span class="field col-sm-4 text-right"><span class="label" i18n="transaction.fee|Transaction fee">Fee </span>{{ tx.fee | number }} <span class="symbol" i18n="shared.sat|sat">sat</span></span>
+  </div>
 
 
-  <div class="row">
+  <div class="row graph-wrapper">
-    <div class="col-sm">
+    <tx-bowtie-graph [tx]="tx" [width]="1112" [height]="346" [network]="network"></tx-bowtie-graph>
-      <table class="table table-borderless table-striped">
+    <div class="above-bow">
-        <tbody>
+      <p class="field pair">
-          <tr *ngIf="tx.status.confirmed; else firstSeen">
+        <span [innerHTML]="'&lrm;' + (tx.size | bytes: 2)"></span>
-            <td i18n="block.timestamp">Timestamp</td>
+        <span [innerHTML]="'&lrm;' + (tx.weight | wuBytes: 2)"></span>
-            <td>
+      </p>
-              &lrm;{{ tx.status.block_time * 1000 | date:'yyyy-MM-dd HH:mm' }}
+      <p class="field" *ngIf="!isCoinbase(tx)">
-            </td>
+        {{ tx.feePerVsize | feeRounding }} <span class="symbol" i18n="shared.sat-vbyte|sat/vB">sat/vB</span>
-          </tr>
+      </p>
-          <ng-template #firstSeen>
-            <tr>
-              <td i18n="transaction.first-seen|Transaction first seen">First seen</td>
-              <td *ngIf="transactionTime > 0; else notSeen">
-                &lrm;{{ transactionTime * 1000 | date:'yyyy-MM-dd HH:mm' }}
-              </td>
-              <ng-template #notSeen>
-                <td>?</td>
-              </ng-template>
-            </tr>
-          </ng-template>
-          <tr>
-            <td class="td-width" i18n="dashboard.latest-transactions.amount">Amount</td>
-            <td>
-              <ng-template [ngIf]="(network === 'liquid' || network === 'liquidtestnet') && haveBlindedOutputValues(tx)" [ngIfElse]="defaultAmount" i18n="shared.confidential">Confidential</ng-template>
-              <ng-template #defaultAmount>
-                <app-amount [satoshis]="getTotalTxOutput(tx)"></app-amount>
-              </ng-template>
-            </td>
-          </tr>
-          <tr>
-            <td i18n="block.size">Size</td>
-            <td [innerHTML]="'&lrm;' + (tx.size | bytes: 2)"></td>
-          </tr>
-          <tr>
-            <td i18n="block.weight">Weight</td>
-            <td [innerHTML]="'&lrm;' + (tx.weight | wuBytes: 2)"></td>
-          </tr>
-          <tr>
-            <td i18n="transaction.inputs">Inputs</td>
-            <td *ngIf="!isCoinbase(tx); else coinbaseInputs">{{ tx.vin.length }}</td>
-            <ng-template #coinbaseInputs>
-              <td i18n="transactions-list.coinbase">Coinbase</td>
-            </ng-template>
-          </tr>
-        </tbody>
-      </table>
     </div>
-
+    <div class="overlaid">
-    <div class="col-sm">
+      <ng-container [ngSwitch]="extraData">
-      <table class="table table-borderless table-striped">
+        <table class="opreturns" *ngSwitchCase="'coinbase'">
-        <tbody>
+          <tbody>
-          <tr>
-            <td class="td-width" i18n="transaction.fee|Transaction fee">Fee</td>
-            <td>{{ tx.fee | number }} <span class="symbol" i18n="shared.sat|sat">sat</span></td>
-          </tr>
-          <tr *ngIf="!cpfpInfo || (!cpfpInfo.bestDescendant && !cpfpInfo.ancestors.length); else cpfpFee">
-            <td i18n="transaction.fee-rate|Transaction fee rate">Fee rate</td>
-            <td>
-              {{ tx.feePerVsize | feeRounding }} <span class="symbol" i18n="shared.sat-vbyte|sat/vB">sat/vB</span>
-              <ng-template [ngIf]="tx.status.confirmed">
-                &nbsp;
-                <app-tx-fee-rating *ngIf="tx.fee && ((cpfpInfo && !cpfpInfo.bestDescendant && !cpfpInfo.ancestors.length) || !cpfpInfo)" [tx]="tx"></app-tx-fee-rating>
-              </ng-template>
-            </td>
-          </tr>
-          <ng-template #cpfpFee>
             <tr>
-              <td i18n="transaction.effective-fee-rate|Effective transaction fee rate">Effective fee rate</td>
+              <td class="label">Coinbase</td>
-              <td>
+              <td class="message">{{ tx.vin[0].scriptsig | hex2ascii }}</td>
-                <div class="effective-fee-container">
-                  {{ tx.effectiveFeePerVsize | feeRounding }} <span class="symbol" i18n="shared.sat-vbyte|sat/vB">sat/vB</span>
-                  <ng-template [ngIf]="tx.status.confirmed">
-                    <app-tx-fee-rating class="d-none d-lg-inline ml-2" *ngIf="tx.fee" [tx]="tx"></app-tx-fee-rating>
-                  </ng-template>
-                </div>
-              </td>
             </tr>
-          </ng-template>
+          </tbody>
-          <tr>
+        </table>
-            <td i18n="transaction.vsize|Transaction Virtual Size">Virtual size</td>
+        <table class="opreturns" *ngSwitchCase="'opreturn'">
-            <td [innerHTML]="'&lrm;' + (tx.weight / 4 | vbytes: 2)"></td>
+          <tbody>
-          </tr>
+            <ng-container *ngFor="let vout of opReturns.slice(0,3)">
-          <tr>
+              <tr>
-            <td i18n="transaction.locktime">Locktime</td>
+                <td class="label">OP_RETURN</td>
-            <td [innerHTML]="'&lrm;' + (tx.locktime | number)"></td>
+                <td *ngIf="vout.scriptpubkey_asm !== 'OP_RETURN'" class="message">{{ vout.scriptpubkey_asm | hex2ascii }}</td>
-          </tr>
+              </tr>
-          <tr>
+            </ng-container>
-            <td i18n="transaction.outputs">Outputs</td>
+          </tbody>
-            <td>{{ tx.vout.length }}</td>
+        </table>
-          </tr>
+      </ng-container>
-        </tbody>
-      </table>
     </div>
   </div>
 </div>

frontend/src/app/components/transaction/transaction-preview.component.scss

@@ -10,26 +10,10 @@
 	font-size: 28px;
 }
 
-.btn-small-height {
-	line-height: 1.1;
-}
-
-.arrow-green {
-	color: #1a9436;
-}
-
-.arrow-red {
-	color: #dc3545;
-}
-
 .row {
 	flex-direction: row;
 }
 
-.effective-fee-container {
-	display: inline-block;
-}
-
 .title {
   h2 {
     line-height: 1;
@@ -46,8 +30,9 @@
   display: flex;
   flex-direction: row;
   justify-content: space-between;
-  align-items: center;
+  align-items: baseline;
-  margin-bottom: 10px;
+  margin-bottom: 2px;
+  max-width: 100%;
 
   h1 {
     font-size: 52px;
@@ -58,6 +43,43 @@
   .features {
     font-size: 24px;
   }
+
+  & > * {
+    flex-grow: 0;
+    flex-shrink: 0;
+  }
+
+  .tx-link {
+    flex-grow: 1;
+    flex-shrink: 1;
+    margin: 0 1em;
+    overflow: hidden;
+    white-space: nowrap;
+    display: flex;
+    flex-direction: row;
+    align-items: baseline;
+
+    .truncated {
+      flex-grow: 1;
+      flex-shrink: 1;
+      overflow: hidden;
+      text-overflow: ellipsis;
+      margin-right: -2px;
+    }
+
+    .last-four {
+      flex-shrink: 0;
+      flex-grow: 0;
+    }
+  }
+
+  .features {
+    align-self: center;
+  }
+}
+
+.top-data {
+  font-size: 28px;
 }
 
 .table {
@@ -68,8 +90,76 @@
   }
 }
 
+.field {
+  font-size: 32px;
+  margin: 0;
+
+  ::ng-deep .symbol {
+    font-size: 24px;
+  }
+
+  .label {
+    color: #ffffff66;
+  }
+
+  &.pair > *:first-child {
+    margin-right: 1em;
+  }
+}
+
 .tx-link {
-  display: inline-block;
+  display: inline;
   font-size: 28px;
   margin-bottom: 6px;
 }
+
+.graph-wrapper {
+  position: relative;
+  background: #181b2d;
+  padding: 10px;
+  padding-bottom: 0;
+
+  .above-bow {
+    position: absolute;
+    top: 20px;
+    left: 0;
+    right: 0;
+    margin: auto;
+    text-align: center;
+  }
+
+  .overlaid {
+    position: absolute;
+    bottom: 0;
+    left: 0;
+    right: 0;
+    width: 100%;
+    text-align: left;
+    font-size: 28px;
+    max-width: 90%;
+    margin: auto;
+    overflow: hidden;
+
+    .opreturns {
+      width: auto;
+      margin: auto;
+      table-layout: auto;
+      background: #2d3348af;
+      border-top-left-radius: 5px;
+      border-top-right-radius: 5px;
+
+      td {
+        padding: 10px 10px;
+
+        &.message {
+          overflow: hidden;
+          display: inline-block;
+          vertical-align: bottom;
+          text-overflow: ellipsis;
+          white-space: nowrap;
+          text-align: left;
+        }
+      }
+    }
+  }
+}

frontend/src/app/components/transaction/transaction-preview.component.ts

@@ -7,10 +7,9 @@ import {
   catchError,
   retryWhen,
   delay,
-  map
 } from 'rxjs/operators';
 import { Transaction, Vout } from '../../interfaces/electrs.interface';
-import { of, merge, Subscription, Observable, Subject, timer, combineLatest, from } from 'rxjs';
+import { of, merge, Subscription, Observable, Subject, from } from 'rxjs';
 import { StateService } from '../../services/state.service';
 import { OpenGraphService } from 'src/app/services/opengraph.service';
 import { ApiService } from 'src/app/services/api.service';
@@ -37,6 +36,10 @@ export class TransactionPreviewComponent implements OnInit, OnDestroy {
   showCpfpDetails = false;
   fetchCpfp$ = new Subject<string>();
   liquidUnblinding = new LiquidUnblinding();
+  isLiquid = false;
+  totalValue: number;
+  opReturns: Vout[];
+  extraData: 'none' | 'coinbase' | 'opreturn';
 
   constructor(
     private route: ActivatedRoute,
@@ -49,7 +52,12 @@ export class TransactionPreviewComponent implements OnInit, OnDestroy {
 
   ngOnInit() {
     this.stateService.networkChanged$.subscribe(
-      (network) => (this.network = network)
+      (network) => {
+        this.network = network;
+        if (this.network === 'liquid' || this.network == 'liquidtestnet') {
+          this.isLiquid = true;
+        }
+      }
     );
 
     this.fetchCpfpSubscription = this.fetchCpfp$
@@ -152,6 +160,9 @@ export class TransactionPreviewComponent implements OnInit, OnDestroy {
           this.tx.feePerVsize = tx.fee / (tx.weight / 4);
           this.isLoadingTx = false;
           this.error = undefined;
+          this.totalValue = this.tx.vout.reduce((acc, v) => v.value + acc, 0);
+          this.opReturns = this.getOpReturns(this.tx);
+          this.extraData = this.chooseExtraData();
 
           if (!tx.status.confirmed && tx.firstSeen) {
             this.transactionTime = tx.firstSeen;
@@ -217,6 +228,20 @@ export class TransactionPreviewComponent implements OnInit, OnDestroy {
     return tx.vout.map((v: Vout) => v.value || 0).reduce((a: number, b: number) => a + b);
   }
 
+  getOpReturns(tx: Transaction): Vout[] {
+    return tx.vout.filter((v) => v.scriptpubkey_type === 'op_return' && v.scriptpubkey_asm !== 'OP_RETURN');
+  }
+
+  chooseExtraData(): 'none' | 'opreturn' | 'coinbase' {
+    if (this.isCoinbase(this.tx)) {
+      return 'coinbase';
+    } else if (this.opReturns?.length) {
+      return 'opreturn';
+    } else {
+      return 'none';
+    }
+  }
+
   ngOnDestroy() {
     this.subscription.unsubscribe();
     this.fetchCpfpSubscription.unsubscribe();

frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.html

@@ -0,0 +1,44 @@
+<svg *ngIf="inputs && outputs" class="bowtie" [attr.height]="(height + 10) + 'px'" [attr.width]="width + 'px'">
+  <defs>
+    <marker id="input-arrow" viewBox="-5 -5 10 10"
+        refX="0" refY="0"
+        markerUnits="strokeWidth"
+        markerWidth="1.5" markerHeight="1"
+        orient="auto">
+      <path d="M -5 -5 L 0 0 L -5 5 L 1 5 L 1 -5 Z" stroke-width="0" [attr.fill]="gradient[0]"/>
+    </marker>
+    <marker id="output-arrow" viewBox="-5 -5 10 10"
+        refX="0" refY="0"
+        markerUnits="strokeWidth"
+        markerWidth="1.5" markerHeight="1"
+        orient="auto">
+      <path d="M 1 -5 L 0 -5 L -5 0 L 0 5 L 1 5 Z" stroke-width="0" [attr.fill]="gradient[0]"/>
+    </marker>
+    <marker id="fee-arrow" viewBox="-5 -5 10 10"
+        refX="0" refY="0"
+        markerUnits="strokeWidth"
+        markerWidth="1.5" markerHeight="1"
+        orient="auto">
+    </marker>
+    <linearGradient id="input-gradient" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" [attr.stop-color]="gradient[0]" />
+      <stop offset="100%" [attr.stop-color]="gradient[1]" />
+    </linearGradient>
+    <linearGradient id="output-gradient" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" [attr.stop-color]="gradient[1]" />
+      <stop offset="100%" [attr.stop-color]="gradient[0]" />
+    </linearGradient>
+    <linearGradient id="fee-gradient" x1="0%" y1="0%" x2="100%" y2="0%">
+      <stop offset="0%" [attr.stop-color]="gradient[1]" />
+      <stop offset="50%" [attr.stop-color]="gradient[1]" />
+      <stop offset="100%" stop-color="transparent" />
+    </linearGradient>
+  </defs>
+  <path [attr.d]="middle.path" class="line middle" [style]="middle.style"/>
+  <ng-container *ngFor="let input of inputs">
+    <path [attr.d]="input.path" class="line {{input.class}}" [style]="input.style" attr.marker-start="url(#{{input.class}}-arrow)"/>
+  </ng-container>
+  <ng-container *ngFor="let output of outputs">
+    <path [attr.d]="output.path" class="line {{output.class}}" [style]="output.style" attr.marker-start="url(#{{output.class}}-arrow)" />
+  </ng-container>
+</svg>

frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.scss

@@ -0,0 +1,15 @@
+.bowtie {
+  .line {
+    fill: none;
+
+    &.input {
+      stroke: url(#input-gradient);
+    }
+    &.output {
+      stroke: url(#output-gradient);
+    }
+    &.fee {
+      stroke: url(#fee-gradient);
+    }
+  }
+}

frontend/src/app/components/tx-bowtie-graph/tx-bowtie-graph.component.ts

@@ -0,0 +1,169 @@
+import { Component, OnInit, Input, OnChanges } from '@angular/core';
+import { Transaction } from '../../interfaces/electrs.interface';
+
+interface SvgLine {
+  path: string;
+  style: string;
+  class?: string;
+}
+
+@Component({
+  selector: 'tx-bowtie-graph',
+  templateUrl: './tx-bowtie-graph.component.html',
+  styleUrls: ['./tx-bowtie-graph.component.scss'],
+})
+export class TxBowtieGraphComponent implements OnInit, OnChanges {
+  @Input() tx: Transaction;
+  @Input() network: string;
+  @Input() width = 1200;
+  @Input() height = 600;
+  @Input() combinedWeight = 100;
+  @Input() minWeight = 2; //
+  @Input() maxStrands = 24; // number of inputs/outputs to keep fully on-screen.
+
+  inputs: SvgLine[];
+  outputs: SvgLine[];
+  middle: SvgLine;
+  isLiquid: boolean = false;
+
+  gradientColors = {
+    '': ['#9339f4', '#105fb0'],
+    bisq: ['#9339f4', '#105fb0'],
+    // liquid: ['#116761', '#183550'],
+    liquid: ['#09a197', '#0f62af'],
+    // 'liquidtestnet': ['#494a4a', '#272e46'],
+    'liquidtestnet': ['#d2d2d2', '#979797'],
+    // testnet: ['#1d486f', '#183550'],
+    testnet: ['#4edf77', '#10a0af'],
+    // signet: ['#6f1d5d', '#471850'],
+    signet: ['#d24fc8', '#a84fd2'],
+  };
+
+  gradient: string[] = ['#105fb0', '#105fb0'];
+
+  ngOnInit(): void {
+    this.isLiquid = (this.network === 'liquid' || this.network === 'liquidtestnet');
+    this.gradient = this.gradientColors[this.network];
+    this.initGraph();
+  }
+
+  ngOnChanges(): void {
+    this.isLiquid = (this.network === 'liquid' || this.network === 'liquidtestnet');
+    this.gradient = this.gradientColors[this.network];
+    this.initGraph();
+  }
+
+  initGraph(): void {
+    const totalValue = this.calcTotalValue(this.tx);
+    const voutWithFee = this.tx.vout.map(v => { return { type: v.scriptpubkey_type === 'fee' ? 'fee' : 'output', value: v?.value }; });
+
+    if (this.tx.fee && !this.isLiquid) {
+      voutWithFee.unshift({ type: 'fee', value: this.tx.fee });
+    }
+
+    this.inputs = this.initLines('in', this.tx.vin.map(v => { return {type: 'input', value: v?.prevout?.value }; }), totalValue, this.maxStrands);
+    this.outputs = this.initLines('out', voutWithFee, totalValue, this.maxStrands);
+
+    this.middle = {
+      path: `M ${(this.width / 2) - 50} ${(this.height / 2) + 0.5} L ${(this.width / 2) + 50} ${(this.height / 2) + 0.5}`,
+      style: `stroke-width: ${this.combinedWeight + 0.5}; stroke: ${this.gradient[1]}`
+    };
+  }
+
+  calcTotalValue(tx: Transaction): number {
+    const totalOutput = this.tx.vout.reduce((acc, v) => (v.value == null ? 0 : v.value) + acc, 0);
+    // simple sum of outputs + fee for bitcoin
+    if (!this.isLiquid) {
+      return this.tx.fee ? totalOutput + this.tx.fee : totalOutput;
+    } else {
+      const totalInput = this.tx.vin.reduce((acc, v) => (v?.prevout?.value == null ? 0 : v.prevout.value) + acc, 0);
+      const confidentialInputCount = this.tx.vin.reduce((acc, v) => acc + (v?.prevout?.value == null ? 1 : 0), 0);
+      const confidentialOutputCount = this.tx.vout.reduce((acc, v) => acc + (v.value == null ? 1 : 0), 0);
+
+      // if there are unknowns on both sides, the total is indeterminate, so we'll just fudge it
+      if (confidentialInputCount && confidentialOutputCount) {
+        const knownInputCount = (tx.vin.length - confidentialInputCount) || 1;
+        const knownOutputCount = (tx.vout.length - confidentialOutputCount) || 1;
+        // assume confidential inputs/outputs have the same average value as the known ones
+        const adjustedTotalInput = totalInput + ((totalInput / knownInputCount) * confidentialInputCount);
+        const adjustedTotalOutput = totalOutput + ((totalOutput / knownOutputCount) * confidentialOutputCount);
+        return Math.max(adjustedTotalInput, adjustedTotalOutput) || 1;
+      } else {
+        // otherwise knowing the actual total of one side suffices
+        return Math.max(totalInput, totalOutput) || 1;
+      }
+    }
+  }
+
+  initLines(side: 'in' | 'out', xputs: { type: string, value: number | void }[], total: number, maxVisibleStrands: number): SvgLine[] {
+    const lines = [];
+    let unknownCount = 0;
+    let unknownTotal = total == null ? this.combinedWeight : total;
+    xputs.forEach(put => {
+      if (put.value == null) {
+        unknownCount++;
+      } else {
+        unknownTotal -= put.value as number;
+      }
+    });
+    const unknownShare = unknownTotal / unknownCount;
+
+    // conceptual weights
+    const weights = xputs.map((put): number => this.combinedWeight * (put.value == null ? unknownShare : put.value as number) / total);
+    // actual displayed line thicknesses
+    const minWeights = weights.map((w) => Math.max(this.minWeight - 1, w) + 1);
+    const visibleStrands = Math.min(maxVisibleStrands, xputs.length);
+    const visibleWeight = minWeights.slice(0, visibleStrands).reduce((acc, v) => v + acc, 0);
+    const gaps = visibleStrands - 1;
+
+    const innerTop = (this.height / 2) - (this.combinedWeight / 2);
+    const innerBottom = innerTop + this.combinedWeight;
+    // tracks the visual bottom of the endpoints of the previous line
+    let lastOuter = 0;
+    let lastInner = innerTop;
+    // gap between strands
+    const spacing = (this.height - visibleWeight) / gaps;
+
+    for (let i = 0; i < xputs.length; i++) {
+      const weight = weights[i];
+      const minWeight = minWeights[i];
+      // set the vertical position of the (center of the) outer side of the line
+      let outer = lastOuter + (minWeight / 2);
+      const inner = Math.min(innerBottom + (minWeight / 2), Math.max(innerTop + (minWeight / 2), lastInner + (weight / 2)));
+
+      // special case to center single input/outputs
+      if (xputs.length === 1) {
+        outer = (this.height / 2);
+      }
+
+      lastOuter += minWeight + spacing;
+      lastInner += weight;
+      lines.push({
+        path: this.makePath(side, outer, inner, minWeight),
+        style: this.makeStyle(minWeight, xputs[i].type),
+        class: xputs[i].type
+      });
+    }
+
+    return lines;
+  }
+
+  makePath(side: 'in' | 'out', outer: number, inner: number, weight: number): string {
+    const start = side === 'in' ? (weight * 0.5) : this.width - (weight * 0.5);
+    const center =  this.width / 2 + (side === 'in' ? -45 : 45 );
+    const midpoint = (start + center) / 2;
+    // correct for svg horizontal gradient bug
+    if (Math.round(outer) === Math.round(inner)) {
+      outer -= 1;
+    }
+    return `M ${start} ${outer} C ${midpoint} ${outer}, ${midpoint} ${inner}, ${center} ${inner}`;
+  }
+
+  makeStyle(minWeight, type): string {
+    if (type === 'fee') {
+      return `stroke-width: ${minWeight}; stroke: url(#fee-gradient)`;
+    } else {
+      return `stroke-width: ${minWeight}`;
+    }
+  }
+}

frontend/src/app/lightning/channel/channel-preview.component.html

@@ -58,7 +58,7 @@
       </table>
     </div>
     <div class="col-md map-col">
-      <app-nodes-channels-map *ngIf="!error" [style]="'channelpage'" [channel]="channelGeo" [fitContainer]="true" (readyEvent)="onMapReady()"></app-nodes-channels-map>
+      <app-nodes-channels-map *ngIf="!error" [style]="'channelpage'" [channel]="channelGeo" [fitContainer]="true" [placeholder]="true" (readyEvent)="onMapReady()"></app-nodes-channels-map>
     </div>
   </div>
 </div>

frontend/src/app/lightning/node/node-preview.component.html

@@ -52,7 +52,7 @@
       </table>
     </div>
     <div class="col-md map-col">
-      <app-nodes-channels-map *ngIf="!error" [style]="'nodepage'" [publicKey]="node.public_key" [fitContainer]="true" (readyEvent)="onMapReady()"></app-nodes-channels-map>
+      <app-nodes-channels-map *ngIf="!error" [style]="'nodepage'" [publicKey]="node.public_key" [fitContainer]="true" [placeholder]="true" [hasLocation]="!!node.as_number" (readyEvent)="onMapReady()"></app-nodes-channels-map>
     </div>
   </div>
 </div>

frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts

@@ -22,6 +22,7 @@ export class NodesChannelsMap implements OnInit {
   @Input() channel: any[] = [];
   @Input() fitContainer = false;
   @Input() hasLocation = true;
+  @Input() placeholder = false;
   @Output() readyEvent = new EventEmitter();
 
   channelsObservable: Observable<any>; 
@@ -201,11 +202,26 @@ export class NodesChannelsMap implements OnInit {
 
   prepareChartOptions(nodes, channels) {
     let title: object;
-    if (channels.length === 0) {
+    if (channels.length === 0 && !this.placeholder) {
       this.chartOptions = null;
       return;
     }
 
+    // empty map fallback
+    if (channels.length === 0 && this.placeholder) {
+      title = {
+        textStyle: {
+          color: 'white',
+          fontSize: 18
+        },
+        text: $localize`No geolocation data available`,
+        left: 'center',
+        top: 'center'
+      };
+      this.zoom = 1.5;
+      this.center = [0, 20];
+    }
+
     this.chartOptions = {
       silent: this.style === 'widget',
       title: title ?? undefined,

frontend/src/app/services/state.service.ts

@@ -153,7 +153,12 @@ export class StateService {
     if (this.env.BASE_MODULE !== 'mempool' && this.env.BASE_MODULE !== 'liquid') {
       return;
     }
-    const networkMatches = url.match(/^\/(bisq|testnet|liquidtestnet|liquid|signet)/);
+    // horrible network regex breakdown:
+    // /^\/                                         starts with a forward slash...
+    // (?:[a-z]{2}(?:-[A-Z]{2})?\/)?                optional locale prefix (non-capturing)
+    // (?:preview\/)?                               optional "preview" prefix (non-capturing)
+    // (bisq|testnet|liquidtestnet|liquid|signet)/  network string (captured as networkMatches[1])
+    const networkMatches = url.match(/^\/(?:[a-z]{2}(?:-[A-Z]{2})?\/)?(?:preview\/)?(bisq|testnet|liquidtestnet|liquid|signet)/);
     switch (networkMatches && networkMatches[1]) {
       case 'liquid':
         if (this.network !== 'liquid') {

frontend/src/app/shared/shared.module.ts

@@ -63,6 +63,7 @@ import { StatusViewComponent } from '../components/status-view/status-view.compo
 import { FeesBoxComponent } from '../components/fees-box/fees-box.component';
 import { DifficultyComponent } from '../components/difficulty/difficulty.component';
 import { TermsOfServiceComponent } from '../components/terms-of-service/terms-of-service.component';
+import { TxBowtieGraphComponent } from '../components/tx-bowtie-graph/tx-bowtie-graph.component';
 import { PrivacyPolicyComponent } from '../components/privacy-policy/privacy-policy.component';
 import { TrademarkPolicyComponent } from '../components/trademark-policy/trademark-policy.component';
 import { PushTransactionComponent } from '../components/push-transaction/push-transaction.component';
@@ -138,6 +139,7 @@ import { GeolocationComponent } from '../shared/components/geolocation/geolocati
     StatusViewComponent,
     FeesBoxComponent,
     DifficultyComponent,
+    TxBowtieGraphComponent,
     TermsOfServiceComponent,
     PrivacyPolicyComponent,
     TrademarkPolicyComponent,
@@ -242,6 +244,7 @@ import { GeolocationComponent } from '../shared/components/geolocation/geolocati
     StatusViewComponent,
     FeesBoxComponent,
     DifficultyComponent,
+    TxBowtieGraphComponent,
     TermsOfServiceComponent,
     PrivacyPolicyComponent,
     TrademarkPolicyComponent,

production/install

@@ -1287,9 +1287,9 @@ case $OS in
         osPackageInstall ${CLN_PKG}
 
         echo "[*] Installing Core Lightning mainnet Cronjob"
-        crontab_cln+='@reboot sleep 30 ; screen -dmS main lightningd --alias `hostname` --bitcoin-datadir /bitcoin\n'
+        crontab_cln+='@reboot sleep 60 ; screen -dmS main lightningd --alias `hostname` --bitcoin-datadir /bitcoin\n'
-        crontab_cln+='@reboot sleep 60 ; screen -dmS sig lightningd --alias `hostname` --bitcoin-datadir /bitcoin --network signet\n'
         crontab_cln+='@reboot sleep 90 ; screen -dmS tes lightningd --alias `hostname` --bitcoin-datadir /bitcoin --network testnet\n'
+        crontab_cln+='@reboot sleep 120 ; screen -dmS sig lightningd --alias `hostname` --bitcoin-datadir /bitcoin --network signet\n'
         echo "${crontab_cln}" | crontab -u "${CLN_USER}" -
     ;;
     Debian)

production/mempool-build-all

@@ -12,7 +12,10 @@ ELEMENTS_RPC_USER=$(grep '^rpcuser' /elements/elements.conf | cut -d '=' -f2)
 ELEMENTS_RPC_PASS=$(grep '^rpcpassword' /elements/elements.conf | cut -d '=' -f2)
 
 # get mysql credentials
-. /mempool/mysql_credentials
+MYSQL_CRED_FILE=${HOME}/mempool/mysql_credentials
+if [ -f "${MYSQL_CRED_FILE}" ];then
+    . ${MYSQL_CRED_FILE}
+}
 
 if [ -f "${LOCKFILE}" ];then
     echo "upgrade already running? check lockfile ${LOCKFILE}"
@@ -63,6 +66,19 @@ build_frontend()
     npm run build || exit 1
 }
 
+build_unfurler()
+{
+    local site="$1"
+    echo "[*] Building unfurler for ${site}"
+    [ -z "${HASH}" ] && exit 1
+    cd "$HOME/${site}/unfurler" || exit 1
+    if [ ! -e "config.json" ];then
+        cp "${HOME}/mempool/production/unfurler-config.${site}.json" "config.json"
+    fi
+    PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true npm install || exit 1
+    npm run build || exit 1
+}
+
 build_backend()
 {
     local site="$1"
@@ -128,6 +144,11 @@ for repo in $backend_repos;do
     update_repo "${repo}"
 done
 
+# build unfurlers
+for repo in mainnet liquid;do
+    build_unfurler "${repo}"
+done
+
 # build backends
 for repo in $backend_repos;do
     build_backend "${repo}"

production/mempool-kill-all

@@ -1,2 +1,8 @@
 #!/usr/bin/env zsh
-killall sh node
+killall sh
+killall node
+killall chrome
+killall xinit
+for pid in `ps uaxww|grep warmer|grep zsh|awk '{print $2}'`;do
+    kill $pid
+done

production/mempool-start-all

@@ -2,7 +2,29 @@
 export NVM_DIR="$HOME/.nvm"
 source "$NVM_DIR/nvm.sh"
 
+# start all mempool backends that exist
 for site in mainnet mainnet-lightning testnet testnet-lightning signet signet-lightning bisq liquid liquidtestnet;do
     cd "${HOME}/${site}/backend/" && \
+    echo "starting mempool backend: ${site}" && \
     screen -dmS "${site}" sh -c 'while true;do npm run start-production;sleep 1;done'
 done
+
+# only start unfurler if GPU present
+if pciconf -lv|grep -i nvidia >/dev/null 2>&1;then
+    export DISPLAY=:0
+    screen -dmS x startx
+    sleep 3
+    for site in mainnet liquid;do
+        cd "$HOME/${site}/unfurler" && \
+        echo "starting mempool unfurler: ${site}" && \
+        screen -dmS "unfurler-${site}" sh -c 'while true;do npm run unfurler;sleep 2;done'
+    done
+fi
+
+# start nginx warm cacher
+for site in mainnet;do
+    echo "starting mempool cache warmer: ${site}"
+    screen -dmS "warmer-${site}" $HOME/mempool/production/nginx-cache-warmer
+done
+
+exit 0

production/nginx-cache-warmer

@@ -2,6 +2,12 @@
 hostname=$(hostname)
 slugs=(`curl -sSL https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json | jq -r '.slugs[]'`)
 
+warm()
+{
+	echo "$1"
+	curl -i -s "$1" | head -1
+}
+
 while true
 do for url in / \
 	'/api/v1/blocks' \
@@ -81,14 +87,14 @@ do for url in / \
 	'/api/v1/lightning/channels-geo?style=graph' \
 
 	do
-		curl -s "https://${hostname}${url}" >/dev/null
+		warm "https://${hostname}${url}"
 	done
 
 	for slug in $slugs
 	do
-		curl -s "https://${hostname}/api/v1/mining/pool/${slug}" >/dev/null
+		warm "https://${hostname}/api/v1/mining/pool/${slug}"
-		curl -s "https://${hostname}/api/v1/mining/pool/${slug}/hashrate" >/dev/null
+		warm "https://${hostname}/api/v1/mining/pool/${slug}/hashrate"
-		curl -s "https://${hostname}/api/v1/mining/pool/${slug}/blocks" >/dev/null
+		warm "https://${hostname}/api/v1/mining/pool/${slug}/blocks"
 	done
 
 	sleep 10

production/unfurl-build

@@ -1,62 +0,0 @@
-#!/usr/bin/env zsh
-PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin:$HOME/bin
-HOSTNAME=$(hostname)
-LOCATION=$(hostname|cut -d . -f2)
-LOCKFILE="${HOME}/lock"
-REF=$(echo "${1:=origin/master}"|sed -e 's!:!/!')
-
-if [ -f "${LOCKFILE}" ];then
-    echo "upgrade already running? check lockfile ${LOCKFILE}"
-    exit 1
-fi
-
-# on exit, remove lockfile but preserve exit code
-trap "rv=\$?; rm -f "${LOCKFILE}"; exit \$rv" INT TERM EXIT
-
-# create lockfile
-touch "${LOCKFILE}"
-
-# notify logged in users
-echo "Upgrading unfurler to ${REF}" | wall
-
-update_repo()
-{
-    echo "[*] Upgrading unfurler to ${REF}"
-    cd "$HOME/unfurl/unfurler" || exit 1
-
-    git fetch origin || exit 1
-    for remote in origin;do
-        git remote add "${remote}" "https://github.com/${remote}/mempool" >/dev/null 2>&1
-        git fetch "${remote}" || exit 1
-    done
-
-    if [ $(git tag -l "${REF}") ];then
-        git reset --hard "tags/${REF}" || exit 1
-    elif [ $(git branch -r -l "origin/${REF}") ];then
-        git reset --hard "origin/${REF}" || exit 1
-    else
-        git reset --hard "${REF}" || exit 1
-    fi
-    export HASH=$(git rev-parse HEAD)
-}
-
-build_backend()
-{
-    echo "[*] Building backend for unfurler"
-    [ -z "${HASH}" ] && exit 1
-    cd "$HOME/unfurl/unfurler" || exit 1
-    if [ ! -e "config.json" ];then
-        cp "${HOME}/unfurl/production/mempool-config.unfurl.json" "config.json"
-    fi
-    npm install || exit 1
-    npm run build || exit 1
-}
-
-update_repo
-build_backend
-
-# notify everyone
-echo "${HOSTNAME} unfurl updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general mempool.dev
-echo "${HOSTNAME} unfurl updated to \`${REF}\` @ \`${HASH}\`" | /usr/local/bin/keybase chat send --nonblock --channel general "mempool.ops.${LOCATION}"
-
-exit 0

production/unfurl-kill

@@ -1,2 +0,0 @@
-#!/usr/bin/env zsh
-killall sh node

production/unfurl-start

@@ -1,6 +0,0 @@
-#!/usr/bin/env zsh
-export NVM_DIR="$HOME/.nvm"
-source "$NVM_DIR/nvm.sh"
-
-cd "${HOME}/unfurl/unfurler/" && \
-screen -dmS "unfurl" sh -c 'while true;do npm run start-production;sleep 1;done'

production/unfurler-config.liquid.json

@@ -0,0 +1,17 @@
+{
+  "SERVER": {
+    "HOST": "https://liquid.network",
+    "HTTP_PORT": 8002
+  },
+  "MEMPOOL": {
+    "HTTP_HOST": "https://liquid.network",
+    "HTTP_PORT": 443,
+    "NETWORK": "liquid"
+  },
+  "PUPPETEER": {
+    "CLUSTER_SIZE": 8,
+    "EXEC_PATH": "/usr/local/bin/chrome",
+    "MAX_PAGE_AGE": 86400,
+    "RENDER_TIMEOUT": 3000
+  }
+}

production/unfurler-config.mainnet.json

@@ -0,0 +1,17 @@
+{
+  "SERVER": {
+    "HOST": "https://mempool.space",
+    "HTTP_PORT": 8001
+  },
+  "MEMPOOL": {
+    "HTTP_HOST": "https://mempool.space",
+    "HTTP_PORT": 443,
+    "NETWORK": "bitcoin"
+  },
+  "PUPPETEER": {
+    "CLUSTER_SIZE": 8,
+    "EXEC_PATH": "/usr/local/bin/chrome",
+    "MAX_PAGE_AGE": 86400,
+    "RENDER_TIMEOUT": 3000
+  }
+}

unfurler/package.json

@@ -11,7 +11,7 @@
     "tsc": "./node_modules/typescript/bin/tsc",
     "build": "npm run tsc",
     "start": "node --max-old-space-size=2048 dist/index.js",
-    "start-production": "node --max-old-space-size=4096 dist/index.js",
+    "unfurler": "node --max-old-space-size=4096 dist/index.js",
     "lint": "./node_modules/.bin/eslint . --ext .ts",
     "lint:fix": "./node_modules/.bin/eslint . --ext .ts --fix",
     "prettier": "./node_modules/.bin/prettier --write \"src/**/*.{js,ts}\""

unfurler/puppeteer.config.json

@@ -41,6 +41,6 @@
     "--use-mock-keychain",
     "--ignore-gpu-blacklist",
     "--ignore-gpu-blocklist",
-    "--use-gl=swiftshader"
+    "--use-gl=egl"
   ]
 }