Merge branch 'master' into gotta-unfurl-fast

backend/src/tasks/lightning/sync-tasks/stats-importer.ts

@@ -4,6 +4,8 @@ import { XMLParser } from 'fast-xml-parser';
 import logger from '../../../logger';
 import fundingTxFetcher from './funding-tx-fetcher';
 import config from '../../../config';
+import { ILightningApi } from '../../../api/lightning/lightning-api.interface';
+import { isIP } from 'net';
 
 const fsPromises = promises;
 
@@ -48,7 +50,7 @@ class LightningStatsImporter {
   /**
    * Generate LN network stats for one day
    */
-  public async computeNetworkStats(timestamp: number, networkGraph): Promise<unknown> {
+  public async computeNetworkStats(timestamp: number, networkGraph: ILightningApi.NetworkGraph): Promise<unknown> {
     // Node counts and network shares
     let clearnetNodes = 0;
     let torNodes = 0;
@@ -61,8 +63,8 @@ class LightningStatsImporter {
       let isUnnanounced = true;
 
       for (const socket of (node.addresses ?? [])) {
-        hasOnion = hasOnion || ['torv2', 'torv3'].includes(socket.network);
+        hasOnion = hasOnion || ['torv2', 'torv3'].includes(socket.network) || socket.addr.indexOf('onion') !== -1;
-        hasClearnet = hasClearnet || ['ipv4', 'ipv6'].includes(socket.network);
+        hasClearnet = hasClearnet || ['ipv4', 'ipv6'].includes(socket.network) || [4, 6].includes(isIP(socket.addr.split(':')[0]));
       }
       if (hasOnion && hasClearnet) {
         clearnetTorNodes++;
@@ -127,22 +129,28 @@ class LightningStatsImporter {
 
       if (channel.node1_policy !== undefined) { // Coming from the node
         for (const policy of [channel.node1_policy, channel.node2_policy]) {
-          if (policy && policy.fee_rate_milli_msat < 5000) {
+          if (policy && parseInt(policy.fee_rate_milli_msat, 10) < 5000) {
             avgFeeRate += parseInt(policy.fee_rate_milli_msat, 10);
             feeRates.push(parseInt(policy.fee_rate_milli_msat, 10));
           }  
-          if (policy && policy.fee_base_msat < 5000) {
+          if (policy && parseInt(policy.fee_base_msat, 10) < 5000) {
             avgBaseFee += parseInt(policy.fee_base_msat, 10);
             baseFees.push(parseInt(policy.fee_base_msat, 10));
           }
         }
       } else { // Coming from the historical import
+        // @ts-ignore
         if (channel.fee_rate_milli_msat < 5000) {
+          // @ts-ignore
           avgFeeRate += parseInt(channel.fee_rate_milli_msat, 10);
+          // @ts-ignore
           feeRates.push(parseInt(channel.fee_rate_milli_msat), 10);
-        }  
+        }
+        // @ts-ignore
         if (channel.fee_base_msat < 5000) {
+          // @ts-ignore
           avgBaseFee += parseInt(channel.fee_base_msat, 10);
+          // @ts-ignore
           baseFees.push(parseInt(channel.fee_base_msat), 10);
         }
       }

frontend/src/app/docs/api-docs/api-docs.component.html

@@ -46,12 +46,14 @@
                 </ng-container>
                 <ng-template #liquid_link_example>
                   <ng-container *ngIf="item.httpRequestMethod === 'GET' && network.val === 'liquid' && item.codeExample.hasOwnProperty('liquid');else default_link_example">
-                    <a [href]="wrapUrl(network.val, item.codeExample.liquid)" target="_blank" rel="nofollow">{{ item.httpRequestMethod }} {{ baseNetworkUrl }}/api{{ item.urlString }}</a>
+                    <a [href]="wrapUrl(network.val, item.codeExample.liquid)" target="_blank" rel="nofollow" *ngIf="item.fragment !== 'get-cpfp'">{{ item.httpRequestMethod }} {{ baseNetworkUrl }}/api{{ item.urlString }}</a>
+                    <p *ngIf="item.fragment === 'get-cpfp'">{{ item.httpRequestMethod }} {{ baseNetworkUrl }}/api{{ item.urlString }}</p>
                   </ng-container>
                 </ng-template>
                 <ng-template #default_link_example>
                   <ng-container *ngIf="item.httpRequestMethod === 'GET'">
-                    <a [href]="wrapUrl(network.val, item.codeExample.default)" target="_blank" rel="nofollow">{{ item.httpRequestMethod }} {{ baseNetworkUrl }}/api{{ item.urlString }}</a>
+                    <a [href]="wrapUrl(network.val, item.codeExample.default)" target="_blank" rel="nofollow" *ngIf="item.fragment !== 'get-cpfp'">{{ item.httpRequestMethod }} {{ baseNetworkUrl }}/api{{ item.urlString }}</a>
+                    <p *ngIf="item.fragment === 'get-cpfp'">{{ item.httpRequestMethod }} {{ baseNetworkUrl }}/api{{ item.urlString }}</p>
                   </ng-container>
                 </ng-template>
                 <div *ngIf="item.httpRequestMethod === 'POST'">{{ item.httpRequestMethod }} {{ item.urlString }}</div>

frontend/src/app/lightning/lightning-dashboard/lightning-dashboard.component.html

@@ -37,25 +37,18 @@
       <div class="card graph-card">
         <div class="card-body pl-2 pr-2">
           <app-nodes-per-isp-chart [widget]="true"></app-nodes-per-isp-chart>
-          <div class="mt-1"><a [attr.data-cy]="'pool-distribution-view-more'" [routerLink]="['/graphs/lightning/nodes-per-isp' | relativeUrl]" i18n="dashboard.view-more">View more &raquo;</a></div>
+          <div class="mt-2"><a [attr.data-cy]="'pool-distribution-view-more'" [routerLink]="['/graphs/lightning/nodes-per-isp' | relativeUrl]" i18n="dashboard.view-more">View more &raquo;</a></div>
         </div>
       </div>
     </div>
 
-    <!-- <div class="col">
-      <div class="card">
-        <div class="card-body">
-          <app-nodes-networks-chart [widget]=true></app-nodes-networks-chart>
-          <div class="mt-1"><a [routerLink]="['/graphs/lightning/nodes-networks' | relativeUrl]" i18n="dashboard.view-more">View more &raquo;</a></div>
-        </div>
-      </div>
-    </div> -->
-
     <div class="col">
       <div class="card graph-card">
-        <div class="card-body pl-2 pr-2">
+        <div class="card-body pl-2 pr-2 pt-1">
+          <h5 class="card-title mt-3" i18n="lightning.network-history">Lightning network history</h5>
           <app-lightning-statistics-chart [widget]=true></app-lightning-statistics-chart>
-          <div class="mt-1"><a [routerLink]="['/graphs/lightning/capacity' | relativeUrl]" i18n="dashboard.view-more">View more &raquo;</a></div>
+          <app-nodes-networks-chart [widget]=true></app-nodes-networks-chart>
+          <div class="mt-1"><a [routerLink]="['/graphs/lightning/nodes-networks' | relativeUrl]" i18n="dashboard.view-more">View more &raquo;</a></div>
         </div>
       </div>
     </div>

frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.scss

@@ -51,8 +51,7 @@
 }
 .chart-widget {
   width: 100%;
-  height: 100%;
+  height: 145px;
-  max-height: 270px;
 }
 
 .formRadioGroup {

frontend/src/app/lightning/nodes-networks-chart/nodes-networks-chart.component.ts

@@ -1,5 +1,5 @@
 import { ChangeDetectionStrategy, Component, Inject, Input, LOCALE_ID, OnInit, HostBinding } from '@angular/core';
-import { EChartsOption} from 'echarts';
+import { EChartsOption, graphic} from 'echarts';
 import { Observable } from 'rxjs';
 import { map, share, startWith, switchMap, tap } from 'rxjs/operators';
 import { formatNumber } from '@angular/common';
@@ -9,6 +9,7 @@ import { MiningService } from 'src/app/services/mining.service';
 import { download } from 'src/app/shared/graphs.utils';
 import { SeoService } from 'src/app/services/seo.service';
 import { LightningApiService } from '../lightning-api.service';
+import { AmountShortenerPipe } from 'src/app/shared/pipes/amount-shortener.pipe';
 
 @Component({
   selector: 'app-nodes-networks-chart',
@@ -26,7 +27,7 @@ import { LightningApiService } from '../lightning-api.service';
 })
 export class NodesNetworksChartComponent implements OnInit {
   @Input() right: number | string = 45;
-  @Input() left: number | string = 55;
+  @Input() left: number | string = 45;
   @Input() widget = false;
 
   miningWindowPreference: string;
@@ -51,7 +52,8 @@ export class NodesNetworksChartComponent implements OnInit {
     private lightningApiService: LightningApiService,
     private formBuilder: FormBuilder,
     private storageService: StorageService,
-    private miningService: MiningService
+    private miningService: MiningService,
+    private amountShortenerPipe: AmountShortenerPipe,
   ) {
   }
 
@@ -82,11 +84,17 @@ export class NodesNetworksChartComponent implements OnInit {
             .pipe(
               tap((response) => {
                 const data = response.body;
-                this.prepareChartOptions({
+                const chartData = {
                   tor_nodes: data.map(val => [val.added * 1000, val.tor_nodes]),
                   clearnet_nodes: data.map(val => [val.added * 1000, val.clearnet_nodes]),
                   unannounced_nodes: data.map(val => [val.added * 1000, val.unannounced_nodes]),
-                });
+                };
+                let maxYAxis = 0;
+                for (const day of data) {
+                  maxYAxis = Math.max(maxYAxis, day.tor_nodes + day.clearnet_nodes + day.unannounced_nodes);
+                }
+                maxYAxis = Math.ceil(maxYAxis / 3000) * 3000;
+                this.prepareChartOptions(chartData, maxYAxis);
                 this.isLoading = false;
               }),
               map((response) => {
@@ -100,7 +108,7 @@ export class NodesNetworksChartComponent implements OnInit {
       );
   }
 
-  prepareChartOptions(data) {
+  prepareChartOptions(data, maxYAxis) {
     let title: object;
     if (data.tor_nodes.length === 0) {
       title = {
@@ -110,24 +118,30 @@ export class NodesNetworksChartComponent implements OnInit {
         },
         text: $localize`:@@23555386d8af1ff73f297e89dd4af3f4689fb9dd:Indexing blocks`,
         left: 'center',
-        top: 'center'
+        top: 'top',
+      };
+    } else if (this.widget) {
+      title = {
+        textStyle: {
+          color: 'grey',
+          fontSize: 11
+        },
+        text: $localize`Nodes per network`,
+        left: 'center',
+        top: 11,
+        zlevel: 10,
       };
     }
 
     this.chartOptions = {
       title: title,
       animation: false,
-      color: [
-        '#D81B60',
-        '#039BE5',
-        '#7CB342',
-        '#FFB300',
-      ],
       grid: {
-        top: 40,
+        height: this.widget ? 100 : undefined,
-        bottom: this.widget ? 30 : 70,
+        top: this.widget ? 10 : 40,
-        right: this.right,
+        bottom: this.widget ? 0 : 70,
-        left: this.left,
+        right: (this.isMobile() && this.widget) ? 35 : this.right,
+        left: (this.isMobile() && this.widget) ? 40 :this.left,
       },
       tooltip: {
         show: !this.isMobile() || !this.widget,
@@ -171,7 +185,7 @@ export class NodesNetworksChartComponent implements OnInit {
           hideOverlap: true,
         }
       },
-      legend: data.tor_nodes.length === 0 ? undefined : {
+      legend: this.widget || data.tor_nodes.length === 0 ? undefined : {
         padding: 10,
         data: [
           {
@@ -207,7 +221,7 @@ export class NodesNetworksChartComponent implements OnInit {
             icon: 'roundRect',
           },
         ],
-        selected: JSON.parse(this.storageService.getValue('nodes_networks_legend'))  ?? {
+        selected: this.widget ? undefined : JSON.parse(this.storageService.getValue('nodes_networks_legend'))  ?? {
           'Total': true,
           'Tor': true,
           'Clearnet': true,
@@ -218,13 +232,14 @@ export class NodesNetworksChartComponent implements OnInit {
         {
           type: 'value',
           position: 'left',
-          min: (value) => {
-            return value.min * 0.9;
-          },
           axisLabel: {
             color: 'rgb(110, 112, 121)',
-            formatter: (val) => {
+            formatter: (val: number): string => {
-              return `${formatNumber(Math.round(val * 100) / 100, this.locale, '1.0-0')}`;
+              if (this.widget) {
+                return `${this.amountShortenerPipe.transform(val, 0)}`;
+              } else {
+                return `${formatNumber(Math.round(val), this.locale, '1.0-0')}`;
+              }
             }
           },
           splitLine: {
@@ -232,8 +247,35 @@ export class NodesNetworksChartComponent implements OnInit {
               type: 'dotted',
               color: '#ffffff66',
               opacity: 0.25,
+            },
+          },
+          max: maxYAxis,
+          min: 0,
+          interval: 3000,
+        },
+        {
+          type: 'value',
+          position: 'right',
+          axisLabel: {
+            color: 'rgb(110, 112, 121)',
+            formatter: (val: number): string => {
+              if (this.widget) {
+                return `${this.amountShortenerPipe.transform(val, 0)}`;
+              } else {
+                return `${formatNumber(Math.round(val), this.locale, '1.0-0')}`;
+              }
             }
           },
+          splitLine: {
+            lineStyle: {
+              type: 'dotted',
+              color: '#ffffff66',
+              opacity: 0.25,
+            },
+          },
+          max: maxYAxis,
+          min: 0,
+          interval: 3000,
         }
       ],
       series: data.tor_nodes.length === 0 ? [] : [
@@ -252,7 +294,12 @@ export class NodesNetworksChartComponent implements OnInit {
             opacity: 0.5,
           },
           stack: 'Total',
-          color: '#FDD835',
+          color: new graphic.LinearGradient(0, 0.75, 0, 1, [
+            { offset: 0, color: '#D81B60' },
+            { offset: 1, color: '#D81B60AA' },
+          ]),
+
+          smooth: true,
         },
         {
           zlevel: 1,
@@ -269,11 +316,15 @@ export class NodesNetworksChartComponent implements OnInit {
             opacity: 0.5,
           },
           stack: 'Total',
-          color: '#00ACC1',
+          color: new graphic.LinearGradient(0, 0.75, 0, 1, [
+            { offset: 0, color: '#FFB300' },
+            { offset: 1, color: '#FFB300AA' },
+          ]),
+          smooth: true,
         },
         {
           zlevel: 1,
-          yAxisIndex: 0,
+          yAxisIndex: 1,
           name: $localize`Tor`,
           showSymbol: false,
           symbol: 'none',
@@ -286,7 +337,11 @@ export class NodesNetworksChartComponent implements OnInit {
             opacity: 0.5,
           },
           stack: 'Total',
-          color: '#7D4698',
+          color: new graphic.LinearGradient(0, 0.75, 0, 1, [
+            { offset: 0, color: '#7D4698' },
+            { offset: 1, color: '#7D4698AA' },
+          ]),
+          smooth: true,
         },
       ],
       dataZoom: this.widget ? null : [{

frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html

@@ -8,18 +8,18 @@
           {{ stats.taggedISP }}
         </p>
       </div>
-      <div class="item">
-        <h5 class="card-title d-inline-block" i18n="lightning.tagged-capacity">Tagged capacity</h5>
-        <p class="card-text" i18n-ngbTooltip="mining.blocks-count-desc">
-          <app-amount [satoshis]="stats.taggedCapacity" [digitsInfo]="'1.2-2'" [noFiat]="true"></app-amount>
-        </p>
-      </div>
       <div class="item">
         <h5 class="card-title d-inline-block" i18n="lightning.tagged-nodes">Tagged nodes</h5>
         <p class="card-text" i18n-ngbTooltip="mining.pools-count-desc">
           {{ stats.taggedNodeCount }}
         </p>
       </div>
+      <div class="item">
+        <h5 class="card-title d-inline-block" i18n="lightning.tagged-capacity">Tagged capacity</h5>
+        <p class="card-text" i18n-ngbTooltip="mining.blocks-count-desc">
+          <app-amount [satoshis]="stats.taggedCapacity" [digitsInfo]="'1.2-2'" [noFiat]="true"></app-amount>
+        </p>
+      </div>
     </div>
   </div>
 

frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.scss

@@ -51,7 +51,7 @@
 }
 .chart-widget {
   width: 100%;
-  height: 320px;
+  height: 145px;
 }
 
 .formRadioGroup {

frontend/src/app/lightning/statistics-chart/lightning-statistics-chart.component.ts

@@ -9,6 +9,7 @@ import { StorageService } from 'src/app/services/storage.service';
 import { MiningService } from 'src/app/services/mining.service';
 import { download } from 'src/app/shared/graphs.utils';
 import { LightningApiService } from '../lightning-api.service';
+import { AmountShortenerPipe } from 'src/app/shared/pipes/amount-shortener.pipe';
 
 @Component({
   selector: 'app-lightning-statistics-chart',
@@ -25,7 +26,7 @@ import { LightningApiService } from '../lightning-api.service';
 })
 export class LightningStatisticsChartComponent implements OnInit {
   @Input() right: number | string = 45;
-  @Input() left: number | string = 55;
+  @Input() left: number | string = 45;
   @Input() widget = false;
 
   miningWindowPreference: string;
@@ -51,6 +52,7 @@ export class LightningStatisticsChartComponent implements OnInit {
     private formBuilder: FormBuilder,
     private storageService: StorageService,
     private miningService: MiningService,
+    private amountShortenerPipe: AmountShortenerPipe,
   ) {
   }
 
@@ -105,24 +107,39 @@ export class LightningStatisticsChartComponent implements OnInit {
           color: 'grey',
           fontSize: 15
         },
-        text: `Indexing in progess`,
+        text: $localize`Indexing in progess`,
         left: 'center',
         top: 'center'
       };
+    } else if (this.widget) {
+      title = {
+        textStyle: {
+          color: 'grey',
+          fontSize: 11
+        },
+        text: $localize`Channels & Capacity`,
+        left: 'center',
+        top: 11,
+        zlevel: 10,
+      };
     }
 
     this.chartOptions = {
       title: title,
       animation: false,
       color: [
-        '#FDD835',
+        '#FFB300',
-        '#D81B60',
+        new graphic.LinearGradient(0, 0.75, 0, 1, [
+          { offset: 0, color: '#D81B60' },
+          { offset: 1, color: '#D81B60AA' },
+        ]),
       ],
       grid: {
-        top: 40,
+        height: this.widget ? 100 : undefined,
-        bottom: this.widget ? 30 : 70,
+        top: this.widget ? 10 : 40,
-        right: this.right,
+        bottom: this.widget ? 0 : 70,
-        left: this.left,
+        right: (this.isMobile() && this.widget) ? 35 : this.right,
+        left: (this.isMobile() && this.widget) ? 40 :this.left,
       },
       tooltip: {
         show: !this.isMobile(),
@@ -166,7 +183,7 @@ export class LightningStatisticsChartComponent implements OnInit {
           hideOverlap: true,
         }
       },
-      legend: data.channel_count.length === 0 ? undefined : {
+      legend: this.widget || data.channel_count.length === 0 ? undefined : {
         padding: 10,
         data: [
           {
@@ -178,7 +195,7 @@ export class LightningStatisticsChartComponent implements OnInit {
             icon: 'roundRect',
           },
           {
-            name: 'Capacity (BTC)',
+            name: 'Capacity',
             inactiveColor: 'rgb(110, 112, 121)',
             textStyle: {
               color: 'white',
@@ -188,17 +205,20 @@ export class LightningStatisticsChartComponent implements OnInit {
         ],
         selected: JSON.parse(this.storageService.getValue('sizes_ln_legend'))  ?? {
           'Channels': true,
-          'Capacity (BTC)': true,
+          'Capacity': true,
         }
       },
       yAxis: data.channel_count.length === 0 ? undefined : [
         {
-          min: 0,
           type: 'value',
           axisLabel: {
             color: 'rgb(110, 112, 121)',
-            formatter: (val) => {
+            formatter: (val: number): string => {
-              return `${formatNumber(Math.round(val), this.locale, '1.0-0')}`;
+              if (this.widget) {
+                return `${this.amountShortenerPipe.transform(val, 0)}`;
+              } else {
+                return `${formatNumber(Math.round(val), this.locale, '1.0-0')}`;
+              }
             }
           },
           splitLine: {
@@ -208,6 +228,7 @@ export class LightningStatisticsChartComponent implements OnInit {
               opacity: 0.25,
             }
           },
+          minInterval: this.widget ? 20000 : undefined,
         },
         {
           min: 0,
@@ -215,8 +236,12 @@ export class LightningStatisticsChartComponent implements OnInit {
           position: 'right',
           axisLabel: {
             color: 'rgb(110, 112, 121)',
-            formatter: (val) => {
+            formatter: (val: number): string => {
-              return `${formatNumber(Math.round(val / 100000000), this.locale, '1.0-0')}`;
+              if (this.widget) {
+                return `${this.amountShortenerPipe.transform(Math.round(val / 100000000), 0)}`;
+              } else {
+                return `${formatNumber(Math.round(val / 100000000), this.locale, '1.0-0')}`;
+              }
             }
           },
           splitLine: {
@@ -244,20 +269,49 @@ export class LightningStatisticsChartComponent implements OnInit {
               opacity: 1,
               width: 1,
             },
-          }
+          },
+          smooth: true,
         },
         {
           zlevel: 0,
           yAxisIndex: 1,
-          name: 'Capacity (BTC)',
+          name: $localize`Capacity`,
           showSymbol: false,
           symbol: 'none',
           stack: 'Total',
           data: data.capacity,
-          areaStyle: {},
+          areaStyle: {
+            opacity: 0.5,
+          },
           type: 'line',
+          smooth: true,
         }
       ],
+      dataZoom: this.widget ? null : [{
+        type: 'inside',
+        realtime: true,
+        zoomLock: true,
+        maxSpan: 100,
+        minSpan: 5,
+        moveOnMouseMove: false,
+      }, {
+        showDetail: false,
+        show: true,
+        type: 'slider',
+        brushSelect: false,
+        realtime: true,
+        left: 20,
+        right: 15,
+        selectedDataBackground: {
+          lineStyle: {
+            color: '#fff',
+            opacity: 0.45,
+          },
+          areaStyle: {
+            opacity: 0,
+          }
+        },
+      }],
     };
   }
 

frontend/src/app/shared/pipes/amount-shortener.pipe.ts

@@ -5,7 +5,7 @@ import { Pipe, PipeTransform } from '@angular/core';
 })
 export class AmountShortenerPipe implements PipeTransform {
   transform(num: number, ...args: any[]): unknown {
-    const digits = args[0] || 1;
+    const digits = args[0] ?? 1;
     const unit = args[1] || undefined;
 
     if (num < 1000) {