Merge branch 'master' into nymkappa/bugfix/stats-import

backend/src/api/explorer/nodes.api.ts

@@ -168,64 +168,115 @@ class NodesApi {
     }
   }
 
-  public async $getNodesISPRanking(groupBy: string, showTor: boolean) {
+  public async $getNodesISPRanking() {
     try {
-      const orderBy = groupBy === 'capacity' ? `CAST(SUM(capacity) as INT)` : `COUNT(DISTINCT nodes.public_key)`;
+      let query = '';
-      
-      // Clearnet
-      let query = `SELECT GROUP_CONCAT(DISTINCT(nodes.as_number)) as ispId, geo_names.names as names,
-          COUNT(DISTINCT nodes.public_key) as nodesCount, CAST(SUM(capacity) as INT) as capacity
-        FROM nodes
-        JOIN geo_names ON geo_names.id = nodes.as_number
-        JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key
-        GROUP BY geo_names.names
-        ORDER BY ${orderBy} DESC
-      `;      
-      const [nodesCountPerAS]: any = await DB.query(query);
 
-      let total = 0;
+      // List all channels and the two linked ISP
-      const nodesPerAs: any[] = [];
+      query = `
+        SELECT short_id, capacity,
+          channels.node1_public_key AS node1PublicKey, isp1.names AS isp1, isp1.id as isp1ID,
+          channels.node2_public_key AS node2PublicKey, isp2.names AS isp2, isp2.id as isp2ID
+        FROM channels
+        JOIN nodes node1 ON node1.public_key = channels.node1_public_key
+        JOIN nodes node2 ON node2.public_key = channels.node2_public_key
+        JOIN geo_names isp1 ON isp1.id = node1.as_number
+        JOIN geo_names isp2 ON isp2.id = node2.as_number
+        WHERE channels.status = 1
+        ORDER BY short_id DESC
+      `;
+      const [channelsIsp]: any = await DB.query(query);
 
-      for (const asGroup of nodesCountPerAS) {
+      // Sum channels capacity and node count per ISP
-        if (groupBy === 'capacity') {
+      const ispList = {};
-          total += asGroup.capacity;
+      for (const channel of channelsIsp) {
-        } else {
+        const isp1 = JSON.parse(channel.isp1);
-          total += asGroup.nodesCount;
+        const isp2 = JSON.parse(channel.isp2);
+
+        if (!ispList[isp1]) {
+          ispList[isp1] = {
+            id: channel.isp1ID,
+            capacity: 0,
+            channels: 0,
+            nodes: {},
+          };
         }
+        if (!ispList[isp2]) {
+          ispList[isp2] = {
+            id: channel.isp2ID,
+            capacity: 0,
+            channels: 0,
+            nodes: {},
+          };
+        }
+        
+        ispList[isp1].capacity += channel.capacity;
+        ispList[isp1].channels += 1;
+        ispList[isp1].nodes[channel.node1PublicKey] = true;
+        ispList[isp2].capacity += channel.capacity;
+        ispList[isp2].channels += 1;
+        ispList[isp2].nodes[channel.node2PublicKey] = true;
       }
 
-      // Tor
+      const ispRanking: any[] = [];
-      if (showTor) {
+      for (const isp of Object.keys(ispList)) {
-        query = `SELECT COUNT(DISTINCT nodes.public_key) as nodesCount, CAST(SUM(capacity) as INT) as capacity
+        ispRanking.push([
-          FROM nodes
+          ispList[isp].id,
-          JOIN channels ON channels.node1_public_key = nodes.public_key OR channels.node2_public_key = nodes.public_key
+          isp,
-          ORDER BY ${orderBy} DESC
+          ispList[isp].capacity,
-        `;      
+          ispList[isp].channels,
-        const [nodesCountTor]: any = await DB.query(query);
+          Object.keys(ispList[isp].nodes).length,
-
+        ]);
-        total += groupBy === 'capacity' ? nodesCountTor[0].capacity : nodesCountTor[0].nodesCount;
-        nodesPerAs.push({
-          ispId: null,
-          name: 'Tor',
-          count: nodesCountTor[0].nodesCount,
-          share: Math.floor((groupBy === 'capacity' ? nodesCountTor[0].capacity : nodesCountTor[0].nodesCount) / total * 10000) / 100,
-          capacity: nodesCountTor[0].capacity,
-        });
       }
 
-      for (const as of nodesCountPerAS) {
+      // Total active channels capacity
-        nodesPerAs.push({
+      query = `SELECT SUM(capacity) AS capacity FROM channels WHERE status = 1`;
-          ispId: as.ispId,
+      const [totalCapacity]: any = await DB.query(query);
-          name: JSON.parse(as.names),
-          count: as.nodesCount,
-          share: Math.floor((groupBy === 'capacity' ? as.capacity : as.nodesCount) / total * 10000) / 100,
-          capacity: as.capacity,
-        });
-      }
 
-      return nodesPerAs;
+      // Get the total capacity of all channels which have at least one node on clearnet
+      query = `
+        SELECT SUM(capacity) as capacity
+        FROM (
+          SELECT capacity, GROUP_CONCAT(socket1.type, socket2.type) as networks
+          FROM channels
+          JOIN nodes_sockets socket1 ON node1_public_key = socket1.public_key
+          JOIN nodes_sockets socket2 ON node2_public_key = socket2.public_key
+          AND channels.status = 1
+          GROUP BY short_id
+        ) channels_tmp
+        WHERE channels_tmp.networks LIKE '%ipv%'
+      `;
+      const [clearnetCapacity]: any = await DB.query(query);
+
+      // Get the total capacity of all channels which have both nodes on Tor 
+      query = `
+        SELECT SUM(capacity) as capacity
+        FROM (
+          SELECT capacity, GROUP_CONCAT(socket1.type, socket2.type) as networks
+          FROM channels
+          JOIN nodes_sockets socket1 ON node1_public_key = socket1.public_key
+          JOIN nodes_sockets socket2 ON node2_public_key = socket2.public_key
+          AND channels.status = 1
+          GROUP BY short_id
+        ) channels_tmp
+        WHERE channels_tmp.networks NOT LIKE '%ipv%' AND
+          channels_tmp.networks NOT LIKE '%dns%' AND
+          channels_tmp.networks NOT LIKE '%websocket%'
+      `;
+      const [torCapacity]: any = await DB.query(query);
+
+      const clearnetCapacityValue = parseInt(clearnetCapacity[0].capacity, 10);
+      const torCapacityValue = parseInt(torCapacity[0].capacity, 10);
+      const unknownCapacityValue = parseInt(totalCapacity[0].capacity) - clearnetCapacityValue - torCapacityValue;
+
+      return {
+        clearnetCapacity: clearnetCapacityValue,
+        torCapacity: torCapacityValue,
+        unknownCapacity: unknownCapacityValue,
+        ispRanking: ispRanking,
+      };
     } catch (e) {
-      logger.err(`Cannot get nodes grouped by AS. Reason: ${e instanceof Error ? e.message : e}`);
+      logger.err(`Cannot get LN ISP ranking. Reason: ${e instanceof Error ? e.message : e}`);
       throw e;
     }
   }

backend/src/api/explorer/nodes.routes.ts

@@ -71,15 +71,7 @@ class NodesRoutes {
 
   private async $getISPRanking(req: Request, res: Response): Promise<void> {
     try {
-      const groupBy = req.query.groupBy as string;
+      const nodesPerAs = await nodesApi.$getNodesISPRanking();
-      const showTor = req.query.showTor as string === 'true' ? true : false;
-
-      if (!['capacity', 'node-count'].includes(groupBy)) {
-        res.status(400).send(`groupBy must be one of 'capacity' or 'node-count'`);
-        return;
-      }
-
-      const nodesPerAs = await nodesApi.$getNodesISPRanking(groupBy, showTor);
 
       res.header('Pragma', 'public');
       res.header('Cache-control', 'public');

frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.html

@@ -3,21 +3,24 @@
   <div *ngIf="widget">
     <div class="pool-distribution" *ngIf="(nodesPerAsObservable$ | async) as stats; else loadingReward">
       <div class="item">
-        <h5 class="card-title d-inline-block" i18n="lightning.tagged-isp">Tagged ISPs</h5>
+        <h5 class="card-title d-inline-block" i18n="lightning.clearnet-capacity">Clearnet capacity</h5>
-        <p class="card-text">
+        <p class="card-text" i18n-ngbTooltip="lightning.clearnet-capacity-desc"
-          {{ stats.taggedISP }}
+          ngbTooltip="How much liquidity is running on nodes advertising at least one clearnet IP address" placement="bottom">
+          <app-amount [satoshis]="stats.clearnetCapacity" [digitsInfo]="'1.2-2'" [noFiat]="true"></app-amount>
         </p>
       </div>
       <div class="item">
-        <h5 class="card-title d-inline-block" i18n="lightning.tagged-nodes">Tagged nodes</h5>
+        <h5 class="card-title d-inline-block">Unknown capacity</h5>
-        <p class="card-text" i18n-ngbTooltip="mining.pools-count-desc">
+        <p class="card-text" i18n-ngbTooltip="lightning.unknown-capacity-desc"
-          {{ stats.taggedNodeCount }}
+        ngbTooltip="How much liquidity is running on nodes which ISP was not identifiable" placement="bottom">
+          <app-amount [satoshis]="stats.unknownCapacity" [digitsInfo]="'1.2-2'" [noFiat]="true"></app-amount>
         </p>
       </div>
       <div class="item">
-        <h5 class="card-title d-inline-block" i18n="lightning.tagged-capacity">Tagged capacity</h5>
+        <h5 class="card-title d-inline-block">Tor capacity</h5>
-        <p class="card-text" i18n-ngbTooltip="mining.blocks-count-desc">
+        <p class="card-text" i18n-ngbTooltip="lightning.tor-capacity-desc"
-          <app-amount [satoshis]="stats.taggedCapacity" [digitsInfo]="'1.2-2'" [noFiat]="true"></app-amount>
+        ngbTooltip="How much liquidity is running on nodes advertising only Tor addresses" placement="bottom">
+          <app-amount [satoshis]="stats.torCapacity" [digitsInfo]="'1.2-2'" [noFiat]="true"></app-amount>
         </p>
       </div>
     </div>
@@ -25,13 +28,13 @@
 
   <div class="card-header" *ngIf="!widget">
     <div class="d-flex d-md-block align-items-baseline" style="margin-bottom: -5px">
-      <span i18n="lightning.nodes-per-isp">Lightning nodes per ISP</span>
+      <span i18n="lightning.top-100-isp-ln">Top 100 ISP hosting LN nodes</span>
       <button class="btn p-0 pl-2" style="margin: 0 0 4px 0px" (click)="onSaveChart()">
         <fa-icon [icon]="['fas', 'download']" [fixedWidth]="true"></fa-icon>
       </button>
     </div>
     <small class="d-block" style="color: #ffffff66; min-height: 25px" i18n="lightning.tor-nodes-excluded">
-      <span *ngIf="!(showTorObservable$ | async)">(Tor nodes excluded)</span>
+      <span>(Tor nodes excluded)</span>
     </small>
   </div>
 
@@ -44,9 +47,8 @@
       <div class="spinner-border text-light"></div>
     </div>
 
-    <div class="d-flex toggle" *ngIf="!widget">
+    <div class="d-flex justify-content-md-end toggle" *ngIf="!widget">
-      <app-toggle [textLeft]="'Show Tor'" [textRight]="" (toggleStatusChanged)="onTorToggleStatusChanged($event)"></app-toggle>
+      <app-toggle [textLeft]="'Sort by nodes'" [textRight]="'capacity'" [checked]="true" (toggleStatusChanged)="onGroupToggleStatusChanged($event)"></app-toggle>
-      <app-toggle [textLeft]="'Nodes'" [textRight]="'Capacity'" (toggleStatusChanged)="onGroupToggleStatusChanged($event)"></app-toggle>
     </div>
 
     <table class="table table-borderless text-center m-auto" style="max-width: 900px"  *ngIf="!widget">
@@ -59,16 +61,15 @@
           <th class="capacity text-right pr-0" i18n="lightning.capacity">Capacity</th>
         </tr>
       </thead>
-      <tbody [attr.data-cy]="'pools-table'" *ngIf="(nodesPerAsObservable$ | async) as asList">
+      <tbody [attr.data-cy]="'pools-table'" *ngIf="(nodesPerAsObservable$ | async) as result">
-        <tr *ngFor="let asEntry of asList.data">
+        <tr *ngFor="let isp of result.ispRanking">
-          <td class="rank text-left pl-0">{{ asEntry.rank }}</td>
+          <td class="rank text-left pl-0">{{ isp[5] }}</td>
           <td class="name text-left text-truncate">
-            <a *ngIf="asEntry.ispId" [routerLink]="[('/lightning/nodes/isp/' + asEntry.ispId) | relativeUrl]">{{ asEntry.name }}</a>
+            <a [routerLink]="[('/lightning/nodes/isp/' + isp[0]) | relativeUrl]">{{ isp[1] }}</a>
-            <span *ngIf="!asEntry.ispId">{{ asEntry.name }}</span>
           </td>
-          <td class="share text-right">{{ asEntry.share }}%</td>
+          <td class="share text-right">{{ sortBy === 'capacity' ? isp[7] : isp[6] }}%</td>
-          <td class="nodes text-right">{{ asEntry.count }}</td>
+          <td class="nodes text-right">{{ isp[4] }}</td>
-          <td class="capacity text-right pr-0"><app-amount [satoshis]="asEntry.capacity" [digitsInfo]="'1.2-2'" [noFiat]="true"></app-amount></td>
+          <td class="capacity text-right pr-0"><app-amount [satoshis]="isp[2]" [digitsInfo]="'1.2-2'" [noFiat]="true"></app-amount></td>
         </tr>
       </tbody>
     </table>

frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.scss

@@ -149,7 +149,8 @@
 }
 
 .name {
-  width: 25%;
+  width: 35%;
+  max-width: 300px;
   @media (max-width: 576px) {
     width: 70%;
     max-width: 150px;
@@ -159,14 +160,14 @@
 }
 
 .share {
-  width: 20%;
+  width: 15%;
   @media (max-width: 576px) {
     display: none
   }
 }
 
 .nodes {
-  width: 20%;
+  width: 15%;
   @media (max-width: 576px) {
     width: 10%;
   }

frontend/src/app/lightning/nodes-per-isp-chart/nodes-per-isp-chart.component.ts

@@ -26,14 +26,15 @@ export class NodesPerISPChartComponent implements OnInit {
     renderer: 'svg',
   };
   timespan = '';
+  sortBy = 'capacity';
+  showUnknown = false;
   chartInstance = undefined;
 
   @HostBinding('attr.dir') dir = 'ltr';
 
   nodesPerAsObservable$: Observable<any>;
-  showTorObservable$: Observable<boolean>;
+  sortBySubject = new Subject<boolean>();
-  groupBySubject = new Subject<boolean>();
+  showUnknownSubject = new Subject<boolean>();
-  showTorSubject = new Subject<boolean>();
 
   constructor(
     private apiService: ApiService,
@@ -48,32 +49,49 @@ export class NodesPerISPChartComponent implements OnInit {
   ngOnInit(): void {
     this.seoService.setTitle($localize`Lightning nodes per ISP`);
 
-    this.showTorObservable$ = this.showTorSubject.asObservable();
-
     this.nodesPerAsObservable$ = combineLatest([
-      this.groupBySubject.pipe(startWith(false)),
+      this.sortBySubject.pipe(startWith(true)),
-      this.showTorSubject.pipe(startWith(false)),
     ])
       .pipe(
         switchMap((selectedFilters) => {
-          return this.apiService.getNodesPerAs(
+          this.sortBy = selectedFilters[0] ? 'capacity' : 'node-count';
-            selectedFilters[0] ? 'capacity' : 'node-count',
+          return this.apiService.getNodesPerIsp()
-            selectedFilters[1] // Show Tor nodes
-          )
             .pipe(
-              tap(data => {
+              tap(() => {
                 this.isLoading = false;
-                this.prepareChartOptions(data);
               }),
               map(data => {
-                for (let i = 0; i < data.length; ++i) {
+                let nodeCount = 0;
-                  data[i].rank = i + 1;
+                let totalCapacity = 0;
+
+                for (let i = 0; i < data.ispRanking.length; ++i) {
+                  nodeCount += data.ispRanking[i][4];
+                  totalCapacity += data.ispRanking[i][2];
+                  data.ispRanking[i][5] = i;
                 }
+                for (let i = 0; i < data.ispRanking.length; ++i) {
+                  data.ispRanking[i][6] = Math.round(data.ispRanking[i][4] / nodeCount * 10000) / 100;
+                  data.ispRanking[i][7] = Math.round(data.ispRanking[i][2] / totalCapacity * 10000) / 100;
+                }
+
+                if (selectedFilters[0] === true) {
+                  data.ispRanking.sort((a, b) => b[7] - a[7]);
+                } else {
+                  data.ispRanking.sort((a, b) => b[6] - a[6]);
+                }
+
+                for (let i = 0; i < data.ispRanking.length; ++i) {
+                  data.ispRanking[i][5] = i + 1;
+                }
+
+                this.prepareChartOptions(data.ispRanking);
+
                 return {
-                  taggedISP: data.length,
+                  taggedISP: data.ispRanking.length,
-                  taggedCapacity: data.reduce((partialSum, isp) => partialSum + isp.capacity, 0),
+                  clearnetCapacity: data.clearnetCapacity,
-                  taggedNodeCount: data.reduce((partialSum, isp) => partialSum + isp.count, 0),
+                  unknownCapacity: data.unknownCapacity,
-                  data: data.slice(0, 100),
+                  torCapacity: data.torCapacity,
+                  ispRanking: data.ispRanking.slice(0, 100),
                 };
               })
             );
@@ -82,22 +100,22 @@ export class NodesPerISPChartComponent implements OnInit {
       );
 
     if (this.widget) {
-      this.showTorSubject.next(false);
+      this.sortBySubject.next(false);  
-      this.groupBySubject.next(false);  
     }
   }
 
-  generateChartSerieData(as): PieSeriesOption[] {
+  generateChartSerieData(ispRanking): PieSeriesOption[] {
     let shareThreshold = 0.5;
     if (this.widget && isMobile() || isMobile()) {
       shareThreshold = 1;
     } else if (this.widget) {
       shareThreshold = 0.75;
     }
-    
+
     const data: object[] = [];
     let totalShareOther = 0;
-    let totalNodeOther = 0;
+    let nodeCountOther = 0;
+    let capacityOther = 0;
 
     let edgeDistance: string | number = '10%';
     if (isMobile() && this.widget) {
@@ -106,18 +124,19 @@ export class NodesPerISPChartComponent implements OnInit {
       edgeDistance = 10;
     }
 
-    as.forEach((as) => {
+    ispRanking.forEach((isp) => {
-      if (as.share < shareThreshold) {
+      if ((this.sortBy === 'capacity' ? isp[7] : isp[6]) < shareThreshold) {
-        totalShareOther += as.share;
+        totalShareOther += this.sortBy === 'capacity' ? isp[7] : isp[6];
-        totalNodeOther += as.count;
+        nodeCountOther += isp[4];
+        capacityOther += isp[2];
         return;
       }
       data.push({
         itemStyle: {
-          color: as.ispId === null ? '#7D4698' : undefined,
+          color: isp[0] === null ? '#7D4698' : undefined,
         },
-        value: as.share,
+        value: this.sortBy === 'capacity' ? isp[7] : isp[6],
-        name: as.name + (isMobile() || this.widget ? `` : ` (${as.share}%)`),
+        name: isp[1].replace('&', '') + (isMobile() || this.widget ? `` : ` (${this.sortBy === 'capacity' ? isp[7] : isp[6]}%)`),
         label: {
           overflow: 'truncate',
           width: isMobile() ? 75 : this.widget ? 125 : 250,
@@ -135,13 +154,13 @@ export class NodesPerISPChartComponent implements OnInit {
           },
           borderColor: '#000',
           formatter: () => {
-            return `<b style="color: white">${as.name} (${as.share}%)</b><br>` +
+            return `<b style="color: white">${isp[1]} (${isp[6]}%)</b><br>` +
-              $localize`${as.count.toString()} nodes<br>` +
+              $localize`${isp[4].toString()} nodes<br>` +
-              $localize`${this.amountShortenerPipe.transform(as.capacity / 100000000, 2)} BTC capacity`
+              $localize`${this.amountShortenerPipe.transform(isp[2] / 100000000, 2)} BTC`
             ;
           }
         },
-        data: as.ispId,
+        data: isp[0],
       } as PieSeriesOption);
     });
 
@@ -167,8 +186,9 @@ export class NodesPerISPChartComponent implements OnInit {
         },
         borderColor: '#000',
         formatter: () => {
-          return `<b style="color: white">${'Other'} (${totalShareOther.toFixed(2)}%)</b><br>` +
+          return `<b style="color: white">Other (${totalShareOther.toFixed(2)}%)</b><br>` +
-            totalNodeOther.toString() + ` nodes`;
+            $localize`${nodeCountOther.toString()} nodes<br>` +
+            $localize`${this.amountShortenerPipe.transform(capacityOther / 100000000, 2)} BTC`;
         }
       },
       data: 9999 as any,
@@ -177,7 +197,7 @@ export class NodesPerISPChartComponent implements OnInit {
     return data;
   }
 
-  prepareChartOptions(as): void {
+  prepareChartOptions(ispRanking): void {
     let pieSize = ['20%', '80%']; // Desktop
     if (isMobile() && !this.widget) {
       pieSize = ['15%', '60%'];
@@ -194,11 +214,11 @@ export class NodesPerISPChartComponent implements OnInit {
       series: [
         {
           zlevel: 0,
-          minShowLabelAngle: 1.8,
+          minShowLabelAngle: 0.9,
           name: 'Lightning nodes',
           type: 'pie',
           radius: pieSize,
-          data: this.generateChartSerieData(as),
+          data: this.generateChartSerieData(ispRanking),
           labelLine: {
             lineStyle: {
               width: 2,
@@ -259,16 +279,8 @@ export class NodesPerISPChartComponent implements OnInit {
     this.chartInstance.setOption(this.chartOptions);
   }
 
-  onTorToggleStatusChanged(e): void {
-    this.showTorSubject.next(e);
-  }
-
   onGroupToggleStatusChanged(e): void {
-    this.groupBySubject.next(e);
+    this.sortBySubject.next(e);
-  }
-
-  isEllipsisActive(e) {
-    return (e.offsetWidth < e.scrollWidth);
   }
 }
 

frontend/src/app/services/api.service.ts

@@ -255,9 +255,8 @@ export class ApiService {
     return this.httpClient.get<any[]>(this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/search', { params });
   }
 
-  getNodesPerAs(groupBy: 'capacity' | 'node-count', showTorNodes: boolean): Observable<any> {
+  getNodesPerIsp(): Observable<any> {
-    return this.httpClient.get<any[]>(this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/nodes/isp-ranking'
+    return this.httpClient.get<any[]>(this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/nodes/isp-ranking');
-      + `?groupBy=${groupBy}&showTor=${showTorNodes}`);
   }
 
   getNodeForCountry$(country: string): Observable<any> {

frontend/src/app/shared/components/toggle/toggle.component.html

@@ -1,7 +1,7 @@
 <div class="d-flex align-items-center">
   <span style="margin-bottom: 0.5rem">{{ textLeft }}</span>&nbsp;
   <label class="switch">
-    <input type="checkbox" (change)="onToggleStatusChanged($event)">
+    <input type="checkbox" [checked]="checked" (change)="onToggleStatusChanged($event)">
     <span class="slider round"></span>
   </label>
   &nbsp;<span style="margin-bottom: 0.5rem">{{ textRight }}</span>

frontend/src/app/shared/components/toggle/toggle.component.ts

@@ -10,6 +10,7 @@ export class ToggleComponent implements AfterViewInit {
   @Output() toggleStatusChanged = new EventEmitter<boolean>();
   @Input() textLeft: string;
   @Input() textRight: string;
+  @Input() checked: boolean = false;
 
   ngAfterViewInit(): void {
     this.toggleStatusChanged.emit(false);