Merge branch 'master' into nymkappa/bugfix/db-deadlock

backend/src/api/explorer/channels.api.ts

@@ -17,14 +17,25 @@ class ChannelsApi {
     }
   }
 
-  public async $getAllChannelsGeo(publicKey?: string): Promise<any[]> {
+  public async $getAllChannelsGeo(publicKey?: string, style?: string): Promise<any[]> {
     try {
-      const params: string[] = [];
+      let select: string;
-      let query = `SELECT nodes_1.public_key as node1_public_key, nodes_1.alias AS node1_alias,
+      if (style === 'widget') {
+        select = `
+          nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude,
+          nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude
+        `;
+      } else {
+        select = `
+          nodes_1.public_key as node1_public_key, nodes_1.alias AS node1_alias,
           nodes_1.latitude AS node1_latitude, nodes_1.longitude AS node1_longitude,
           nodes_2.public_key as node2_public_key, nodes_2.alias AS node2_alias,
-        nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude,
+          nodes_2.latitude AS node2_latitude, nodes_2.longitude AS node2_longitude
-        channels.capacity
+        `;
+      }
+
+      const params: string[] = [];
+      let query = `SELECT ${select}
         FROM channels
         JOIN nodes AS nodes_1 on nodes_1.public_key = channels.node1_public_key
         JOIN nodes AS nodes_2 on nodes_2.public_key = channels.node2_public_key
@@ -36,13 +47,30 @@ class ChannelsApi {
         query += ' AND (nodes_1.public_key = ? OR nodes_2.public_key = ?)';
         params.push(publicKey);
         params.push(publicKey);
+      } else {
+        query += ` AND channels.capacity > 1000000
+          GROUP BY nodes_1.public_key, nodes_2.public_key
+          ORDER BY channels.capacity DESC
+          LIMIT 10000
+        `;        
       }
 
       const [rows]: any = await DB.query(query, params);
-      return rows.map((row) => [
+      return rows.map((row) => {
-        row.node1_public_key, row.node1_alias, row.node1_longitude, row.node1_latitude,
+        if (style === 'widget') {
-        row.node2_public_key, row.node2_alias, row.node2_longitude, row.node2_latitude,
+          return [
-        row.capacity]);
+            row.node1_longitude, row.node1_latitude,
+            row.node2_longitude, row.node2_latitude,
+          ];
+        } else {
+          return [
+            row.node1_public_key, row.node1_alias,
+            row.node1_longitude, row.node1_latitude,
+            row.node2_public_key, row.node2_alias,
+            row.node2_longitude, row.node2_latitude,
+          ];
+        }
+      });
     } catch (e) {
       logger.err('$getAllChannelsGeo error: ' + (e instanceof Error ? e.message : e));
       throw e;

backend/src/api/explorer/channels.routes.ts

@@ -102,7 +102,11 @@ class ChannelsRoutes {
 
   private async $getAllChannelsGeo(req: Request, res: Response) {
     try {
-      const channels = await channelsApi.$getAllChannelsGeo(req.params?.publicKey);
+      const style: string = typeof req.query.style === 'string' ? req.query.style : '';
+      const channels = await channelsApi.$getAllChannelsGeo(req.params?.publicKey, style);
+      res.header('Pragma', 'public');
+      res.header('Cache-control', 'public');
+      res.setHeader('Expires', new Date(Date.now() + 1000 * 60).toUTCString());
       res.json(channels);
     } catch (e) {
       res.status(500).send(e instanceof Error ? e.message : e);

backend/src/tasks/lightning/network-sync.service.ts

@@ -98,7 +98,7 @@ class NetworkSyncService {
       const [closedChannelsRaw]: any[] = await DB.query(`SELECT id FROM channels WHERE status = 2`);
       const closedChannels = {};
       for (const closedChannel of closedChannelsRaw) {
-        closedChannels[Common.channelShortIdToIntegerId(closedChannel.id)] = true;
+        closedChannels[closedChannel.id] = true;
       }
 
       let progress = 0;

frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.html

@@ -1,3 +1,4 @@
+<div [style]="style === 'widget' ? 'height: 250px' : ''">
   <div *ngIf="channelsObservable | async">
     <div *ngIf="chartOptions" [class]="'full-container ' + style + (fitContainer ? ' fit-container' : '')">
       <div *ngIf="style === 'graph'" class="card-header">
@@ -7,10 +8,15 @@
         <small style="color: #ffffff66" i18n="lightning.tor-nodes-excluded">(Tor nodes excluded)</small>
       </div>
 
-    <div class="chart" echarts [initOpts]="chartInitOptions" [options]="chartOptions" (chartInit)="onChartInit($event)"
+      <div class="chart" [class]="style" echarts [initOpts]="chartInitOptions" [options]="chartOptions" (chartInit)="onChartInit($event)"
         (chartFinished)="onChartFinished($event)">
       </div>
     </div>
 
     <div *ngIf="!chartOptions && style === 'nodepage'" style="padding-top: 30px"></div>
   </div>
+
+  <div class="text-center loading-spinner" [class]="style" *ngIf="isLoading">
+    <div class="spinner-border text-light"></div>
+  </div>
+</div>

frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.scss

@@ -10,7 +10,7 @@
 .full-container {
   padding: 0px 15px;
   width: 100%;
-  min-height: 500px;
+  min-height: 600px;
   height: calc(100% - 150px);
 
   @media (max-width: 992px) {
@@ -18,17 +18,20 @@
     padding-bottom: 100px;
   }
 }
-
 .full-container.nodepage {
+  min-height: 400px;
+  margin-top: 25px;
+  margin-bottom: 25px;
+}
+.full-container.channelpage {
+  min-height: 400px;
   margin-top: 25px;
   margin-bottom: 25px;
 }
-
 .full-container.widget {
   height: 250px;
   min-height: 250px;
 }
-
 .full-container.fit-container {
   margin: 0;
   padding: 0;
@@ -41,25 +44,6 @@
   }
 }
 
-.widget {
-  width: 90vw;
-  margin-left: auto;
-  margin-right: auto;
-  height: 250px;
-  -webkit-mask: linear-gradient(0deg, #11131f00 5%, #11131fff 25%);
-  @media (max-width: 767.98px) {
-    width: 100vw;
-  }
-}
-
-.widget > .chart {
-  min-height: 250px;
-  -webkit-mask: linear-gradient(180deg, #11131f00 0%, #11131fff 20%);
-  @media (max-width: 767.98px) {
-    padding-bottom: 0px;
-  }
-}
-
 .chart {
   min-height: 500px;
   width: 100%;
@@ -80,3 +64,47 @@
     padding-bottom: 55px;
   }
 }
+.chart.graph {
+  min-height: 600px;
+}
+.chart.nodepage {
+  min-height: 400px;
+}
+.chart.channelpage {
+  min-height: 400px;
+}
+
+.widget {
+  width: 90vw;
+  margin-left: auto;
+  margin-right: auto;
+  height: 250px;
+  -webkit-mask: linear-gradient(0deg, #11131f00 5%, #11131fff 25%);
+  @media (max-width: 767.98px) {
+    width: 100vw;
+  }
+}
+.widget > .chart {
+  min-height: 250px;
+  -webkit-mask: linear-gradient(180deg, #11131f00 0%, #11131fff 20%);
+  @media (max-width: 767.98px) {
+    padding-bottom: 0px;
+  }
+}
+
+.loading-spinner {
+  position: absolute;
+  top: 50%;
+  left: calc(50% - 15px);
+  z-index: 100;
+}
+.loading-spinner.widget {
+  position: absolute;
+  top: 200px;
+  z-index: 100;
+  width: 100%;
+  left: 0;
+  @media (max-width: 767.98px) {
+    top: 250px;
+  }
+}

frontend/src/app/lightning/nodes-channels-map/nodes-channels-map.component.ts

@@ -32,6 +32,7 @@ export class NodesChannelsMap implements OnInit {
   channelColor = '#466d9d';
   channelCurve = 0;
   nodeSize = 4;
+  isLoading = true;
 
   chartInstance = undefined;
   chartOptions: EChartsOption = {};
@@ -74,7 +75,7 @@ export class NodesChannelsMap implements OnInit {
        switchMap((params: ParamMap) => {
         return zip(
           this.assetsService.getWorldMapJson$,
-          this.style !== 'channelpage' ? this.apiService.getChannelsGeo$(params.get('public_key') ?? undefined) : [''],
+          this.style !== 'channelpage' ? this.apiService.getChannelsGeo$(params.get('public_key') ?? undefined, this.style) : [''],
           [params.get('public_key') ?? undefined]
         ).pipe(tap((data) => {
           registerMap('world', data[0]);
@@ -93,11 +94,13 @@ export class NodesChannelsMap implements OnInit {
             }
           }
           for (const channel of geoloc) {
-            if (!thisNodeGPS && data[2] === channel[0]) {
+            if (this.style === 'nodepage' && !thisNodeGPS) {
+              if (data[2] === channel[0]) {
                 thisNodeGPS = [channel[2], channel[3]];
-            } else if (!thisNodeGPS && data[2] === channel[4]) {
+              } else if (data[2] === channel[4]) {
                 thisNodeGPS = [channel[6], channel[7]];
               }
+            }
 
             // 0 - node1 pubkey
             // 1 - node1 alias
@@ -105,48 +108,68 @@ export class NodesChannelsMap implements OnInit {
             // 4 - node2 pubkey
             // 5 - node2 alias
             // 6,7 - node2 GPS
+            const node1PubKey = 0;
+            const node1Alias = 1;
+            let node1GpsLat = 2;
+            let node1GpsLgt = 3;
+            const node2PubKey = 4;
+            const node2Alias = 5;
+            let node2GpsLat = 6;
+            let node2GpsLgt = 7;
+            let node1UniqueId = channel[node1PubKey];
+            let node2UniqueId = channel[node2PubKey];
+            if (this.style === 'widget') {
+              node1GpsLat = 0;
+              node1GpsLgt = 1;
+              node2GpsLat = 2;
+              node2GpsLgt = 3;
+              node1UniqueId = channel[node1GpsLat].toString() + channel[node1GpsLgt].toString();
+              node2UniqueId = channel[node2GpsLat].toString() + channel[node2GpsLgt].toString();
+            }
 
             // We add a bit of noise so nodes at the same location are not all
             // on top of each other
             let random = Math.random() * 2 * Math.PI;
             let random2 = Math.random() * 0.01;
             
-            if (!nodesPubkeys[channel[0]]) {
+            if (!nodesPubkeys[node1UniqueId]) {
               nodes.push([
-                channel[2] + random2 * Math.cos(random),
+                channel[node1GpsLat] + random2 * Math.cos(random),
-                channel[3] + random2 * Math.sin(random),
+                channel[node1GpsLgt] + random2 * Math.sin(random),
                 1,
-                channel[0],
+                channel[node1PubKey],
-                channel[1]
+                channel[node1Alias]
               ]);
-              nodesPubkeys[channel[0]] = nodes[nodes.length - 1];
+              nodesPubkeys[node1UniqueId] = nodes[nodes.length - 1];
             }
 
             random = Math.random() * 2 * Math.PI;
             random2 = Math.random() * 0.01;
 
-            if (!nodesPubkeys[channel[4]]) {
+            if (!nodesPubkeys[node2UniqueId]) {
               nodes.push([
-                channel[6] + random2 * Math.cos(random),
+                channel[node2GpsLat] + random2 * Math.cos(random),
-                channel[7] + random2 * Math.sin(random),
+                channel[node2GpsLgt] + random2 * Math.sin(random),
                 1,
-                channel[4],
+                channel[node2PubKey],
-                channel[5]
+                channel[node2Alias]
               ]);
-              nodesPubkeys[channel[4]] = nodes[nodes.length - 1];
+              nodesPubkeys[node2UniqueId] = nodes[nodes.length - 1];
             }
 
             const channelLoc = [];
-            channelLoc.push(nodesPubkeys[channel[0]].slice(0, 2));            
+            channelLoc.push(nodesPubkeys[node1UniqueId].slice(0, 2));            
-            channelLoc.push(nodesPubkeys[channel[4]].slice(0, 2));
+            channelLoc.push(nodesPubkeys[node2UniqueId].slice(0, 2));
             channelsLoc.push(channelLoc);
           }
+
           if (this.style === 'nodepage' && thisNodeGPS) {
             this.center = [thisNodeGPS[0], thisNodeGPS[1]];
             this.zoom = 10;
             this.channelWidth = 1;
             this.channelOpacity = 1;
           }
+
           if (this.style === 'channelpage' && this.channel.length > 0) {
             this.channelWidth = 2;
             this.channelOpacity = 1;
@@ -238,7 +261,7 @@ export class NodesChannelsMap implements OnInit {
         },
         {
           large: false,
-          progressive: 200,
+          progressive: this.style === 'widget' ? 500 : 200,
           silent: true,
           type: 'lines',
           coordinateSystem: 'geo',
@@ -266,6 +289,10 @@ export class NodesChannelsMap implements OnInit {
 
     this.chartInstance = ec;
 
+    this.chartInstance.on('finished', () => {
+      this.isLoading = false;
+    });
+    
     if (this.style === 'widget') {
       this.chartInstance.getZr().on('click', (e) => {
         this.zone.run(() => {

frontend/src/app/services/api.service.ts

@@ -271,10 +271,11 @@ export class ApiService {
     return this.httpClient.get<any[]>(this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/nodes/countries');
   }
 
-  getChannelsGeo$(publicKey?: string): Observable<any> {
+  getChannelsGeo$(publicKey?: string, style?: 'graph' | 'nodepage' | 'widget' | 'channelpage'): Observable<any> {
     return this.httpClient.get<any[]>(
       this.apiBaseUrl + this.apiBasePath + '/api/v1/lightning/channels-geo' +
-        (publicKey !== undefined ? `/${publicKey}` : '')
+        (publicKey !== undefined ? `/${publicKey}`   : '') +
+        (style     !== undefined ? `?style=${style}` : '')
     );
   }
 }

production/nginx-cache-warmer

@@ -77,6 +77,8 @@ do for url in / \
 	'/api/v1/mining/difficulty-adjustments/2y' \
 	'/api/v1/mining/difficulty-adjustments/3y' \
 	'/api/v1/mining/difficulty-adjustments/all' \
+	'/api/v1/lightning/channels-geo?style=widget' \
+	'/api/v1/lightning/channels-geo?style=graph' \
 
 	do
 		curl -s "https://${hostname}${url}" >/dev/null