Merge pull request #2670 from mononaut/expose-node-tlv-data

Show node tlv data & liquidity ads

backend/src/api/database-migration.ts

@@ -4,7 +4,7 @@ import logger from '../logger';
 import { Common } from './common';
 
 class DatabaseMigration {
-  private static currentVersion = 42;
+  private static currentVersion = 43;
   private queryTimeout = 120000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -356,6 +356,10 @@ class DatabaseMigration {
     if (databaseSchemaVersion < 42 && isBitcoin === true) {
       await this.$executeQuery('ALTER TABLE `channels` ADD closing_resolved tinyint(1) DEFAULT 0');
     }
+
+    if (databaseSchemaVersion < 43 && isBitcoin === true) {
+      await this.$executeQuery(this.getCreateLNNodeRecordsTableQuery(), await this.$checkIfTableExists('nodes_records'));
+    }
   }
 
   /**
@@ -791,6 +795,19 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getCreateLNNodeRecordsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS nodes_records (
+      public_key varchar(66) NOT NULL,
+      type int(10) unsigned NOT NULL,
+      payload blob NOT NULL,
+      UNIQUE KEY public_key_type (public_key, type),
+      INDEX (public_key),
+      FOREIGN KEY (public_key)
+        REFERENCES nodes (public_key)
+        ON DELETE CASCADE
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
   public async $truncateIndexedData(tables: string[]) {
     const allowedTables = ['blocks', 'hashrates', 'prices'];
 

backend/src/api/explorer/nodes.api.ts

@@ -105,6 +105,18 @@ class NodesApi {
         node.closed_channel_count = rows[0].closed_channel_count;
       }
 
+      // Custom records
+      query = `
+        SELECT type, payload
+        FROM nodes_records
+        WHERE public_key = ?
+      `;
+      [rows] = await DB.query(query, [public_key]);
+      node.custom_records = {};
+      for (const record of rows) {
+        node.custom_records[record.type] = Buffer.from(record.payload, 'binary').toString('hex');
+      }
+
       return node;
     } catch (e) {
       logger.err(`Cannot get node information for ${public_key}. Reason: ${(e instanceof Error ? e.message : e)}`);

backend/src/api/lightning/clightning/clightning-convert.ts

@@ -7,6 +7,15 @@ import { Common } from '../../common';
  * Convert a clightning "listnode" entry to a lnd node entry
  */
 export function convertNode(clNode: any): ILightningApi.Node {
+  let custom_records: { [type: number]: string } | undefined = undefined;
+  if (clNode.option_will_fund) {
+    try {
+      custom_records = { '1': Buffer.from(clNode.option_will_fund.compact_lease || '', 'hex').toString('base64') };
+    } catch (e) {
+      logger.err(`Cannot decode option_will_fund compact_lease for ${clNode.nodeid}). Reason: ` + (e instanceof Error ? e.message : e));
+      custom_records = undefined;
+    }
+  }
   return {
     alias: clNode.alias ?? '',
     color: `#${clNode.color ?? ''}`,
@@ -23,6 +32,7 @@ export function convertNode(clNode: any): ILightningApi.Node {
       };
     }) ?? [],
     last_update: clNode?.last_timestamp ?? 0,
+    custom_records
   };
 }
 

backend/src/api/lightning/lightning-api.interface.ts

@@ -49,6 +49,7 @@ export namespace ILightningApi {
     }[];
     color: string;
     features: { [key: number]: Feature };
+    custom_records?: { [type: number]: string };
   }
 
   export interface Info {

backend/src/repositories/NodeRecordsRepository.ts

@@ -0,0 +1,67 @@
+import { ResultSetHeader, RowDataPacket } from 'mysql2';
+import DB from '../database';
+import logger from '../logger';
+
+export interface NodeRecord {
+  publicKey: string; // node public key
+  type: number; // TLV extension record type
+  payload: string; // base64 record payload
+}
+
+class NodesRecordsRepository {
+  public async $saveRecord(record: NodeRecord): Promise<void> {
+    try {
+      const payloadBytes = Buffer.from(record.payload, 'base64');
+      await DB.query(`
+        INSERT INTO nodes_records(public_key, type, payload)
+        VALUE (?, ?, ?)
+        ON DUPLICATE KEY UPDATE
+          payload = ?
+      `, [record.publicKey, record.type, payloadBytes, payloadBytes]);
+    } catch (e: any) {
+      if (e.errno !== 1062) { // ER_DUP_ENTRY - Not an issue, just ignore this
+        logger.err(`Cannot save node record (${[record.publicKey, record.type, record.payload]}) into db. Reason: ` + (e instanceof Error ? e.message : e));
+        // We don't throw, not a critical issue if we miss some nodes records
+      }
+    }
+   }
+
+  public async $getRecordTypes(publicKey: string): Promise<any> {
+    try {
+      const query = `
+        SELECT type FROM nodes_records
+        WHERE public_key = ?
+      `;
+      const [rows] = await DB.query<RowDataPacket[][]>(query, [publicKey]);
+      return rows.map(row => row['type']);
+    } catch (e) {
+      logger.err(`Cannot retrieve custom records for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
+  public async $deleteUnusedRecords(publicKey: string, recordTypes: number[]): Promise<number> {
+    try {
+      let query;
+      if (recordTypes.length) {
+        query = `
+          DELETE FROM nodes_records
+          WHERE public_key = ?
+          AND type NOT IN (${recordTypes.map(type => `${type}`).join(',')})
+        `;
+      } else {
+        query = `
+          DELETE FROM nodes_records
+          WHERE public_key = ?
+        `;
+      }
+      const [result] = await DB.query<ResultSetHeader>(query, [publicKey]);
+      return result.affectedRows;
+    } catch (e) {
+      logger.err(`Cannot delete unused custom records for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e));
+      return 0;
+    }
+  }
+}
+
+export default new NodesRecordsRepository();

backend/src/tasks/lightning/network-sync.service.ts

@@ -13,6 +13,7 @@ import fundingTxFetcher from './sync-tasks/funding-tx-fetcher';
 import NodesSocketsRepository from '../../repositories/NodesSocketsRepository';
 import { Common } from '../../api/common';
 import blocks from '../../api/blocks';
+import NodeRecordsRepository from '../../repositories/NodeRecordsRepository';
 
 class NetworkSyncService {
   loggerTimer = 0;
@@ -63,6 +64,7 @@ class NetworkSyncService {
     let progress = 0;
 
     let deletedSockets = 0;
+    let deletedRecords = 0;
     const graphNodesPubkeys: string[] = [];
     for (const node of nodes) {
       const latestUpdated = await channelsApi.$getLatestChannelUpdateForNode(node.pub_key);
@@ -84,8 +86,23 @@ class NetworkSyncService {
         addresses.push(socket.addr);
       }
       deletedSockets += await NodesSocketsRepository.$deleteUnusedSockets(node.pub_key, addresses);
+
+      const oldRecordTypes = await NodeRecordsRepository.$getRecordTypes(node.pub_key);
+      const customRecordTypes: number[] = [];
+      for (const [type, payload] of Object.entries(node.custom_records || {})) {
+        const numericalType = parseInt(type);
+        await NodeRecordsRepository.$saveRecord({
+          publicKey: node.pub_key,
+          type: numericalType,
+          payload,
+        });
+        customRecordTypes.push(numericalType);
+      }
+      if (oldRecordTypes.reduce((changed, type) => changed || customRecordTypes.indexOf(type) === -1, false)) {
+        deletedRecords += await NodeRecordsRepository.$deleteUnusedRecords(node.pub_key, customRecordTypes);
+      }
     }
-    logger.info(`${progress} nodes updated. ${deletedSockets} sockets deleted`);
+    logger.info(`${progress} nodes updated. ${deletedSockets} sockets deleted. ${deletedRecords} custom records deleted.`);
 
     // If a channel if not present in the graph, mark it as inactive
     await nodesApi.$setNodesInactive(graphNodesPubkeys);

frontend/src/app/lightning/node/liquidity-ad.ts

@@ -0,0 +1,31 @@
+export interface ILiquidityAd {
+  funding_weight: number;
+  lease_fee_basis: number; // lease fee rate in parts-per-thousandth
+  lease_fee_base_sat: number; // fixed lease fee in sats
+  channel_fee_max_rate: number; // max routing fee rate in parts-per-thousandth
+  channel_fee_max_base: number; // max routing base fee in milli-sats
+  compact_lease?: string;
+}
+
+export function parseLiquidityAdHex(compact_lease: string): ILiquidityAd | false {
+  if (!compact_lease || compact_lease.length < 20 || compact_lease.length > 28) {
+    return false;
+  }
+  try {
+    const liquidityAd: ILiquidityAd = {
+      funding_weight: parseInt(compact_lease.slice(0, 4), 16),
+      lease_fee_basis: parseInt(compact_lease.slice(4, 8), 16),
+      channel_fee_max_rate: parseInt(compact_lease.slice(8, 12), 16),
+      lease_fee_base_sat: parseInt(compact_lease.slice(12, 20), 16),
+      channel_fee_max_base: compact_lease.length > 20 ? parseInt(compact_lease.slice(20), 16) : 0,
+    }
+    if (Object.values(liquidityAd).reduce((valid: boolean, value: number): boolean => (valid && !isNaN(value) && value >= 0), true)) {
+      liquidityAd.compact_lease = compact_lease;
+      return liquidityAd;
+    } else {
+      return false;
+    }
+  } catch (err) {
+    return false;
+  }
+}

frontend/src/app/lightning/node/node.component.html

@@ -125,6 +125,93 @@
     <app-clipboard [button]="true" [text]="node.socketsObject[selectedSocketIndex].socket" [leftPadding]="false"></app-clipboard>
   </div>
 
+  <div *ngIf="hasDetails" [hidden]="!showDetails" id="details" class="details mt-3">
+    <div class="box">
+      <ng-template [ngIf]="liquidityAd">
+        <div class="detail-section">
+          <h5 class="mb-3" i18n="node.liquidity-ad">Liquidity ad</h5>
+          <div class="row">
+            <div class="col-md">
+              <table class="table table-borderless table-striped">
+                <tbody>
+                  <tr>
+                    <td class="label" i18n="liquidity-ad.lease-fee-rate|Liquidity ad lease fee rate">Lease fee rate</td>
+                    <td>
+                      <span class="d-inline-block">
+                        {{ liquidityAd.lease_fee_basis !== null ? ((liquidityAd.lease_fee_basis * 1000) | amountShortener : 2 : undefined : true) : '-' }} <span class="symbol">ppm {{ liquidityAd.lease_fee_basis !== null ? '(' + (liquidityAd.lease_fee_basis / 10 | amountShortener : 2 : undefined : true) + '%)' : '' }}</span>
+                      </span>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td class="label" i18n="liquidity-ad.lease-base-fee">Lease base fee</td>
+                    <td>
+                      <app-sats [valueOverride]="liquidityAd.lease_fee_base_sat === null ? '- ' : undefined" [satoshis]="liquidityAd.lease_fee_base_sat"></app-sats>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td class="label" i18n="liquidity-ad.funding-weight">Funding weight</td>
+                    <td [innerHTML]="'&lrm;' + (liquidityAd.funding_weight | wuBytes: 2)"></td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+            <div class="col-md">
+              <table class="table table-borderless table-striped">
+                <tbody>
+                  <tr>
+                    <td class="label" i18n="liquidity-ad.channel-fee-rate|Liquidity ad channel fee rate">Channel fee rate</td>
+                    <td>
+                      <span class="d-inline-block">
+                        {{ liquidityAd.channel_fee_max_rate !== null ? ((liquidityAd.channel_fee_max_rate * 1000) | amountShortener : 2 : undefined : true) : '-' }} <span class="symbol">ppm {{ liquidityAd.channel_fee_max_rate !== null ? '(' + (liquidityAd.channel_fee_max_rate / 10 | amountShortener : 2 : undefined : true) + '%)' : '' }}</span>
+                      </span>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td class="label" i18n="liquidity-ad.channel-base-fee">Channel base fee</td>
+                    <td>
+                      <span *ngIf="liquidityAd.channel_fee_max_base !== null">
+                        {{ liquidityAd.channel_fee_max_base | amountShortener : 0 }}
+                        <span class="symbol" i18n="shared.m-sats">mSats</span>
+                      </span>
+                      <span *ngIf="liquidityAd.channel_fee_max_base === null">
+                        -
+                      </span>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td class="label" i18n="liquidity-ad.compact-lease">Compact lease</td>
+                    <td class="compact-lease">{{ liquidityAd.compact_lease }}</td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+          </div>
+        </div>
+      </ng-template>
+      <ng-template [ngIf]="tlvRecords?.length">
+        <div class="detail-section">
+          <h5 class="mb-3" i18n="node.tlv.records">TLV extension records</h5>
+          <div class="row">
+            <div class="col">
+              <table class="table table-borderless table-striped">
+                <tbody>
+                  <tr *ngFor="let recordItem of tlvRecords">
+                    <td class="tlv-type">{{ recordItem.type }}</td>
+                    <td class="tlv-payload">{{ recordItem.payload }}</td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+          </div>
+        </div>
+      </ng-template>
+    </div>
+  </div>
+
+  <div *ngIf="hasDetails" class="text-right mt-3">
+    <button type="button" class="btn btn-outline-info btn-sm btn-details" (click)="toggleShowDetails()" i18n="node.details|Node Details">Details</button>
+  </div>
+
   <div *ngIf="!error">
     <div class="row" *ngIf="node.as_number && node.active_channel_count">
       <div class="col-sm">

frontend/src/app/lightning/node/node.component.scss

@@ -72,3 +72,32 @@ app-fiat {
     height: 28px !important;
   };
 }
+
+.details {
+
+  .detail-section {
+    margin-bottom: 1.5rem;
+    &:last-child {
+      margin-bottom: 0;
+    }
+  }
+
+  .tlv-type {
+    font-size: 12px;
+    color: #ffffff66;
+  }
+
+  .tlv-payload {
+    font-size: 12px;
+    width: 100%;
+    word-break: break-all;
+    white-space: normal;
+    font-family: "Courier New", Courier, monospace;
+  }
+
+  .compact-lease {
+    word-break: break-all;
+    white-space: normal;
+    font-family: "Courier New", Courier, monospace;
+  }
+}

frontend/src/app/lightning/node/node.component.ts

@@ -1,10 +1,16 @@
 import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core';
 import { ActivatedRoute, ParamMap } from '@angular/router';
 import { Observable } from 'rxjs';
-import { catchError, map, switchMap } from 'rxjs/operators';
+import { catchError, map, switchMap, tap } from 'rxjs/operators';
 import { SeoService } from '../../services/seo.service';
 import { LightningApiService } from '../lightning-api.service';
 import { GeolocationData } from '../../shared/components/geolocation/geolocation.component';
+import { ILiquidityAd, parseLiquidityAdHex } from './liquidity-ad';
+
+interface CustomRecord {
+  type: string;
+  payload: string;
+}
 
 @Component({
   selector: 'app-node',
@@ -24,6 +30,10 @@ export class NodeComponent implements OnInit {
   channelListLoading = false;
   clearnetSocketCount = 0;
   torSocketCount = 0;
+  hasDetails = false;
+  showDetails = false;
+  liquidityAd: ILiquidityAd;
+  tlvRecords: CustomRecord[];
 
   constructor(
     private lightningApiService: LightningApiService,
@@ -36,6 +46,8 @@ export class NodeComponent implements OnInit {
       .pipe(
         switchMap((params: ParamMap) => {
           this.publicKey = params.get('public_key');
+          this.tlvRecords = [];
+          this.liquidityAd = null;
           return this.lightningApiService.getNode$(params.get('public_key'));
         }),
         map((node) => {
@@ -79,6 +91,26 @@ export class NodeComponent implements OnInit {
 
           return node;
         }),
+        tap((node) => {
+          this.hasDetails = Object.keys(node.custom_records).length > 0;
+          for (const [type, payload] of Object.entries(node.custom_records)) {
+            if (typeof payload !== 'string') {
+              break;
+            }
+
+            let parsed = false;
+            if (type === '1') {
+              const ad = parseLiquidityAdHex(payload);
+              if (ad) {
+                parsed = true;
+                this.liquidityAd = ad;
+              }
+            }
+            if (!parsed) {
+              this.tlvRecords.push({ type, payload });
+            }
+          }
+        }),
         catchError(err => {
           this.error = err;
           return [{
@@ -89,6 +121,10 @@ export class NodeComponent implements OnInit {
       );
   }
 
+  toggleShowDetails(): void {
+    this.showDetails = !this.showDetails;
+  }
+
   changeSocket(index: number) {
     this.selectedSocketIndex = index;
   }