Merge pull request #2670 from mononaut/expose-node-tlv-data

Show node tlv data & liquidity ads
2022-11-21 21:03:29 +09:00 · 2022-11-21 21:03:29 +09:00 · a0b6719105
commit a0b6719105
parent d36b239dbe c2ab0bc715
10 changed files with 310 additions and 3 deletions
--- a/backend/src/api/database-migration.ts
+++ b/backend/src/api/database-migration.ts
@ -4,7 +4,7 @@ import logger from '../logger';
 import { Common } from './common';

 class DatabaseMigration {
-  private static currentVersion = 42;
+  private static currentVersion = 43;
  private queryTimeout = 120000;
  private statisticsAddedIndexed = false;
  private uniqueLogs: string[] = [];
@ -356,6 +356,10 @@ class DatabaseMigration {
    if (databaseSchemaVersion < 42 && isBitcoin === true) {
      await this.$executeQuery('ALTER TABLE `channels` ADD closing_resolved tinyint(1) DEFAULT 0');
    }
+
+    if (databaseSchemaVersion < 43 && isBitcoin === true) {
+      await this.$executeQuery(this.getCreateLNNodeRecordsTableQuery(), await this.$checkIfTableExists('nodes_records'));
+    }
  }

  /**
@ -791,6 +795,19 @@ class DatabaseMigration {
    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
  }

+  private getCreateLNNodeRecordsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS nodes_records (
+      public_key varchar(66) NOT NULL,
+      type int(10) unsigned NOT NULL,
+      payload blob NOT NULL,
+      UNIQUE KEY public_key_type (public_key, type),
+      INDEX (public_key),
+      FOREIGN KEY (public_key)
+        REFERENCES nodes (public_key)
+        ON DELETE CASCADE
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
  public async $truncateIndexedData(tables: string[]) {
    const allowedTables = ['blocks', 'hashrates', 'prices'];

--- a/backend/src/api/explorer/nodes.api.ts
+++ b/backend/src/api/explorer/nodes.api.ts
@ -105,6 +105,18 @@ class NodesApi {
        node.closed_channel_count = rows[0].closed_channel_count;
      }

+      // Custom records
+      query = `
+        SELECT type, payload
+        FROM nodes_records
+        WHERE public_key = ?
+      `;
+      [rows] = await DB.query(query, [public_key]);
+      node.custom_records = {};
+      for (const record of rows) {
+        node.custom_records[record.type] = Buffer.from(record.payload, 'binary').toString('hex');
+      }
+
      return node;
    } catch (e) {
      logger.err(`Cannot get node information for ${public_key}. Reason: ${(e instanceof Error ? e.message : e)}`);
--- a/backend/src/api/lightning/clightning/clightning-convert.ts
+++ b/backend/src/api/lightning/clightning/clightning-convert.ts
@ -7,6 +7,15 @@ import { Common } from '../../common';
 * Convert a clightning "listnode" entry to a lnd node entry
 */
 export function convertNode(clNode: any): ILightningApi.Node {
+  let custom_records: { [type: number]: string } | undefined = undefined;
+  if (clNode.option_will_fund) {
+    try {
+      custom_records = { '1': Buffer.from(clNode.option_will_fund.compact_lease || '', 'hex').toString('base64') };
+    } catch (e) {
+      logger.err(`Cannot decode option_will_fund compact_lease for ${clNode.nodeid}). Reason: ` + (e instanceof Error ? e.message : e));
+      custom_records = undefined;
+    }
+  }
  return {
    alias: clNode.alias ?? '',
    color: `#${clNode.color ?? ''}`,
@ -23,6 +32,7 @@ export function convertNode(clNode: any): ILightningApi.Node {
      };
    }) ?? [],
    last_update: clNode?.last_timestamp ?? 0,
+    custom_records
  };
 }

--- a/backend/src/api/lightning/lightning-api.interface.ts
+++ b/backend/src/api/lightning/lightning-api.interface.ts
@ -49,6 +49,7 @@ export namespace ILightningApi {
    }[];
    color: string;
    features: { [key: number]: Feature };
+    custom_records?: { [type: number]: string };
  }

  export interface Info {
--- a/backend/src/repositories/NodeRecordsRepository.ts
+++ b/backend/src/repositories/NodeRecordsRepository.ts
@ -0,0 +1,67 @@
+import { ResultSetHeader, RowDataPacket } from 'mysql2';
+import DB from '../database';
+import logger from '../logger';
+
+export interface NodeRecord {
+  publicKey: string; // node public key
+  type: number; // TLV extension record type
+  payload: string; // base64 record payload
+}
+
+class NodesRecordsRepository {
+  public async $saveRecord(record: NodeRecord): Promise<void> {
+    try {
+      const payloadBytes = Buffer.from(record.payload, 'base64');
+      await DB.query(`
+        INSERT INTO nodes_records(public_key, type, payload)
+        VALUE (?, ?, ?)
+        ON DUPLICATE KEY UPDATE
+          payload = ?
+      `, [record.publicKey, record.type, payloadBytes, payloadBytes]);
+    } catch (e: any) {
+      if (e.errno !== 1062) { // ER_DUP_ENTRY - Not an issue, just ignore this
+        logger.err(`Cannot save node record (${[record.publicKey, record.type, record.payload]}) into db. Reason: ` + (e instanceof Error ? e.message : e));
+        // We don't throw, not a critical issue if we miss some nodes records
+      }
+    }
+   }
+
+  public async $getRecordTypes(publicKey: string): Promise<any> {
+    try {
+      const query = `
+        SELECT type FROM nodes_records
+        WHERE public_key = ?
+      `;
+      const [rows] = await DB.query<RowDataPacket[][]>(query, [publicKey]);
+      return rows.map(row => row['type']);
+    } catch (e) {
+      logger.err(`Cannot retrieve custom records for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e));
+      return [];
+    }
+  }
+
+  public async $deleteUnusedRecords(publicKey: string, recordTypes: number[]): Promise<number> {
+    try {
+      let query;
+      if (recordTypes.length) {
+        query = `
+          DELETE FROM nodes_records
+          WHERE public_key = ?
+          AND type NOT IN (${recordTypes.map(type => `${type}`).join(',')})
+        `;
+      } else {
+        query = `
+          DELETE FROM nodes_records
+          WHERE public_key = ?
+        `;
+      }
+      const [result] = await DB.query<ResultSetHeader>(query, [publicKey]);
+      return result.affectedRows;
+    } catch (e) {
+      logger.err(`Cannot delete unused custom records for ${publicKey} from db. Reason: ` + (e instanceof Error ? e.message : e));
+      return 0;
+    }
+  }
+}
+
+export default new NodesRecordsRepository();
--- a/backend/src/tasks/lightning/network-sync.service.ts
+++ b/backend/src/tasks/lightning/network-sync.service.ts
@ -13,6 +13,7 @@ import fundingTxFetcher from './sync-tasks/funding-tx-fetcher';
 import NodesSocketsRepository from '../../repositories/NodesSocketsRepository';
 import { Common } from '../../api/common';
 import blocks from '../../api/blocks';
+import NodeRecordsRepository from '../../repositories/NodeRecordsRepository';

 class NetworkSyncService {
  loggerTimer = 0;
@ -63,6 +64,7 @@ class NetworkSyncService {
    let progress = 0;

    let deletedSockets = 0;
+    let deletedRecords = 0;
    const graphNodesPubkeys: string[] = [];
    for (const node of nodes) {
      const latestUpdated = await channelsApi.$getLatestChannelUpdateForNode(node.pub_key);
@ -84,8 +86,23 @@ class NetworkSyncService {
        addresses.push(socket.addr);
      }
      deletedSockets += await NodesSocketsRepository.$deleteUnusedSockets(node.pub_key, addresses);
+
+      const oldRecordTypes = await NodeRecordsRepository.$getRecordTypes(node.pub_key);
+      const customRecordTypes: number[] = [];
+      for (const [type, payload] of Object.entries(node.custom_records || {})) {
+        const numericalType = parseInt(type);
+        await NodeRecordsRepository.$saveRecord({
+          publicKey: node.pub_key,
+          type: numericalType,
+          payload,
+        });
+        customRecordTypes.push(numericalType);
+      }
+      if (oldRecordTypes.reduce((changed, type) => changed || customRecordTypes.indexOf(type) === -1, false)) {
+        deletedRecords += await NodeRecordsRepository.$deleteUnusedRecords(node.pub_key, customRecordTypes);
+      }
    }
-    logger.info(`${progress} nodes updated. ${deletedSockets} sockets deleted`);
+    logger.info(`${progress} nodes updated. ${deletedSockets} sockets deleted. ${deletedRecords} custom records deleted.`);

    // If a channel if not present in the graph, mark it as inactive
    await nodesApi.$setNodesInactive(graphNodesPubkeys);
--- a/frontend/src/app/lightning/node/liquidity-ad.ts
+++ b/frontend/src/app/lightning/node/liquidity-ad.ts
@ -0,0 +1,31 @@
+export interface ILiquidityAd {
+  funding_weight: number;
+  lease_fee_basis: number; // lease fee rate in parts-per-thousandth
+  lease_fee_base_sat: number; // fixed lease fee in sats
+  channel_fee_max_rate: number; // max routing fee rate in parts-per-thousandth
+  channel_fee_max_base: number; // max routing base fee in milli-sats
+  compact_lease?: string;
+}
+
+export function parseLiquidityAdHex(compact_lease: string): ILiquidityAd | false {
+  if (!compact_lease || compact_lease.length < 20 || compact_lease.length > 28) {
+    return false;
+  }
+  try {
+    const liquidityAd: ILiquidityAd = {
+      funding_weight: parseInt(compact_lease.slice(0, 4), 16),
+      lease_fee_basis: parseInt(compact_lease.slice(4, 8), 16),
+      channel_fee_max_rate: parseInt(compact_lease.slice(8, 12), 16),
+      lease_fee_base_sat: parseInt(compact_lease.slice(12, 20), 16),
+      channel_fee_max_base: compact_lease.length > 20 ? parseInt(compact_lease.slice(20), 16) : 0,
+    }
+    if (Object.values(liquidityAd).reduce((valid: boolean, value: number): boolean => (valid && !isNaN(value) && value >= 0), true)) {
+      liquidityAd.compact_lease = compact_lease;
+      return liquidityAd;
+    } else {
+      return false;
+    }
+  } catch (err) {
+    return false;
+  }
+}
--- a/frontend/src/app/lightning/node/node.component.html
+++ b/frontend/src/app/lightning/node/node.component.html
@ -125,6 +125,93 @@
    <app-clipboard [button]="true" [text]="node.socketsObject[selectedSocketIndex].socket" [leftPadding]="false"></app-clipboard>
  </div>

+  <div *ngIf="hasDetails" [hidden]="!showDetails" id="details" class="details mt-3">
+    <div class="box">
+      <ng-template [ngIf]="liquidityAd">
+        <div class="detail-section">
+          <h5 class="mb-3" i18n="node.liquidity-ad">Liquidity ad</h5>
+          <div class="row">
+            <div class="col-md">
+              <table class="table table-borderless table-striped">
+                <tbody>
+                  <tr>
+                    <td class="label" i18n="liquidity-ad.lease-fee-rate|Liquidity ad lease fee rate">Lease fee rate</td>
+                    <td>
+                      <span class="d-inline-block">
+                        {{ liquidityAd.lease_fee_basis !== null ? ((liquidityAd.lease_fee_basis * 1000) | amountShortener : 2 : undefined : true) : '-' }} <span class="symbol">ppm {{ liquidityAd.lease_fee_basis !== null ? '(' + (liquidityAd.lease_fee_basis / 10 | amountShortener : 2 : undefined : true) + '%)' : '' }}</span>
+                      </span>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td class="label" i18n="liquidity-ad.lease-base-fee">Lease base fee</td>
+                    <td>
+                      <app-sats [valueOverride]="liquidityAd.lease_fee_base_sat === null ? '- ' : undefined" [satoshis]="liquidityAd.lease_fee_base_sat"></app-sats>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td class="label" i18n="liquidity-ad.funding-weight">Funding weight</td>
+                    <td [innerHTML]="'&lrm;' + (liquidityAd.funding_weight | wuBytes: 2)"></td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+            <div class="col-md">
+              <table class="table table-borderless table-striped">
+                <tbody>
+                  <tr>
+                    <td class="label" i18n="liquidity-ad.channel-fee-rate|Liquidity ad channel fee rate">Channel fee rate</td>
+                    <td>
+                      <span class="d-inline-block">
+                        {{ liquidityAd.channel_fee_max_rate !== null ? ((liquidityAd.channel_fee_max_rate * 1000) | amountShortener : 2 : undefined : true) : '-' }} <span class="symbol">ppm {{ liquidityAd.channel_fee_max_rate !== null ? '(' + (liquidityAd.channel_fee_max_rate / 10 | amountShortener : 2 : undefined : true) + '%)' : '' }}</span>
+                      </span>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td class="label" i18n="liquidity-ad.channel-base-fee">Channel base fee</td>
+                    <td>
+                      <span *ngIf="liquidityAd.channel_fee_max_base !== null">
+                        {{ liquidityAd.channel_fee_max_base | amountShortener : 0 }}
+                        <span class="symbol" i18n="shared.m-sats">mSats</span>
+                      </span>
+                      <span *ngIf="liquidityAd.channel_fee_max_base === null">
+                        -
+                      </span>
+                    </td>
+                  </tr>
+                  <tr>
+                    <td class="label" i18n="liquidity-ad.compact-lease">Compact lease</td>
+                    <td class="compact-lease">{{ liquidityAd.compact_lease }}</td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+          </div>
+        </div>
+      </ng-template>
+      <ng-template [ngIf]="tlvRecords?.length">
+        <div class="detail-section">
+          <h5 class="mb-3" i18n="node.tlv.records">TLV extension records</h5>
+          <div class="row">
+            <div class="col">
+              <table class="table table-borderless table-striped">
+                <tbody>
+                  <tr *ngFor="let recordItem of tlvRecords">
+                    <td class="tlv-type">{{ recordItem.type }}</td>
+                    <td class="tlv-payload">{{ recordItem.payload }}</td>
+                  </tr>
+                </tbody>
+              </table>
+            </div>
+          </div>
+        </div>
+      </ng-template>
+    </div>
+  </div>
+
+  <div *ngIf="hasDetails" class="text-right mt-3">
+    <button type="button" class="btn btn-outline-info btn-sm btn-details" (click)="toggleShowDetails()" i18n="node.details|Node Details">Details</button>
+  </div>
+
  <div *ngIf="!error">
    <div class="row" *ngIf="node.as_number && node.active_channel_count">
      <div class="col-sm">
--- a/frontend/src/app/lightning/node/node.component.scss
+++ b/frontend/src/app/lightning/node/node.component.scss
@ -72,3 +72,32 @@ app-fiat {
    height: 28px !important;
  };
 }
+
+.details {
+
+  .detail-section {
+    margin-bottom: 1.5rem;
+    &:last-child {
+      margin-bottom: 0;
+    }
+  }
+
+  .tlv-type {
+    font-size: 12px;
+    color: #ffffff66;
+  }
+
+  .tlv-payload {
+    font-size: 12px;
+    width: 100%;
+    word-break: break-all;
+    white-space: normal;
+    font-family: "Courier New", Courier, monospace;
+  }
+
+  .compact-lease {
+    word-break: break-all;
+    white-space: normal;
+    font-family: "Courier New", Courier, monospace;
+  }
+}
--- a/frontend/src/app/lightning/node/node.component.ts
+++ b/frontend/src/app/lightning/node/node.component.ts
@ -1,10 +1,16 @@
 import { ChangeDetectionStrategy, Component, OnInit } from '@angular/core';
 import { ActivatedRoute, ParamMap } from '@angular/router';
 import { Observable } from 'rxjs';
-import { catchError, map, switchMap } from 'rxjs/operators';
+import { catchError, map, switchMap, tap } from 'rxjs/operators';
 import { SeoService } from '../../services/seo.service';
 import { LightningApiService } from '../lightning-api.service';
 import { GeolocationData } from '../../shared/components/geolocation/geolocation.component';
+import { ILiquidityAd, parseLiquidityAdHex } from './liquidity-ad';
+
+interface CustomRecord {
+  type: string;
+  payload: string;
+}

@Component({
  selector: 'app-node',
@ -24,6 +30,10 @@ export class NodeComponent implements OnInit {
  channelListLoading = false;
  clearnetSocketCount = 0;
  torSocketCount = 0;
+  hasDetails = false;
+  showDetails = false;
+  liquidityAd: ILiquidityAd;
+  tlvRecords: CustomRecord[];

  constructor(
    private lightningApiService: LightningApiService,
@ -36,6 +46,8 @@ export class NodeComponent implements OnInit {
      .pipe(
        switchMap((params: ParamMap) => {
          this.publicKey = params.get('public_key');
+          this.tlvRecords = [];
+          this.liquidityAd = null;
          return this.lightningApiService.getNode$(params.get('public_key'));
        }),
        map((node) => {
@ -79,6 +91,26 @@ export class NodeComponent implements OnInit {

          return node;
        }),
+        tap((node) => {
+          this.hasDetails = Object.keys(node.custom_records).length > 0;
+          for (const [type, payload] of Object.entries(node.custom_records)) {
+            if (typeof payload !== 'string') {
+              break;
+            }
+
+            let parsed = false;
+            if (type === '1') {
+              const ad = parseLiquidityAdHex(payload);
+              if (ad) {
+                parsed = true;
+                this.liquidityAd = ad;
+              }
+            }
+            if (!parsed) {
+              this.tlvRecords.push({ type, payload });
+            }
+          }
+        }),
        catchError(err => {
          this.error = err;
          return [{
@ -89,6 +121,10 @@ export class NodeComponent implements OnInit {
      );
  }

+  toggleShowDetails(): void {
+    this.showDetails = !this.showDetails;
+  }
+
  changeSocket(index: number) {
    this.selectedSocketIndex = index;
  }