From 3f83e517f05828af08175cb417d8a2d4b87b8cf4 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Fri, 29 Jul 2022 08:08:22 +0200 Subject: [PATCH 1/6] Create CLightningClient class --- backend/src/rpc-api/core-lightning/jsonrpc.ts | 249 ++++++++++++++++++ 1 file changed, 249 insertions(+) create mode 100644 backend/src/rpc-api/core-lightning/jsonrpc.ts diff --git a/backend/src/rpc-api/core-lightning/jsonrpc.ts b/backend/src/rpc-api/core-lightning/jsonrpc.ts new file mode 100644 index 000000000..037dfff75 --- /dev/null +++ b/backend/src/rpc-api/core-lightning/jsonrpc.ts @@ -0,0 +1,249 @@ +'use strict'; + +const methods = [ + 'addgossip', + 'autocleaninvoice', + 'check', + 'checkmessage', + 'close', + 'connect', + 'createinvoice', + 'createinvoicerequest', + 'createoffer', + 'createonion', + 'decode', + 'decodepay', + 'delexpiredinvoice', + 'delinvoice', + 'delpay', + 'dev-listaddrs', + 'dev-rescan-outputs', + 'disableoffer', + 'disconnect', + 'estimatefees', + 'feerates', + 'fetchinvoice', + 'fundchannel', + 'fundchannel_cancel', + 'fundchannel_complete', + 'fundchannel_start', + 'fundpsbt', + 'getchaininfo', + 'getinfo', + 'getlog', + 'getrawblockbyheight', + 'getroute', + 'getsharedsecret', + 'getutxout', + 'help', + 'invoice', + 'keysend', + 'legacypay', + 'listchannels', + 'listconfigs', + 'listforwards', + 'listfunds', + 'listinvoices', + 'listnodes', + 'listoffers', + 'listpays', + 'listpeers', + 'listsendpays', + 'listtransactions', + 'multifundchannel', + 'multiwithdraw', + 'newaddr', + 'notifications', + 'offer', + 'offerout', + 'openchannel_abort', + 'openchannel_bump', + 'openchannel_init', + 'openchannel_signed', + 'openchannel_update', + 'pay', + 'payersign', + 'paystatus', + 'ping', + 'plugin', + 'reserveinputs', + 'sendinvoice', + 'sendonion', + 'sendonionmessage', + 'sendpay', + 'sendpsbt', + 'sendrawtransaction', + 'setchannelfee', + 'signmessage', + 'signpsbt', + 'stop', + 'txdiscard', + 'txprepare', + 'txsend', + 'unreserveinputs', + 'utxopsbt', + 'waitanyinvoice', + 'waitblockheight', + 'waitinvoice', + 'waitsendpay', + 'withdraw' +]; + + +import EventEmitter from 'events'; +import { existsSync, statSync } from 'fs'; +import { createConnection, Socket } from 'net'; +import { homedir } from 'os'; +import path from 'path'; +import { createInterface, Interface } from 'readline'; +import logger from '../../logger'; + +class LightningError extends Error { + type: string = 'lightning'; + message: string = 'lightning-client error'; + + constructor(error) { + super(); + this.type = error.type; + this.message = error.message; + } +} + +const defaultRpcPath = path.join(homedir(), '.lightning') + , fStat = (...p) => statSync(path.join(...p)) + , fExists = (...p) => existsSync(path.join(...p)) + +class CLightningClient extends EventEmitter { + private rpcPath: string; + private reconnectWait: number; + private reconnectTimeout; + private reqcount: number; + private client: Socket; + private rl: Interface; + private clientConnectionPromise: Promise; + + constructor(rpcPath = defaultRpcPath) { + if (!path.isAbsolute(rpcPath)) { + throw new Error('The rpcPath must be an absolute path'); + } + + if (!fExists(rpcPath) || !fStat(rpcPath).isSocket()) { + // network directory provided, use the lightning-rpc within in + if (fExists(rpcPath, 'lightning-rpc')) { + rpcPath = path.join(rpcPath, 'lightning-rpc'); + } + + // main data directory provided, default to using the bitcoin mainnet subdirectory + // to be removed in v0.2.0 + else if (fExists(rpcPath, 'bitcoin', 'lightning-rpc')) { + logger.warn(`[CLightningClient] ${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`) + logger.warn(`[CLightningClient] specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`) + rpcPath = path.join(rpcPath, 'bitcoin', 'lightning-rpc') + } + } + + logger.debug(`[CLightningClient] Connecting to ${rpcPath}`); + + super(); + this.rpcPath = rpcPath; + this.reconnectWait = 0.5; + this.reconnectTimeout = null; + this.reqcount = 0; + + const _self = this; + + this.client = createConnection(rpcPath); + this.rl = createInterface({ input: this.client }) + + this.clientConnectionPromise = new Promise(resolve => { + _self.client.on('connect', () => { + logger.debug(`[CLightningClient] Lightning client connected`); + _self.reconnectWait = 1; + resolve(); + }); + + _self.client.on('end', () => { + logger.err('[CLightningClient] Lightning client connection closed, reconnecting'); + _self.increaseWaitTime(); + _self.reconnect(); + }); + + _self.client.on('error', error => { + logger.err(`[CLightningClient] Lightning client connection error: ${error}`); + _self.emit('error', error); + _self.increaseWaitTime(); + _self.reconnect(); + }); + }); + + this.rl.on('line', line => { + line = line.trim(); + if (!line) { + return; + } + const data = JSON.parse(line); + logger.debug(`[CLightningClient] #${data.id} <-- ${JSON.stringify(data.error || data.result)}`); + _self.emit('res:' + data.id, data); + }); + } + + increaseWaitTime(): void { + if (this.reconnectWait >= 16) { + this.reconnectWait = 16; + } else { + this.reconnectWait *= 2; + } + } + + reconnect(): void { + const _self = this; + + if (this.reconnectTimeout) { + return; + } + + this.reconnectTimeout = setTimeout(() => { + logger.debug('[CLightningClient] Trying to reconnect...'); + + _self.client.connect(_self.rpcPath); + _self.reconnectTimeout = null; + }, this.reconnectWait * 1000); + } + + call(method, args = []): Promise { + const _self = this; + + const callInt = ++this.reqcount; + const sendObj = { + jsonrpc: '2.0', + method, + params: args, + id: '' + callInt + }; + + logger.debug(`[CLightningClient] #${callInt} --> ${method} ${args}`); + + // Wait for the client to connect + return this.clientConnectionPromise + .then(() => new Promise((resolve, reject) => { + // Wait for a response + this.once('res:' + callInt, res => res.error == null + ? resolve(res.result) + : reject(new LightningError(res.error)) + ); + + // Send the command + _self.client.write(JSON.stringify(sendObj)); + })); + } +} + +const protify = s => s.replace(/-([a-z])/g, m => m[1].toUpperCase()); + +methods.forEach(k => { + CLightningClient.prototype[protify(k)] = function (...args: any) { + return this.call(k, args); + }; +}); + +export default new CLightningClient(); From a94403b3a1bd3ec65435d1b4067ad8f90a8bb2f8 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Fri, 29 Jul 2022 16:33:07 +0200 Subject: [PATCH 2/6] Wrote some utility functions to convert clightning output to our db schema --- .../lightning/clightning/clightning-client.ts | 4 + .../clightning/clightning-convert.ts | 95 +++++++++++++++++++ .../lightning/clightning}/jsonrpc.ts | 12 +-- backend/src/config.ts | 8 ++ 4 files changed, 113 insertions(+), 6 deletions(-) create mode 100644 backend/src/api/lightning/clightning/clightning-client.ts create mode 100644 backend/src/api/lightning/clightning/clightning-convert.ts rename backend/src/{rpc-api/core-lightning => api/lightning/clightning}/jsonrpc.ts (94%) diff --git a/backend/src/api/lightning/clightning/clightning-client.ts b/backend/src/api/lightning/clightning/clightning-client.ts new file mode 100644 index 000000000..2b974bca0 --- /dev/null +++ b/backend/src/api/lightning/clightning/clightning-client.ts @@ -0,0 +1,4 @@ +import config from '../../../config'; +import CLightningClient from './jsonrpc'; + +export default new CLightningClient(config.CLIGHTNING.SOCKET); diff --git a/backend/src/api/lightning/clightning/clightning-convert.ts b/backend/src/api/lightning/clightning/clightning-convert.ts new file mode 100644 index 000000000..34ef6f942 --- /dev/null +++ b/backend/src/api/lightning/clightning/clightning-convert.ts @@ -0,0 +1,95 @@ +import logger from "../../../logger"; +import { ILightningApi } from "../lightning-api.interface"; + +export function convertNode(clNode: any): ILightningApi.Node { + return { + alias: clNode.alias ?? '', + color: `#${clNode.color ?? ''}`, + features: [], // TODO parse and return clNode.feature + public_key: clNode.nodeid, + sockets: clNode.addresses?.map(addr => `${addr.address}:${addr.port}`) ?? [], + updated_at: new Date((clNode?.last_timestamp ?? 0) * 1000).toUTCString(), + }; +} + +export function convertAndmergeBidirectionalChannels(clChannels: any[]): ILightningApi.Channel[] { + const consolidatedChannelList: ILightningApi.Channel[] = []; + const clChannelsDict = {}; + const clChannelsDictCount = {}; + + for (const clChannel of clChannels) { + if (!clChannelsDict[clChannel.short_channel_id]) { + clChannelsDict[clChannel.short_channel_id] = clChannel; + clChannelsDictCount[clChannel.short_channel_id] = 1; + } else { + consolidatedChannelList.push( + buildBidirectionalChannel(clChannel, clChannelsDict[clChannel.short_channel_id]) + ); + delete clChannelsDict[clChannel.short_channel_id]; + clChannelsDictCount[clChannel.short_channel_id]++; + } + } + const bidirectionalChannelsCount = consolidatedChannelList.length; + + for (const short_channel_id of Object.keys(clChannelsDict)) { + consolidatedChannelList.push(buildUnidirectionalChannel(clChannelsDict[short_channel_id])); + } + const unidirectionalChannelsCount = consolidatedChannelList.length - bidirectionalChannelsCount; + + logger.debug(`clightning knows ${clChannels.length} channels. ` + + `We found ${bidirectionalChannelsCount} bidirectional channels ` + + `and ${unidirectionalChannelsCount} unidirectional channels.`); + + return consolidatedChannelList; +} + +function buildBidirectionalChannel(clChannelA: any, clChannelB: any): ILightningApi.Channel { + const lastUpdate = Math.max(clChannelA.last_update ?? 0, clChannelB.last_update ?? 0); + + return { + id: clChannelA.short_channel_id, + capacity: clChannelA.satoshis, + transaction_id: '', // TODO + transaction_vout: 0, // TODO + updated_at: new Date(lastUpdate * 1000).toUTCString(), + policies: [ + convertPolicy(clChannelA), + convertPolicy(clChannelB) + ] + }; +} + +function buildUnidirectionalChannel(clChannel: any): ILightningApi.Channel { + return { + id: clChannel.short_channel_id, + capacity: clChannel.satoshis, + policies: [convertPolicy(clChannel), getEmptyPolicy()], + transaction_id: '', // TODO + transaction_vout: 0, // TODO + updated_at: new Date((clChannel.last_update ?? 0) * 1000).toUTCString(), + }; +} + +function convertPolicy(clChannel: any): ILightningApi.Policy { + return { + public_key: clChannel.source, + base_fee_mtokens: clChannel.base_fee_millisatoshi, + fee_rate: clChannel.fee_per_millionth, + is_disabled: !clChannel.active, + max_htlc_mtokens: clChannel.htlc_maximum_msat.slice(0, -4), + min_htlc_mtokens: clChannel.htlc_minimum_msat.slice(0, -4), + updated_at: new Date((clChannel.last_update ?? 0) * 1000).toUTCString(), + }; +} + +function getEmptyPolicy(): ILightningApi.Policy { + return { + public_key: 'null', + base_fee_mtokens: '0', + fee_rate: 0, + is_disabled: true, + max_htlc_mtokens: '0', + min_htlc_mtokens: '0', + updated_at: new Date(0).toUTCString(), + }; +} diff --git a/backend/src/rpc-api/core-lightning/jsonrpc.ts b/backend/src/api/lightning/clightning/jsonrpc.ts similarity index 94% rename from backend/src/rpc-api/core-lightning/jsonrpc.ts rename to backend/src/api/lightning/clightning/jsonrpc.ts index 037dfff75..d0b187a54 100644 --- a/backend/src/rpc-api/core-lightning/jsonrpc.ts +++ b/backend/src/api/lightning/clightning/jsonrpc.ts @@ -1,3 +1,5 @@ +// Imported from https://github.com/shesek/lightning-client-js + 'use strict'; const methods = [ @@ -96,7 +98,7 @@ import { createConnection, Socket } from 'net'; import { homedir } from 'os'; import path from 'path'; import { createInterface, Interface } from 'readline'; -import logger from '../../logger'; +import logger from '../../../logger'; class LightningError extends Error { type: string = 'lightning'; @@ -113,7 +115,7 @@ const defaultRpcPath = path.join(homedir(), '.lightning') , fStat = (...p) => statSync(path.join(...p)) , fExists = (...p) => existsSync(path.join(...p)) -class CLightningClient extends EventEmitter { +export default class CLightningClient extends EventEmitter { private rpcPath: string; private reconnectWait: number; private reconnectTimeout; @@ -182,7 +184,7 @@ class CLightningClient extends EventEmitter { return; } const data = JSON.parse(line); - logger.debug(`[CLightningClient] #${data.id} <-- ${JSON.stringify(data.error || data.result)}`); + // logger.debug(`[CLightningClient] #${data.id} <-- ${JSON.stringify(data.error || data.result)}`); _self.emit('res:' + data.id, data); }); } @@ -210,7 +212,7 @@ class CLightningClient extends EventEmitter { }, this.reconnectWait * 1000); } - call(method, args = []): Promise { + call(method, args = []): Promise { const _self = this; const callInt = ++this.reqcount; @@ -245,5 +247,3 @@ methods.forEach(k => { return this.call(k, args); }; }); - -export default new CLightningClient(); diff --git a/backend/src/config.ts b/backend/src/config.ts index d480e6c51..b42a45ab2 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -38,6 +38,9 @@ interface IConfig { MACAROON_PATH: string; REST_API_URL: string; }; + CLIGHTNING: { + SOCKET: string; + }; ELECTRUM: { HOST: string; PORT: number; @@ -186,6 +189,9 @@ const defaults: IConfig = { 'MACAROON_PATH': '', 'REST_API_URL': 'https://localhost:8080', }, + 'CLIGHTNING': { + 'SOCKET': '', + }, 'SOCKS5PROXY': { 'ENABLED': false, 'USE_ONION': true, @@ -226,6 +232,7 @@ class Config implements IConfig { BISQ: IConfig['BISQ']; LIGHTNING: IConfig['LIGHTNING']; LND: IConfig['LND']; + CLIGHTNING: IConfig['CLIGHTNING']; SOCKS5PROXY: IConfig['SOCKS5PROXY']; PRICE_DATA_SERVER: IConfig['PRICE_DATA_SERVER']; EXTERNAL_DATA_SERVER: IConfig['EXTERNAL_DATA_SERVER']; @@ -244,6 +251,7 @@ class Config implements IConfig { this.BISQ = configs.BISQ; this.LIGHTNING = configs.LIGHTNING; this.LND = configs.LND; + this.CLIGHTNING = configs.CLIGHTNING; this.SOCKS5PROXY = configs.SOCKS5PROXY; this.PRICE_DATA_SERVER = configs.PRICE_DATA_SERVER; this.EXTERNAL_DATA_SERVER = configs.EXTERNAL_DATA_SERVER; From eb90434c28f7d19b1e172226438291195f8105f0 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Fri, 29 Jul 2022 17:41:09 +0200 Subject: [PATCH 3/6] Delete historical generation code --- .../lightning/clightning/clightning-client.ts | 265 +++++++++++++++++- .../clightning/clightning-convert.ts | 44 +-- .../src/api/lightning/clightning/jsonrpc.ts | 249 ---------------- .../lightning-api-abstract-factory.ts | 2 - .../api/lightning/lightning-api-factory.ts | 5 +- .../src/tasks/lightning/node-sync.service.ts | 2 +- 6 files changed, 295 insertions(+), 272 deletions(-) delete mode 100644 backend/src/api/lightning/clightning/jsonrpc.ts diff --git a/backend/src/api/lightning/clightning/clightning-client.ts b/backend/src/api/lightning/clightning/clightning-client.ts index 2b974bca0..629092d03 100644 --- a/backend/src/api/lightning/clightning/clightning-client.ts +++ b/backend/src/api/lightning/clightning/clightning-client.ts @@ -1,4 +1,263 @@ -import config from '../../../config'; -import CLightningClient from './jsonrpc'; +// Imported from https://github.com/shesek/lightning-client-js -export default new CLightningClient(config.CLIGHTNING.SOCKET); +'use strict'; + +const methods = [ + 'addgossip', + 'autocleaninvoice', + 'check', + 'checkmessage', + 'close', + 'connect', + 'createinvoice', + 'createinvoicerequest', + 'createoffer', + 'createonion', + 'decode', + 'decodepay', + 'delexpiredinvoice', + 'delinvoice', + 'delpay', + 'dev-listaddrs', + 'dev-rescan-outputs', + 'disableoffer', + 'disconnect', + 'estimatefees', + 'feerates', + 'fetchinvoice', + 'fundchannel', + 'fundchannel_cancel', + 'fundchannel_complete', + 'fundchannel_start', + 'fundpsbt', + 'getchaininfo', + 'getinfo', + 'getlog', + 'getrawblockbyheight', + 'getroute', + 'getsharedsecret', + 'getutxout', + 'help', + 'invoice', + 'keysend', + 'legacypay', + 'listchannels', + 'listconfigs', + 'listforwards', + 'listfunds', + 'listinvoices', + 'listnodes', + 'listoffers', + 'listpays', + 'listpeers', + 'listsendpays', + 'listtransactions', + 'multifundchannel', + 'multiwithdraw', + 'newaddr', + 'notifications', + 'offer', + 'offerout', + 'openchannel_abort', + 'openchannel_bump', + 'openchannel_init', + 'openchannel_signed', + 'openchannel_update', + 'pay', + 'payersign', + 'paystatus', + 'ping', + 'plugin', + 'reserveinputs', + 'sendinvoice', + 'sendonion', + 'sendonionmessage', + 'sendpay', + 'sendpsbt', + 'sendrawtransaction', + 'setchannelfee', + 'signmessage', + 'signpsbt', + 'stop', + 'txdiscard', + 'txprepare', + 'txsend', + 'unreserveinputs', + 'utxopsbt', + 'waitanyinvoice', + 'waitblockheight', + 'waitinvoice', + 'waitsendpay', + 'withdraw' +]; + + +import EventEmitter from 'events'; +import { existsSync, statSync } from 'fs'; +import { createConnection, Socket } from 'net'; +import { homedir } from 'os'; +import path from 'path'; +import { createInterface, Interface } from 'readline'; +import logger from '../../../logger'; +import { AbstractLightningApi } from '../lightning-api-abstract-factory'; +import { ILightningApi } from '../lightning-api.interface'; +import { convertAndmergeBidirectionalChannels, convertNode } from './clightning-convert'; + +class LightningError extends Error { + type: string = 'lightning'; + message: string = 'lightning-client error'; + + constructor(error) { + super(); + this.type = error.type; + this.message = error.message; + } +} + +const defaultRpcPath = path.join(homedir(), '.lightning') + , fStat = (...p) => statSync(path.join(...p)) + , fExists = (...p) => existsSync(path.join(...p)) + +export default class CLightningClient extends EventEmitter implements AbstractLightningApi { + private rpcPath: string; + private reconnectWait: number; + private reconnectTimeout; + private reqcount: number; + private client: Socket; + private rl: Interface; + private clientConnectionPromise: Promise; + + constructor(rpcPath = defaultRpcPath) { + if (!path.isAbsolute(rpcPath)) { + throw new Error('The rpcPath must be an absolute path'); + } + + if (!fExists(rpcPath) || !fStat(rpcPath).isSocket()) { + // network directory provided, use the lightning-rpc within in + if (fExists(rpcPath, 'lightning-rpc')) { + rpcPath = path.join(rpcPath, 'lightning-rpc'); + } + + // main data directory provided, default to using the bitcoin mainnet subdirectory + // to be removed in v0.2.0 + else if (fExists(rpcPath, 'bitcoin', 'lightning-rpc')) { + logger.warn(`[CLightningClient] ${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`) + logger.warn(`[CLightningClient] specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`) + rpcPath = path.join(rpcPath, 'bitcoin', 'lightning-rpc') + } + } + + logger.debug(`[CLightningClient] Connecting to ${rpcPath}`); + + super(); + this.rpcPath = rpcPath; + this.reconnectWait = 0.5; + this.reconnectTimeout = null; + this.reqcount = 0; + + const _self = this; + + this.client = createConnection(rpcPath); + this.rl = createInterface({ input: this.client }) + + this.clientConnectionPromise = new Promise(resolve => { + _self.client.on('connect', () => { + logger.info(`[CLightningClient] Lightning client connected`); + _self.reconnectWait = 1; + resolve(); + }); + + _self.client.on('end', () => { + logger.err('[CLightningClient] Lightning client connection closed, reconnecting'); + _self.increaseWaitTime(); + _self.reconnect(); + }); + + _self.client.on('error', error => { + logger.err(`[CLightningClient] Lightning client connection error: ${error}`); + _self.emit('error', error); + _self.increaseWaitTime(); + _self.reconnect(); + }); + }); + + this.rl.on('line', line => { + line = line.trim(); + if (!line) { + return; + } + const data = JSON.parse(line); + // logger.debug(`[CLightningClient] #${data.id} <-- ${JSON.stringify(data.error || data.result)}`); + _self.emit('res:' + data.id, data); + }); + } + + increaseWaitTime(): void { + if (this.reconnectWait >= 16) { + this.reconnectWait = 16; + } else { + this.reconnectWait *= 2; + } + } + + reconnect(): void { + const _self = this; + + if (this.reconnectTimeout) { + return; + } + + this.reconnectTimeout = setTimeout(() => { + logger.debug('[CLightningClient] Trying to reconnect...'); + + _self.client.connect(_self.rpcPath); + _self.reconnectTimeout = null; + }, this.reconnectWait * 1000); + } + + call(method, args = []): Promise { + const _self = this; + + const callInt = ++this.reqcount; + const sendObj = { + jsonrpc: '2.0', + method, + params: args, + id: '' + callInt + }; + + // logger.debug(`[CLightningClient] #${callInt} --> ${method} ${args}`); + + // Wait for the client to connect + return this.clientConnectionPromise + .then(() => new Promise((resolve, reject) => { + // Wait for a response + this.once('res:' + callInt, res => res.error == null + ? resolve(res.result) + : reject(new LightningError(res.error)) + ); + + // Send the command + _self.client.write(JSON.stringify(sendObj)); + })); + } + + async $getNetworkGraph(): Promise { + const listnodes: any[] = await this.call('listnodes'); + const listchannels: any[] = await this.call('listchannels'); + const channelsList = convertAndmergeBidirectionalChannels(listchannels['channels']); + + return { + nodes: listnodes['nodes'].map(node => convertNode(node)), + channels: channelsList, + }; + } +} + +const protify = s => s.replace(/-([a-z])/g, m => m[1].toUpperCase()); + +methods.forEach(k => { + CLightningClient.prototype[protify(k)] = function (...args: any) { + return this.call(k, args); + }; +}); diff --git a/backend/src/api/lightning/clightning/clightning-convert.ts b/backend/src/api/lightning/clightning/clightning-convert.ts index 34ef6f942..8ceec3b7e 100644 --- a/backend/src/api/lightning/clightning/clightning-convert.ts +++ b/backend/src/api/lightning/clightning/clightning-convert.ts @@ -1,6 +1,8 @@ -import logger from "../../../logger"; -import { ILightningApi } from "../lightning-api.interface"; +import { ILightningApi } from '../lightning-api.interface'; +/** + * Convert a clightning "listnode" entry to a lnd node entry + */ export function convertNode(clNode: any): ILightningApi.Node { return { alias: clNode.alias ?? '', @@ -12,7 +14,10 @@ export function convertNode(clNode: any): ILightningApi.Node { }; } -export function convertAndmergeBidirectionalChannels(clChannels: any[]): ILightningApi.Channel[] { +/** + * Convert clightning "listchannels" response to lnd "describegraph.channels" format + */ + export function convertAndmergeBidirectionalChannels(clChannels: any[]): ILightningApi.Channel[] { const consolidatedChannelList: ILightningApi.Channel[] = []; const clChannelsDict = {}; const clChannelsDictCount = {}; @@ -23,27 +28,24 @@ export function convertAndmergeBidirectionalChannels(clChannels: any[]): ILightn clChannelsDictCount[clChannel.short_channel_id] = 1; } else { consolidatedChannelList.push( - buildBidirectionalChannel(clChannel, clChannelsDict[clChannel.short_channel_id]) + buildFullChannel(clChannel, clChannelsDict[clChannel.short_channel_id]) ); delete clChannelsDict[clChannel.short_channel_id]; clChannelsDictCount[clChannel.short_channel_id]++; } } - const bidirectionalChannelsCount = consolidatedChannelList.length; - for (const short_channel_id of Object.keys(clChannelsDict)) { - consolidatedChannelList.push(buildUnidirectionalChannel(clChannelsDict[short_channel_id])); + consolidatedChannelList.push(buildIncompleteChannel(clChannelsDict[short_channel_id])); } - const unidirectionalChannelsCount = consolidatedChannelList.length - bidirectionalChannelsCount; - - logger.debug(`clightning knows ${clChannels.length} channels. ` + - `We found ${bidirectionalChannelsCount} bidirectional channels ` + - `and ${unidirectionalChannelsCount} unidirectional channels.`); return consolidatedChannelList; } -function buildBidirectionalChannel(clChannelA: any, clChannelB: any): ILightningApi.Channel { +/** + * Convert two clightning "getchannels" entries into a full a lnd "describegraph.channels" format + * In this case, clightning knows the channel policy for both nodes + */ +function buildFullChannel(clChannelA: any, clChannelB: any): ILightningApi.Channel { const lastUpdate = Math.max(clChannelA.last_update ?? 0, clChannelB.last_update ?? 0); return { @@ -59,7 +61,11 @@ function buildBidirectionalChannel(clChannelA: any, clChannelB: any): ILightning }; } -function buildUnidirectionalChannel(clChannel: any): ILightningApi.Channel { +/** + * Convert one clightning "getchannels" entry into a full a lnd "describegraph.channels" format + * In this case, clightning knows the channel policy of only one node + */ + function buildIncompleteChannel(clChannel: any): ILightningApi.Channel { return { id: clChannel.short_channel_id, capacity: clChannel.satoshis, @@ -70,7 +76,10 @@ function buildUnidirectionalChannel(clChannel: any): ILightningApi.Channel { }; } -function convertPolicy(clChannel: any): ILightningApi.Policy { +/** + * Convert a clightning "listnode" response to a lnd channel policy format + */ + function convertPolicy(clChannel: any): ILightningApi.Policy { return { public_key: clChannel.source, base_fee_mtokens: clChannel.base_fee_millisatoshi, @@ -82,7 +91,10 @@ function convertPolicy(clChannel: any): ILightningApi.Policy { }; } -function getEmptyPolicy(): ILightningApi.Policy { +/** + * Create an empty channel policy in lnd format + */ + function getEmptyPolicy(): ILightningApi.Policy { return { public_key: 'null', base_fee_mtokens: '0', diff --git a/backend/src/api/lightning/clightning/jsonrpc.ts b/backend/src/api/lightning/clightning/jsonrpc.ts deleted file mode 100644 index d0b187a54..000000000 --- a/backend/src/api/lightning/clightning/jsonrpc.ts +++ /dev/null @@ -1,249 +0,0 @@ -// Imported from https://github.com/shesek/lightning-client-js - -'use strict'; - -const methods = [ - 'addgossip', - 'autocleaninvoice', - 'check', - 'checkmessage', - 'close', - 'connect', - 'createinvoice', - 'createinvoicerequest', - 'createoffer', - 'createonion', - 'decode', - 'decodepay', - 'delexpiredinvoice', - 'delinvoice', - 'delpay', - 'dev-listaddrs', - 'dev-rescan-outputs', - 'disableoffer', - 'disconnect', - 'estimatefees', - 'feerates', - 'fetchinvoice', - 'fundchannel', - 'fundchannel_cancel', - 'fundchannel_complete', - 'fundchannel_start', - 'fundpsbt', - 'getchaininfo', - 'getinfo', - 'getlog', - 'getrawblockbyheight', - 'getroute', - 'getsharedsecret', - 'getutxout', - 'help', - 'invoice', - 'keysend', - 'legacypay', - 'listchannels', - 'listconfigs', - 'listforwards', - 'listfunds', - 'listinvoices', - 'listnodes', - 'listoffers', - 'listpays', - 'listpeers', - 'listsendpays', - 'listtransactions', - 'multifundchannel', - 'multiwithdraw', - 'newaddr', - 'notifications', - 'offer', - 'offerout', - 'openchannel_abort', - 'openchannel_bump', - 'openchannel_init', - 'openchannel_signed', - 'openchannel_update', - 'pay', - 'payersign', - 'paystatus', - 'ping', - 'plugin', - 'reserveinputs', - 'sendinvoice', - 'sendonion', - 'sendonionmessage', - 'sendpay', - 'sendpsbt', - 'sendrawtransaction', - 'setchannelfee', - 'signmessage', - 'signpsbt', - 'stop', - 'txdiscard', - 'txprepare', - 'txsend', - 'unreserveinputs', - 'utxopsbt', - 'waitanyinvoice', - 'waitblockheight', - 'waitinvoice', - 'waitsendpay', - 'withdraw' -]; - - -import EventEmitter from 'events'; -import { existsSync, statSync } from 'fs'; -import { createConnection, Socket } from 'net'; -import { homedir } from 'os'; -import path from 'path'; -import { createInterface, Interface } from 'readline'; -import logger from '../../../logger'; - -class LightningError extends Error { - type: string = 'lightning'; - message: string = 'lightning-client error'; - - constructor(error) { - super(); - this.type = error.type; - this.message = error.message; - } -} - -const defaultRpcPath = path.join(homedir(), '.lightning') - , fStat = (...p) => statSync(path.join(...p)) - , fExists = (...p) => existsSync(path.join(...p)) - -export default class CLightningClient extends EventEmitter { - private rpcPath: string; - private reconnectWait: number; - private reconnectTimeout; - private reqcount: number; - private client: Socket; - private rl: Interface; - private clientConnectionPromise: Promise; - - constructor(rpcPath = defaultRpcPath) { - if (!path.isAbsolute(rpcPath)) { - throw new Error('The rpcPath must be an absolute path'); - } - - if (!fExists(rpcPath) || !fStat(rpcPath).isSocket()) { - // network directory provided, use the lightning-rpc within in - if (fExists(rpcPath, 'lightning-rpc')) { - rpcPath = path.join(rpcPath, 'lightning-rpc'); - } - - // main data directory provided, default to using the bitcoin mainnet subdirectory - // to be removed in v0.2.0 - else if (fExists(rpcPath, 'bitcoin', 'lightning-rpc')) { - logger.warn(`[CLightningClient] ${rpcPath}/lightning-rpc is missing, using the bitcoin mainnet subdirectory at ${rpcPath}/bitcoin instead.`) - logger.warn(`[CLightningClient] specifying the main lightning data directory is deprecated, please specify the network directory explicitly.\n`) - rpcPath = path.join(rpcPath, 'bitcoin', 'lightning-rpc') - } - } - - logger.debug(`[CLightningClient] Connecting to ${rpcPath}`); - - super(); - this.rpcPath = rpcPath; - this.reconnectWait = 0.5; - this.reconnectTimeout = null; - this.reqcount = 0; - - const _self = this; - - this.client = createConnection(rpcPath); - this.rl = createInterface({ input: this.client }) - - this.clientConnectionPromise = new Promise(resolve => { - _self.client.on('connect', () => { - logger.debug(`[CLightningClient] Lightning client connected`); - _self.reconnectWait = 1; - resolve(); - }); - - _self.client.on('end', () => { - logger.err('[CLightningClient] Lightning client connection closed, reconnecting'); - _self.increaseWaitTime(); - _self.reconnect(); - }); - - _self.client.on('error', error => { - logger.err(`[CLightningClient] Lightning client connection error: ${error}`); - _self.emit('error', error); - _self.increaseWaitTime(); - _self.reconnect(); - }); - }); - - this.rl.on('line', line => { - line = line.trim(); - if (!line) { - return; - } - const data = JSON.parse(line); - // logger.debug(`[CLightningClient] #${data.id} <-- ${JSON.stringify(data.error || data.result)}`); - _self.emit('res:' + data.id, data); - }); - } - - increaseWaitTime(): void { - if (this.reconnectWait >= 16) { - this.reconnectWait = 16; - } else { - this.reconnectWait *= 2; - } - } - - reconnect(): void { - const _self = this; - - if (this.reconnectTimeout) { - return; - } - - this.reconnectTimeout = setTimeout(() => { - logger.debug('[CLightningClient] Trying to reconnect...'); - - _self.client.connect(_self.rpcPath); - _self.reconnectTimeout = null; - }, this.reconnectWait * 1000); - } - - call(method, args = []): Promise { - const _self = this; - - const callInt = ++this.reqcount; - const sendObj = { - jsonrpc: '2.0', - method, - params: args, - id: '' + callInt - }; - - logger.debug(`[CLightningClient] #${callInt} --> ${method} ${args}`); - - // Wait for the client to connect - return this.clientConnectionPromise - .then(() => new Promise((resolve, reject) => { - // Wait for a response - this.once('res:' + callInt, res => res.error == null - ? resolve(res.result) - : reject(new LightningError(res.error)) - ); - - // Send the command - _self.client.write(JSON.stringify(sendObj)); - })); - } -} - -const protify = s => s.replace(/-([a-z])/g, m => m[1].toUpperCase()); - -methods.forEach(k => { - CLightningClient.prototype[protify(k)] = function (...args: any) { - return this.call(k, args); - }; -}); diff --git a/backend/src/api/lightning/lightning-api-abstract-factory.ts b/backend/src/api/lightning/lightning-api-abstract-factory.ts index 026568c6d..e6691b0a4 100644 --- a/backend/src/api/lightning/lightning-api-abstract-factory.ts +++ b/backend/src/api/lightning/lightning-api-abstract-factory.ts @@ -1,7 +1,5 @@ import { ILightningApi } from './lightning-api.interface'; export interface AbstractLightningApi { - $getNetworkInfo(): Promise; $getNetworkGraph(): Promise; - $getInfo(): Promise; } diff --git a/backend/src/api/lightning/lightning-api-factory.ts b/backend/src/api/lightning/lightning-api-factory.ts index ab551095c..fdadd8230 100644 --- a/backend/src/api/lightning/lightning-api-factory.ts +++ b/backend/src/api/lightning/lightning-api-factory.ts @@ -1,9 +1,12 @@ import config from '../../config'; +import CLightningClient from './clightning/clightning-client'; import { AbstractLightningApi } from './lightning-api-abstract-factory'; import LndApi from './lnd/lnd-api'; function lightningApiFactory(): AbstractLightningApi { - switch (config.LIGHTNING.BACKEND) { + switch (config.LIGHTNING.ENABLED === true && config.LIGHTNING.BACKEND) { + case 'cln': + return new CLightningClient(config.CLIGHTNING.SOCKET); case 'lnd': default: return new LndApi(); diff --git a/backend/src/tasks/lightning/node-sync.service.ts b/backend/src/tasks/lightning/node-sync.service.ts index 10cd2d744..d3367d51c 100644 --- a/backend/src/tasks/lightning/node-sync.service.ts +++ b/backend/src/tasks/lightning/node-sync.service.ts @@ -5,9 +5,9 @@ import bitcoinClient from '../../api/bitcoin/bitcoin-client'; import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory'; import config from '../../config'; import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface'; -import lightningApi from '../../api/lightning/lightning-api-factory'; import { ILightningApi } from '../../api/lightning/lightning-api.interface'; import { $lookupNodeLocation } from './sync-tasks/node-locations'; +import lightningApi from '../../api/lightning/lightning-api-factory'; class NodeSyncService { constructor() {} From 80f1ee45b5b8a0198fa572a69effef09e4d4fc95 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Mon, 1 Aug 2022 19:42:33 +0200 Subject: [PATCH 4/6] Rebased using the update lightning interfaces --- .../lightning/clightning/clightning-client.ts | 2 +- .../clightning/clightning-convert.ts | 80 +++++++++---------- .../src/tasks/lightning/node-sync.service.ts | 24 ++++-- 3 files changed, 57 insertions(+), 49 deletions(-) diff --git a/backend/src/api/lightning/clightning/clightning-client.ts b/backend/src/api/lightning/clightning/clightning-client.ts index 629092d03..f5643ed01 100644 --- a/backend/src/api/lightning/clightning/clightning-client.ts +++ b/backend/src/api/lightning/clightning/clightning-client.ts @@ -249,7 +249,7 @@ export default class CLightningClient extends EventEmitter implements AbstractLi return { nodes: listnodes['nodes'].map(node => convertNode(node)), - channels: channelsList, + edges: channelsList, }; } } diff --git a/backend/src/api/lightning/clightning/clightning-convert.ts b/backend/src/api/lightning/clightning/clightning-convert.ts index 8ceec3b7e..008094bf5 100644 --- a/backend/src/api/lightning/clightning/clightning-convert.ts +++ b/backend/src/api/lightning/clightning/clightning-convert.ts @@ -8,14 +8,19 @@ export function convertNode(clNode: any): ILightningApi.Node { alias: clNode.alias ?? '', color: `#${clNode.color ?? ''}`, features: [], // TODO parse and return clNode.feature - public_key: clNode.nodeid, - sockets: clNode.addresses?.map(addr => `${addr.address}:${addr.port}`) ?? [], - updated_at: new Date((clNode?.last_timestamp ?? 0) * 1000).toUTCString(), + pub_key: clNode.nodeid, + addresses: clNode.addresses?.map((addr) => { + return { + network: addr.type, + addr: `${addr.address}:${addr.port}` + }; + }), + last_update: clNode?.last_timestamp ?? 0, }; } /** - * Convert clightning "listchannels" response to lnd "describegraph.channels" format + * Convert clightning "listchannels" response to lnd "describegraph.edges" format */ export function convertAndmergeBidirectionalChannels(clChannels: any[]): ILightningApi.Channel[] { const consolidatedChannelList: ILightningApi.Channel[] = []; @@ -41,67 +46,58 @@ export function convertNode(clNode: any): ILightningApi.Node { return consolidatedChannelList; } +export function convertChannelId(channelId): string { + const s = channelId.split('x').map(part => parseInt(part)); + return BigInt((s[0] << 40) | (s[1] << 16) | s[2]).toString(); +} + /** - * Convert two clightning "getchannels" entries into a full a lnd "describegraph.channels" format + * Convert two clightning "getchannels" entries into a full a lnd "describegraph.edges" format * In this case, clightning knows the channel policy for both nodes */ function buildFullChannel(clChannelA: any, clChannelB: any): ILightningApi.Channel { const lastUpdate = Math.max(clChannelA.last_update ?? 0, clChannelB.last_update ?? 0); return { - id: clChannelA.short_channel_id, + channel_id: clChannelA.short_channel_id, capacity: clChannelA.satoshis, - transaction_id: '', // TODO - transaction_vout: 0, // TODO - updated_at: new Date(lastUpdate * 1000).toUTCString(), - policies: [ - convertPolicy(clChannelA), - convertPolicy(clChannelB) - ] + last_update: lastUpdate, + node1_policy: convertPolicy(clChannelA), + node2_policy: convertPolicy(clChannelB), + chan_point: ':0', // TODO + node1_pub: clChannelA.source, + node2_pub: clChannelB.source, }; } /** - * Convert one clightning "getchannels" entry into a full a lnd "describegraph.channels" format + * Convert one clightning "getchannels" entry into a full a lnd "describegraph.edges" format * In this case, clightning knows the channel policy of only one node */ function buildIncompleteChannel(clChannel: any): ILightningApi.Channel { return { - id: clChannel.short_channel_id, + channel_id: clChannel.short_channel_id, capacity: clChannel.satoshis, - policies: [convertPolicy(clChannel), getEmptyPolicy()], - transaction_id: '', // TODO - transaction_vout: 0, // TODO - updated_at: new Date((clChannel.last_update ?? 0) * 1000).toUTCString(), + last_update: clChannel.last_update ?? 0, + node1_policy: convertPolicy(clChannel), + node2_policy: null, + chan_point: ':0', // TODO + node1_pub: clChannel.source, + node2_pub: clChannel.destination, }; } /** * Convert a clightning "listnode" response to a lnd channel policy format */ - function convertPolicy(clChannel: any): ILightningApi.Policy { + function convertPolicy(clChannel: any): ILightningApi.RoutingPolicy { return { - public_key: clChannel.source, - base_fee_mtokens: clChannel.base_fee_millisatoshi, - fee_rate: clChannel.fee_per_millionth, - is_disabled: !clChannel.active, - max_htlc_mtokens: clChannel.htlc_maximum_msat.slice(0, -4), - min_htlc_mtokens: clChannel.htlc_minimum_msat.slice(0, -4), - updated_at: new Date((clChannel.last_update ?? 0) * 1000).toUTCString(), - }; -} - -/** - * Create an empty channel policy in lnd format - */ - function getEmptyPolicy(): ILightningApi.Policy { - return { - public_key: 'null', - base_fee_mtokens: '0', - fee_rate: 0, - is_disabled: true, - max_htlc_mtokens: '0', - min_htlc_mtokens: '0', - updated_at: new Date(0).toUTCString(), + time_lock_delta: 0, // TODO + min_htlc: clChannel.htlc_minimum_msat.slice(0, -4), + max_htlc_msat: clChannel.htlc_maximum_msat.slice(0, -4), + fee_base_msat: clChannel.base_fee_millisatoshi, + fee_rate_milli_msat: clChannel.fee_per_millionth, + disabled: !clChannel.active, + last_update: clChannel.last_update ?? 0, }; } diff --git a/backend/src/tasks/lightning/node-sync.service.ts b/backend/src/tasks/lightning/node-sync.service.ts index d3367d51c..863ee30da 100644 --- a/backend/src/tasks/lightning/node-sync.service.ts +++ b/backend/src/tasks/lightning/node-sync.service.ts @@ -8,6 +8,7 @@ import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface'; import { ILightningApi } from '../../api/lightning/lightning-api.interface'; import { $lookupNodeLocation } from './sync-tasks/node-locations'; import lightningApi from '../../api/lightning/lightning-api-factory'; +import { convertChannelId } from '../../api/lightning/clightning/clightning-convert'; class NodeSyncService { constructor() {} @@ -320,7 +321,7 @@ class NodeSyncService { ;`; await DB.query(query, [ - channel.channel_id, + this.toIntegerId(channel.channel_id), this.toShortId(channel.channel_id), channel.capacity, txid, @@ -391,8 +392,7 @@ class NodeSyncService { private async $saveNode(node: ILightningApi.Node): Promise { try { - const updatedAt = this.utcDateToMysql(node.last_update); - const sockets = node.addresses.map(a => a.addr).join(','); + const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? ''; const query = `INSERT INTO nodes( public_key, first_seen, @@ -401,15 +401,16 @@ class NodeSyncService { color, sockets ) - VALUES (?, NOW(), ?, ?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = ?, alias = ?, color = ?, sockets = ?;`; + VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?) + ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, color = ?, sockets = ?`; await DB.query(query, [ node.pub_key, - updatedAt, + node.last_update, node.alias, node.color, sockets, - updatedAt, + node.last_update, node.alias, node.color, sockets, @@ -419,8 +420,19 @@ class NodeSyncService { } } + private toIntegerId(id: string): string { + if (config.LIGHTNING.BACKEND === 'lnd') { + return id; + } + return convertChannelId(id); + } + /** Decodes a channel id returned by lnd as uint64 to a short channel id */ private toShortId(id: string): string { + if (config.LIGHTNING.BACKEND === 'cln') { + return id; + } + const n = BigInt(id); return [ n >> 40n, // nth block From 00cd3ee9bf6c74487b30ebb7ec092495059b4198 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 2 Aug 2022 16:18:19 +0200 Subject: [PATCH 5/6] Don't run the ln network update if the graph is emtpy --- backend/src/index.ts | 6 +++--- ...node-sync.service.ts => network-sync.service.ts} | 13 +++++++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) rename backend/src/tasks/lightning/{node-sync.service.ts => network-sync.service.ts} (97%) diff --git a/backend/src/index.ts b/backend/src/index.ts index fa80fb2ad..0f7cc7aa7 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -28,7 +28,7 @@ import nodesRoutes from './api/explorer/nodes.routes'; import channelsRoutes from './api/explorer/channels.routes'; import generalLightningRoutes from './api/explorer/general.routes'; import lightningStatsUpdater from './tasks/lightning/stats-updater.service'; -import nodeSyncService from './tasks/lightning/node-sync.service'; +import networkSyncService from './tasks/lightning/network-sync.service'; import statisticsRoutes from './api/statistics/statistics.routes'; import miningRoutes from './api/mining/mining-routes'; import bisqRoutes from './api/bisq/bisq.routes'; @@ -136,8 +136,8 @@ class Server { } if (config.LIGHTNING.ENABLED) { - nodeSyncService.$startService() - .then(() => lightningStatsUpdater.$startService()); + networkSyncService.$startService() + .then(() => lightningStatsUpdater.$startService()); } this.server.listen(config.MEMPOOL.HTTP_PORT, () => { diff --git a/backend/src/tasks/lightning/node-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts similarity index 97% rename from backend/src/tasks/lightning/node-sync.service.ts rename to backend/src/tasks/lightning/network-sync.service.ts index 863ee30da..826664cf4 100644 --- a/backend/src/tasks/lightning/node-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -10,7 +10,7 @@ import { $lookupNodeLocation } from './sync-tasks/node-locations'; import lightningApi from '../../api/lightning/lightning-api-factory'; import { convertChannelId } from '../../api/lightning/clightning/clightning-convert'; -class NodeSyncService { +class NetworkSyncService { constructor() {} public async $startService() { @@ -28,6 +28,11 @@ class NodeSyncService { logger.info(`Updating nodes and channels...`); const networkGraph = await lightningApi.$getNetworkGraph(); + if (networkGraph.nodes.length === 0 || networkGraph.edges.length === 0) { + logger.info(`LN Network graph is empty, retrying in 10 seconds`); + setTimeout(this.$runUpdater, 10000); + return; + } for (const node of networkGraph.nodes) { await this.$saveNode(node); @@ -376,6 +381,10 @@ class NodeSyncService { } private async $setChannelsInactive(graphChannelsIds: string[]): Promise { + if (graphChannelsIds.length === 0) { + return; + } + try { await DB.query(` UPDATE channels @@ -447,4 +456,4 @@ class NodeSyncService { } } -export default new NodeSyncService(); +export default new NetworkSyncService(); From a25af16f7c0557d1144d035344ee060c56dc04a7 Mon Sep 17 00:00:00 2001 From: nymkappa Date: Tue, 2 Aug 2022 16:39:34 +0200 Subject: [PATCH 6/6] Fetch funding tx for clightning channels --- .../lightning/clightning/clightning-client.ts | 2 +- .../clightning/clightning-convert.ts | 25 +++++++++++++------ backend/src/index.ts | 4 ++- .../sync-tasks/funding-tx-fetcher.ts | 16 ++++++------ 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/backend/src/api/lightning/clightning/clightning-client.ts b/backend/src/api/lightning/clightning/clightning-client.ts index f5643ed01..15f472f2e 100644 --- a/backend/src/api/lightning/clightning/clightning-client.ts +++ b/backend/src/api/lightning/clightning/clightning-client.ts @@ -245,7 +245,7 @@ export default class CLightningClient extends EventEmitter implements AbstractLi async $getNetworkGraph(): Promise { const listnodes: any[] = await this.call('listnodes'); const listchannels: any[] = await this.call('listchannels'); - const channelsList = convertAndmergeBidirectionalChannels(listchannels['channels']); + const channelsList = await convertAndmergeBidirectionalChannels(listchannels['channels']); return { nodes: listnodes['nodes'].map(node => convertNode(node)), diff --git a/backend/src/api/lightning/clightning/clightning-convert.ts b/backend/src/api/lightning/clightning/clightning-convert.ts index 008094bf5..1a267bc65 100644 --- a/backend/src/api/lightning/clightning/clightning-convert.ts +++ b/backend/src/api/lightning/clightning/clightning-convert.ts @@ -1,4 +1,5 @@ import { ILightningApi } from '../lightning-api.interface'; +import FundingTxFetcher from '../../../tasks/lightning/sync-tasks/funding-tx-fetcher'; /** * Convert a clightning "listnode" entry to a lnd node entry @@ -22,7 +23,7 @@ export function convertNode(clNode: any): ILightningApi.Node { /** * Convert clightning "listchannels" response to lnd "describegraph.edges" format */ - export function convertAndmergeBidirectionalChannels(clChannels: any[]): ILightningApi.Channel[] { + export async function convertAndmergeBidirectionalChannels(clChannels: any[]): Promise { const consolidatedChannelList: ILightningApi.Channel[] = []; const clChannelsDict = {}; const clChannelsDictCount = {}; @@ -33,14 +34,14 @@ export function convertNode(clNode: any): ILightningApi.Node { clChannelsDictCount[clChannel.short_channel_id] = 1; } else { consolidatedChannelList.push( - buildFullChannel(clChannel, clChannelsDict[clChannel.short_channel_id]) + await buildFullChannel(clChannel, clChannelsDict[clChannel.short_channel_id]) ); delete clChannelsDict[clChannel.short_channel_id]; clChannelsDictCount[clChannel.short_channel_id]++; } } for (const short_channel_id of Object.keys(clChannelsDict)) { - consolidatedChannelList.push(buildIncompleteChannel(clChannelsDict[short_channel_id])); + consolidatedChannelList.push(await buildIncompleteChannel(clChannelsDict[short_channel_id])); } return consolidatedChannelList; @@ -55,16 +56,20 @@ export function convertChannelId(channelId): string { * Convert two clightning "getchannels" entries into a full a lnd "describegraph.edges" format * In this case, clightning knows the channel policy for both nodes */ -function buildFullChannel(clChannelA: any, clChannelB: any): ILightningApi.Channel { +async function buildFullChannel(clChannelA: any, clChannelB: any): Promise { const lastUpdate = Math.max(clChannelA.last_update ?? 0, clChannelB.last_update ?? 0); - + + const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannelA.short_channel_id); + const parts = clChannelA.short_channel_id.split('x'); + const outputIdx = parts[2]; + return { channel_id: clChannelA.short_channel_id, capacity: clChannelA.satoshis, last_update: lastUpdate, node1_policy: convertPolicy(clChannelA), node2_policy: convertPolicy(clChannelB), - chan_point: ':0', // TODO + chan_point: `${tx.txid}:${outputIdx}`, node1_pub: clChannelA.source, node2_pub: clChannelB.source, }; @@ -74,14 +79,18 @@ function buildFullChannel(clChannelA: any, clChannelB: any): ILightningApi.Chann * Convert one clightning "getchannels" entry into a full a lnd "describegraph.edges" format * In this case, clightning knows the channel policy of only one node */ - function buildIncompleteChannel(clChannel: any): ILightningApi.Channel { + async function buildIncompleteChannel(clChannel: any): Promise { + const tx = await FundingTxFetcher.$fetchChannelOpenTx(clChannel.short_channel_id); + const parts = clChannel.short_channel_id.split('x'); + const outputIdx = parts[2]; + return { channel_id: clChannel.short_channel_id, capacity: clChannel.satoshis, last_update: clChannel.last_update ?? 0, node1_policy: convertPolicy(clChannel), node2_policy: null, - chan_point: ':0', // TODO + chan_point: `${tx.txid}:${outputIdx}`, node1_pub: clChannel.source, node2_pub: clChannel.destination, }; diff --git a/backend/src/index.ts b/backend/src/index.ts index 0f7cc7aa7..976ec12df 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -34,6 +34,7 @@ import miningRoutes from './api/mining/mining-routes'; import bisqRoutes from './api/bisq/bisq.routes'; import liquidRoutes from './api/liquid/liquid.routes'; import bitcoinRoutes from './api/bitcoin/bitcoin.routes'; +import fundingTxFetcher from "./tasks/lightning/sync-tasks/funding-tx-fetcher"; class Server { private wss: WebSocket.Server | undefined; @@ -136,7 +137,8 @@ class Server { } if (config.LIGHTNING.ENABLED) { - networkSyncService.$startService() + fundingTxFetcher.$init() + .then(() => networkSyncService.$startService()) .then(() => lightningStatsUpdater.$startService()); } diff --git a/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts b/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts index 9da721876..926d20c91 100644 --- a/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts +++ b/backend/src/tasks/lightning/sync-tasks/funding-tx-fetcher.ts @@ -1,8 +1,6 @@ import { existsSync, promises } from 'fs'; -import bitcoinApiFactory from '../../../api/bitcoin/bitcoin-api-factory'; import bitcoinClient from '../../../api/bitcoin/bitcoin-client'; import config from '../../../config'; -import DB from '../../../database'; import logger from '../../../logger'; const fsPromises = promises; @@ -16,12 +14,7 @@ class FundingTxFetcher { private channelNewlyProcessed = 0; public fundingTxCache = {}; - async $fetchChannelsFundingTxs(channelIds: string[]): Promise { - if (this.running) { - return; - } - this.running = true; - + async $init(): Promise { // Load funding tx disk cache if (Object.keys(this.fundingTxCache).length === 0 && existsSync(CACHE_FILE_NAME)) { try { @@ -32,6 +25,13 @@ class FundingTxFetcher { } logger.debug(`Imported ${Object.keys(this.fundingTxCache).length} funding tx amount from the disk cache`); } + } + + async $fetchChannelsFundingTxs(channelIds: string[]): Promise { + if (this.running) { + return; + } + this.running = true; const globalTimer = new Date().getTime() / 1000; let cacheTimer = new Date().getTime() / 1000;