import DB from '../../database'; import logger from '../../logger'; import channelsApi from '../../api/explorer/channels.api'; import bitcoinClient from '../../api/bitcoin/bitcoin-client'; import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory'; import config from '../../config'; import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface'; import { ILightningApi } from '../../api/lightning/lightning-api.interface'; import { $lookupNodeLocation } from './sync-tasks/node-locations'; import lightningApi from '../../api/lightning/lightning-api-factory'; import { convertChannelId } from '../../api/lightning/clightning/clightning-convert'; class NetworkSyncService { constructor() {} public async $startService() { logger.info('Starting node sync service'); await this.$runUpdater(); setInterval(async () => { await this.$runUpdater(); }, 1000 * 60 * 60); } private async $runUpdater() { try { logger.info(`Updating nodes and channels...`); const networkGraph = await lightningApi.$getNetworkGraph(); if (networkGraph.nodes.length === 0 || networkGraph.edges.length === 0) { logger.info(`LN Network graph is empty, retrying in 10 seconds`); setTimeout(this.$runUpdater, 10000); return; } for (const node of networkGraph.nodes) { await this.$saveNode(node); } logger.info(`Nodes updated.`); if (config.MAXMIND.ENABLED) { await $lookupNodeLocation(); } const graphChannelsIds: string[] = []; for (const channel of networkGraph.edges) { await this.$saveChannel(channel); graphChannelsIds.push(channel.channel_id); } await this.$setChannelsInactive(graphChannelsIds); logger.info(`Channels updated.`); await this.$findInactiveNodesAndChannels(); await this.$lookUpCreationDateFromChain(); await this.$updateNodeFirstSeen(); await this.$scanForClosedChannels(); if (config.MEMPOOL.BACKEND === 'esplora') { await this.$runClosedChannelsForensics(); } } catch (e) { logger.err('$runUpdater() error: ' + (e instanceof Error ? e.message : e)); } } // This method look up the creation date of the earliest channel of the node // and update the node to that date in order to get the earliest first seen date private async $updateNodeFirstSeen() { try { const [nodes]: any[] = await DB.query(`SELECT nodes.public_key, UNIX_TIMESTAMP(nodes.first_seen) AS first_seen, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node1_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created1, (SELECT UNIX_TIMESTAMP(created) FROM channels WHERE channels.node2_public_key = nodes.public_key ORDER BY created ASC LIMIT 1) AS created2 FROM nodes`); for (const node of nodes) { let lowest = 0; if (node.created1) { if (node.created2 && node.created2 < node.created1) { lowest = node.created2; } else { lowest = node.created1; } } else if (node.created2) { lowest = node.created2; } if (lowest && lowest < node.first_seen) { const query = `UPDATE nodes SET first_seen = FROM_UNIXTIME(?) WHERE public_key = ?`; const params = [lowest, node.public_key]; await DB.query(query, params); } } logger.info(`Node first seen dates scan complete.`); } catch (e) { logger.err('$updateNodeFirstSeen() error: ' + (e instanceof Error ? e.message : e)); } } private async $lookUpCreationDateFromChain() { logger.info(`Running channel creation date lookup...`); try { const channels = await channelsApi.$getChannelsWithoutCreatedDate(); for (const channel of channels) { const transaction = await bitcoinClient.getRawTransaction(channel.transaction_id, 1); await DB.query(`UPDATE channels SET created = FROM_UNIXTIME(?) WHERE channels.id = ?`, [transaction.blocktime, channel.id]); } logger.info(`Channel creation dates scan complete.`); } catch (e) { logger.err('$setCreationDateFromChain() error: ' + (e instanceof Error ? e.message : e)); } } // Looking for channels whos nodes are inactive private async $findInactiveNodesAndChannels(): Promise { logger.info(`Running inactive channels scan...`); try { const [channels]: [{ id: string }[]] = await DB.query(` SELECT channels.id FROM channels WHERE channels.status = 1 AND ( ( SELECT COUNT(*) FROM nodes WHERE nodes.public_key = channels.node1_public_key ) = 0 OR ( SELECT COUNT(*) FROM nodes WHERE nodes.public_key = channels.node2_public_key ) = 0) `); for (const channel of channels) { await this.$updateChannelStatus(channel.id, 0); } logger.info(`Inactive channels scan complete.`); } catch (e) { logger.err('$findInactiveNodesAndChannels() error: ' + (e instanceof Error ? e.message : e)); } } private async $scanForClosedChannels(): Promise { try { logger.info(`Starting closed channels scan...`); const channels = await channelsApi.$getChannelsByStatus(0); for (const channel of channels) { const spendingTx = await bitcoinApi.$getOutspend(channel.transaction_id, channel.transaction_vout); if (spendingTx.spent === true && spendingTx.status?.confirmed === true) { logger.debug('Marking channel: ' + channel.id + ' as closed.'); await DB.query(`UPDATE channels SET status = 2, closing_date = FROM_UNIXTIME(?) WHERE id = ?`, [spendingTx.status.block_time, channel.id]); if (spendingTx.txid && !channel.closing_transaction_id) { await DB.query(`UPDATE channels SET closing_transaction_id = ? WHERE id = ?`, [spendingTx.txid, channel.id]); } } } logger.info(`Closed channels scan complete.`); } catch (e) { logger.err('$scanForClosedChannels() error: ' + (e instanceof Error ? e.message : e)); } } /* 1. Mutually closed 2. Forced closed 3. Forced closed with penalty */ private async $runClosedChannelsForensics(): Promise { if (!config.ESPLORA.REST_API_URL) { return; } try { logger.info(`Started running closed channel forensics...`); const channels = await channelsApi.$getClosedChannelsWithoutReason(); for (const channel of channels) { let reason = 0; // Only Esplora backend can retrieve spent transaction outputs const outspends = await bitcoinApi.$getOutspends(channel.closing_transaction_id); const lightningScriptReasons: number[] = []; for (const outspend of outspends) { if (outspend.spent && outspend.txid) { const spendingTx = await bitcoinApi.$getRawTransaction(outspend.txid); const lightningScript = this.findLightningScript(spendingTx.vin[outspend.vin || 0]); lightningScriptReasons.push(lightningScript); } } if (lightningScriptReasons.length === outspends.length && lightningScriptReasons.filter((r) => r === 1).length === outspends.length) { reason = 1; } else { const filteredReasons = lightningScriptReasons.filter((r) => r !== 1); if (filteredReasons.length) { if (filteredReasons.some((r) => r === 2 || r === 4)) { reason = 3; } else { reason = 2; } } else { /* We can detect a commitment transaction (force close) by reading Sequence and Locktime https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction */ const closingTx = await bitcoinApi.$getRawTransaction(channel.closing_transaction_id); const sequenceHex: string = closingTx.vin[0].sequence.toString(16); const locktimeHex: string = closingTx.locktime.toString(16); if (sequenceHex.substring(0, 2) === '80' && locktimeHex.substring(0, 2) === '20') { reason = 2; // Here we can't be sure if it's a penalty or not } else { reason = 1; } } } if (reason) { logger.debug('Setting closing reason ' + reason + ' for channel: ' + channel.id + '.'); await DB.query(`UPDATE channels SET closing_reason = ? WHERE id = ?`, [reason, channel.id]); } } logger.info(`Closed channels forensics scan complete.`); } catch (e) { logger.err('$runClosedChannelsForensics() error: ' + (e instanceof Error ? e.message : e)); } } private findLightningScript(vin: IEsploraApi.Vin): number { const topElement = vin.witness[vin.witness.length - 2]; if (/^OP_IF OP_PUSHBYTES_33 \w{66} OP_ELSE OP_PUSH(NUM_\d+|BYTES_(1 \w{2}|2 \w{4})) OP_CSV OP_DROP OP_PUSHBYTES_33 \w{66} OP_ENDIF OP_CHECKSIG$/.test(vin.inner_witnessscript_asm)) { // https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction-outputs if (topElement === '01') { // top element is '01' to get in the revocation path // 'Revoked Lightning Force Close'; // Penalty force closed return 2; } else { // top element is '', this is a delayed to_local output // 'Lightning Force Close'; return 3; } } else if ( /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_NOTIF OP_DROP OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm) || /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_IF OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_DROP OP_PUSHBYTES_3 \w{6} OP_CLTV OP_DROP OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm) ) { // https://github.com/lightning/bolts/blob/master/03-transactions.md#offered-htlc-outputs // https://github.com/lightning/bolts/blob/master/03-transactions.md#received-htlc-outputs if (topElement.length === 66) { // top element is a public key // 'Revoked Lightning HTLC'; Penalty force closed return 4; } else if (topElement) { // top element is a preimage // 'Lightning HTLC'; return 5; } else { // top element is '' to get in the expiry of the script // 'Expired Lightning HTLC'; return 6; } } else if (/^OP_PUSHBYTES_33 \w{66} OP_CHECKSIG OP_IFDUP OP_NOTIF OP_PUSHNUM_16 OP_CSV OP_ENDIF$/.test(vin.inner_witnessscript_asm)) { // https://github.com/lightning/bolts/blob/master/03-transactions.md#to_local_anchor-and-to_remote_anchor-output-option_anchors if (topElement) { // top element is a signature // 'Lightning Anchor'; return 7; } else { // top element is '', it has been swept after 16 blocks // 'Swept Lightning Anchor'; return 8; } } return 1; } private async $saveChannel(channel: ILightningApi.Channel): Promise { const [ txid, vout ] = channel.chan_point.split(':'); const policy1: Partial = channel.node1_policy || {}; const policy2: Partial = channel.node2_policy || {}; try { const query = `INSERT INTO channels ( id, short_id, capacity, transaction_id, transaction_vout, updated_at, status, node1_public_key, node1_base_fee_mtokens, node1_cltv_delta, node1_fee_rate, node1_is_disabled, node1_max_htlc_mtokens, node1_min_htlc_mtokens, node1_updated_at, node2_public_key, node2_base_fee_mtokens, node2_cltv_delta, node2_fee_rate, node2_is_disabled, node2_max_htlc_mtokens, node2_min_htlc_mtokens, node2_updated_at ) VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE capacity = ?, updated_at = ?, status = 1, node1_public_key = ?, node1_base_fee_mtokens = ?, node1_cltv_delta = ?, node1_fee_rate = ?, node1_is_disabled = ?, node1_max_htlc_mtokens = ?, node1_min_htlc_mtokens = ?, node1_updated_at = ?, node2_public_key = ?, node2_base_fee_mtokens = ?, node2_cltv_delta = ?, node2_fee_rate = ?, node2_is_disabled = ?, node2_max_htlc_mtokens = ?, node2_min_htlc_mtokens = ?, node2_updated_at = ? ;`; await DB.query(query, [ this.toIntegerId(channel.channel_id), this.toShortId(channel.channel_id), channel.capacity, txid, vout, this.utcDateToMysql(channel.last_update), channel.node1_pub, policy1.fee_base_msat, policy1.time_lock_delta, policy1.fee_rate_milli_msat, policy1.disabled, policy1.max_htlc_msat, policy1.min_htlc, this.utcDateToMysql(policy1.last_update), channel.node2_pub, policy2.fee_base_msat, policy2.time_lock_delta, policy2.fee_rate_milli_msat, policy2.disabled, policy2.max_htlc_msat, policy2.min_htlc, this.utcDateToMysql(policy2.last_update), channel.capacity, this.utcDateToMysql(channel.last_update), channel.node1_pub, policy1.fee_base_msat, policy1.time_lock_delta, policy1.fee_rate_milli_msat, policy1.disabled, policy1.max_htlc_msat, policy1.min_htlc, this.utcDateToMysql(policy1.last_update), channel.node2_pub, policy2.fee_base_msat, policy2.time_lock_delta, policy2.fee_rate_milli_msat, policy2.disabled, policy2.max_htlc_msat, policy2.min_htlc, this.utcDateToMysql(policy2.last_update) ]); } catch (e) { logger.err('$saveChannel() error: ' + (e instanceof Error ? e.message : e)); } } private async $updateChannelStatus(channelId: string, status: number): Promise { try { await DB.query(`UPDATE channels SET status = ? WHERE id = ?`, [status, channelId]); } catch (e) { logger.err('$updateChannelStatus() error: ' + (e instanceof Error ? e.message : e)); } } private async $setChannelsInactive(graphChannelsIds: string[]): Promise { if (graphChannelsIds.length === 0) { return; } try { await DB.query(` UPDATE channels SET status = 0 WHERE short_id NOT IN ( ${graphChannelsIds.map(id => `"${id}"`).join(',')} ) AND status != 2 `); } catch (e) { logger.err('$setChannelsInactive() error: ' + (e instanceof Error ? e.message : e)); } } private async $saveNode(node: ILightningApi.Node): Promise { try { const sockets = (node.addresses?.map(a => a.addr).join(',')) ?? ''; const query = `INSERT INTO nodes( public_key, first_seen, updated_at, alias, color, sockets ) VALUES (?, NOW(), FROM_UNIXTIME(?), ?, ?, ?) ON DUPLICATE KEY UPDATE updated_at = FROM_UNIXTIME(?), alias = ?, color = ?, sockets = ?`; await DB.query(query, [ node.pub_key, node.last_update, node.alias, node.color, sockets, node.last_update, node.alias, node.color, sockets, ]); } catch (e) { logger.err('$saveNode() error: ' + (e instanceof Error ? e.message : e)); } } private toIntegerId(id: string): string { if (config.LIGHTNING.BACKEND === 'lnd') { return id; } return convertChannelId(id); } /** Decodes a channel id returned by lnd as uint64 to a short channel id */ private toShortId(id: string): string { if (config.LIGHTNING.BACKEND === 'cln') { return id; } const n = BigInt(id); return [ n >> 40n, // nth block (n >> 16n) & 0xffffffn, // nth tx of the block n & 0xffffn // nth output of the tx ].join('x'); } private utcDateToMysql(date?: number): string { const d = new Date((date || 0) * 1000); return d.toISOString().split('T')[0] + ' ' + d.toTimeString().split(' ')[0]; } } export default new NetworkSyncService();