diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index a9cb14929..0f43580f6 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 47; + private static currentVersion = 48; private queryTimeout = 900_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -379,7 +379,12 @@ class DatabaseMigration { await this.$executeQuery(this.getCreateCPFPTableQuery(), await this.$checkIfTableExists('cpfp_clusters')); await this.$executeQuery(this.getCreateTransactionsTableQuery(), await this.$checkIfTableExists('transactions')); } -} + + if (databaseSchemaVersion < 48 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `channels` ADD source_checked tinyint(1) DEFAULT 0'); + await this.$executeQuery(this.getCreateChannelsForensicsTableQuery(), await this.$checkIfTableExists('channels_forensics')); + } + } /** * Special case here for the `statistics` table - It appeared that somehow some dbs already had the `added` field indexed @@ -759,6 +764,25 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } + private getCreateChannelsForensicsTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS channels_forensics ( + channel_id bigint(11) unsigned NOT NULL, + closing_fee bigint(20) unsigned DEFAULT 0, + node1_funding_balance bigint(20) unsigned DEFAULT 0, + node2_funding_balance bigint(20) unsigned DEFAULT 0, + node1_closing_balance bigint(20) unsigned DEFAULT 0, + node2_closing_balance bigint(20) unsigned DEFAULT 0, + funding_ratio float unsigned DEFAULT NULL, + closed_by varchar(66) DEFAULT NULL, + single_funded tinyint(1) default 0, + outputs JSON NOT NULL, + PRIMARY KEY (channel_id), + FOREIGN KEY (channel_id) + REFERENCES channels (id) + ON DELETE CASCADE + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } + private getCreateNodesStatsQuery(): string { return `CREATE TABLE IF NOT EXISTS node_stats ( id int(11) unsigned NOT NULL AUTO_INCREMENT, diff --git a/backend/src/api/explorer/channels.api.ts b/backend/src/api/explorer/channels.api.ts index 787bbe521..22d17476f 100644 --- a/backend/src/api/explorer/channels.api.ts +++ b/backend/src/api/explorer/channels.api.ts @@ -128,6 +128,22 @@ class ChannelsApi { } } + public async $getChannelsWithoutSourceChecked(): Promise { + try { + const query = ` + SELECT channels.*, forensics.* + FROM channels + LEFT JOIN channels_forensics AS forensics ON forensics.channel_id = channels.id + WHERE channels.source_checked != 1 + `; + const [rows]: any = await DB.query(query); + return rows; + } catch (e) { + logger.err('$getUnresolvedClosedChannels error: ' + (e instanceof Error ? e.message : e)); + throw e; + } + } + public async $getChannelsWithoutCreatedDate(): Promise { try { const query = `SELECT * FROM channels WHERE created IS NULL`; @@ -145,12 +161,16 @@ class ChannelsApi { SELECT n1.alias AS alias_left, n1.longitude as node1_longitude, n1.latitude as node1_latitude, n2.alias AS alias_right, n2.longitude as node2_longitude, n2.latitude as node2_latitude, channels.*, - ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right + ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right, + forensics.closing_fee as closing_fee, forensics.node1_funding_balance as node1_funding_balance, forensics.node2_funding_balance as node2_funding_balance, + forensics.funding_ratio as funding_ratio, forensics.node1_closing_balance as node1_closing_balance, forensics.node2_closing_balance as node2_closing_balance, + forensics.closed_by as closed_by, forensics.single_funded as single_funded FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key + LEFT JOIN channels_forensics AS forensics ON forensics.channel_id = channels.id WHERE ( ns1.id = ( SELECT MAX(id) @@ -257,6 +277,118 @@ class ChannelsApi { } } + public async $getChannelForensicsByTransactionId(transactionId: string): Promise { + try { + const query = ` + SELECT + channels.id, channels.node1_public_key, channels.node2_public_key, + channels.closing_reason, channels.closing_transaction_id, + forensics.* + FROM channels + LEFT JOIN channels_forensics as forensics ON forensics.channel_id = channels.id + WHERE channels.closing_transaction_id = ? + `; + const [rows]: any = await DB.query(query, [transactionId]); + if (rows.length > 0) { + rows[0].outputs = JSON.parse(rows[0].outputs); + return rows[0]; + } + } catch (e) { + logger.err('$getChannelForensicsByTransactionId error: ' + (e instanceof Error ? e.message : e)); + // don't throw - this data isn't essential + } + } + + public async $updateClosingInfo(channelInfo: { id: string, node1_closing_balance: number, node2_closing_balance: number, closed_by: string | null, closing_fee: number, outputs: ILightningApi.ForensicOutput[]}): Promise { + try { + const query = ` + INSERT INTO channels_forensics + ( + channel_id, + node1_closing_balance, + node2_closing_balance, + closed_by, + closing_fee, + outputs + ) + VALUES (?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + node1_closing_balance = ?, + node2_closing_balance = ?, + closed_by = ?, + closing_fee = ?, + outputs = ? + `; + const jsonOutputs = JSON.stringify(channelInfo.outputs); + await DB.query(query, [ + channelInfo.id, + channelInfo.node1_closing_balance, + channelInfo.node2_closing_balance, + channelInfo.closed_by, + channelInfo.closing_fee, + jsonOutputs, + channelInfo.node1_closing_balance, + channelInfo.node2_closing_balance, + channelInfo.closed_by, + channelInfo.closing_fee, + jsonOutputs + ]); + } catch (e) { + logger.err('$updateClosingInfo error: ' + (e instanceof Error ? e.message : e)); + // don't throw - this data isn't essential + } + } + + public async $updateOpeningInfo(channelInfo: { id: string, node1_funding_balance: number, node2_funding_balance: number, funding_ratio: number, single_funded: boolean | void }): Promise { + try { + const query = ` + INSERT INTO channels_forensics + ( + channel_id, + node1_funding_balance, + node2_funding_balance, + funding_ratio, + single_funded, + outputs + ) + VALUES (?, ?, ?, ?, ?, 'null') + ON DUPLICATE KEY UPDATE + node1_funding_balance = ?, + node2_funding_balance = ?, + funding_ratio = ?, + single_funded = ? + `; + await DB.query(query, [ + channelInfo.id, + channelInfo.node1_funding_balance, + channelInfo.node2_funding_balance, + channelInfo.funding_ratio, + channelInfo.single_funded ? 1 : 0, + channelInfo.node1_funding_balance, + channelInfo.node2_funding_balance, + channelInfo.funding_ratio, + channelInfo.single_funded ? 1 : 0, + ]); + } catch (e) { + logger.err('$updateOpeningInfo error: ' + (e instanceof Error ? e.message : e)); + // don't throw - this data isn't essential + } + } + + public async $markChannelSourceChecked(id: string): Promise { + try { + const query = ` + UPDATE channels + SET source_checked = 1 + WHERE id = ? + `; + await DB.query(query, [id]); + } catch (e) { + logger.err('$markChannelSourceChecked error: ' + (e instanceof Error ? e.message : e)); + // don't throw - this data isn't essential + } + } + public async $getChannelsForNode(public_key: string, index: number, length: number, status: string): Promise { try { let channelStatusFilter; @@ -385,11 +517,15 @@ class ChannelsApi { 'transaction_id': channel.transaction_id, 'transaction_vout': channel.transaction_vout, 'closing_transaction_id': channel.closing_transaction_id, + 'closing_fee': channel.closing_fee, 'closing_reason': channel.closing_reason, 'closing_date': channel.closing_date, 'updated_at': channel.updated_at, 'created': channel.created, 'status': channel.status, + 'funding_ratio': channel.funding_ratio, + 'closed_by': channel.closed_by, + 'single_funded': !!channel.single_funded, 'node_left': { 'alias': channel.alias_left, 'public_key': channel.node1_public_key, @@ -404,6 +540,9 @@ class ChannelsApi { 'updated_at': channel.node1_updated_at, 'longitude': channel.node1_longitude, 'latitude': channel.node1_latitude, + 'funding_balance': channel.node1_funding_balance, + 'closing_balance': channel.node1_closing_balance, + 'initiated_close': channel.closed_by === channel.node1_public_key ? true : undefined, }, 'node_right': { 'alias': channel.alias_right, @@ -419,6 +558,9 @@ class ChannelsApi { 'updated_at': channel.node2_updated_at, 'longitude': channel.node2_longitude, 'latitude': channel.node2_latitude, + 'funding_balance': channel.node2_funding_balance, + 'closing_balance': channel.node2_closing_balance, + 'initiated_close': channel.closed_by === channel.node2_public_key ? true : undefined, }, }; } diff --git a/backend/src/api/lightning/lightning-api.interface.ts b/backend/src/api/lightning/lightning-api.interface.ts index 6e3ea0de3..453e2fffc 100644 --- a/backend/src/api/lightning/lightning-api.interface.ts +++ b/backend/src/api/lightning/lightning-api.interface.ts @@ -83,4 +83,10 @@ export namespace ILightningApi { is_required: boolean; is_known: boolean; } + + export interface ForensicOutput { + node?: 1 | 2; + type: number; + value: number; + } } \ No newline at end of file diff --git a/backend/src/tasks/lightning/network-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts index 9f40a350a..ca10ba919 100644 --- a/backend/src/tasks/lightning/network-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -304,6 +304,233 @@ class NetworkSyncService { logger.err('$scanForClosedChannels() error: ' + (e instanceof Error ? e.message : e)); } } + + private findLightningScript(vin: IEsploraApi.Vin): number { + const topElement = vin.witness[vin.witness.length - 2]; + if (/^OP_IF OP_PUSHBYTES_33 \w{66} OP_ELSE OP_PUSH(NUM_\d+|BYTES_(1 \w{2}|2 \w{4})) OP_CSV OP_DROP OP_PUSHBYTES_33 \w{66} OP_ENDIF OP_CHECKSIG$/.test(vin.inner_witnessscript_asm)) { + // https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction-outputs + if (topElement === '01') { + // top element is '01' to get in the revocation path + // 'Revoked Lightning Force Close'; + // Penalty force closed + return 2; + } else { + // top element is '', this is a delayed to_local output + // 'Lightning Force Close'; + return 3; + } + } else if ( + /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_NOTIF OP_DROP OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm) || + /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_IF OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_DROP OP_PUSHBYTES_3 \w{6} OP_CLTV OP_DROP OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm) + ) { + // https://github.com/lightning/bolts/blob/master/03-transactions.md#offered-htlc-outputs + // https://github.com/lightning/bolts/blob/master/03-transactions.md#received-htlc-outputs + if (topElement.length === 66) { + // top element is a public key + // 'Revoked Lightning HTLC'; Penalty force closed + return 4; + } else if (topElement) { + // top element is a preimage + // 'Lightning HTLC'; + return 5; + } else { + // top element is '' to get in the expiry of the script + // 'Expired Lightning HTLC'; + return 6; + } + } else if (/^OP_PUSHBYTES_33 \w{66} OP_CHECKSIG OP_IFDUP OP_NOTIF OP_PUSHNUM_16 OP_CSV OP_ENDIF$/.test(vin.inner_witnessscript_asm)) { + // https://github.com/lightning/bolts/blob/master/03-transactions.md#to_local_anchor-and-to_remote_anchor-output-option_anchors + if (topElement) { + // top element is a signature + // 'Lightning Anchor'; + return 7; + } else { + // top element is '', it has been swept after 16 blocks + // 'Swept Lightning Anchor'; + return 8; + } + } + return 1; + } + + // If a channel open tx spends funds from a closed channel output, + // we can attribute that output to a specific counterparty + private async $runOpenedChannelsForensics(): Promise { + let progress = 0; + + try { + logger.info(`Started running open channel forensics...`); + const channels = await channelsApi.$getChannelsWithoutSourceChecked(); + + for (const openChannel of channels) { + const openTx = await bitcoinApi.$getRawTransaction(openChannel.transaction_id); + for (const input of openTx.vin) { + const closeChannel = await channelsApi.$getChannelForensicsByTransactionId(input.txid); + if (closeChannel) { + // this input directly spends a channel close output + await this.$attributeChannelBalances(closeChannel, openChannel, input); + } else { + // check if this input spends any swept channel close outputs + await this.$attributeSweptChannelCloses(openChannel, input); + } + } + // calculate how much of the total input value is attributable to the channel open output + openChannel.funding_ratio = openTx.vout[openChannel.transaction_vout].value / ((openTx.vout.reduce((sum, v) => sum + v.value, 0) || 1) + openTx.fee); + // save changes to the opening channel, and mark it as checked + if (openTx.vin.length === 1) { + openChannel.single_funded = true; + } + await channelsApi.$updateOpeningInfo(openChannel); + await channelsApi.$markChannelSourceChecked(openChannel.id); + + ++progress; + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Updating channel opened channel forensics ${progress}/${channels.length}`); + this.loggerTimer = new Date().getTime() / 1000; + } + } + + logger.info(`Open channels forensics scan complete.`); + } catch (e) { + logger.err('$runOpenedChannelsForensics() error: ' + (e instanceof Error ? e.message : e)); + } + } + + // Check if a channel open tx input spends the result of a swept channel close output + private async $attributeSweptChannelCloses(openChannel: ILightningApi.Channel, input: IEsploraApi.Vin): Promise { + const sweepTx = await bitcoinApi.$getRawTransaction(input.txid); + if (!sweepTx) { + logger.err(`couldn't find input transaction for channel forensics ${openChannel.channel_id} ${input.txid}`); + return; + } + const openContribution = sweepTx.vout[input.vout].value; + for (const sweepInput of sweepTx.vin) { + const lnScriptType = this.findLightningScript(sweepInput); + if (lnScriptType > 1) { + const closeChannel = await channelsApi.$getChannelForensicsByTransactionId(sweepInput.txid); + if (closeChannel) { + const initiator = (lnScriptType === 2 || lnScriptType === 4) ? 'remote' : (lnScriptType === 3 ? 'local' : null); + await this.$attributeChannelBalances(closeChannel, openChannel, sweepInput, openContribution, initiator); + } + } + } + } + + private async $attributeChannelBalances( + closeChannel, openChannel, input: IEsploraApi.Vin, openContribution: number | null = null, + initiator: 'remote' | 'local' | null = null + ): Promise { + // figure out which node controls the input/output + let openSide; + let closeLocal; + let closeRemote; + let matched = false; + let ambiguous = false; // if counterparties are the same in both channels, we can't tell them apart + if (openChannel.node1_public_key === closeChannel.node1_public_key) { + openSide = 1; + closeLocal = 1; + closeRemote = 2; + matched = true; + } else if (openChannel.node1_public_key === closeChannel.node2_public_key) { + openSide = 1; + closeLocal = 2; + closeRemote = 1; + matched = true; + } + if (openChannel.node2_public_key === closeChannel.node1_public_key) { + openSide = 2; + closeLocal = 1; + closeRemote = 2; + if (matched) { + ambiguous = true; + } + matched = true; + } else if (openChannel.node2_public_key === closeChannel.node2_public_key) { + openSide = 2; + closeLocal = 2; + closeRemote = 1; + if (matched) { + ambiguous = true; + } + matched = true; + } + + if (matched && !ambiguous) { + // fetch closing channel transaction and perform forensics on the outputs + let closingTx: IEsploraApi.Transaction | undefined; + let outspends: IEsploraApi.Outspend[] | undefined; + try { + closingTx = await bitcoinApi.$getRawTransaction(input.txid); + outspends = await bitcoinApi.$getOutspends(input.txid); + } catch (e) { + logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + closeChannel.closing_transaction_id + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`); + } + if (!outspends || !closingTx) { + return; + } + if (!closeChannel.outputs) { + closeChannel.outputs = closeChannel.outputs || closingTx.vout.map(vout => { + return { + type: 0, + value: vout.value, + }; + }); + } + for (let i = 0; i < outspends.length; i++) { + const outspend = outspends[i]; + const output = closeChannel.outputs[i]; + if (outspend.spent && outspend.txid) { + try { + const spendingTx = await bitcoinApi.$getRawTransaction(outspend.txid); + if (spendingTx) { + output.type = this.findLightningScript(spendingTx.vin[outspend.vin || 0]); + } + } catch (e) { + logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + outspend.txid}. Reason ${e instanceof Error ? e.message : e}`); + } + } else { + output.type = 0; + } + } + + // attribute outputs to each counterparty, and sum up total known balances + closeChannel.outputs[input.vout].node = closeLocal; + const isPenalty = closeChannel.outputs.filter((out) => out.type === 2 || out.type === 4).length > 0; + const normalOutput = [1,3].includes(closeChannel.outputs[input.vout].type); + let localClosingBalance = 0; + let remoteClosingBalance = 0; + for (const output of closeChannel.outputs) { + if (isPenalty) { + // penalty close, so local node takes everything + localClosingBalance += output.value; + } else if (output.node) { + // this output determinstically linked to one of the counterparties + if (output.node === closeLocal) { + localClosingBalance += output.value; + } else { + remoteClosingBalance += output.value; + } + } else if (normalOutput && (output.type === 1 || output.type === 3)) { + // local node had one main output, therefore remote node takes the other + remoteClosingBalance += output.value; + } + } + + openChannel[`node${openSide}_funding_balance`] = openChannel[`node${openSide}_funding_balance`] + (openContribution || closingTx?.vout[input.vout]?.value || 0); + closeChannel[`node${closeLocal}_closing_balance`] = localClosingBalance; + closeChannel[`node${closeRemote}_closing_balance`] = remoteClosingBalance; + closeChannel.closing_fee = closingTx.fee; + + if (initiator) { + const initiatorSide = initiator === 'remote' ? closeRemote : closeLocal; + closeChannel.closed_by = closeChannel[`node${initiatorSide}_public_key`]; + } + + // save changes to the closing channel + await channelsApi.$updateClosingInfo(closeChannel); + } + } } export default new NetworkSyncService();