From 030889250fd225985e524e7c64311843c67a8fb2 Mon Sep 17 00:00:00 2001 From: softsimon Date: Wed, 30 Nov 2022 17:56:53 +0900 Subject: [PATCH 01/13] Mempool GBT config --- backend/mempool-config.sample.json | 3 ++- .../src/__fixtures__/mempool-config.template.json | 3 ++- backend/src/__tests__/config.test.ts | 3 ++- backend/src/api/websocket-handler.ts | 13 ++++++------- backend/src/config.ts | 6 ++++-- 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/backend/mempool-config.sample.json b/backend/mempool-config.sample.json index b0b157c42..6de690ad8 100644 --- a/backend/mempool-config.sample.json +++ b/backend/mempool-config.sample.json @@ -25,7 +25,8 @@ "AUTOMATIC_BLOCK_REINDEXING": false, "POOLS_JSON_URL": "https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json", "POOLS_JSON_TREE_URL": "https://api.github.com/repos/mempool/mining-pools/git/trees/master", - "ADVANCED_TRANSACTION_SELECTION": false, + "ADVANCED_GBT_AUDIT": false, + "ADVANCED_GBT_MEMPOOL": false, "TRANSACTION_INDEXING": false }, "CORE_RPC": { diff --git a/backend/src/__fixtures__/mempool-config.template.json b/backend/src/__fixtures__/mempool-config.template.json index 8b368a43a..7a988a70d 100644 --- a/backend/src/__fixtures__/mempool-config.template.json +++ b/backend/src/__fixtures__/mempool-config.template.json @@ -26,7 +26,8 @@ "INDEXING_BLOCKS_AMOUNT": 14, "POOLS_JSON_TREE_URL": "__POOLS_JSON_TREE_URL__", "POOLS_JSON_URL": "__POOLS_JSON_URL__", - "ADVANCED_TRANSACTION_SELECTION": "__ADVANCED_TRANSACTION_SELECTION__", + "ADVANCED_GBT_AUDIT": "__ADVANCED_GBT_AUDIT__", + "ADVANCED_GBT_MEMPOOL": "__ADVANCED_GBT_MEMPOOL__", "TRANSACTION_INDEXING": "__TRANSACTION_INDEXING__" }, "CORE_RPC": { diff --git a/backend/src/__tests__/config.test.ts b/backend/src/__tests__/config.test.ts index c95888cf2..58cf3a214 100644 --- a/backend/src/__tests__/config.test.ts +++ b/backend/src/__tests__/config.test.ts @@ -38,7 +38,8 @@ describe('Mempool Backend Config', () => { STDOUT_LOG_MIN_PRIORITY: 'debug', POOLS_JSON_TREE_URL: 'https://api.github.com/repos/mempool/mining-pools/git/trees/master', POOLS_JSON_URL: 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json', - ADVANCED_TRANSACTION_SELECTION: false, + ADVANCED_GBT_AUDIT: false, + ADVANCED_GBT_MEMPOOL: false, TRANSACTION_INDEXING: false, }); diff --git a/backend/src/api/websocket-handler.ts b/backend/src/api/websocket-handler.ts index 0499fe842..31224fc0c 100644 --- a/backend/src/api/websocket-handler.ts +++ b/backend/src/api/websocket-handler.ts @@ -250,12 +250,12 @@ class WebsocketHandler { throw new Error('WebSocket.Server is not set'); } - if (config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) { + if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { await mempoolBlocks.makeBlockTemplates(newMempool, 8, null, true); - } - else { + } else { mempoolBlocks.updateMempoolBlocks(newMempool); } + const mBlocks = mempoolBlocks.getMempoolBlocks(); const mBlockDeltas = mempoolBlocks.getMempoolBlockDeltas(); const mempoolInfo = memPool.getMempoolInfo(); @@ -417,9 +417,8 @@ class WebsocketHandler { } const _memPool = memPool.getMempool(); - let matchRate; - if (config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) { + if (config.MEMPOOL.ADVANCED_GBT_AUDIT) { await mempoolBlocks.makeBlockTemplates(_memPool, 2); } else { mempoolBlocks.updateMempoolBlocks(_memPool); @@ -429,7 +428,7 @@ class WebsocketHandler { const projectedBlocks = mempoolBlocks.getMempoolBlocksWithTransactions(); const { censored, added, fresh, score } = Audit.auditBlock(transactions, projectedBlocks, _memPool); - matchRate = Math.round(score * 100 * 100) / 100; + const matchRate = Math.round(score * 100 * 100) / 100; const stripped = projectedBlocks[0]?.transactions ? projectedBlocks[0].transactions.map((tx) => { return { @@ -468,7 +467,7 @@ class WebsocketHandler { delete _memPool[txId]; } - if (config.MEMPOOL.ADVANCED_TRANSACTION_SELECTION) { + if (config.MEMPOOL.ADVANCED_GBT_MEMPOOL) { await mempoolBlocks.makeBlockTemplates(_memPool, 2); } else { mempoolBlocks.updateMempoolBlocks(_memPool); diff --git a/backend/src/config.ts b/backend/src/config.ts index 808e1406b..3a3d2b56d 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -29,7 +29,8 @@ interface IConfig { AUTOMATIC_BLOCK_REINDEXING: boolean; POOLS_JSON_URL: string, POOLS_JSON_TREE_URL: string, - ADVANCED_TRANSACTION_SELECTION: boolean; + ADVANCED_GBT_AUDIT: boolean; + ADVANCED_GBT_MEMPOOL: boolean; TRANSACTION_INDEXING: boolean; }; ESPLORA: { @@ -148,7 +149,8 @@ const defaults: IConfig = { 'AUTOMATIC_BLOCK_REINDEXING': false, 'POOLS_JSON_URL': 'https://raw.githubusercontent.com/mempool/mining-pools/master/pools.json', 'POOLS_JSON_TREE_URL': 'https://api.github.com/repos/mempool/mining-pools/git/trees/master', - 'ADVANCED_TRANSACTION_SELECTION': false, + 'ADVANCED_GBT_AUDIT': false, + 'ADVANCED_GBT_MEMPOOL': false, 'TRANSACTION_INDEXING': false, }, 'ESPLORA': { From a9e766046f270b5f2afd893b2d874ee3c1f98f15 Mon Sep 17 00:00:00 2001 From: wiz Date: Wed, 30 Nov 2022 18:10:47 +0900 Subject: [PATCH 02/13] [ops] Increase nginx max concurrent streams --- production/nginx/http-basic.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/production/nginx/http-basic.conf b/production/nginx/http-basic.conf index 5d707ca42..dc880cad5 100644 --- a/production/nginx/http-basic.conf +++ b/production/nginx/http-basic.conf @@ -35,3 +35,5 @@ gzip_types application/javascript application/json application/ld+json applicati # limit request body size client_max_body_size 10m; +# need to bump this up for about page sponsor images lol +http2_max_concurrent_streams 256; From cf89ded14d138f03c9e8676edcbea8d12faf87e7 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Thu, 10 Nov 2022 12:21:19 -0600 Subject: [PATCH 03/13] detect links between channel close and open txs --- backend/src/api/database-migration.ts | 28 ++- backend/src/api/explorer/channels.api.ts | 144 ++++++++++- .../api/lightning/lightning-api.interface.ts | 6 + .../tasks/lightning/network-sync.service.ts | 227 ++++++++++++++++++ 4 files changed, 402 insertions(+), 3 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index a9cb14929..0f43580f6 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -4,7 +4,7 @@ import logger from '../logger'; import { Common } from './common'; class DatabaseMigration { - private static currentVersion = 47; + private static currentVersion = 48; private queryTimeout = 900_000; private statisticsAddedIndexed = false; private uniqueLogs: string[] = []; @@ -379,7 +379,12 @@ class DatabaseMigration { await this.$executeQuery(this.getCreateCPFPTableQuery(), await this.$checkIfTableExists('cpfp_clusters')); await this.$executeQuery(this.getCreateTransactionsTableQuery(), await this.$checkIfTableExists('transactions')); } -} + + if (databaseSchemaVersion < 48 && isBitcoin === true) { + await this.$executeQuery('ALTER TABLE `channels` ADD source_checked tinyint(1) DEFAULT 0'); + await this.$executeQuery(this.getCreateChannelsForensicsTableQuery(), await this.$checkIfTableExists('channels_forensics')); + } + } /** * Special case here for the `statistics` table - It appeared that somehow some dbs already had the `added` field indexed @@ -759,6 +764,25 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } + private getCreateChannelsForensicsTableQuery(): string { + return `CREATE TABLE IF NOT EXISTS channels_forensics ( + channel_id bigint(11) unsigned NOT NULL, + closing_fee bigint(20) unsigned DEFAULT 0, + node1_funding_balance bigint(20) unsigned DEFAULT 0, + node2_funding_balance bigint(20) unsigned DEFAULT 0, + node1_closing_balance bigint(20) unsigned DEFAULT 0, + node2_closing_balance bigint(20) unsigned DEFAULT 0, + funding_ratio float unsigned DEFAULT NULL, + closed_by varchar(66) DEFAULT NULL, + single_funded tinyint(1) default 0, + outputs JSON NOT NULL, + PRIMARY KEY (channel_id), + FOREIGN KEY (channel_id) + REFERENCES channels (id) + ON DELETE CASCADE + ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; + } + private getCreateNodesStatsQuery(): string { return `CREATE TABLE IF NOT EXISTS node_stats ( id int(11) unsigned NOT NULL AUTO_INCREMENT, diff --git a/backend/src/api/explorer/channels.api.ts b/backend/src/api/explorer/channels.api.ts index 787bbe521..22d17476f 100644 --- a/backend/src/api/explorer/channels.api.ts +++ b/backend/src/api/explorer/channels.api.ts @@ -128,6 +128,22 @@ class ChannelsApi { } } + public async $getChannelsWithoutSourceChecked(): Promise { + try { + const query = ` + SELECT channels.*, forensics.* + FROM channels + LEFT JOIN channels_forensics AS forensics ON forensics.channel_id = channels.id + WHERE channels.source_checked != 1 + `; + const [rows]: any = await DB.query(query); + return rows; + } catch (e) { + logger.err('$getUnresolvedClosedChannels error: ' + (e instanceof Error ? e.message : e)); + throw e; + } + } + public async $getChannelsWithoutCreatedDate(): Promise { try { const query = `SELECT * FROM channels WHERE created IS NULL`; @@ -145,12 +161,16 @@ class ChannelsApi { SELECT n1.alias AS alias_left, n1.longitude as node1_longitude, n1.latitude as node1_latitude, n2.alias AS alias_right, n2.longitude as node2_longitude, n2.latitude as node2_latitude, channels.*, - ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right + ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right, + forensics.closing_fee as closing_fee, forensics.node1_funding_balance as node1_funding_balance, forensics.node2_funding_balance as node2_funding_balance, + forensics.funding_ratio as funding_ratio, forensics.node1_closing_balance as node1_closing_balance, forensics.node2_closing_balance as node2_closing_balance, + forensics.closed_by as closed_by, forensics.single_funded as single_funded FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key + LEFT JOIN channels_forensics AS forensics ON forensics.channel_id = channels.id WHERE ( ns1.id = ( SELECT MAX(id) @@ -257,6 +277,118 @@ class ChannelsApi { } } + public async $getChannelForensicsByTransactionId(transactionId: string): Promise { + try { + const query = ` + SELECT + channels.id, channels.node1_public_key, channels.node2_public_key, + channels.closing_reason, channels.closing_transaction_id, + forensics.* + FROM channels + LEFT JOIN channels_forensics as forensics ON forensics.channel_id = channels.id + WHERE channels.closing_transaction_id = ? + `; + const [rows]: any = await DB.query(query, [transactionId]); + if (rows.length > 0) { + rows[0].outputs = JSON.parse(rows[0].outputs); + return rows[0]; + } + } catch (e) { + logger.err('$getChannelForensicsByTransactionId error: ' + (e instanceof Error ? e.message : e)); + // don't throw - this data isn't essential + } + } + + public async $updateClosingInfo(channelInfo: { id: string, node1_closing_balance: number, node2_closing_balance: number, closed_by: string | null, closing_fee: number, outputs: ILightningApi.ForensicOutput[]}): Promise { + try { + const query = ` + INSERT INTO channels_forensics + ( + channel_id, + node1_closing_balance, + node2_closing_balance, + closed_by, + closing_fee, + outputs + ) + VALUES (?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + node1_closing_balance = ?, + node2_closing_balance = ?, + closed_by = ?, + closing_fee = ?, + outputs = ? + `; + const jsonOutputs = JSON.stringify(channelInfo.outputs); + await DB.query(query, [ + channelInfo.id, + channelInfo.node1_closing_balance, + channelInfo.node2_closing_balance, + channelInfo.closed_by, + channelInfo.closing_fee, + jsonOutputs, + channelInfo.node1_closing_balance, + channelInfo.node2_closing_balance, + channelInfo.closed_by, + channelInfo.closing_fee, + jsonOutputs + ]); + } catch (e) { + logger.err('$updateClosingInfo error: ' + (e instanceof Error ? e.message : e)); + // don't throw - this data isn't essential + } + } + + public async $updateOpeningInfo(channelInfo: { id: string, node1_funding_balance: number, node2_funding_balance: number, funding_ratio: number, single_funded: boolean | void }): Promise { + try { + const query = ` + INSERT INTO channels_forensics + ( + channel_id, + node1_funding_balance, + node2_funding_balance, + funding_ratio, + single_funded, + outputs + ) + VALUES (?, ?, ?, ?, ?, 'null') + ON DUPLICATE KEY UPDATE + node1_funding_balance = ?, + node2_funding_balance = ?, + funding_ratio = ?, + single_funded = ? + `; + await DB.query(query, [ + channelInfo.id, + channelInfo.node1_funding_balance, + channelInfo.node2_funding_balance, + channelInfo.funding_ratio, + channelInfo.single_funded ? 1 : 0, + channelInfo.node1_funding_balance, + channelInfo.node2_funding_balance, + channelInfo.funding_ratio, + channelInfo.single_funded ? 1 : 0, + ]); + } catch (e) { + logger.err('$updateOpeningInfo error: ' + (e instanceof Error ? e.message : e)); + // don't throw - this data isn't essential + } + } + + public async $markChannelSourceChecked(id: string): Promise { + try { + const query = ` + UPDATE channels + SET source_checked = 1 + WHERE id = ? + `; + await DB.query(query, [id]); + } catch (e) { + logger.err('$markChannelSourceChecked error: ' + (e instanceof Error ? e.message : e)); + // don't throw - this data isn't essential + } + } + public async $getChannelsForNode(public_key: string, index: number, length: number, status: string): Promise { try { let channelStatusFilter; @@ -385,11 +517,15 @@ class ChannelsApi { 'transaction_id': channel.transaction_id, 'transaction_vout': channel.transaction_vout, 'closing_transaction_id': channel.closing_transaction_id, + 'closing_fee': channel.closing_fee, 'closing_reason': channel.closing_reason, 'closing_date': channel.closing_date, 'updated_at': channel.updated_at, 'created': channel.created, 'status': channel.status, + 'funding_ratio': channel.funding_ratio, + 'closed_by': channel.closed_by, + 'single_funded': !!channel.single_funded, 'node_left': { 'alias': channel.alias_left, 'public_key': channel.node1_public_key, @@ -404,6 +540,9 @@ class ChannelsApi { 'updated_at': channel.node1_updated_at, 'longitude': channel.node1_longitude, 'latitude': channel.node1_latitude, + 'funding_balance': channel.node1_funding_balance, + 'closing_balance': channel.node1_closing_balance, + 'initiated_close': channel.closed_by === channel.node1_public_key ? true : undefined, }, 'node_right': { 'alias': channel.alias_right, @@ -419,6 +558,9 @@ class ChannelsApi { 'updated_at': channel.node2_updated_at, 'longitude': channel.node2_longitude, 'latitude': channel.node2_latitude, + 'funding_balance': channel.node2_funding_balance, + 'closing_balance': channel.node2_closing_balance, + 'initiated_close': channel.closed_by === channel.node2_public_key ? true : undefined, }, }; } diff --git a/backend/src/api/lightning/lightning-api.interface.ts b/backend/src/api/lightning/lightning-api.interface.ts index 6e3ea0de3..453e2fffc 100644 --- a/backend/src/api/lightning/lightning-api.interface.ts +++ b/backend/src/api/lightning/lightning-api.interface.ts @@ -83,4 +83,10 @@ export namespace ILightningApi { is_required: boolean; is_known: boolean; } + + export interface ForensicOutput { + node?: 1 | 2; + type: number; + value: number; + } } \ No newline at end of file diff --git a/backend/src/tasks/lightning/network-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts index 9f40a350a..ca10ba919 100644 --- a/backend/src/tasks/lightning/network-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -304,6 +304,233 @@ class NetworkSyncService { logger.err('$scanForClosedChannels() error: ' + (e instanceof Error ? e.message : e)); } } + + private findLightningScript(vin: IEsploraApi.Vin): number { + const topElement = vin.witness[vin.witness.length - 2]; + if (/^OP_IF OP_PUSHBYTES_33 \w{66} OP_ELSE OP_PUSH(NUM_\d+|BYTES_(1 \w{2}|2 \w{4})) OP_CSV OP_DROP OP_PUSHBYTES_33 \w{66} OP_ENDIF OP_CHECKSIG$/.test(vin.inner_witnessscript_asm)) { + // https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction-outputs + if (topElement === '01') { + // top element is '01' to get in the revocation path + // 'Revoked Lightning Force Close'; + // Penalty force closed + return 2; + } else { + // top element is '', this is a delayed to_local output + // 'Lightning Force Close'; + return 3; + } + } else if ( + /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_NOTIF OP_DROP OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm) || + /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_IF OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_DROP OP_PUSHBYTES_3 \w{6} OP_CLTV OP_DROP OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm) + ) { + // https://github.com/lightning/bolts/blob/master/03-transactions.md#offered-htlc-outputs + // https://github.com/lightning/bolts/blob/master/03-transactions.md#received-htlc-outputs + if (topElement.length === 66) { + // top element is a public key + // 'Revoked Lightning HTLC'; Penalty force closed + return 4; + } else if (topElement) { + // top element is a preimage + // 'Lightning HTLC'; + return 5; + } else { + // top element is '' to get in the expiry of the script + // 'Expired Lightning HTLC'; + return 6; + } + } else if (/^OP_PUSHBYTES_33 \w{66} OP_CHECKSIG OP_IFDUP OP_NOTIF OP_PUSHNUM_16 OP_CSV OP_ENDIF$/.test(vin.inner_witnessscript_asm)) { + // https://github.com/lightning/bolts/blob/master/03-transactions.md#to_local_anchor-and-to_remote_anchor-output-option_anchors + if (topElement) { + // top element is a signature + // 'Lightning Anchor'; + return 7; + } else { + // top element is '', it has been swept after 16 blocks + // 'Swept Lightning Anchor'; + return 8; + } + } + return 1; + } + + // If a channel open tx spends funds from a closed channel output, + // we can attribute that output to a specific counterparty + private async $runOpenedChannelsForensics(): Promise { + let progress = 0; + + try { + logger.info(`Started running open channel forensics...`); + const channels = await channelsApi.$getChannelsWithoutSourceChecked(); + + for (const openChannel of channels) { + const openTx = await bitcoinApi.$getRawTransaction(openChannel.transaction_id); + for (const input of openTx.vin) { + const closeChannel = await channelsApi.$getChannelForensicsByTransactionId(input.txid); + if (closeChannel) { + // this input directly spends a channel close output + await this.$attributeChannelBalances(closeChannel, openChannel, input); + } else { + // check if this input spends any swept channel close outputs + await this.$attributeSweptChannelCloses(openChannel, input); + } + } + // calculate how much of the total input value is attributable to the channel open output + openChannel.funding_ratio = openTx.vout[openChannel.transaction_vout].value / ((openTx.vout.reduce((sum, v) => sum + v.value, 0) || 1) + openTx.fee); + // save changes to the opening channel, and mark it as checked + if (openTx.vin.length === 1) { + openChannel.single_funded = true; + } + await channelsApi.$updateOpeningInfo(openChannel); + await channelsApi.$markChannelSourceChecked(openChannel.id); + + ++progress; + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Updating channel opened channel forensics ${progress}/${channels.length}`); + this.loggerTimer = new Date().getTime() / 1000; + } + } + + logger.info(`Open channels forensics scan complete.`); + } catch (e) { + logger.err('$runOpenedChannelsForensics() error: ' + (e instanceof Error ? e.message : e)); + } + } + + // Check if a channel open tx input spends the result of a swept channel close output + private async $attributeSweptChannelCloses(openChannel: ILightningApi.Channel, input: IEsploraApi.Vin): Promise { + const sweepTx = await bitcoinApi.$getRawTransaction(input.txid); + if (!sweepTx) { + logger.err(`couldn't find input transaction for channel forensics ${openChannel.channel_id} ${input.txid}`); + return; + } + const openContribution = sweepTx.vout[input.vout].value; + for (const sweepInput of sweepTx.vin) { + const lnScriptType = this.findLightningScript(sweepInput); + if (lnScriptType > 1) { + const closeChannel = await channelsApi.$getChannelForensicsByTransactionId(sweepInput.txid); + if (closeChannel) { + const initiator = (lnScriptType === 2 || lnScriptType === 4) ? 'remote' : (lnScriptType === 3 ? 'local' : null); + await this.$attributeChannelBalances(closeChannel, openChannel, sweepInput, openContribution, initiator); + } + } + } + } + + private async $attributeChannelBalances( + closeChannel, openChannel, input: IEsploraApi.Vin, openContribution: number | null = null, + initiator: 'remote' | 'local' | null = null + ): Promise { + // figure out which node controls the input/output + let openSide; + let closeLocal; + let closeRemote; + let matched = false; + let ambiguous = false; // if counterparties are the same in both channels, we can't tell them apart + if (openChannel.node1_public_key === closeChannel.node1_public_key) { + openSide = 1; + closeLocal = 1; + closeRemote = 2; + matched = true; + } else if (openChannel.node1_public_key === closeChannel.node2_public_key) { + openSide = 1; + closeLocal = 2; + closeRemote = 1; + matched = true; + } + if (openChannel.node2_public_key === closeChannel.node1_public_key) { + openSide = 2; + closeLocal = 1; + closeRemote = 2; + if (matched) { + ambiguous = true; + } + matched = true; + } else if (openChannel.node2_public_key === closeChannel.node2_public_key) { + openSide = 2; + closeLocal = 2; + closeRemote = 1; + if (matched) { + ambiguous = true; + } + matched = true; + } + + if (matched && !ambiguous) { + // fetch closing channel transaction and perform forensics on the outputs + let closingTx: IEsploraApi.Transaction | undefined; + let outspends: IEsploraApi.Outspend[] | undefined; + try { + closingTx = await bitcoinApi.$getRawTransaction(input.txid); + outspends = await bitcoinApi.$getOutspends(input.txid); + } catch (e) { + logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + closeChannel.closing_transaction_id + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`); + } + if (!outspends || !closingTx) { + return; + } + if (!closeChannel.outputs) { + closeChannel.outputs = closeChannel.outputs || closingTx.vout.map(vout => { + return { + type: 0, + value: vout.value, + }; + }); + } + for (let i = 0; i < outspends.length; i++) { + const outspend = outspends[i]; + const output = closeChannel.outputs[i]; + if (outspend.spent && outspend.txid) { + try { + const spendingTx = await bitcoinApi.$getRawTransaction(outspend.txid); + if (spendingTx) { + output.type = this.findLightningScript(spendingTx.vin[outspend.vin || 0]); + } + } catch (e) { + logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + outspend.txid}. Reason ${e instanceof Error ? e.message : e}`); + } + } else { + output.type = 0; + } + } + + // attribute outputs to each counterparty, and sum up total known balances + closeChannel.outputs[input.vout].node = closeLocal; + const isPenalty = closeChannel.outputs.filter((out) => out.type === 2 || out.type === 4).length > 0; + const normalOutput = [1,3].includes(closeChannel.outputs[input.vout].type); + let localClosingBalance = 0; + let remoteClosingBalance = 0; + for (const output of closeChannel.outputs) { + if (isPenalty) { + // penalty close, so local node takes everything + localClosingBalance += output.value; + } else if (output.node) { + // this output determinstically linked to one of the counterparties + if (output.node === closeLocal) { + localClosingBalance += output.value; + } else { + remoteClosingBalance += output.value; + } + } else if (normalOutput && (output.type === 1 || output.type === 3)) { + // local node had one main output, therefore remote node takes the other + remoteClosingBalance += output.value; + } + } + + openChannel[`node${openSide}_funding_balance`] = openChannel[`node${openSide}_funding_balance`] + (openContribution || closingTx?.vout[input.vout]?.value || 0); + closeChannel[`node${closeLocal}_closing_balance`] = localClosingBalance; + closeChannel[`node${closeRemote}_closing_balance`] = remoteClosingBalance; + closeChannel.closing_fee = closingTx.fee; + + if (initiator) { + const initiatorSide = initiator === 'remote' ? closeRemote : closeLocal; + closeChannel.closed_by = closeChannel[`node${initiatorSide}_public_key`]; + } + + // save changes to the closing channel + await channelsApi.$updateClosingInfo(closeChannel); + } + } } export default new NetworkSyncService(); From 0c96a11150583c90f5ffb3261c835b8fe0cd668f Mon Sep 17 00:00:00 2001 From: Mononaut Date: Thu, 10 Nov 2022 18:32:18 -0600 Subject: [PATCH 04/13] display channel close forensics results --- .../src/app/interfaces/node-api.interface.ts | 6 +- .../channel-close-box.component.html | 19 ++++++ .../channel-close-box.component.scss | 9 +++ .../channel-close-box.component.spec.ts | 25 ++++++++ .../channel-close-box.component.ts | 58 +++++++++++++++++++ .../lightning/channel/channel.component.html | 11 ++++ .../lightning/channel/channel.component.ts | 5 ++ .../src/app/lightning/lightning.module.ts | 3 + 8 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.html create mode 100644 frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.scss create mode 100644 frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.spec.ts create mode 100644 frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.ts diff --git a/frontend/src/app/interfaces/node-api.interface.ts b/frontend/src/app/interfaces/node-api.interface.ts index d32e641f7..2e6b94988 100644 --- a/frontend/src/app/interfaces/node-api.interface.ts +++ b/frontend/src/app/interfaces/node-api.interface.ts @@ -217,8 +217,8 @@ export interface IChannel { updated_at: string; created: string; status: number; - node_left: Node, - node_right: Node, + node_left: INode, + node_right: INode, } @@ -236,4 +236,6 @@ export interface INode { updated_at: string; longitude: number; latitude: number; + funding_balance?: number; + closing_balance?: number; } diff --git a/frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.html b/frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.html new file mode 100644 index 000000000..ae59767ff --- /dev/null +++ b/frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.html @@ -0,0 +1,19 @@ +
+ + + + + + + + + + + + + + + + +
Starting balance{{ minStartingBalance | number : '1.0-0' }} - {{ maxStartingBalance | number : '1.0-0' }}?
Closing balance{{ minClosingBalance | number : '1.0-0' }} - {{ maxClosingBalance | number : '1.0-0' }}?
+
\ No newline at end of file diff --git a/frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.scss b/frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.scss new file mode 100644 index 000000000..a42871308 --- /dev/null +++ b/frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.scss @@ -0,0 +1,9 @@ +.box { + margin-top: 20px; +} + +@media (max-width: 768px) { + .box { + margin-bottom: 20px; + } +} \ No newline at end of file diff --git a/frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.spec.ts b/frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.spec.ts new file mode 100644 index 000000000..eea4ee99c --- /dev/null +++ b/frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.spec.ts @@ -0,0 +1,25 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { ChannelCloseBoxComponent } from './channel-close-box.component'; + +describe('ChannelCloseBoxComponent', () => { + let component: ChannelCloseBoxComponent; + let fixture: ComponentFixture; + + beforeEach(async () => { + await TestBed.configureTestingModule({ + declarations: [ ChannelCloseBoxComponent ] + }) + .compileComponents(); + }); + + beforeEach(() => { + fixture = TestBed.createComponent(ChannelCloseBoxComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.ts b/frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.ts new file mode 100644 index 000000000..05cc31434 --- /dev/null +++ b/frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.ts @@ -0,0 +1,58 @@ +import { ChangeDetectionStrategy, Component, Input, OnChanges, SimpleChanges } from '@angular/core'; + +@Component({ + selector: 'app-channel-close-box', + templateUrl: './channel-close-box.component.html', + styleUrls: ['./channel-close-box.component.scss'], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class ChannelCloseBoxComponent implements OnChanges { + @Input() channel: any; + @Input() local: any; + @Input() remote: any; + + showStartingBalance: boolean = false; + showClosingBalance: boolean = false; + minStartingBalance: number; + maxStartingBalance: number; + minClosingBalance: number; + maxClosingBalance: number; + + constructor() { } + + ngOnChanges(changes: SimpleChanges): void { + if (this.channel && this.local && this.remote) { + this.showStartingBalance = (this.local.funding_balance || this.remote.funding_balance) && this.channel.funding_ratio; + this.showClosingBalance = this.local.closing_balance || this.remote.closing_balance; + + if (this.channel.single_funded) { + if (this.local.funding_balance) { + this.minStartingBalance = this.channel.capacity; + this.maxStartingBalance = this.channel.capacity; + } else if (this.remote.funding_balance) { + this.minStartingBalance = 0; + this.maxStartingBalance = 0; + } + } else { + this.minStartingBalance = clampRound(0, this.channel.capacity, this.local.funding_balance * this.channel.funding_ratio); + this.maxStartingBalance = clampRound(0, this.channel.capacity, this.channel.capacity - (this.remote.funding_balance * this.channel.funding_ratio)); + } + + const closingCapacity = this.channel.capacity - this.channel.closing_fee; + this.minClosingBalance = clampRound(0, closingCapacity, this.local.closing_balance); + this.maxClosingBalance = clampRound(0, closingCapacity, closingCapacity - this.remote.closing_balance); + + // margin of error to account for 2 x 330 sat anchor outputs + if (Math.abs(this.minClosingBalance - this.maxClosingBalance) <= 660) { + this.maxClosingBalance = this.minClosingBalance; + } + } else { + this.showStartingBalance = false; + this.showClosingBalance = false; + } + } +} + +function clampRound(min: number, max: number, value: number): number { + return Math.max(0, Math.min(max, Math.round(value))); +} diff --git a/frontend/src/app/lightning/channel/channel.component.html b/frontend/src/app/lightning/channel/channel.component.html index c25af5377..f52b85762 100644 --- a/frontend/src/app/lightning/channel/channel.component.html +++ b/frontend/src/app/lightning/channel/channel.component.html @@ -48,6 +48,15 @@ Capacity + + Closed by + + + {{ channel.node_left.alias }} + {{ channel.node_right.alias }} + + + @@ -59,9 +68,11 @@
+
+
diff --git a/frontend/src/app/lightning/channel/channel.component.ts b/frontend/src/app/lightning/channel/channel.component.ts index d64d388ea..379e8a005 100644 --- a/frontend/src/app/lightning/channel/channel.component.ts +++ b/frontend/src/app/lightning/channel/channel.component.ts @@ -78,4 +78,9 @@ export class ChannelComponent implements OnInit { ); } + showCloseBoxes(channel: IChannel): boolean { + return !!(channel.node_left.funding_balance || channel.node_left.closing_balance + || channel.node_right.funding_balance || channel.node_right.closing_balance); + } + } diff --git a/frontend/src/app/lightning/lightning.module.ts b/frontend/src/app/lightning/lightning.module.ts index fa2f1a1ec..5d67433c7 100644 --- a/frontend/src/app/lightning/lightning.module.ts +++ b/frontend/src/app/lightning/lightning.module.ts @@ -12,6 +12,7 @@ import { ChannelsListComponent } from './channels-list/channels-list.component'; import { ChannelComponent } from './channel/channel.component'; import { LightningWrapperComponent } from './lightning-wrapper/lightning-wrapper.component'; import { ChannelBoxComponent } from './channel/channel-box/channel-box.component'; +import { ChannelCloseBoxComponent } from './channel/channel-close-box/channel-close-box.component'; import { ClosingTypeComponent } from './channel/closing-type/closing-type.component'; import { LightningStatisticsChartComponent } from './statistics-chart/lightning-statistics-chart.component'; import { NodeStatisticsChartComponent } from './node-statistics-chart/node-statistics-chart.component'; @@ -45,6 +46,7 @@ import { GroupComponent } from './group/group.component'; ChannelComponent, LightningWrapperComponent, ChannelBoxComponent, + ChannelCloseBoxComponent, ClosingTypeComponent, LightningStatisticsChartComponent, NodesNetworksChartComponent, @@ -81,6 +83,7 @@ import { GroupComponent } from './group/group.component'; ChannelComponent, LightningWrapperComponent, ChannelBoxComponent, + ChannelCloseBoxComponent, ClosingTypeComponent, LightningStatisticsChartComponent, NodesNetworksChartComponent, From 8f0830f6d1624a1b6020d8939fdd2c0c92839b36 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 20 Nov 2022 19:11:14 +0900 Subject: [PATCH 05/13] detect channels opened from change outputs --- backend/src/api/explorer/channels.api.ts | 26 ++- .../tasks/lightning/network-sync.service.ts | 154 +++++++++--------- 2 files changed, 105 insertions(+), 75 deletions(-) diff --git a/backend/src/api/explorer/channels.api.ts b/backend/src/api/explorer/channels.api.ts index 22d17476f..37edf11d0 100644 --- a/backend/src/api/explorer/channels.api.ts +++ b/backend/src/api/explorer/channels.api.ts @@ -277,7 +277,7 @@ class ChannelsApi { } } - public async $getChannelForensicsByTransactionId(transactionId: string): Promise { + public async $getChannelForensicsByClosingId(transactionId: string): Promise { try { const query = ` SELECT @@ -294,7 +294,29 @@ class ChannelsApi { return rows[0]; } } catch (e) { - logger.err('$getChannelForensicsByTransactionId error: ' + (e instanceof Error ? e.message : e)); + logger.err('$getChannelForensicsByClosingId error: ' + (e instanceof Error ? e.message : e)); + // don't throw - this data isn't essential + } + } + + public async $getChannelForensicsByOpeningId(transactionId: string): Promise { + try { + const query = ` + SELECT + channels.id, channels.node1_public_key, channels.node2_public_key, + channels.status, channels.transaction_id, + forensics.* + FROM channels + LEFT JOIN channels_forensics as forensics ON forensics.channel_id = channels.id + WHERE channels.transaction_id = ? + `; + const [rows]: any = await DB.query(query, [transactionId]); + if (rows.length > 0) { + rows[0].outputs = JSON.parse(rows[0].outputs); + return rows[0]; + } + } catch (e) { + logger.err('$getChannelForensicsByOpeningId error: ' + (e instanceof Error ? e.message : e)); // don't throw - this data isn't essential } } diff --git a/backend/src/tasks/lightning/network-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts index ca10ba919..ac7c7a2ca 100644 --- a/backend/src/tasks/lightning/network-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -306,7 +306,7 @@ class NetworkSyncService { } private findLightningScript(vin: IEsploraApi.Vin): number { - const topElement = vin.witness[vin.witness.length - 2]; + const topElement = vin.witness ? vin.witness[vin.witness.length - 2] : ''; if (/^OP_IF OP_PUSHBYTES_33 \w{66} OP_ELSE OP_PUSH(NUM_\d+|BYTES_(1 \w{2}|2 \w{4})) OP_CSV OP_DROP OP_PUSHBYTES_33 \w{66} OP_ENDIF OP_CHECKSIG$/.test(vin.inner_witnessscript_asm)) { // https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction-outputs if (topElement === '01') { @@ -325,7 +325,7 @@ class NetworkSyncService { ) { // https://github.com/lightning/bolts/blob/master/03-transactions.md#offered-htlc-outputs // https://github.com/lightning/bolts/blob/master/03-transactions.md#received-htlc-outputs - if (topElement.length === 66) { + if (topElement?.length === 66) { // top element is a public key // 'Revoked Lightning HTLC'; Penalty force closed return 4; @@ -365,28 +365,35 @@ class NetworkSyncService { for (const openChannel of channels) { const openTx = await bitcoinApi.$getRawTransaction(openChannel.transaction_id); for (const input of openTx.vin) { - const closeChannel = await channelsApi.$getChannelForensicsByTransactionId(input.txid); + const closeChannel = await channelsApi.$getChannelForensicsByClosingId(input.txid); if (closeChannel) { // this input directly spends a channel close output await this.$attributeChannelBalances(closeChannel, openChannel, input); } else { - // check if this input spends any swept channel close outputs - await this.$attributeSweptChannelCloses(openChannel, input); + const prevOpenChannel = await channelsApi.$getChannelForensicsByOpeningId(input.txid); + if (prevOpenChannel) { + await this.$attributeChannelBalances(prevOpenChannel, openChannel, input, null, null, true); + } else { + // check if this input spends any swept channel close outputs + await this.$attributeSweptChannelCloses(openChannel, input); + } } } // calculate how much of the total input value is attributable to the channel open output openChannel.funding_ratio = openTx.vout[openChannel.transaction_vout].value / ((openTx.vout.reduce((sum, v) => sum + v.value, 0) || 1) + openTx.fee); // save changes to the opening channel, and mark it as checked - if (openTx.vin.length === 1) { + if (openTx?.vin?.length === 1) { openChannel.single_funded = true; } - await channelsApi.$updateOpeningInfo(openChannel); + if (openChannel.node1_funding_balance || openChannel.node2_funding_balance || openChannel.node1_closing_balance || openChannel.node2_closing_balance || openChannel.closed_by) { + await channelsApi.$updateOpeningInfo(openChannel); + } await channelsApi.$markChannelSourceChecked(openChannel.id); ++progress; const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); if (elapsedSeconds > 10) { - logger.info(`Updating channel opened channel forensics ${progress}/${channels.length}`); + logger.info(`Updating opened channel forensics ${progress}/${channels?.length}`); this.loggerTimer = new Date().getTime() / 1000; } } @@ -408,7 +415,7 @@ class NetworkSyncService { for (const sweepInput of sweepTx.vin) { const lnScriptType = this.findLightningScript(sweepInput); if (lnScriptType > 1) { - const closeChannel = await channelsApi.$getChannelForensicsByTransactionId(sweepInput.txid); + const closeChannel = await channelsApi.$getChannelForensicsByClosingId(sweepInput.txid); if (closeChannel) { const initiator = (lnScriptType === 2 || lnScriptType === 4) ? 'remote' : (lnScriptType === 3 ? 'local' : null); await this.$attributeChannelBalances(closeChannel, openChannel, sweepInput, openContribution, initiator); @@ -418,8 +425,8 @@ class NetworkSyncService { } private async $attributeChannelBalances( - closeChannel, openChannel, input: IEsploraApi.Vin, openContribution: number | null = null, - initiator: 'remote' | 'local' | null = null + prevChannel, openChannel, input: IEsploraApi.Vin, openContribution: number | null = null, + initiator: 'remote' | 'local' | null = null, linkedOpenings: boolean = false ): Promise { // figure out which node controls the input/output let openSide; @@ -427,18 +434,18 @@ class NetworkSyncService { let closeRemote; let matched = false; let ambiguous = false; // if counterparties are the same in both channels, we can't tell them apart - if (openChannel.node1_public_key === closeChannel.node1_public_key) { + if (openChannel.node1_public_key === prevChannel.node1_public_key) { openSide = 1; closeLocal = 1; closeRemote = 2; matched = true; - } else if (openChannel.node1_public_key === closeChannel.node2_public_key) { + } else if (openChannel.node1_public_key === prevChannel.node2_public_key) { openSide = 1; closeLocal = 2; closeRemote = 1; matched = true; } - if (openChannel.node2_public_key === closeChannel.node1_public_key) { + if (openChannel.node2_public_key === prevChannel.node1_public_key) { openSide = 2; closeLocal = 1; closeRemote = 2; @@ -446,7 +453,7 @@ class NetworkSyncService { ambiguous = true; } matched = true; - } else if (openChannel.node2_public_key === closeChannel.node2_public_key) { + } else if (openChannel.node2_public_key === prevChannel.node2_public_key) { openSide = 2; closeLocal = 2; closeRemote = 1; @@ -458,77 +465,78 @@ class NetworkSyncService { if (matched && !ambiguous) { // fetch closing channel transaction and perform forensics on the outputs - let closingTx: IEsploraApi.Transaction | undefined; + let prevChannelTx: IEsploraApi.Transaction | undefined; let outspends: IEsploraApi.Outspend[] | undefined; try { - closingTx = await bitcoinApi.$getRawTransaction(input.txid); + prevChannelTx = await bitcoinApi.$getRawTransaction(input.txid); outspends = await bitcoinApi.$getOutspends(input.txid); } catch (e) { - logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + closeChannel.closing_transaction_id + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`); + logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + input.txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`); } - if (!outspends || !closingTx) { + if (!outspends || !prevChannelTx) { return; } - if (!closeChannel.outputs) { - closeChannel.outputs = closeChannel.outputs || closingTx.vout.map(vout => { - return { - type: 0, - value: vout.value, - }; - }); - } - for (let i = 0; i < outspends.length; i++) { - const outspend = outspends[i]; - const output = closeChannel.outputs[i]; - if (outspend.spent && outspend.txid) { - try { - const spendingTx = await bitcoinApi.$getRawTransaction(outspend.txid); - if (spendingTx) { - output.type = this.findLightningScript(spendingTx.vin[outspend.vin || 0]); - } - } catch (e) { - logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + outspend.txid}. Reason ${e instanceof Error ? e.message : e}`); - } - } else { - output.type = 0; + if (!linkedOpenings) { + if (!prevChannel.outputs) { + prevChannel.outputs = prevChannel.outputs || prevChannelTx.vout.map(vout => { + return { + type: 0, + value: vout.value, + }; + }); } - } - - // attribute outputs to each counterparty, and sum up total known balances - closeChannel.outputs[input.vout].node = closeLocal; - const isPenalty = closeChannel.outputs.filter((out) => out.type === 2 || out.type === 4).length > 0; - const normalOutput = [1,3].includes(closeChannel.outputs[input.vout].type); - let localClosingBalance = 0; - let remoteClosingBalance = 0; - for (const output of closeChannel.outputs) { - if (isPenalty) { - // penalty close, so local node takes everything - localClosingBalance += output.value; - } else if (output.node) { - // this output determinstically linked to one of the counterparties - if (output.node === closeLocal) { - localClosingBalance += output.value; + for (let i = 0; i < outspends?.length; i++) { + const outspend = outspends[i]; + const output = prevChannel.outputs[i]; + if (outspend.spent && outspend.txid) { + try { + const spendingTx = await bitcoinApi.$getRawTransaction(outspend.txid); + if (spendingTx) { + output.type = this.findLightningScript(spendingTx.vin[outspend.vin || 0]); + } + } catch (e) { + logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + outspend.txid}. Reason ${e instanceof Error ? e.message : e}`); + } } else { + output.type = 0; + } + } + + // attribute outputs to each counterparty, and sum up total known balances + prevChannel.outputs[input.vout].node = closeLocal; + const isPenalty = prevChannel.outputs.filter((out) => out.type === 2 || out.type === 4)?.length > 0; + const normalOutput = [1,3].includes(prevChannel.outputs[input.vout].type); + let localClosingBalance = 0; + let remoteClosingBalance = 0; + for (const output of prevChannel.outputs) { + if (isPenalty) { + // penalty close, so local node takes everything + localClosingBalance += output.value; + } else if (output.node) { + // this output determinstically linked to one of the counterparties + if (output.node === closeLocal) { + localClosingBalance += output.value; + } else { + remoteClosingBalance += output.value; + } + } else if (normalOutput && (output.type === 1 || output.type === 3)) { + // local node had one main output, therefore remote node takes the other remoteClosingBalance += output.value; } - } else if (normalOutput && (output.type === 1 || output.type === 3)) { - // local node had one main output, therefore remote node takes the other - remoteClosingBalance += output.value; } + prevChannel[`node${closeLocal}_closing_balance`] = localClosingBalance; + prevChannel[`node${closeRemote}_closing_balance`] = remoteClosingBalance; + prevChannel.closing_fee = prevChannelTx.fee; + + if (initiator && !linkedOpenings) { + const initiatorSide = initiator === 'remote' ? closeRemote : closeLocal; + prevChannel.closed_by = prevChannel[`node${initiatorSide}_public_key`]; + } + + // save changes to the closing channel + await channelsApi.$updateClosingInfo(prevChannel); } - - openChannel[`node${openSide}_funding_balance`] = openChannel[`node${openSide}_funding_balance`] + (openContribution || closingTx?.vout[input.vout]?.value || 0); - closeChannel[`node${closeLocal}_closing_balance`] = localClosingBalance; - closeChannel[`node${closeRemote}_closing_balance`] = remoteClosingBalance; - closeChannel.closing_fee = closingTx.fee; - - if (initiator) { - const initiatorSide = initiator === 'remote' ? closeRemote : closeLocal; - closeChannel.closed_by = closeChannel[`node${initiatorSide}_public_key`]; - } - - // save changes to the closing channel - await channelsApi.$updateClosingInfo(closeChannel); + openChannel[`node${openSide}_funding_balance`] = openChannel[`node${openSide}_funding_balance`] + (openContribution || prevChannelTx?.vout[input.vout]?.value || 0); } } } From 35ae67217717e7043f0417aeb797a4674decc8be Mon Sep 17 00:00:00 2001 From: Mononaut Date: Sun, 20 Nov 2022 19:18:33 +0900 Subject: [PATCH 06/13] break long-running forensics tasks --- backend/src/tasks/lightning/network-sync.service.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/backend/src/tasks/lightning/network-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts index ac7c7a2ca..7ff49f68b 100644 --- a/backend/src/tasks/lightning/network-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -31,6 +31,7 @@ class NetworkSyncService { } private async $runTasks(): Promise { + const taskStartTime = Date.now(); try { logger.info(`Updating nodes and channels`); @@ -57,7 +58,7 @@ class NetworkSyncService { logger.err('$runTasks() error: ' + (e instanceof Error ? e.message : e)); } - setTimeout(() => { this.$runTasks(); }, 1000 * config.LIGHTNING.GRAPH_REFRESH_INTERVAL); + setTimeout(() => { this.$runTasks(); }, Math.max(1, (1000 * config.LIGHTNING.GRAPH_REFRESH_INTERVAL) - (Date.now() - taskStartTime))); } /** @@ -353,9 +354,10 @@ class NetworkSyncService { return 1; } - // If a channel open tx spends funds from a closed channel output, + // If a channel open tx spends funds from a another channel transaction, // we can attribute that output to a specific counterparty private async $runOpenedChannelsForensics(): Promise { + const runTimer = Date.now(); let progress = 0; try { @@ -396,6 +398,9 @@ class NetworkSyncService { logger.info(`Updating opened channel forensics ${progress}/${channels?.length}`); this.loggerTimer = new Date().getTime() / 1000; } + if (Date.now() - runTimer > (config.LIGHTNING.GRAPH_REFRESH_INTERVAL * 1000)) { + break; + } } logger.info(`Open channels forensics scan complete.`); From dc7d5bc94d46090c26b8fffdfa4bcb9f383ad86a Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 21 Nov 2022 09:14:06 +0900 Subject: [PATCH 07/13] handle batched channel opens. infer funding balances in both directions. --- backend/src/api/explorer/channels.api.ts | 30 ++++++------ .../tasks/lightning/network-sync.service.ts | 47 ++++++++++++------- 2 files changed, 45 insertions(+), 32 deletions(-) diff --git a/backend/src/api/explorer/channels.api.ts b/backend/src/api/explorer/channels.api.ts index 37edf11d0..7f76a2971 100644 --- a/backend/src/api/explorer/channels.api.ts +++ b/backend/src/api/explorer/channels.api.ts @@ -282,7 +282,7 @@ class ChannelsApi { const query = ` SELECT channels.id, channels.node1_public_key, channels.node2_public_key, - channels.closing_reason, channels.closing_transaction_id, + channels.closing_reason, channels.closing_transaction_id, channels.capacity, forensics.* FROM channels LEFT JOIN channels_forensics as forensics ON forensics.channel_id = channels.id @@ -304,7 +304,7 @@ class ChannelsApi { const query = ` SELECT channels.id, channels.node1_public_key, channels.node2_public_key, - channels.status, channels.transaction_id, + channels.status, channels.transaction_id, channels.capacity, forensics.* FROM channels LEFT JOIN channels_forensics as forensics ON forensics.channel_id = channels.id @@ -312,8 +312,10 @@ class ChannelsApi { `; const [rows]: any = await DB.query(query, [transactionId]); if (rows.length > 0) { - rows[0].outputs = JSON.parse(rows[0].outputs); - return rows[0]; + return rows.map(row => { + row.outputs = JSON.parse(row.outputs); + return row; + }); } } catch (e) { logger.err('$getChannelForensicsByOpeningId error: ' + (e instanceof Error ? e.message : e)); @@ -344,15 +346,15 @@ class ChannelsApi { const jsonOutputs = JSON.stringify(channelInfo.outputs); await DB.query(query, [ channelInfo.id, - channelInfo.node1_closing_balance, - channelInfo.node2_closing_balance, + channelInfo.node1_closing_balance || 0, + channelInfo.node2_closing_balance || 0, channelInfo.closed_by, - channelInfo.closing_fee, + channelInfo.closing_fee || 0, jsonOutputs, - channelInfo.node1_closing_balance, - channelInfo.node2_closing_balance, + channelInfo.node1_closing_balance || 0, + channelInfo.node2_closing_balance || 0, channelInfo.closed_by, - channelInfo.closing_fee, + channelInfo.closing_fee || 0, jsonOutputs ]); } catch (e) { @@ -382,12 +384,12 @@ class ChannelsApi { `; await DB.query(query, [ channelInfo.id, - channelInfo.node1_funding_balance, - channelInfo.node2_funding_balance, + channelInfo.node1_funding_balance || 0, + channelInfo.node2_funding_balance || 0, channelInfo.funding_ratio, channelInfo.single_funded ? 1 : 0, - channelInfo.node1_funding_balance, - channelInfo.node2_funding_balance, + channelInfo.node1_funding_balance || 0, + channelInfo.node2_funding_balance || 0, channelInfo.funding_ratio, channelInfo.single_funded ? 1 : 0, ]); diff --git a/backend/src/tasks/lightning/network-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts index 7ff49f68b..4c92fb3bb 100644 --- a/backend/src/tasks/lightning/network-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -372,9 +372,12 @@ class NetworkSyncService { // this input directly spends a channel close output await this.$attributeChannelBalances(closeChannel, openChannel, input); } else { - const prevOpenChannel = await channelsApi.$getChannelForensicsByOpeningId(input.txid); - if (prevOpenChannel) { - await this.$attributeChannelBalances(prevOpenChannel, openChannel, input, null, null, true); + const prevOpenChannels = await channelsApi.$getChannelForensicsByOpeningId(input.txid); + if (prevOpenChannels?.length) { + // this input spends a channel open change output + for (const prevOpenChannel of prevOpenChannels) { + await this.$attributeChannelBalances(prevOpenChannel, openChannel, input, null, null, true); + } } else { // check if this input spends any swept channel close outputs await this.$attributeSweptChannelCloses(openChannel, input); @@ -435,33 +438,33 @@ class NetworkSyncService { ): Promise { // figure out which node controls the input/output let openSide; - let closeLocal; - let closeRemote; + let prevLocal; + let prevRemote; let matched = false; let ambiguous = false; // if counterparties are the same in both channels, we can't tell them apart if (openChannel.node1_public_key === prevChannel.node1_public_key) { openSide = 1; - closeLocal = 1; - closeRemote = 2; + prevLocal = 1; + prevRemote = 2; matched = true; } else if (openChannel.node1_public_key === prevChannel.node2_public_key) { openSide = 1; - closeLocal = 2; - closeRemote = 1; + prevLocal = 2; + prevRemote = 1; matched = true; } if (openChannel.node2_public_key === prevChannel.node1_public_key) { openSide = 2; - closeLocal = 1; - closeRemote = 2; + prevLocal = 1; + prevRemote = 2; if (matched) { ambiguous = true; } matched = true; } else if (openChannel.node2_public_key === prevChannel.node2_public_key) { openSide = 2; - closeLocal = 2; - closeRemote = 1; + prevLocal = 2; + prevRemote = 1; if (matched) { ambiguous = true; } @@ -508,7 +511,7 @@ class NetworkSyncService { } // attribute outputs to each counterparty, and sum up total known balances - prevChannel.outputs[input.vout].node = closeLocal; + prevChannel.outputs[input.vout].node = prevLocal; const isPenalty = prevChannel.outputs.filter((out) => out.type === 2 || out.type === 4)?.length > 0; const normalOutput = [1,3].includes(prevChannel.outputs[input.vout].type); let localClosingBalance = 0; @@ -519,7 +522,7 @@ class NetworkSyncService { localClosingBalance += output.value; } else if (output.node) { // this output determinstically linked to one of the counterparties - if (output.node === closeLocal) { + if (output.node === prevLocal) { localClosingBalance += output.value; } else { remoteClosingBalance += output.value; @@ -529,17 +532,25 @@ class NetworkSyncService { remoteClosingBalance += output.value; } } - prevChannel[`node${closeLocal}_closing_balance`] = localClosingBalance; - prevChannel[`node${closeRemote}_closing_balance`] = remoteClosingBalance; + prevChannel[`node${prevLocal}_closing_balance`] = localClosingBalance; + prevChannel[`node${prevRemote}_closing_balance`] = remoteClosingBalance; prevChannel.closing_fee = prevChannelTx.fee; if (initiator && !linkedOpenings) { - const initiatorSide = initiator === 'remote' ? closeRemote : closeLocal; + const initiatorSide = initiator === 'remote' ? prevRemote : prevLocal; prevChannel.closed_by = prevChannel[`node${initiatorSide}_public_key`]; } // save changes to the closing channel await channelsApi.$updateClosingInfo(prevChannel); + } else { + if (prevChannelTx.vin.length <= 1) { + prevChannel[`node${prevLocal}_funding_balance`] = prevChannel.capacity; + prevChannel.single_funded = true; + prevChannel.funding_ratio = 1; + // save changes to the closing channel + await channelsApi.$updateOpeningInfo(prevChannel); + } } openChannel[`node${openSide}_funding_balance`] = openChannel[`node${openSide}_funding_balance`] + (openContribution || prevChannelTx?.vout[input.vout]?.value || 0); } From 5e1f54e862c1fe3c42faa1e81fb6098983ec3203 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 21 Nov 2022 09:16:42 +0900 Subject: [PATCH 08/13] hide closing balances if channel still open --- .../channel/channel-close-box/channel-close-box.component.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.html b/frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.html index ae59767ff..b5615324b 100644 --- a/frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.html +++ b/frontend/src/app/lightning/channel/channel-close-box/channel-close-box.component.html @@ -8,7 +8,7 @@ {{ minStartingBalance | number : '1.0-0' }} - {{ maxStartingBalance | number : '1.0-0' }} ? - + Closing balance {{ minClosingBalance | number : '1.0-0' }} - {{ maxClosingBalance | number : '1.0-0' }} From 609f68eb243a365ee65eb924a56a6a3a4bd250e0 Mon Sep 17 00:00:00 2001 From: Mononaut Date: Thu, 24 Nov 2022 15:37:05 +0900 Subject: [PATCH 09/13] move linked channel scan into forensics task, add backend throttling --- .../src/tasks/lightning/forensics.service.ts | 272 ++++++++++++++++-- .../tasks/lightning/network-sync.service.ts | 250 ---------------- 2 files changed, 252 insertions(+), 270 deletions(-) diff --git a/backend/src/tasks/lightning/forensics.service.ts b/backend/src/tasks/lightning/forensics.service.ts index 9b999fca1..af79a270a 100644 --- a/backend/src/tasks/lightning/forensics.service.ts +++ b/backend/src/tasks/lightning/forensics.service.ts @@ -5,13 +5,16 @@ import bitcoinApi from '../../api/bitcoin/bitcoin-api-factory'; import config from '../../config'; import { IEsploraApi } from '../../api/bitcoin/esplora-api.interface'; import { Common } from '../../api/common'; +import { ILightningApi } from '../../api/lightning/lightning-api.interface'; const throttleDelay = 20; //ms +const tempCacheSize = 10000; class ForensicsService { loggerTimer = 0; closedChannelsScanBlock = 0; txCache: { [txid: string]: IEsploraApi.Transaction } = {}; + tempCached: string[] = []; constructor() {} @@ -29,6 +32,7 @@ class ForensicsService { if (config.MEMPOOL.BACKEND === 'esplora') { await this.$runClosedChannelsForensics(false); + await this.$runOpenedChannelsForensics(); } } catch (e) { @@ -95,16 +99,9 @@ class ForensicsService { const lightningScriptReasons: number[] = []; for (const outspend of outspends) { if (outspend.spent && outspend.txid) { - let spendingTx: IEsploraApi.Transaction | undefined = this.txCache[outspend.txid]; + let spendingTx = await this.fetchTransaction(outspend.txid); if (!spendingTx) { - try { - spendingTx = await bitcoinApi.$getRawTransaction(outspend.txid); - await Common.sleep$(throttleDelay); - this.txCache[outspend.txid] = spendingTx; - } catch (e) { - logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + outspend.txid}. Reason ${e instanceof Error ? e.message : e}`); - continue; - } + continue; } cached.push(spendingTx.txid); const lightningScript = this.findLightningScript(spendingTx.vin[outspend.vin || 0]); @@ -124,16 +121,9 @@ class ForensicsService { We can detect a commitment transaction (force close) by reading Sequence and Locktime https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction */ - let closingTx: IEsploraApi.Transaction | undefined = this.txCache[channel.closing_transaction_id]; + let closingTx = await this.fetchTransaction(channel.closing_transaction_id, true); if (!closingTx) { - try { - closingTx = await bitcoinApi.$getRawTransaction(channel.closing_transaction_id); - await Common.sleep$(throttleDelay); - this.txCache[channel.closing_transaction_id] = closingTx; - } catch (e) { - logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + channel.closing_transaction_id}. Reason ${e instanceof Error ? e.message : e}`); - continue; - } + continue; } cached.push(closingTx.txid); const sequenceHex: string = closingTx.vin[0].sequence.toString(16); @@ -174,7 +164,7 @@ class ForensicsService { } private findLightningScript(vin: IEsploraApi.Vin): number { - const topElement = vin.witness[vin.witness.length - 2]; + const topElement = vin.witness?.length > 2 ? vin.witness[vin.witness.length - 2] : null; if (/^OP_IF OP_PUSHBYTES_33 \w{66} OP_ELSE OP_PUSH(NUM_\d+|BYTES_(1 \w{2}|2 \w{4})) OP_CSV OP_DROP OP_PUSHBYTES_33 \w{66} OP_ENDIF OP_CHECKSIG$/.test(vin.inner_witnessscript_asm)) { // https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction-outputs if (topElement === '01') { @@ -193,7 +183,7 @@ class ForensicsService { ) { // https://github.com/lightning/bolts/blob/master/03-transactions.md#offered-htlc-outputs // https://github.com/lightning/bolts/blob/master/03-transactions.md#received-htlc-outputs - if (topElement.length === 66) { + if (topElement?.length === 66) { // top element is a public key // 'Revoked Lightning HTLC'; Penalty force closed return 4; @@ -220,6 +210,248 @@ class ForensicsService { } return 1; } + + // If a channel open tx spends funds from a another channel transaction, + // we can attribute that output to a specific counterparty + private async $runOpenedChannelsForensics(): Promise { + const runTimer = Date.now(); + let progress = 0; + + try { + logger.info(`Started running open channel forensics...`); + const channels = await channelsApi.$getChannelsWithoutSourceChecked(); + + for (const openChannel of channels) { + let openTx = await this.fetchTransaction(openChannel.transaction_id, true); + if (!openTx) { + continue; + } + for (const input of openTx.vin) { + const closeChannel = await channelsApi.$getChannelForensicsByClosingId(input.txid); + if (closeChannel) { + // this input directly spends a channel close output + await this.$attributeChannelBalances(closeChannel, openChannel, input); + } else { + const prevOpenChannels = await channelsApi.$getChannelForensicsByOpeningId(input.txid); + if (prevOpenChannels?.length) { + // this input spends a channel open change output + for (const prevOpenChannel of prevOpenChannels) { + await this.$attributeChannelBalances(prevOpenChannel, openChannel, input, null, null, true); + } + } else { + // check if this input spends any swept channel close outputs + await this.$attributeSweptChannelCloses(openChannel, input); + } + } + } + // calculate how much of the total input value is attributable to the channel open output + openChannel.funding_ratio = openTx.vout[openChannel.transaction_vout].value / ((openTx.vout.reduce((sum, v) => sum + v.value, 0) || 1) + openTx.fee); + // save changes to the opening channel, and mark it as checked + if (openTx?.vin?.length === 1) { + openChannel.single_funded = true; + } + if (openChannel.node1_funding_balance || openChannel.node2_funding_balance || openChannel.node1_closing_balance || openChannel.node2_closing_balance || openChannel.closed_by) { + await channelsApi.$updateOpeningInfo(openChannel); + } + await channelsApi.$markChannelSourceChecked(openChannel.id); + + ++progress; + const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); + if (elapsedSeconds > 10) { + logger.info(`Updating opened channel forensics ${progress}/${channels?.length}`); + this.loggerTimer = new Date().getTime() / 1000; + this.truncateTempCache(); + } + if (Date.now() - runTimer > (config.LIGHTNING.FORENSICS_INTERVAL * 1000)) { + break; + } + } + + logger.info(`Open channels forensics scan complete.`); + } catch (e) { + logger.err('$runOpenedChannelsForensics() error: ' + (e instanceof Error ? e.message : e)); + } finally { + this.clearTempCache(); + } + } + + // Check if a channel open tx input spends the result of a swept channel close output + private async $attributeSweptChannelCloses(openChannel: ILightningApi.Channel, input: IEsploraApi.Vin): Promise { + let sweepTx = await this.fetchTransaction(input.txid, true); + if (!sweepTx) { + logger.err(`couldn't find input transaction for channel forensics ${openChannel.channel_id} ${input.txid}`); + return; + } + const openContribution = sweepTx.vout[input.vout].value; + for (const sweepInput of sweepTx.vin) { + const lnScriptType = this.findLightningScript(sweepInput); + if (lnScriptType > 1) { + const closeChannel = await channelsApi.$getChannelForensicsByClosingId(sweepInput.txid); + if (closeChannel) { + const initiator = (lnScriptType === 2 || lnScriptType === 4) ? 'remote' : (lnScriptType === 3 ? 'local' : null); + await this.$attributeChannelBalances(closeChannel, openChannel, sweepInput, openContribution, initiator); + } + } + } + } + + private async $attributeChannelBalances( + prevChannel, openChannel, input: IEsploraApi.Vin, openContribution: number | null = null, + initiator: 'remote' | 'local' | null = null, linkedOpenings: boolean = false + ): Promise { + // figure out which node controls the input/output + let openSide; + let prevLocal; + let prevRemote; + let matched = false; + let ambiguous = false; // if counterparties are the same in both channels, we can't tell them apart + if (openChannel.node1_public_key === prevChannel.node1_public_key) { + openSide = 1; + prevLocal = 1; + prevRemote = 2; + matched = true; + } else if (openChannel.node1_public_key === prevChannel.node2_public_key) { + openSide = 1; + prevLocal = 2; + prevRemote = 1; + matched = true; + } + if (openChannel.node2_public_key === prevChannel.node1_public_key) { + openSide = 2; + prevLocal = 1; + prevRemote = 2; + if (matched) { + ambiguous = true; + } + matched = true; + } else if (openChannel.node2_public_key === prevChannel.node2_public_key) { + openSide = 2; + prevLocal = 2; + prevRemote = 1; + if (matched) { + ambiguous = true; + } + matched = true; + } + + if (matched && !ambiguous) { + // fetch closing channel transaction and perform forensics on the outputs + let prevChannelTx = await this.fetchTransaction(input.txid, true); + let outspends: IEsploraApi.Outspend[] | undefined; + try { + outspends = await bitcoinApi.$getOutspends(input.txid); + await Common.sleep$(throttleDelay); + } catch (e) { + logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + input.txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`); + } + if (!outspends || !prevChannelTx) { + return; + } + if (!linkedOpenings) { + if (!prevChannel.outputs) { + prevChannel.outputs = prevChannel.outputs || prevChannelTx.vout.map(vout => { + return { + type: 0, + value: vout.value, + }; + }); + } + for (let i = 0; i < outspends?.length; i++) { + const outspend = outspends[i]; + const output = prevChannel.outputs[i]; + if (outspend.spent && outspend.txid) { + try { + const spendingTx = await this.fetchTransaction(outspend.txid, true); + if (spendingTx) { + output.type = this.findLightningScript(spendingTx.vin[outspend.vin || 0]); + } + } catch (e) { + logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + outspend.txid}. Reason ${e instanceof Error ? e.message : e}`); + } + } else { + output.type = 0; + } + } + + // attribute outputs to each counterparty, and sum up total known balances + prevChannel.outputs[input.vout].node = prevLocal; + const isPenalty = prevChannel.outputs.filter((out) => out.type === 2 || out.type === 4)?.length > 0; + const normalOutput = [1,3].includes(prevChannel.outputs[input.vout].type); + let localClosingBalance = 0; + let remoteClosingBalance = 0; + for (const output of prevChannel.outputs) { + if (isPenalty) { + // penalty close, so local node takes everything + localClosingBalance += output.value; + } else if (output.node) { + // this output determinstically linked to one of the counterparties + if (output.node === prevLocal) { + localClosingBalance += output.value; + } else { + remoteClosingBalance += output.value; + } + } else if (normalOutput && (output.type === 1 || output.type === 3)) { + // local node had one main output, therefore remote node takes the other + remoteClosingBalance += output.value; + } + } + prevChannel[`node${prevLocal}_closing_balance`] = localClosingBalance; + prevChannel[`node${prevRemote}_closing_balance`] = remoteClosingBalance; + prevChannel.closing_fee = prevChannelTx.fee; + + if (initiator && !linkedOpenings) { + const initiatorSide = initiator === 'remote' ? prevRemote : prevLocal; + prevChannel.closed_by = prevChannel[`node${initiatorSide}_public_key`]; + } + + // save changes to the closing channel + await channelsApi.$updateClosingInfo(prevChannel); + } else { + if (prevChannelTx.vin.length <= 1) { + prevChannel[`node${prevLocal}_funding_balance`] = prevChannel.capacity; + prevChannel.single_funded = true; + prevChannel.funding_ratio = 1; + // save changes to the closing channel + await channelsApi.$updateOpeningInfo(prevChannel); + } + } + openChannel[`node${openSide}_funding_balance`] = openChannel[`node${openSide}_funding_balance`] + (openContribution || prevChannelTx?.vout[input.vout]?.value || 0); + } + } + + async fetchTransaction(txid: string, temp: boolean = false): Promise { + let tx = this.txCache[txid]; + if (!tx) { + try { + tx = await bitcoinApi.$getRawTransaction(txid); + this.txCache[txid] = tx; + if (temp) { + this.tempCached.push(txid); + } + await Common.sleep$(throttleDelay); + } catch (e) { + logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`); + return null; + } + } + return tx; + } + + clearTempCache(): void { + for (const txid of this.tempCached) { + delete this.txCache[txid]; + } + this.tempCached = []; + } + + truncateTempCache(): void { + if (this.tempCached.length > tempCacheSize) { + const removed = this.tempCached.splice(0, this.tempCached.length - tempCacheSize); + for (const txid of removed) { + delete this.txCache[txid]; + } + } + } } export default new ForensicsService(); diff --git a/backend/src/tasks/lightning/network-sync.service.ts b/backend/src/tasks/lightning/network-sync.service.ts index 4c92fb3bb..c5e5a102d 100644 --- a/backend/src/tasks/lightning/network-sync.service.ts +++ b/backend/src/tasks/lightning/network-sync.service.ts @@ -305,256 +305,6 @@ class NetworkSyncService { logger.err('$scanForClosedChannels() error: ' + (e instanceof Error ? e.message : e)); } } - - private findLightningScript(vin: IEsploraApi.Vin): number { - const topElement = vin.witness ? vin.witness[vin.witness.length - 2] : ''; - if (/^OP_IF OP_PUSHBYTES_33 \w{66} OP_ELSE OP_PUSH(NUM_\d+|BYTES_(1 \w{2}|2 \w{4})) OP_CSV OP_DROP OP_PUSHBYTES_33 \w{66} OP_ENDIF OP_CHECKSIG$/.test(vin.inner_witnessscript_asm)) { - // https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction-outputs - if (topElement === '01') { - // top element is '01' to get in the revocation path - // 'Revoked Lightning Force Close'; - // Penalty force closed - return 2; - } else { - // top element is '', this is a delayed to_local output - // 'Lightning Force Close'; - return 3; - } - } else if ( - /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_NOTIF OP_DROP OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm) || - /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_IF OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_DROP OP_PUSHBYTES_3 \w{6} OP_CLTV OP_DROP OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm) - ) { - // https://github.com/lightning/bolts/blob/master/03-transactions.md#offered-htlc-outputs - // https://github.com/lightning/bolts/blob/master/03-transactions.md#received-htlc-outputs - if (topElement?.length === 66) { - // top element is a public key - // 'Revoked Lightning HTLC'; Penalty force closed - return 4; - } else if (topElement) { - // top element is a preimage - // 'Lightning HTLC'; - return 5; - } else { - // top element is '' to get in the expiry of the script - // 'Expired Lightning HTLC'; - return 6; - } - } else if (/^OP_PUSHBYTES_33 \w{66} OP_CHECKSIG OP_IFDUP OP_NOTIF OP_PUSHNUM_16 OP_CSV OP_ENDIF$/.test(vin.inner_witnessscript_asm)) { - // https://github.com/lightning/bolts/blob/master/03-transactions.md#to_local_anchor-and-to_remote_anchor-output-option_anchors - if (topElement) { - // top element is a signature - // 'Lightning Anchor'; - return 7; - } else { - // top element is '', it has been swept after 16 blocks - // 'Swept Lightning Anchor'; - return 8; - } - } - return 1; - } - - // If a channel open tx spends funds from a another channel transaction, - // we can attribute that output to a specific counterparty - private async $runOpenedChannelsForensics(): Promise { - const runTimer = Date.now(); - let progress = 0; - - try { - logger.info(`Started running open channel forensics...`); - const channels = await channelsApi.$getChannelsWithoutSourceChecked(); - - for (const openChannel of channels) { - const openTx = await bitcoinApi.$getRawTransaction(openChannel.transaction_id); - for (const input of openTx.vin) { - const closeChannel = await channelsApi.$getChannelForensicsByClosingId(input.txid); - if (closeChannel) { - // this input directly spends a channel close output - await this.$attributeChannelBalances(closeChannel, openChannel, input); - } else { - const prevOpenChannels = await channelsApi.$getChannelForensicsByOpeningId(input.txid); - if (prevOpenChannels?.length) { - // this input spends a channel open change output - for (const prevOpenChannel of prevOpenChannels) { - await this.$attributeChannelBalances(prevOpenChannel, openChannel, input, null, null, true); - } - } else { - // check if this input spends any swept channel close outputs - await this.$attributeSweptChannelCloses(openChannel, input); - } - } - } - // calculate how much of the total input value is attributable to the channel open output - openChannel.funding_ratio = openTx.vout[openChannel.transaction_vout].value / ((openTx.vout.reduce((sum, v) => sum + v.value, 0) || 1) + openTx.fee); - // save changes to the opening channel, and mark it as checked - if (openTx?.vin?.length === 1) { - openChannel.single_funded = true; - } - if (openChannel.node1_funding_balance || openChannel.node2_funding_balance || openChannel.node1_closing_balance || openChannel.node2_closing_balance || openChannel.closed_by) { - await channelsApi.$updateOpeningInfo(openChannel); - } - await channelsApi.$markChannelSourceChecked(openChannel.id); - - ++progress; - const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer); - if (elapsedSeconds > 10) { - logger.info(`Updating opened channel forensics ${progress}/${channels?.length}`); - this.loggerTimer = new Date().getTime() / 1000; - } - if (Date.now() - runTimer > (config.LIGHTNING.GRAPH_REFRESH_INTERVAL * 1000)) { - break; - } - } - - logger.info(`Open channels forensics scan complete.`); - } catch (e) { - logger.err('$runOpenedChannelsForensics() error: ' + (e instanceof Error ? e.message : e)); - } - } - - // Check if a channel open tx input spends the result of a swept channel close output - private async $attributeSweptChannelCloses(openChannel: ILightningApi.Channel, input: IEsploraApi.Vin): Promise { - const sweepTx = await bitcoinApi.$getRawTransaction(input.txid); - if (!sweepTx) { - logger.err(`couldn't find input transaction for channel forensics ${openChannel.channel_id} ${input.txid}`); - return; - } - const openContribution = sweepTx.vout[input.vout].value; - for (const sweepInput of sweepTx.vin) { - const lnScriptType = this.findLightningScript(sweepInput); - if (lnScriptType > 1) { - const closeChannel = await channelsApi.$getChannelForensicsByClosingId(sweepInput.txid); - if (closeChannel) { - const initiator = (lnScriptType === 2 || lnScriptType === 4) ? 'remote' : (lnScriptType === 3 ? 'local' : null); - await this.$attributeChannelBalances(closeChannel, openChannel, sweepInput, openContribution, initiator); - } - } - } - } - - private async $attributeChannelBalances( - prevChannel, openChannel, input: IEsploraApi.Vin, openContribution: number | null = null, - initiator: 'remote' | 'local' | null = null, linkedOpenings: boolean = false - ): Promise { - // figure out which node controls the input/output - let openSide; - let prevLocal; - let prevRemote; - let matched = false; - let ambiguous = false; // if counterparties are the same in both channels, we can't tell them apart - if (openChannel.node1_public_key === prevChannel.node1_public_key) { - openSide = 1; - prevLocal = 1; - prevRemote = 2; - matched = true; - } else if (openChannel.node1_public_key === prevChannel.node2_public_key) { - openSide = 1; - prevLocal = 2; - prevRemote = 1; - matched = true; - } - if (openChannel.node2_public_key === prevChannel.node1_public_key) { - openSide = 2; - prevLocal = 1; - prevRemote = 2; - if (matched) { - ambiguous = true; - } - matched = true; - } else if (openChannel.node2_public_key === prevChannel.node2_public_key) { - openSide = 2; - prevLocal = 2; - prevRemote = 1; - if (matched) { - ambiguous = true; - } - matched = true; - } - - if (matched && !ambiguous) { - // fetch closing channel transaction and perform forensics on the outputs - let prevChannelTx: IEsploraApi.Transaction | undefined; - let outspends: IEsploraApi.Outspend[] | undefined; - try { - prevChannelTx = await bitcoinApi.$getRawTransaction(input.txid); - outspends = await bitcoinApi.$getOutspends(input.txid); - } catch (e) { - logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + input.txid + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`); - } - if (!outspends || !prevChannelTx) { - return; - } - if (!linkedOpenings) { - if (!prevChannel.outputs) { - prevChannel.outputs = prevChannel.outputs || prevChannelTx.vout.map(vout => { - return { - type: 0, - value: vout.value, - }; - }); - } - for (let i = 0; i < outspends?.length; i++) { - const outspend = outspends[i]; - const output = prevChannel.outputs[i]; - if (outspend.spent && outspend.txid) { - try { - const spendingTx = await bitcoinApi.$getRawTransaction(outspend.txid); - if (spendingTx) { - output.type = this.findLightningScript(spendingTx.vin[outspend.vin || 0]); - } - } catch (e) { - logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + outspend.txid}. Reason ${e instanceof Error ? e.message : e}`); - } - } else { - output.type = 0; - } - } - - // attribute outputs to each counterparty, and sum up total known balances - prevChannel.outputs[input.vout].node = prevLocal; - const isPenalty = prevChannel.outputs.filter((out) => out.type === 2 || out.type === 4)?.length > 0; - const normalOutput = [1,3].includes(prevChannel.outputs[input.vout].type); - let localClosingBalance = 0; - let remoteClosingBalance = 0; - for (const output of prevChannel.outputs) { - if (isPenalty) { - // penalty close, so local node takes everything - localClosingBalance += output.value; - } else if (output.node) { - // this output determinstically linked to one of the counterparties - if (output.node === prevLocal) { - localClosingBalance += output.value; - } else { - remoteClosingBalance += output.value; - } - } else if (normalOutput && (output.type === 1 || output.type === 3)) { - // local node had one main output, therefore remote node takes the other - remoteClosingBalance += output.value; - } - } - prevChannel[`node${prevLocal}_closing_balance`] = localClosingBalance; - prevChannel[`node${prevRemote}_closing_balance`] = remoteClosingBalance; - prevChannel.closing_fee = prevChannelTx.fee; - - if (initiator && !linkedOpenings) { - const initiatorSide = initiator === 'remote' ? prevRemote : prevLocal; - prevChannel.closed_by = prevChannel[`node${initiatorSide}_public_key`]; - } - - // save changes to the closing channel - await channelsApi.$updateClosingInfo(prevChannel); - } else { - if (prevChannelTx.vin.length <= 1) { - prevChannel[`node${prevLocal}_funding_balance`] = prevChannel.capacity; - prevChannel.single_funded = true; - prevChannel.funding_ratio = 1; - // save changes to the closing channel - await channelsApi.$updateOpeningInfo(prevChannel); - } - } - openChannel[`node${openSide}_funding_balance`] = openChannel[`node${openSide}_funding_balance`] + (openContribution || prevChannelTx?.vout[input.vout]?.value || 0); - } - } } export default new NetworkSyncService(); From ded11892f5faea6628ab6ec94d4e5097339447cf Mon Sep 17 00:00:00 2001 From: Mononaut Date: Fri, 25 Nov 2022 15:54:44 +0900 Subject: [PATCH 10/13] merge forensics columns into main channels table --- backend/src/api/database-migration.ts | 29 ++---- backend/src/api/explorer/channels.api.ts | 89 ++++++------------- .../src/tasks/lightning/forensics.service.ts | 10 +-- 3 files changed, 39 insertions(+), 89 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index 0f43580f6..a2977d3ba 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -382,7 +382,15 @@ class DatabaseMigration { if (databaseSchemaVersion < 48 && isBitcoin === true) { await this.$executeQuery('ALTER TABLE `channels` ADD source_checked tinyint(1) DEFAULT 0'); - await this.$executeQuery(this.getCreateChannelsForensicsTableQuery(), await this.$checkIfTableExists('channels_forensics')); + await this.$executeQuery('ALTER TABLE `channels` ADD closing_fee bigint(20) unsigned DEFAULT 0'); + await this.$executeQuery('ALTER TABLE `channels` ADD node1_funding_balance bigint(20) unsigned DEFAULT 0'); + await this.$executeQuery('ALTER TABLE `channels` ADD node2_funding_balance bigint(20) unsigned DEFAULT 0'); + await this.$executeQuery('ALTER TABLE `channels` ADD node1_closing_balance bigint(20) unsigned DEFAULT 0'); + await this.$executeQuery('ALTER TABLE `channels` ADD node2_closing_balance bigint(20) unsigned DEFAULT 0'); + await this.$executeQuery('ALTER TABLE `channels` ADD funding_ratio float unsigned DEFAULT NULL'); + await this.$executeQuery('ALTER TABLE `channels` ADD closed_by varchar(66) DEFAULT NULL'); + await this.$executeQuery('ALTER TABLE `channels` ADD single_funded tinyint(1) DEFAULT 0'); + await this.$executeQuery('ALTER TABLE `channels` ADD outputs JSON DEFAULT "[]"'); } } @@ -764,25 +772,6 @@ class DatabaseMigration { ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; } - private getCreateChannelsForensicsTableQuery(): string { - return `CREATE TABLE IF NOT EXISTS channels_forensics ( - channel_id bigint(11) unsigned NOT NULL, - closing_fee bigint(20) unsigned DEFAULT 0, - node1_funding_balance bigint(20) unsigned DEFAULT 0, - node2_funding_balance bigint(20) unsigned DEFAULT 0, - node1_closing_balance bigint(20) unsigned DEFAULT 0, - node2_closing_balance bigint(20) unsigned DEFAULT 0, - funding_ratio float unsigned DEFAULT NULL, - closed_by varchar(66) DEFAULT NULL, - single_funded tinyint(1) default 0, - outputs JSON NOT NULL, - PRIMARY KEY (channel_id), - FOREIGN KEY (channel_id) - REFERENCES channels (id) - ON DELETE CASCADE - ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; - } - private getCreateNodesStatsQuery(): string { return `CREATE TABLE IF NOT EXISTS node_stats ( id int(11) unsigned NOT NULL AUTO_INCREMENT, diff --git a/backend/src/api/explorer/channels.api.ts b/backend/src/api/explorer/channels.api.ts index 7f76a2971..e761288a3 100644 --- a/backend/src/api/explorer/channels.api.ts +++ b/backend/src/api/explorer/channels.api.ts @@ -131,9 +131,8 @@ class ChannelsApi { public async $getChannelsWithoutSourceChecked(): Promise { try { const query = ` - SELECT channels.*, forensics.* + SELECT channels.* FROM channels - LEFT JOIN channels_forensics AS forensics ON forensics.channel_id = channels.id WHERE channels.source_checked != 1 `; const [rows]: any = await DB.query(query); @@ -161,16 +160,12 @@ class ChannelsApi { SELECT n1.alias AS alias_left, n1.longitude as node1_longitude, n1.latitude as node1_latitude, n2.alias AS alias_right, n2.longitude as node2_longitude, n2.latitude as node2_latitude, channels.*, - ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right, - forensics.closing_fee as closing_fee, forensics.node1_funding_balance as node1_funding_balance, forensics.node2_funding_balance as node2_funding_balance, - forensics.funding_ratio as funding_ratio, forensics.node1_closing_balance as node1_closing_balance, forensics.node2_closing_balance as node2_closing_balance, - forensics.closed_by as closed_by, forensics.single_funded as single_funded + ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right FROM channels LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key - LEFT JOIN channels_forensics AS forensics ON forensics.channel_id = channels.id WHERE ( ns1.id = ( SELECT MAX(id) @@ -277,15 +272,12 @@ class ChannelsApi { } } - public async $getChannelForensicsByClosingId(transactionId: string): Promise { + public async $getChannelByClosingId(transactionId: string): Promise { try { const query = ` SELECT - channels.id, channels.node1_public_key, channels.node2_public_key, - channels.closing_reason, channels.closing_transaction_id, channels.capacity, - forensics.* + channels.* FROM channels - LEFT JOIN channels_forensics as forensics ON forensics.channel_id = channels.id WHERE channels.closing_transaction_id = ? `; const [rows]: any = await DB.query(query, [transactionId]); @@ -294,20 +286,17 @@ class ChannelsApi { return rows[0]; } } catch (e) { - logger.err('$getChannelForensicsByClosingId error: ' + (e instanceof Error ? e.message : e)); + logger.err('$getChannelByClosingId error: ' + (e instanceof Error ? e.message : e)); // don't throw - this data isn't essential } } - public async $getChannelForensicsByOpeningId(transactionId: string): Promise { + public async $getChannelsByOpeningId(transactionId: string): Promise { try { const query = ` SELECT - channels.id, channels.node1_public_key, channels.node2_public_key, - channels.status, channels.transaction_id, channels.capacity, - forensics.* + channels.* FROM channels - LEFT JOIN channels_forensics as forensics ON forensics.channel_id = channels.id WHERE channels.transaction_id = ? `; const [rows]: any = await DB.query(query, [transactionId]); @@ -318,7 +307,7 @@ class ChannelsApi { }); } } catch (e) { - logger.err('$getChannelForensicsByOpeningId error: ' + (e instanceof Error ? e.message : e)); + logger.err('$getChannelsByOpeningId error: ' + (e instanceof Error ? e.message : e)); // don't throw - this data isn't essential } } @@ -326,36 +315,21 @@ class ChannelsApi { public async $updateClosingInfo(channelInfo: { id: string, node1_closing_balance: number, node2_closing_balance: number, closed_by: string | null, closing_fee: number, outputs: ILightningApi.ForensicOutput[]}): Promise { try { const query = ` - INSERT INTO channels_forensics - ( - channel_id, - node1_closing_balance, - node2_closing_balance, - closed_by, - closing_fee, - outputs - ) - VALUES (?, ?, ?, ?, ?, ?) - ON DUPLICATE KEY UPDATE + UPDATE channels SET node1_closing_balance = ?, node2_closing_balance = ?, closed_by = ?, closing_fee = ?, outputs = ? + WHERE channels.id = ? `; - const jsonOutputs = JSON.stringify(channelInfo.outputs); await DB.query(query, [ + channelInfo.node1_closing_balance || 0, + channelInfo.node2_closing_balance || 0, + channelInfo.closed_by, + channelInfo.closing_fee || 0, + JSON.stringify(channelInfo.outputs), channelInfo.id, - channelInfo.node1_closing_balance || 0, - channelInfo.node2_closing_balance || 0, - channelInfo.closed_by, - channelInfo.closing_fee || 0, - jsonOutputs, - channelInfo.node1_closing_balance || 0, - channelInfo.node2_closing_balance || 0, - channelInfo.closed_by, - channelInfo.closing_fee || 0, - jsonOutputs ]); } catch (e) { logger.err('$updateClosingInfo error: ' + (e instanceof Error ? e.message : e)); @@ -366,32 +340,19 @@ class ChannelsApi { public async $updateOpeningInfo(channelInfo: { id: string, node1_funding_balance: number, node2_funding_balance: number, funding_ratio: number, single_funded: boolean | void }): Promise { try { const query = ` - INSERT INTO channels_forensics - ( - channel_id, - node1_funding_balance, - node2_funding_balance, - funding_ratio, - single_funded, - outputs - ) - VALUES (?, ?, ?, ?, ?, 'null') - ON DUPLICATE KEY UPDATE - node1_funding_balance = ?, - node2_funding_balance = ?, - funding_ratio = ?, - single_funded = ? + UPDATE channels SET + node1_funding_balance = ?, + node2_funding_balance = ?, + funding_ratio = ?, + single_funded = ? + WHERE channels.id = ? `; await DB.query(query, [ + channelInfo.node1_funding_balance || 0, + channelInfo.node2_funding_balance || 0, + channelInfo.funding_ratio, + channelInfo.single_funded ? 1 : 0, channelInfo.id, - channelInfo.node1_funding_balance || 0, - channelInfo.node2_funding_balance || 0, - channelInfo.funding_ratio, - channelInfo.single_funded ? 1 : 0, - channelInfo.node1_funding_balance || 0, - channelInfo.node2_funding_balance || 0, - channelInfo.funding_ratio, - channelInfo.single_funded ? 1 : 0, ]); } catch (e) { logger.err('$updateOpeningInfo error: ' + (e instanceof Error ? e.message : e)); diff --git a/backend/src/tasks/lightning/forensics.service.ts b/backend/src/tasks/lightning/forensics.service.ts index af79a270a..7083ad072 100644 --- a/backend/src/tasks/lightning/forensics.service.ts +++ b/backend/src/tasks/lightning/forensics.service.ts @@ -227,12 +227,12 @@ class ForensicsService { continue; } for (const input of openTx.vin) { - const closeChannel = await channelsApi.$getChannelForensicsByClosingId(input.txid); + const closeChannel = await channelsApi.$getChannelByClosingId(input.txid); if (closeChannel) { // this input directly spends a channel close output await this.$attributeChannelBalances(closeChannel, openChannel, input); } else { - const prevOpenChannels = await channelsApi.$getChannelForensicsByOpeningId(input.txid); + const prevOpenChannels = await channelsApi.$getChannelsByOpeningId(input.txid); if (prevOpenChannels?.length) { // this input spends a channel open change output for (const prevOpenChannel of prevOpenChannels) { @@ -286,7 +286,7 @@ class ForensicsService { for (const sweepInput of sweepTx.vin) { const lnScriptType = this.findLightningScript(sweepInput); if (lnScriptType > 1) { - const closeChannel = await channelsApi.$getChannelForensicsByClosingId(sweepInput.txid); + const closeChannel = await channelsApi.$getChannelByClosingId(sweepInput.txid); if (closeChannel) { const initiator = (lnScriptType === 2 || lnScriptType === 4) ? 'remote' : (lnScriptType === 3 ? 'local' : null); await this.$attributeChannelBalances(closeChannel, openChannel, sweepInput, openContribution, initiator); @@ -348,8 +348,8 @@ class ForensicsService { return; } if (!linkedOpenings) { - if (!prevChannel.outputs) { - prevChannel.outputs = prevChannel.outputs || prevChannelTx.vout.map(vout => { + if (!prevChannel.outputs || !prevChannel.outputs.length) { + prevChannel.outputs = prevChannelTx.vout.map(vout => { return { type: 0, value: vout.value, From ba10df69b7a72c07965a5e44055dbb581d823c1f Mon Sep 17 00:00:00 2001 From: Mononaut Date: Mon, 28 Nov 2022 10:41:48 +0900 Subject: [PATCH 11/13] improve precision of output attribution for mutual closes --- backend/src/tasks/lightning/forensics.service.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/tasks/lightning/forensics.service.ts b/backend/src/tasks/lightning/forensics.service.ts index 7083ad072..7acb36e89 100644 --- a/backend/src/tasks/lightning/forensics.service.ts +++ b/backend/src/tasks/lightning/forensics.service.ts @@ -377,6 +377,7 @@ class ForensicsService { prevChannel.outputs[input.vout].node = prevLocal; const isPenalty = prevChannel.outputs.filter((out) => out.type === 2 || out.type === 4)?.length > 0; const normalOutput = [1,3].includes(prevChannel.outputs[input.vout].type); + const mutualClose = ((prevChannel.status === 2 || prevChannel.status === 'closed') && prevChannel.closing_reason === 1); let localClosingBalance = 0; let remoteClosingBalance = 0; for (const output of prevChannel.outputs) { @@ -390,7 +391,7 @@ class ForensicsService { } else { remoteClosingBalance += output.value; } - } else if (normalOutput && (output.type === 1 || output.type === 3)) { + } else if (normalOutput && (output.type === 1 || output.type === 3 || (mutualClose && prevChannel.outputs.length === 2))) { // local node had one main output, therefore remote node takes the other remoteClosingBalance += output.value; } From afc5c6786bf4e17b100dad6525c713731c182826 Mon Sep 17 00:00:00 2001 From: softsimon Date: Wed, 30 Nov 2022 22:07:47 +0900 Subject: [PATCH 12/13] Remove annoying frontend console log --- .../components/bisq-master-page/bisq-master-page.component.ts | 1 - .../liquid-master-page/liquid-master-page.component.ts | 1 - frontend/src/app/components/master-page/master-page.component.ts | 1 - 3 files changed, 3 deletions(-) diff --git a/frontend/src/app/components/bisq-master-page/bisq-master-page.component.ts b/frontend/src/app/components/bisq-master-page/bisq-master-page.component.ts index 941d9e21e..0ab0259bd 100644 --- a/frontend/src/app/components/bisq-master-page/bisq-master-page.component.ts +++ b/frontend/src/app/components/bisq-master-page/bisq-master-page.component.ts @@ -30,7 +30,6 @@ export class BisqMasterPageComponent implements OnInit { this.connectionState$ = this.stateService.connectionState$; this.urlLanguage = this.languageService.getLanguageForUrl(); this.navigationService.subnetPaths.subscribe((paths) => { - console.log('network paths updated...'); this.networkPaths = paths; }); } diff --git a/frontend/src/app/components/liquid-master-page/liquid-master-page.component.ts b/frontend/src/app/components/liquid-master-page/liquid-master-page.component.ts index c57673529..e5d0aaa0a 100644 --- a/frontend/src/app/components/liquid-master-page/liquid-master-page.component.ts +++ b/frontend/src/app/components/liquid-master-page/liquid-master-page.component.ts @@ -33,7 +33,6 @@ export class LiquidMasterPageComponent implements OnInit { this.network$ = merge(of(''), this.stateService.networkChanged$); this.urlLanguage = this.languageService.getLanguageForUrl(); this.navigationService.subnetPaths.subscribe((paths) => { - console.log('network paths updated...'); this.networkPaths = paths; }); } diff --git a/frontend/src/app/components/master-page/master-page.component.ts b/frontend/src/app/components/master-page/master-page.component.ts index 8f7b4fecc..34c525108 100644 --- a/frontend/src/app/components/master-page/master-page.component.ts +++ b/frontend/src/app/components/master-page/master-page.component.ts @@ -35,7 +35,6 @@ export class MasterPageComponent implements OnInit { this.urlLanguage = this.languageService.getLanguageForUrl(); this.subdomain = this.enterpriseService.getSubdomain(); this.navigationService.subnetPaths.subscribe((paths) => { - console.log('network paths updated...'); this.networkPaths = paths; }); } From 8117b9799c8285edf02ba4d70689d5551ab22948 Mon Sep 17 00:00:00 2001 From: softsimon Date: Wed, 30 Nov 2022 22:14:15 +0900 Subject: [PATCH 13/13] Adding production config to enable gbt audit. --- production/mempool-config.mainnet.json | 2 ++ production/mempool-config.signet.json | 2 ++ production/mempool-config.testnet.json | 2 ++ 3 files changed, 6 insertions(+) diff --git a/production/mempool-config.mainnet.json b/production/mempool-config.mainnet.json index 06a14d223..ab2fa69c1 100644 --- a/production/mempool-config.mainnet.json +++ b/production/mempool-config.mainnet.json @@ -10,6 +10,8 @@ "POLL_RATE_MS": 1000, "INDEXING_BLOCKS_AMOUNT": -1, "BLOCKS_SUMMARIES_INDEXING": true, + "ADVANCED_GBT_AUDIT": true, + "ADVANCED_GBT_MEMPOOL": false, "USE_SECOND_NODE_FOR_MINFEE": true }, "SYSLOG" : { diff --git a/production/mempool-config.signet.json b/production/mempool-config.signet.json index f42c4dc50..313a09679 100644 --- a/production/mempool-config.signet.json +++ b/production/mempool-config.signet.json @@ -7,6 +7,8 @@ "SPAWN_CLUSTER_PROCS": 0, "API_URL_PREFIX": "/api/v1/", "INDEXING_BLOCKS_AMOUNT": -1, + "ADVANCED_GBT_AUDIT": true, + "ADVANCED_GBT_MEMPOOL": false, "POLL_RATE_MS": 1000 }, "SYSLOG" : { diff --git a/production/mempool-config.testnet.json b/production/mempool-config.testnet.json index cc63f93bf..908df7886 100644 --- a/production/mempool-config.testnet.json +++ b/production/mempool-config.testnet.json @@ -7,6 +7,8 @@ "SPAWN_CLUSTER_PROCS": 0, "API_URL_PREFIX": "/api/v1/", "INDEXING_BLOCKS_AMOUNT": -1, + "ADVANCED_GBT_AUDIT": true, + "ADVANCED_GBT_MEMPOOL": false, "POLL_RATE_MS": 1000 }, "SYSLOG" : {