detect links between channel close and open txs

backend/src/api/database-migration.ts

@@ -4,7 +4,7 @@ import logger from '../logger';
 import { Common } from './common';
 
 class DatabaseMigration {
-  private static currentVersion = 47;
+  private static currentVersion = 48;
   private queryTimeout = 900_000;
   private statisticsAddedIndexed = false;
   private uniqueLogs: string[] = [];
@@ -379,6 +379,11 @@ class DatabaseMigration {
       await this.$executeQuery(this.getCreateCPFPTableQuery(), await this.$checkIfTableExists('cpfp_clusters'));
       await this.$executeQuery(this.getCreateTransactionsTableQuery(), await this.$checkIfTableExists('transactions'));
     }
+
+    if (databaseSchemaVersion < 48 && isBitcoin === true) {
+      await this.$executeQuery('ALTER TABLE `channels` ADD source_checked tinyint(1) DEFAULT 0');
+      await this.$executeQuery(this.getCreateChannelsForensicsTableQuery(), await this.$checkIfTableExists('channels_forensics'));
+    }
   }
 
   /**
@@ -759,6 +764,25 @@ class DatabaseMigration {
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
   }
 
+  private getCreateChannelsForensicsTableQuery(): string {
+    return `CREATE TABLE IF NOT EXISTS channels_forensics (
+      channel_id bigint(11) unsigned NOT NULL,
+      closing_fee bigint(20) unsigned DEFAULT 0,
+      node1_funding_balance bigint(20) unsigned DEFAULT 0,
+      node2_funding_balance bigint(20) unsigned DEFAULT 0,
+      node1_closing_balance bigint(20) unsigned DEFAULT 0,
+      node2_closing_balance bigint(20) unsigned DEFAULT 0,
+      funding_ratio float unsigned DEFAULT NULL,
+      closed_by varchar(66) DEFAULT NULL,
+      single_funded tinyint(1) default 0,
+      outputs JSON NOT NULL,
+      PRIMARY KEY (channel_id),
+      FOREIGN KEY (channel_id)
+        REFERENCES channels (id)
+        ON DELETE CASCADE
+    ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;
+  }
+
   private getCreateNodesStatsQuery(): string {
     return `CREATE TABLE IF NOT EXISTS node_stats (
       id int(11) unsigned NOT NULL AUTO_INCREMENT,

backend/src/api/explorer/channels.api.ts

@@ -128,6 +128,22 @@ class ChannelsApi {
     }
   }
 
+  public async $getChannelsWithoutSourceChecked(): Promise<any[]> {
+    try {
+      const query = `
+        SELECT channels.*, forensics.*
+        FROM channels
+        LEFT JOIN channels_forensics AS forensics ON forensics.channel_id = channels.id
+        WHERE channels.source_checked != 1
+      `;
+      const [rows]: any = await DB.query(query);
+      return rows;
+    } catch (e) {
+      logger.err('$getUnresolvedClosedChannels error: ' + (e instanceof Error ? e.message : e));
+      throw e;
+    }
+  }
+
   public async $getChannelsWithoutCreatedDate(): Promise<any[]> {
     try {
       const query = `SELECT * FROM channels WHERE created IS NULL`;
@@ -145,12 +161,16 @@ class ChannelsApi {
         SELECT n1.alias AS alias_left, n1.longitude as node1_longitude, n1.latitude as node1_latitude,
           n2.alias AS alias_right, n2.longitude as node2_longitude, n2.latitude as node2_latitude,
           channels.*,
-          ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right
+          ns1.channels AS channels_left, ns1.capacity AS capacity_left, ns2.channels AS channels_right, ns2.capacity AS capacity_right,
+          forensics.closing_fee as closing_fee, forensics.node1_funding_balance as node1_funding_balance, forensics.node2_funding_balance as node2_funding_balance,
+          forensics.funding_ratio as funding_ratio, forensics.node1_closing_balance as node1_closing_balance, forensics.node2_closing_balance as node2_closing_balance,
+          forensics.closed_by as closed_by, forensics.single_funded as single_funded
         FROM channels
         LEFT JOIN nodes AS n1 ON n1.public_key = channels.node1_public_key
         LEFT JOIN nodes AS n2 ON n2.public_key = channels.node2_public_key
         LEFT JOIN node_stats AS ns1 ON ns1.public_key = channels.node1_public_key
         LEFT JOIN node_stats AS ns2 ON ns2.public_key = channels.node2_public_key
+        LEFT JOIN channels_forensics AS forensics ON forensics.channel_id = channels.id
         WHERE (
           ns1.id = (
             SELECT MAX(id)
@@ -257,6 +277,118 @@ class ChannelsApi {
     }
   }
 
+  public async $getChannelForensicsByTransactionId(transactionId: string): Promise<any> {
+    try {
+      const query = `
+        SELECT
+          channels.id, channels.node1_public_key, channels.node2_public_key,
+          channels.closing_reason, channels.closing_transaction_id,
+          forensics.*
+        FROM channels
+        LEFT JOIN channels_forensics as forensics ON forensics.channel_id = channels.id
+        WHERE channels.closing_transaction_id = ?
+      `;
+      const [rows]: any = await DB.query(query, [transactionId]);
+      if (rows.length > 0) {
+        rows[0].outputs = JSON.parse(rows[0].outputs);
+        return rows[0];
+      }
+    } catch (e) {
+      logger.err('$getChannelForensicsByTransactionId error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $updateClosingInfo(channelInfo: { id: string, node1_closing_balance: number, node2_closing_balance: number, closed_by: string | null, closing_fee: number, outputs: ILightningApi.ForensicOutput[]}): Promise<void> {
+    try {
+      const query = `
+        INSERT INTO channels_forensics
+        (
+          channel_id,
+          node1_closing_balance,
+          node2_closing_balance,
+          closed_by,
+          closing_fee,
+          outputs
+        )
+        VALUES (?, ?, ?, ?, ?, ?)
+        ON DUPLICATE KEY UPDATE 
+          node1_closing_balance = ?,
+          node2_closing_balance = ?,
+          closed_by = ?,
+          closing_fee = ?,
+          outputs = ?
+      `;
+      const jsonOutputs = JSON.stringify(channelInfo.outputs);
+      await DB.query<ResultSetHeader>(query, [
+        channelInfo.id,
+        channelInfo.node1_closing_balance,
+        channelInfo.node2_closing_balance,
+        channelInfo.closed_by,
+        channelInfo.closing_fee,
+        jsonOutputs,
+        channelInfo.node1_closing_balance,
+        channelInfo.node2_closing_balance,
+        channelInfo.closed_by,
+        channelInfo.closing_fee,
+        jsonOutputs
+      ]);
+    } catch (e) {
+      logger.err('$updateClosingInfo error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $updateOpeningInfo(channelInfo: { id: string, node1_funding_balance: number, node2_funding_balance: number, funding_ratio: number, single_funded: boolean | void }): Promise<void> {
+    try {
+      const query = `
+      INSERT INTO channels_forensics
+      (
+        channel_id,
+        node1_funding_balance,
+        node2_funding_balance,
+        funding_ratio,
+        single_funded,
+        outputs
+      )
+      VALUES (?, ?, ?, ?, ?, 'null')
+      ON DUPLICATE KEY UPDATE
+        node1_funding_balance = ?,
+        node2_funding_balance = ?,
+        funding_ratio = ?,
+        single_funded = ?
+      `;
+      await DB.query<ResultSetHeader>(query, [
+        channelInfo.id,
+        channelInfo.node1_funding_balance,
+        channelInfo.node2_funding_balance,
+        channelInfo.funding_ratio,
+        channelInfo.single_funded ? 1 : 0,
+        channelInfo.node1_funding_balance,
+        channelInfo.node2_funding_balance,
+        channelInfo.funding_ratio,
+        channelInfo.single_funded ? 1 : 0,
+      ]);
+    } catch (e) {
+      logger.err('$updateOpeningInfo error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
+  public async $markChannelSourceChecked(id: string): Promise<void> {
+    try {
+      const query = `
+        UPDATE channels
+        SET source_checked = 1
+        WHERE id = ?
+      `;
+      await DB.query<ResultSetHeader>(query, [id]);
+    } catch (e) {
+      logger.err('$markChannelSourceChecked error: ' + (e instanceof Error ? e.message : e));
+      // don't throw - this data isn't essential
+    }
+  }
+
   public async $getChannelsForNode(public_key: string, index: number, length: number, status: string): Promise<any[]> {
     try {
       let channelStatusFilter;
@@ -385,11 +517,15 @@ class ChannelsApi {
       'transaction_id': channel.transaction_id,
       'transaction_vout': channel.transaction_vout,
       'closing_transaction_id': channel.closing_transaction_id,
+      'closing_fee': channel.closing_fee,
       'closing_reason': channel.closing_reason,
       'closing_date': channel.closing_date,
       'updated_at': channel.updated_at,
       'created': channel.created,
       'status': channel.status,
+      'funding_ratio': channel.funding_ratio,
+      'closed_by': channel.closed_by,
+      'single_funded': !!channel.single_funded,
       'node_left': {
         'alias': channel.alias_left,
         'public_key': channel.node1_public_key,
@@ -404,6 +540,9 @@ class ChannelsApi {
         'updated_at': channel.node1_updated_at,
         'longitude': channel.node1_longitude,
         'latitude': channel.node1_latitude,
+        'funding_balance': channel.node1_funding_balance,
+        'closing_balance': channel.node1_closing_balance,
+        'initiated_close': channel.closed_by === channel.node1_public_key ? true : undefined,
       },
       'node_right': {
         'alias': channel.alias_right,
@@ -419,6 +558,9 @@ class ChannelsApi {
         'updated_at': channel.node2_updated_at,
         'longitude': channel.node2_longitude,
         'latitude': channel.node2_latitude,
+        'funding_balance': channel.node2_funding_balance,
+        'closing_balance': channel.node2_closing_balance,
+        'initiated_close': channel.closed_by === channel.node2_public_key ? true : undefined,
       },
     };
   }

backend/src/api/lightning/lightning-api.interface.ts

@@ -83,4 +83,10 @@ export namespace ILightningApi {
     is_required: boolean;
     is_known: boolean;
   }
+
+  export interface ForensicOutput {
+    node?: 1 | 2;
+    type: number;
+    value: number;
+  }
 }

backend/src/tasks/lightning/network-sync.service.ts

@@ -304,6 +304,233 @@ class NetworkSyncService {
       logger.err('$scanForClosedChannels() error: ' + (e instanceof Error ? e.message : e));
     }
   }
+
+  private findLightningScript(vin: IEsploraApi.Vin): number {
+    const topElement = vin.witness[vin.witness.length - 2];
+      if (/^OP_IF OP_PUSHBYTES_33 \w{66} OP_ELSE OP_PUSH(NUM_\d+|BYTES_(1 \w{2}|2 \w{4})) OP_CSV OP_DROP OP_PUSHBYTES_33 \w{66} OP_ENDIF OP_CHECKSIG$/.test(vin.inner_witnessscript_asm)) {
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#commitment-transaction-outputs
+        if (topElement === '01') {
+          // top element is '01' to get in the revocation path
+          // 'Revoked Lightning Force Close';
+          // Penalty force closed
+          return 2;
+        } else {
+          // top element is '', this is a delayed to_local output
+          // 'Lightning Force Close';
+          return 3;
+        }
+      } else if (
+        /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_NOTIF OP_DROP OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm) ||
+        /^OP_DUP OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUAL OP_IF OP_CHECKSIG OP_ELSE OP_PUSHBYTES_33 \w{66} OP_SWAP OP_SIZE OP_PUSHBYTES_1 20 OP_EQUAL OP_IF OP_HASH160 OP_PUSHBYTES_20 \w{40} OP_EQUALVERIFY OP_PUSHNUM_2 OP_SWAP OP_PUSHBYTES_33 \w{66} OP_PUSHNUM_2 OP_CHECKMULTISIG OP_ELSE OP_DROP OP_PUSHBYTES_3 \w{6} OP_CLTV OP_DROP OP_CHECKSIG OP_ENDIF (OP_PUSHNUM_1 OP_CSV OP_DROP |)OP_ENDIF$/.test(vin.inner_witnessscript_asm)
+      ) {
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#offered-htlc-outputs
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#received-htlc-outputs
+        if (topElement.length === 66) {
+          // top element is a public key
+          // 'Revoked Lightning HTLC'; Penalty force closed
+          return 4;
+        } else if (topElement) {
+          // top element is a preimage
+          // 'Lightning HTLC';
+          return 5;
+        } else {
+          // top element is '' to get in the expiry of the script
+          // 'Expired Lightning HTLC';
+          return 6;
+        }
+      } else if (/^OP_PUSHBYTES_33 \w{66} OP_CHECKSIG OP_IFDUP OP_NOTIF OP_PUSHNUM_16 OP_CSV OP_ENDIF$/.test(vin.inner_witnessscript_asm)) {
+        // https://github.com/lightning/bolts/blob/master/03-transactions.md#to_local_anchor-and-to_remote_anchor-output-option_anchors
+        if (topElement) {
+          // top element is a signature
+          // 'Lightning Anchor';
+          return 7;
+        } else {
+          // top element is '', it has been swept after 16 blocks
+          // 'Swept Lightning Anchor';
+          return 8;
+        }
+      }
+      return 1;
+  }
+
+  // If a channel open tx spends funds from a closed channel output,
+  // we can attribute that output to a specific counterparty
+  private async $runOpenedChannelsForensics(): Promise<void> {
+    let progress = 0;
+
+    try {
+      logger.info(`Started running open channel forensics...`);
+      const channels = await channelsApi.$getChannelsWithoutSourceChecked();
+
+      for (const openChannel of channels) {
+        const openTx = await bitcoinApi.$getRawTransaction(openChannel.transaction_id);
+        for (const input of openTx.vin) {
+          const closeChannel = await channelsApi.$getChannelForensicsByTransactionId(input.txid);
+          if (closeChannel) {
+            // this input directly spends a channel close output
+            await this.$attributeChannelBalances(closeChannel, openChannel, input);
+          } else {
+            // check if this input spends any swept channel close outputs
+            await this.$attributeSweptChannelCloses(openChannel, input);
+          }
+        }
+        // calculate how much of the total input value is attributable to the channel open output
+        openChannel.funding_ratio = openTx.vout[openChannel.transaction_vout].value / ((openTx.vout.reduce((sum, v) => sum + v.value, 0) || 1) + openTx.fee);
+        // save changes to the opening channel, and mark it as checked
+        if (openTx.vin.length === 1) {
+          openChannel.single_funded = true;
+        }
+        await channelsApi.$updateOpeningInfo(openChannel);
+        await channelsApi.$markChannelSourceChecked(openChannel.id);
+
+        ++progress;
+        const elapsedSeconds = Math.round((new Date().getTime() / 1000) - this.loggerTimer);
+        if (elapsedSeconds > 10) {
+          logger.info(`Updating channel opened channel forensics ${progress}/${channels.length}`);
+          this.loggerTimer = new Date().getTime() / 1000;
+        }
+      }
+
+      logger.info(`Open channels forensics scan complete.`);
+    } catch (e) {
+      logger.err('$runOpenedChannelsForensics() error: ' + (e instanceof Error ? e.message : e));
+    }
+  }
+
+  // Check if a channel open tx input spends the result of a swept channel close output
+  private async $attributeSweptChannelCloses(openChannel: ILightningApi.Channel, input: IEsploraApi.Vin): Promise<void> {
+    const sweepTx = await bitcoinApi.$getRawTransaction(input.txid);
+    if (!sweepTx) {
+      logger.err(`couldn't find input transaction for channel forensics ${openChannel.channel_id} ${input.txid}`);
+      return;
+    }
+    const openContribution = sweepTx.vout[input.vout].value;
+    for (const sweepInput of sweepTx.vin) {
+      const lnScriptType = this.findLightningScript(sweepInput);
+      if (lnScriptType > 1) {
+        const closeChannel = await channelsApi.$getChannelForensicsByTransactionId(sweepInput.txid);
+        if (closeChannel) {
+          const initiator = (lnScriptType === 2 || lnScriptType === 4) ? 'remote' : (lnScriptType === 3 ? 'local' : null);
+          await this.$attributeChannelBalances(closeChannel, openChannel, sweepInput, openContribution, initiator);
+        }
+      }
+    }
+  }
+
+  private async $attributeChannelBalances(
+    closeChannel, openChannel, input: IEsploraApi.Vin, openContribution: number | null = null,
+    initiator: 'remote' | 'local' | null = null
+  ): Promise<void> {
+    // figure out which node controls the input/output
+    let openSide;
+    let closeLocal;
+    let closeRemote;
+    let matched = false;
+    let ambiguous = false; // if counterparties are the same in both channels, we can't tell them apart
+    if (openChannel.node1_public_key === closeChannel.node1_public_key) {
+      openSide = 1;
+      closeLocal = 1;
+      closeRemote = 2;
+      matched = true;
+    } else if (openChannel.node1_public_key === closeChannel.node2_public_key) {
+      openSide = 1;
+      closeLocal = 2;
+      closeRemote = 1;
+      matched = true;
+    }
+    if (openChannel.node2_public_key === closeChannel.node1_public_key) {
+      openSide = 2;
+      closeLocal = 1;
+      closeRemote = 2;
+      if (matched) {
+        ambiguous = true;
+      }
+      matched = true;
+    } else if (openChannel.node2_public_key === closeChannel.node2_public_key) {
+      openSide = 2;
+      closeLocal = 2;
+      closeRemote = 1;
+      if (matched) {
+        ambiguous = true;
+      }
+      matched = true;
+    }
+
+    if (matched && !ambiguous) {
+      // fetch closing channel transaction and perform forensics on the outputs
+      let closingTx: IEsploraApi.Transaction | undefined;
+      let outspends: IEsploraApi.Outspend[] | undefined;
+      try {
+        closingTx = await bitcoinApi.$getRawTransaction(input.txid);
+        outspends = await bitcoinApi.$getOutspends(input.txid);
+      } catch (e) {
+        logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + closeChannel.closing_transaction_id + '/outspends'}. Reason ${e instanceof Error ? e.message : e}`);
+      }
+      if (!outspends || !closingTx) {
+        return;
+      }
+      if (!closeChannel.outputs) {
+        closeChannel.outputs = closeChannel.outputs || closingTx.vout.map(vout => {
+          return {
+            type: 0,
+            value: vout.value,
+          };
+        });
+      }
+      for (let i = 0; i < outspends.length; i++) {
+        const outspend = outspends[i];
+        const output = closeChannel.outputs[i];
+        if (outspend.spent && outspend.txid) {
+          try {
+            const spendingTx = await bitcoinApi.$getRawTransaction(outspend.txid);
+            if (spendingTx) {
+              output.type = this.findLightningScript(spendingTx.vin[outspend.vin || 0]);
+            }
+          } catch (e) {
+            logger.err(`Failed to call ${config.ESPLORA.REST_API_URL + '/tx/' + outspend.txid}. Reason ${e instanceof Error ? e.message : e}`);
+          }
+        } else {
+          output.type = 0;
+        }
+      }
+
+      // attribute outputs to each counterparty, and sum up total known balances
+      closeChannel.outputs[input.vout].node = closeLocal;
+      const isPenalty = closeChannel.outputs.filter((out) => out.type === 2 || out.type === 4).length > 0;
+      const normalOutput = [1,3].includes(closeChannel.outputs[input.vout].type);
+      let localClosingBalance = 0;
+      let remoteClosingBalance = 0;
+      for (const output of closeChannel.outputs) {
+        if (isPenalty) {
+          // penalty close, so local node takes everything
+          localClosingBalance += output.value;
+        } else if (output.node) {
+          // this output determinstically linked to one of the counterparties
+          if (output.node === closeLocal) {
+            localClosingBalance += output.value;
+          } else {
+            remoteClosingBalance += output.value;
+          }
+        } else if (normalOutput && (output.type === 1 || output.type === 3)) {
+          // local node had one main output, therefore remote node takes the other
+          remoteClosingBalance += output.value;
+        }
+      }
+
+      openChannel[`node${openSide}_funding_balance`] = openChannel[`node${openSide}_funding_balance`] + (openContribution || closingTx?.vout[input.vout]?.value || 0);
+      closeChannel[`node${closeLocal}_closing_balance`] = localClosingBalance;
+      closeChannel[`node${closeRemote}_closing_balance`] = remoteClosingBalance;
+      closeChannel.closing_fee = closingTx.fee;
+
+      if (initiator) {
+        const initiatorSide = initiator === 'remote' ? closeRemote : closeLocal;
+        closeChannel.closed_by = closeChannel[`node${initiatorSide}_public_key`];
+      }
+
+      // save changes to the closing channel
+      await channelsApi.$updateClosingInfo(closeChannel);
+    }
+  }
 }
 
 export default new NetworkSyncService();