feat(lightning): the group node redials the LSP, and says why it was dropped

lightning-kmp never reconnects by itself. The identity's own node has Phoenix's connection
daemon for that; the group's node connected once, so a single drop left it offline for the
rest of the term.

It now redials after every drop, waiting two seconds and doubling up to five minutes. A
connection that ends within a minute counts as a failure, so an LSP that hangs up on our
features isn't redialled every two seconds. Each drop is logged with its reason. When the
LSP's init doesn't offer a feature we require, the log names it. The node waits up to a
second for that init, because the socket can read as closed before the init's event arrives.

Run against ACINQ's mainnet LSP on an emulator, every drop reads: "Connection closed: the
LSP does not offer simple_taproot_channels, which this node requires". The plan doc now
records that against Phase 0's first step.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Kgothatso NgakoandClaude Opus 5.5 committed 2026-10-26 22:37:54 +02:00
1 parent 3d891e7c13
commit 3e4de464fe
4 files changed
+178 -4

No files matched your search

@@ -18,10 +18,12 @@ import fr.acinq.bitcoin.Satoshi
import fr.acinq.bitcoin.utils.Either
import fr.acinq.lightning.Feature
import fr.acinq.lightning.FeatureSupport
import fr.acinq.lightning.Features
import fr.acinq.lightning.Lightning
import fr.acinq.lightning.MilliSatoshi
import fr.acinq.lightning.NodeParams
import fr.acinq.lightning.PaymentEvents
import fr.acinq.lightning.PeerConnected
import fr.acinq.lightning.WalletParams
import fr.acinq.lightning.blockchain.Feerates
import fr.acinq.lightning.blockchain.electrum.ElectrumWatcher
@@ -45,7 +47,9 @@ import fr.acinq.lightning.io.WrappedChannelCommand
import fr.acinq.lightning.logging.LoggerFactory
import fr.acinq.lightning.payment.Bolt11Invoice
import fr.acinq.lightning.payment.LiquidityPolicy
import fr.acinq.lightning.utils.Connection
import fr.acinq.lightning.utils.msat
import fr.acinq.lightning.wire.Init
import kotlin.time.Clock
import kotlin.time.Duration
import kotlin.time.Duration.Companion.minutes
@@ -55,6 +59,9 @@ import kotlinx.coroutines.Job
import kotlinx.coroutines.delay
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.first
import kotlinx.coroutines.flow.filterNotNull
import kotlinx.coroutines.withTimeoutOrNull
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
@@ -83,6 +90,8 @@ class GroupLightningNode(
/** How many pool counters must hold round one before the node connects; 0 for a test with no co-signers. */
private val minimumPool: Int = MINIMUM_POOL,
private val poolWait: Duration = POOL_WAIT,
/** How long to wait before redialling the LSP after [failures] attempts in a row that did not hold. */
private val reconnectAfter: (failures: Int) -> Duration = ::reconnectDelay,
) {
private val logger = Logger.withTag("GroupLightningNode")
private val chain = wallet.statement.chain
@@ -99,8 +108,11 @@ class GroupLightningNode(
/** What this class needs of a running node. */
interface PeerHandle {
/** Dials the LSP; true once the handshake is done, false when it could not be. */
suspend fun connect(): Boolean
fun disconnect()
/** Suspends until the connection to the LSP is down -- at once, when it is down already -- and says why. */
suspend fun awaitDisconnect(): String
/** Each channel's local commitment number, by channel id: what the stopped message reports. */
fun commitmentNumbers(): Map<String, Long>
/** The node's channels, as a snapshot reports them. */
@@ -135,6 +147,7 @@ class GroupLightningNode(
private var peer: PeerHandle? = null
private var transport: GroupIcebergTransport? = null
private var watching: Job? = null
private var connecting: Job? = null
private var lastReported: Triple<List<WalletWire.ChannelView>, List<WalletWire.PaymentView>, List<GroupSweep.Input>>? = null
private var lastReportedAt: kotlin.time.Instant? = null
@@ -174,7 +187,7 @@ class GroupLightningNode(
this.transport = transport
peer = handle
_state.value = State.Running(term)
handle.connect()
stayConnected(handle, runCatching { handle.connect() })
lastReported = null
report(term, force = true)
watchForHandover(term)
@@ -192,6 +205,9 @@ class GroupLightningNode(
private suspend fun halt() {
val running = _state.value as? State.Running ?: return
val numbers = peer?.commitmentNumbers().orEmpty()
// First, or the disconnect below would read as a drop and be redialled.
connecting?.cancel()
connecting = null
peer?.disconnect()
transport?.stop()
peer = null
@@ -275,6 +291,38 @@ class GroupLightningNode(
}
}
/**
* Redials the LSP each time the connection ends, logging why it ended, until the node stops. lightning-kmp
* does not reconnect by itself, and the identity's own node has Phoenix's connection daemon to do it; this
* one had nothing, so a single drop left it offline for the rest of the term.
*
* A connection that ends inside [STABLE_CONNECTION] counts against the backoff like one that never came
* up: an LSP that completes the handshake and hangs up on reading our features would otherwise be redialled
* every few seconds for as long as the term lasts.
*/
private fun stayConnected(handle: PeerHandle, first: Result<Boolean>) {
connecting?.cancel()
connecting = scope.launch {
var attempt = first
var since = now()
var failures = 0
while (isActive) {
val reason = attempt.fold({ handle.awaitDisconnect() }, { it.message ?: it::class.simpleName.orEmpty() })
val connected = attempt.getOrDefault(false)
val lasted = now() - since
failures = if (connected && lasted >= STABLE_CONNECTION) 1 else failures + 1
val wait = reconnectAfter(failures)
logger.w(
if (connected) "The group's node lost the LSP after $lasted: $reason. Reconnecting in $wait"
else "The group's node could not reach the LSP: $reason. Trying again in $wait"
)
delay(wait)
since = now()
attempt = runCatching { handle.connect() }
}
}
}
/** Stops the node as soon as the room's newest operator statement names a later term. */
private fun watchForHandover(term: Long) {
watching?.cancel()
@@ -325,6 +373,34 @@ class GroupLightningNode(
/** How many of the newest payments a snapshot carries; members keep every one they have seen. */
const val REPORTED_PAYMENTS = 50
/** How long a connection to the LSP has to last to reset the backoff. */
val STABLE_CONNECTION: Duration = 1.minutes
val RECONNECT_FIRST: Duration = 2.seconds
val RECONNECT_MAX: Duration = 5.minutes
/** How long a drop's reason waits for the LSP's init, which can reach the node after the socket closes. */
val INIT_GRACE: Duration = 1.seconds
/** [RECONNECT_FIRST], doubling with each attempt in a row that did not hold, up to [RECONNECT_MAX]. */
fun reconnectDelay(failures: Int): Duration =
if (failures >= 20) RECONNECT_MAX else (RECONNECT_FIRST * (1 shl (failures - 1).coerceAtLeast(0))).coerceAtMost(RECONNECT_MAX)
/**
* Why a connection to the LSP ended, from the socket's reason and the features each side sent. The
* features come first when they explain it: an LSP that does not offer a feature we require hangs up
* after the Init exchange, and the socket only says it was closed.
*/
internal fun dropReason(ours: Features, theirs: Features?, closed: Connection.CLOSED): String {
val socket = closed.reason?.message ?: "closed by this node"
if (theirs == null) return "$socket, before the LSP sent its features"
val missing = ours.activated.filter { (feature, support) -> support == FeatureSupport.Mandatory && !theirs.hasFeature(feature) }.keys.map { it.rfcName }
val unknownToUs = theirs.unknown.filter { it.bitIndex % 2 == 0 }.map { "bit ${it.bitIndex}" }
return when {
missing.isNotEmpty() -> "$socket: the LSP does not offer ${missing.joinToString()}, which this node requires"
unknownToUs.isNotEmpty() -> "$socket: the LSP requires ${unknownToUs.joinToString()}, which this node does not know"
else -> socket
}
}
/**
* Builds real lightning-kmp nodes: simple taproot channels only, peer storage on, zero-conf with the
* LSP alone, on the chain's Electrum connection, persisted in the databases [databasesFor] opens for
@@ -379,9 +455,12 @@ class GroupLightningNode(
}
}
val payments = mutableListOf<WalletWire.PaymentView>()
// The LSP's features as it sent them on the current connection, for saying why one ended.
val lspInit = MutableStateFlow<Init?>(null)
scope.launch {
nodeParams.nodeEvents.collect { event ->
when (event) {
is PeerConnected -> lspInit.value = event.theirInit
is PaymentEvents.PaymentReceived -> payments += paymentView(event.payment, incoming = true)
is PaymentEvents.PaymentSent -> payments += paymentView(event.payment, incoming = false)
else -> {}
@@ -389,8 +468,18 @@ class GroupLightningNode(
}
}
object : PeerHandle {
override suspend fun connect(): Boolean = peer.connect(connectTimeout = 10.seconds, handshakeTimeout = 10.seconds)
override suspend fun connect(): Boolean {
lspInit.value = null
return peer.connect(connectTimeout = 10.seconds, handshakeTimeout = 10.seconds)
}
override fun disconnect() = peer.disconnect()
override suspend fun awaitDisconnect(): String {
val closed = peer.connectionState.first { it is Connection.CLOSED } as Connection.CLOSED
// An LSP that hangs up on our features does so milliseconds after its own init, and the
// socket reads as closed before the peer's event for that init has reached us: wait for it.
val theirs = withTimeoutOrNull(INIT_GRACE) { lspInit.filterNotNull().first() }
return dropReason(nodeParams.features.initFeatures(), theirs?.features, closed)
}
override fun commitmentNumbers(): Map<String, Long> = peer.channelsFlow.value.mapNotNull { (id, state) ->
(state as? ChannelStateWithCommitments)?.let { id.toHex() to it.commitments.localCommitIndex }
}.toMap()
@@ -12,6 +12,7 @@ import fr.acinq.lightning.Lightning.randomBytes32
import fr.acinq.lightning.MilliSatoshi
import fr.acinq.lightning.payment.Bolt11Invoice
import fr.acinq.lightning.payment.OfferManager
import kotlinx.coroutines.channels.Channel
/**
* A stand-in for the group's lightning-kmp node: no LSP to connect to, channels and payments as a test
@@ -21,6 +22,10 @@ import fr.acinq.lightning.payment.OfferManager
class FakeGroupNode(private val configuration: GroupLightningNode.NodeConfiguration) : GroupLightningNode.PeerHandle {
var connected = false
var disconnected = false
/** How many times the node has dialled the LSP. */
var connects = 0
/** A reason sent here ends the current connection, as the LSP hanging up would, with that reason. */
val drops = Channel<String>(Channel.UNLIMITED)
var channels = listOf<WalletWire.ChannelView>()
val payments = mutableListOf<WalletWire.PaymentView>()
val invoices = mutableListOf<Bolt11Invoice>()
@@ -32,8 +37,9 @@ class FakeGroupNode(private val configuration: GroupLightningNode.NodeConfigurat
val forceClosed = mutableListOf<String>()
var heldOutputs = listOf<GroupSweep.Input>()
override suspend fun connect(): Boolean { connected = true; return true }
override suspend fun connect(): Boolean { connected = true; connects++; return true }
override fun disconnect() { disconnected = true }
override suspend fun awaitDisconnect(): String = drops.receive().also { connected = false }
override fun commitmentNumbers(): Map<String, Long> = mapOf("ab".repeat(32) to 7L)
override fun channels(): List<WalletWire.ChannelView> = channels
override fun payments(): List<WalletWire.PaymentView> = payments.toList()
@@ -8,7 +8,13 @@ import com.machankura.bennu.compose.wallet.lightning.GroupLightningNodes
import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair
import com.vitorpamplona.quartz.nip01Core.core.toHexKey
import fr.acinq.bitcoin.PublicKey
import fr.acinq.lightning.Feature
import fr.acinq.lightning.FeatureSupport
import fr.acinq.lightning.Features
import fr.acinq.lightning.Lightning.randomBytes32
import fr.acinq.lightning.UnknownFeature
import fr.acinq.lightning.io.TcpSocket
import fr.acinq.lightning.utils.Connection
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.SupervisorJob
@@ -21,6 +27,7 @@ import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertIs
import kotlin.test.assertTrue
import kotlin.time.Duration
import kotlin.time.Clock
import kotlin.time.Duration.Companion.milliseconds
import kotlin.time.Duration.Companion.minutes
@@ -73,12 +80,22 @@ class GroupLightningNodeJvmTest {
)
}
private suspend fun node(device: GroupLightningFixture.Device) = GroupLightningNode(
private suspend fun node(device: GroupLightningFixture.Device, reconnectAfter: (Int) -> Duration = { 10.milliseconds }) = GroupLightningNode(
device.db, device.room, device.publicKey, GroupLightningWalletManager.walletFor(device.db, fixture.roomId, fixture.chain)!!, scope, factory,
handoverCheck = 100.milliseconds,
minimumPool = 0,
reconnectAfter = reconnectAfter,
)
private suspend fun runningNode(reconnectAfter: (Int) -> Duration = { 10.milliseconds }): Pair<GroupLightningNode, FakeGroupNode> {
val all = ready()
elect(all, all[1])
heardFromRoom(all[1])
val node = node(all[1], reconnectAfter)
assertIs<GroupLightningNode.State.Running>(node.start())
return node to peers.single()
}
@Test
fun `the node runs on the group's keys`() = runBlocking<Unit> {
val all = ready()
@@ -156,4 +173,64 @@ class GroupLightningNodeJvmTest {
assertIs<GroupLightningNode.State.Refused>(node.start())
assertTrue(peers.isEmpty(), "it never connected")
}
@Test
fun `a node the LSP drops dials it again`() = runBlocking<Unit> {
val (node, peer) = runningNode()
assertEquals(1, peer.connects)
peer.drops.send("Connection closed")
withTimeout(5.seconds) { while (peer.connects < 2) delay(10) }
peer.drops.send("Connection closed")
withTimeout(5.seconds) { while (peer.connects < 3) delay(10) }
assertIs<GroupLightningNode.State.Running>(node.state.value)
node.stop()
}
@Test
fun `connections that drop at once back off further each time`() = runBlocking<Unit> {
val failures = mutableListOf<Int>()
val (node, peer) = runningNode(reconnectAfter = { failures += it; 10.milliseconds })
repeat(3) { i ->
peer.drops.send("Connection closed")
withTimeout(5.seconds) { while (peer.connects < i + 2) delay(10) }
}
assertEquals(listOf(1, 2, 3), failures)
node.stop()
}
@Test
fun `a stopped node does not dial again`() = runBlocking<Unit> {
val (node, peer) = runningNode()
node.stop()
peer.drops.send("Connection closed")
delay(200)
assertEquals(1, peer.connects)
assertTrue(peer.disconnected)
}
@Test
fun `the backoff doubles from two seconds to five minutes`() {
assertEquals(
listOf(2.seconds, 4.seconds, 8.seconds, 16.seconds, 32.seconds, 64.seconds, 128.seconds, 256.seconds, 5.minutes, 5.minutes),
(1..10).map(GroupLightningNode::reconnectDelay),
)
assertEquals(5.minutes, GroupLightningNode.reconnectDelay(1_000))
}
@Test
fun `a drop names the feature the LSP would not offer`() {
val ours = Features(Feature.SimpleTaprootChannels to FeatureSupport.Mandatory, Feature.VariableLengthOnion to FeatureSupport.Mandatory)
val closed = Connection.CLOSED(TcpSocket.IOException.ConnectionClosed())
assertEquals(
"Connection closed: the LSP does not offer simple_taproot_channels, which this node requires",
GroupLightningNode.dropReason(ours, Features(Feature.VariableLengthOnion to FeatureSupport.Mandatory), closed),
)
assertEquals(
"Connection closed: the LSP requires bit 600, which this node does not know",
GroupLightningNode.dropReason(ours, Features(ours.activated, setOf(UnknownFeature(600), UnknownFeature(601))), closed),
)
assertEquals("Connection closed", GroupLightningNode.dropReason(ours, ours, closed))
assertEquals("Connection closed, before the LSP sent its features", GroupLightningNode.dropReason(ours, null, closed))
assertEquals("closed by this node", GroupLightningNode.dropReason(ours, ours, Connection.CLOSED(null)))
}
}