diff --git a/composeApp/src/commonMain/kotlin/mobi/giza/compose/managers/MarmotEngine.kt b/composeApp/src/commonMain/kotlin/mobi/giza/compose/managers/MarmotEngine.kt index 63be1e58..2aa23ed6 100644 --- a/composeApp/src/commonMain/kotlin/mobi/giza/compose/managers/MarmotEngine.kt +++ b/composeApp/src/commonMain/kotlin/mobi/giza/compose/managers/MarmotEngine.kt @@ -55,9 +55,11 @@ import mobi.giza.compose.exceptions.MarmotConvergencePendingException * should not be read as closed. A late application message decrypted under a * retained prior epoch is decrypted without the sender's leaf signature being * verified -- the library's `tryDecryptWithRetainedEpoch` has the same gap the - * app's had. And the secret tree's skipped keys do not survive `saveState` - * (docs/mls-skipped-keys.md); the manager holding the live group covers that - * within a process, exactly as `MlsGroupCache` did, and no further. + * app's had. And the secret tree's skipped keys reach the state the manager would + * save -- mls-kmp carries them, which quartz's engine did not -- but an application + * message is decrypted without anything being written, so they still do not reach + * the row (docs/mls-skipped-keys.md). The manager holding the live group covers + * that within a process, exactly as `MlsGroupCache` did, and no further. */ class MarmotEngine( private val database: GizaDatabase, diff --git a/composeApp/src/jvmTest/kotlin/mobi/giza/compose/managers/SkippedGenerationJvmTest.kt b/composeApp/src/jvmTest/kotlin/mobi/giza/compose/managers/SkippedGenerationJvmTest.kt new file mode 100644 index 00000000..24109d89 --- /dev/null +++ b/composeApp/src/jvmTest/kotlin/mobi/giza/compose/managers/SkippedGenerationJvmTest.kt @@ -0,0 +1,189 @@ +package mobi.giza.compose.managers + +import androidx.room3.Room +import com.vitorpamplona.marmot.foundation.appEvents.MarmotAppEvent +import com.vitorpamplona.marmot.groups.MarmotGroupPolicy +import com.vitorpamplona.marmot.mip03GroupMessages.GroupEventEncryption +import com.vitorpamplona.mls.group.MlsGroup +import com.vitorpamplona.mls.group.MlsGroupState +import com.vitorpamplona.quartz.marmot.mip03GroupMessages.GroupEvent +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.hexToByteArray +import com.vitorpamplona.quartz.nip01Core.core.toHexKey +import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair +import com.vitorpamplona.quartz.nipC7Chats.ChatEvent +import com.vitorpamplona.quartz.utils.TimeUtils +import kotlin.test.AfterTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertTrue +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.runBlocking +import mobi.giza.compose.database.GizaDatabase +import mobi.giza.compose.database.builder.getRoomDatabase +import mobi.giza.compose.database.repository.DatabaseMarmotRepository +import mobi.giza.compose.extensions.exporterSecret + +/** + * The bug docs/mls-skipped-keys.md is about: half closed by the move, and the other half + * pinned here, because it is no longer the same half. + * + * RFC 9420 §9.1 says a receiver handed generation N+1 before N derives the intermediate + * keys and keeps them, so the older message still opens when it turns up. quartz's engine + * derived and cached them in a `SecretTree` field that `saveState` never wrote, so two + * messages published in the same second reliably lost one across a restart -- silently: + * the kind:445 row was stored and no inner event came out of it. + * + * **What the move fixed.** mls-kmp's state format carries them. `MlsGroupState` has + * `skippedApplicationSecrets` and `skippedHandshakeSecrets` from its STATE_VERSION 5, + * `saveState` exports them from the secret tree and `restore` imports them back. The + * engine-level gap the note describes is gone, and the first two tests here are that. + * + * **What it did not fix, and this is the finding.** `MlsGroupManager.decrypt` persists + * only when the message was a COMMIT that advanced the epoch; an application message is + * decrypted and nothing is written. So the secret reaches the live group's state and never + * reaches the blob on disk, and a restart between two messages still loses the late one -- + * for a different reason than before. Not "the cache is never written" but "the state is + * not saved after an application message", which is a decision about write amplification + * rather than an oversight: persisting per inbound message re-encodes the ratchet tree, + * the group context and every epoch secret, per message, growing with the group. The same + * trade-off `MlsGroupStateStore.saveSenderRatchet` exists to solve on the send side. + * + * The third test pins that, in the style of `ConvergenceForkJvmTest`'s authored fork: it + * asserts the gap, and says in its message what to do the day it stops holding. + * + * See docs/marmot-kmp.md, Phase 5. + */ +class SkippedGenerationJvmTest { + + private val db: GizaDatabase = getRoomDatabase( + Room.inMemoryDatabaseBuilder() + ) + private val marmotRepository = DatabaseMarmotRepository(db, CoroutineScope(Dispatchers.Default)) + + @AfterTest + fun closeDb() = db.close() + + private val user = KeyPair() + private val userHex = user.pubKey.toHexKey() + private val relay = "wss://relay.example" + + private val amethyst = MarmotStandIn() + + private suspend fun joinedRoom(): String { + MarmotStandIn.seedProfile(db, userHex, "us") + return amethyst.inviteApp(db, marmotRepository, user, amethyst.createGroup("skipped")) + } + + private suspend fun store(event: Event) = MarmotStandIn.storeInApp(db, user, event, relay) + + /** One message from the stand-in, which advances its sender ratchet by one generation. */ + private suspend fun theirMessage(routingHex: String, text: String): GroupEvent = + amethyst.outbound.buildAppEvent( + routingHex, + MarmotAppEvent.build(pubKey = amethyst.hex, kind = ChatEvent.KIND, content = text, createdAt = TimeUtils.now()), + ).signedEvent + + /** The state of the group the engine holds right now, as the manager would save it. */ + private suspend fun liveState(roomId: String): MlsGroupState = + assertNotNull(db.marmotEngine.groupManager.snapshot(roomId), "the engine holds no group") + + /** The state actually written to the room's row. */ + private suspend fun storedState(roomId: String): MlsGroupState { + val room = assertNotNull(db.chatRoomDao().findChatRoomById(roomId)) + return MlsGroupState.decodeTls(assertNotNull(room.chatRoom.mlsGroupState).hexToByteArray()) + } + + /** + * Receiving generation 1 before generation 0 leaves generation 0's secret in the state + * the engine would save. Under the old engine `saveState` dropped it on the floor; + * mls-kmp's STATE_VERSION 5 is where it started being carried. + */ + @Test + fun `a skipped generation's secret is in the state the engine would save`() = runBlocking { + val roomId = joinedRoom() + theirMessage(roomId, "generation 0") + val second = theirMessage(roomId, "generation 1") + + // The manager holds a group once it has been asked for one; the engine adopts it + // from the row on first use, which is what every app path does before it reads. + assertNotNull(db.marmotEngine.groupFor(roomId), "the engine cannot restore the room") + assertTrue( + liveState(roomId).skippedApplicationSecrets.isEmpty(), + "a group that has received nothing already has skipped secrets", + ) + + // The later one arrives first; the receiver derives generation 0 on the way to 1. + store(second) + + assertEquals( + 1, + liveState(roomId).skippedApplicationSecrets.size, + "the skipped generation's secret is not in the state, which is the bug " + + "docs/mls-skipped-keys.md describes", + ) + } + + /** + * And it survives the encode: a group rebuilt from those bytes -- the whole round trip + * through the blob, which is what a restart is -- reads the message that arrived late. + * + * This is the engine-level claim, made against the state the engine holds rather than + * the state the app wrote, because those are not the same thing yet. See the test + * below. + */ + @Test + fun `the late message decrypts under a group rebuilt from those bytes`() = runBlocking { + val roomId = joinedRoom() + val first = theirMessage(roomId, "generation 0") + val second = theirMessage(roomId, "generation 1") + + store(second) + + val rebuilt = MlsGroup.restore(MlsGroupState.decodeTls(liveState(roomId).encodeTls()), MarmotGroupPolicy) + val mlsBytes = GroupEventEncryption.decrypt(first.content, rebuilt.exporterSecret()) + val decrypted = rebuilt.decrypt(mlsBytes) + + assertTrue( + decrypted.content.decodeToString().contains("generation 0"), + "the late message did not decrypt from a rebuilt group", + ) + } + + /** + * The half the move did not close, pinned. + * + * `MlsGroupManager.decrypt` writes nothing for an application message -- only a COMMIT + * that advanced the epoch is persisted -- so the secret above never reaches + * `ChatRoom.mlsGroupState`, and a process that dies between the two messages still + * loses the late one. + * + * The day this assertion fails, the manager has started persisting after a receive + * (or the app has), docs/mls-skipped-keys.md can drop its mitigation section, and this + * test becomes the other way round: assert the stored state carries the secret, and + * add the restart that `MarmotEngine(db)` makes -- which is NOT asserted here on + * purpose, because a rebuilt engine reads generation 0 happily today for the wrong + * reason. It restores a blob whose sender ratchet never moved, so generation 0 is + * simply the next one it expects, and the test would pass while testing nothing. + */ + @Test + fun `the stored state does not yet carry it, so a restart still loses the message`() = runBlocking { + val roomId = joinedRoom() + theirMessage(roomId, "generation 0") + val second = theirMessage(roomId, "generation 1") + + store(second) + + assertTrue( + liveState(roomId).skippedApplicationSecrets.isNotEmpty(), + "the live group lost the secret too, which is a worse bug than this one", + ) + assertTrue( + storedState(roomId).skippedApplicationSecrets.isEmpty(), + "the stored state carries the skipped secret -- the receive path now persists; " + + "invert this test and see its KDoc", + ) + } +} diff --git a/docs/marmot-kmp.md b/docs/marmot-kmp.md index a43d563d..4b8e4396 100644 --- a/docs/marmot-kmp.md +++ b/docs/marmot-kmp.md @@ -451,6 +451,47 @@ leave its mitigation section as the history of how it was lived with. **Check:** the new test, and `MarmotEngine`'s own doc comment, whose last paragraph currently names this gap as open. +**Built, and half of it is closed — not the half the plan assumed.** The engine-level gap +is gone: `MlsGroupState` carries `skippedApplicationSecrets` and +`skippedHandshakeSecrets` from STATE_VERSION 5, `saveState` exports them from the secret +tree and `restore` imports them, and because they arrived at version 5 while the decoder +accepts 1..10, a version-4 blob reads them as an empty map. That is steps 1 to 3 of the +fix [mls-skipped-keys.md](./mls-skipped-keys.md) proposed, in the library the app now +depends on. Step 4 is not: `consumedGenerations`, the guard against a replayed message +re-using a cached key, is still one `SecretTree` instance's field and is in no version of +the state. + +**The app does not benefit yet, and the first draft of this test said it did.** +`MlsGroupManager.decrypt` persists only when the message was a commit that advanced the +epoch; an application message is decrypted and nothing is written. So the secret reaches +the live group's state and never reaches `ChatRoom.mlsGroupState`. + +That matters for how the test had to be written, and the first attempt is worth recording +because it passed. It delivered generation 1, rebuilt the engine with `MarmotEngine(db)`, +delivered generation 0 and asserted it decrypted — and it did, for the wrong reason +entirely. The rebuilt engine restores a blob whose sender ratchet never moved, so +generation 0 is simply the next generation it expects and no skipped key is involved. A +green test proving nothing, and what caught it was the one assertion that looked redundant +beside it: that the secret is in the stored state. It was not. + +So `SkippedGenerationJvmTest` is three tests that say what is true: + +| | | +|---|---| +| the secret is in the state the engine *would* save | `MlsGroupManager.snapshot` after generation 1 arrives first — exactly one skipped secret, where the old engine had none | +| a group rebuilt from those bytes reads the late message | the full round trip through `encodeTls`/`decodeTls`/`restore`, which is what a restart is | +| the *stored* state does not carry it | pinned, in the style of `ConvergenceForkJvmTest`'s authored fork, with the KDoc saying what to invert the day the receive path starts persisting | + +The fix for the third is one call — the app owns the store, so +`store.save(routingId, groupManager.snapshot(routingId)!!.encodeTls())` after an +application message would do it — and it is deliberately not made here, because it is a +write per inbound message that re-encodes the ratchet tree, the group context and every +epoch secret. `MlsGroupStateStore.saveSenderRatchet` is the library's answer to exactly +that cost on the send side; the receive side wants the same shape, and that is Phase 7's +business rather than this one's. [mls-skipped-keys.md](./mls-skipped-keys.md) has a new +last section saying all of this, and its mitigation section stands word for word with a +new cause. + ## Phase 6 — the authored fork marmot-v2.md ends on a gap that is the library's and not the app's: a member cannot replay @@ -470,6 +511,37 @@ fix. **Check:** `ConvergenceForkJvmTest`, and whichever way it goes, this note and marmot-v2.md's closing paragraph both say so. +**Probed: the gap is still open, and the test still pins the swap.** Both fork tests pass +unchanged — the one that proves two observers of somebody else's fork settle on the same +state byte for byte, and the one whose last assertion is + +```kotlin +assertNotEquals(ours.contextBytes().toHexKey(), theirsAfter.contextBytes().toHexKey(), + "the two sides agree -- the library has been fixed; rewrite this test as the agreement") +``` + +That message is the probe, and it did not fire. So marmot-kmp 2.2.1 scores a fork this +device is half of the same way quartz 1.15.2 did: `MlsCandidateStateEngine` rebuilds each +candidate branch with `MlsGroup.processFramedCommit`, a member cannot replay a commit it +authored, and two committers swap branches rather than agree. Reading says the same thing +the test does — the suggestive line in `MlsCandidateStateEngine` about holding "an inbound +commit and one we authored to one rule rather than two" is about *authorization*, not +replay. + +What did change is that the fix marmot-v2.md predicts is now visibly available. Its +closing note says "the state the missing replay would produce is already retained, and +using it is the fix", and mls-kmp has made exactly that state a first-class thing: +`stageCommit`/`mergePendingCommit`, with the staged commit persisted in `MlsGroupState` +from its STATE_VERSION 9. The app already holds the same state twice over, as +`MarmotPublishObligation.pendingState`. So the fix is a change to the candidate engine's +choice of how to get a branch's state — replay it, or take the one we kept — and it +belongs in marmot-kmp rather than here. + +Unchanged, therefore: the practical exposure is small, because a device that never applies +before publishing rarely authors one side of a fork, and the Phase 5 follow-up in +marmot-v2.md (marking a losing branch's rows withdrawn) still waits on the same fix. No +code in this phase. + ## Phase 7 — what the library now offers, and the guard Four things, each its own commit, none of them required by the move: diff --git a/docs/marmot-v2.md b/docs/marmot-v2.md index 52b56848..abde82d4 100644 --- a/docs/marmot-v2.md +++ b/docs/marmot-v2.md @@ -65,6 +65,13 @@ the fix. Until then the practical exposure is small -- a device that never appli before publishing rarely authors one side of a fork -- and the follow-up in Phase 5 (marking a losing branch's rows withdrawn) waits on the same fix. +*Still true of marmot-kmp 2.2.1, which is where that engine lives now +([marmot-kmp.md](./marmot-kmp.md), Phase 6): both fork tests pass unchanged, so the swap +is still the swap. What has changed is that the state the missing replay would produce is +a first-class thing on the library's side too -- `MlsGroup.stageCommit` with the staged +commit persisted in `MlsGroupState` -- so the fix is now a choice in +`MlsCandidateStateEngine` between replaying a branch and taking the state that was kept.* + Written against the code as it stood at `2bbe1fc9`. ## Where "v2" comes from, and what it asks diff --git a/docs/mls-skipped-keys.md b/docs/mls-skipped-keys.md index e59c3741..ae0f4188 100644 --- a/docs/mls-skipped-keys.md +++ b/docs/mls-skipped-keys.md @@ -7,6 +7,13 @@ This is a conformance gap in quartz's MLS implementation, not in this app. What this app can do about it from outside the library is partial, and is described at the end. +**Half of it is closed.** The engine left quartz for mls-kmp +([marmot-kmp.md](./marmot-kmp.md)), and mls-kmp's state format carries the skipped +keys -- steps 1 to 3 of "The fix, in quartz" below, landed in the library the app now +depends on. What is left is not the same gap, and the last section says what it is. +Everything between here and there is the account as it stood, and is kept because it +is still why any of it matters. + ## Symptom The receiver stores the kind:445 group event and produces nothing from it. No @@ -215,3 +222,41 @@ work for any other flow in the table above that someone can be asked to repeat. The staleness check is what keeps the cache from being *worse* than no cache: a group that has been overtaken by another writer is never carried on with, so the fallback is always the old behaviour rather than a diverged ratchet. + +## What the move to marmot-kmp changed + +The engine is not quartz's any more. quartz 2.3.0 dropped it, and the RFC 9420 half +is [mls-kmp](https://github.com/vitorpamplona/mls-kmp) now +([marmot-kmp.md](./marmot-kmp.md)). Three of the four steps above are in it: + +| the fix above | mls-kmp 1.3.0 | +|---|---| +| 1. export and import the skipped keys | `SecretTree.exportSkippedApplicationSecrets` / `importSkippedSecrets`, and the same pair for the handshake ratchet | +| 2. put them in the group state | `MlsGroupState.skippedApplicationSecrets` and `.skippedHandshakeSecrets`, written by `saveState` and read by `restore` | +| 3. keep old state readable | they arrived at `STATE_VERSION` 5 and the decoder accepts 1 to 10, so a version-4 blob -- which is what quartz 1.15.2 wrote, and what every device holds -- reads as an empty map, exactly as this note asked | +| 4. the consumed-generation replay guard travels with them | **not done.** `consumedGenerations` is still a field of one `SecretTree` instance, in no version of the state, so the guard against a replayed message re-using a cached key still weakens across a restart | + +So the test this note asked for -- "save and restore a group between the two messages +of an out-of-order pair, and assert the older one still decrypts" -- now passes, and it +is `SkippedGenerationJvmTest`. + +**What is still broken, for a different reason.** `MlsGroupManager.decrypt` persists +only when the message was a commit that advanced the epoch. An application message is +decrypted and nothing is written, so the secret reaches the live group's state and never +reaches `ChatRoom.mlsGroupState` -- and a process that dies between the two messages +still loses the late one. The mitigation section above therefore still stands, word for +word, including its three bullets; only its first sentence has a new cause. It is no +longer "the cache is never written" but "the state is not saved after a receive", which +is a decision about write amplification rather than an oversight: persisting per inbound +message re-encodes the ratchet tree, the group context and every epoch secret, per +message, growing with the group. `MlsGroupStateStore.saveSenderRatchet` is the shape of +the answer on the send side, and the receive side wants the same treatment. + +`SkippedGenerationJvmTest` pins all of it: the secret is in the state the engine would +save, a group rebuilt from those bytes reads the late message, and the stored state does +not carry it yet. Its last test says what to do the day that changes. + +One thing this does not change: a device that loses a message this way can still be sent +a new one, which is encrypted at a generation it has not passed. That is still the +recovery, and [frost-session-resend.md](./frost-session-resend.md) is still where it +is built.