diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/archive/ArchiveEvent.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/archive/ArchiveEvent.kt new file mode 100644 index 00000000..08a731c2 --- /dev/null +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/archive/ArchiveEvent.kt @@ -0,0 +1,287 @@ +package press.mantra.compose.nostr.archive + +import androidx.compose.runtime.Immutable +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.core.Kind +import com.vitorpamplona.quartz.nip01Core.core.TagArrayBuilder +import com.vitorpamplona.quartz.nip01Core.signers.EventTemplate +import com.vitorpamplona.quartz.nip01Core.signers.eventTemplate +import com.vitorpamplona.quartz.nip31Alts.alt +import com.vitorpamplona.quartz.utils.TimeUtils +import kotlinx.serialization.json.jsonArray +import press.mantra.compose.network.serialization.CommonJson +import press.mantra.compose.nostr.archive.tags.ArchiveIdTag +import press.mantra.compose.nostr.archive.tags.ArchivePageTag +import press.mantra.compose.nostr.archive.tags.ArchiveRecipientTag +import press.mantra.compose.nostr.nip30303.ArtifactEvent +import press.mantra.compose.nostr.nip30303.ArtifactVersionEvent +import press.mantra.compose.nostr.nip30303.ChapterEvent +import press.mantra.compose.nostr.nip30303.ChunkEvent +import press.mantra.compose.nostr.nip30303.DialectEvent +import press.mantra.compose.nostr.nip30303.TranslationArtifactVersionEvent +import press.mantra.compose.nostr.nip30303.TranslationChapterEvent +import press.mantra.compose.nostr.nip30303.TranslationChunkEvent +import press.mantra.compose.nostr.nip30303.TranslationEvent + +/** + * 30327 + * + * A page of the group's signed record, sent to a member who does not have it. + * + * A member added after the work was done sees none of it and never will. MLS + * gives them no history -- correctly, that is forward secrecy -- but the sharper + * problem is that group-signed events never travel at all: `FrostSigningManager` + * applies a finished signature locally on every device that took part, because + * the outbound pipeline would re-author a rumor as its sender and strip the + * group's signature off. A member who was not in the session has nothing to + * derive the event from, and no later message carries it. This is that path. + * + * Each payload travels whole, keeping its own id, author and signature, so the + * receiver checks it rather than believing it -- see [isSignedByRoom] on + * `GroupKeyStateEvent`. The sender of an archive is therefore not trusted, which + * is what lets any member answer. + * + * See docs/member-archive.md for the whole design, including the two things this + * deliberately does not do: it never carries the chat, and it cannot make its + * recipient able to *sign* anything. + * + * ### Why not a [press.mantra.compose.nostr.nip30303.SubmissionEvent] each + * + * The envelope fits and the meaning does not. A submission is an *act* -- this + * member is putting this event in front of this group -- and an archive asserts + * nothing; it re-delivers what the group already agreed. On one kind a + * four-hundred-event backfill is indistinguishable from four hundred new + * submissions and every device has to guess which it is looking at. It would + * also be one inner event and one kind:445 per payload, where a page is one, and + * the submission arm of `ChatMessage.applyInnerEvent` files a chat line per + * payload, which an archive must not. + */ +@Immutable +class ArchiveEvent( + id: HexKey, + pubKey: HexKey, + createdAt: Long, + tags: Array>, + content: String, + sig: HexKey, +) : Event(id, pubKey, createdAt, KIND, tags, content, sig) { + + /** + * The signed events this page carries, or null when the content is not a + * readable page. + * + * Nothing here is verified yet. Parsing says the framing is intact; + * `ArchiveManager` is what asks whether the group signed each one, and it + * asks per payload rather than per page. + */ + fun payloads(): List? = decodePage(content) + + /** Which archive this page belongs to, or null if it names none. */ + fun archiveId(): String? = tags.firstNotNullOfOrNull(ArchiveIdTag::parse)?.archiveId + + /** Where this page sits in that archive, or null if it does not say. */ + fun page(): ArchivePageTag? = tags.firstNotNullOfOrNull(ArchivePageTag::parse) + + /** Who should act on this page. A hint -- see [ArchiveRecipientTag]. */ + fun recipient(): HexKey? = tags.firstNotNullOfOrNull(ArchiveRecipientTag::parse)?.pubKey + + companion object { + /** + * Sits past `GroupKeyStateEvent` at 30326, in the Marmot inner-event + * space it shares with the nip30303 document kinds and the FROST signing + * family. + * + * 30313 was free next to the document kinds and is not used, on + * `FrostSigningEvents`' own advice: the DKG's 30310-30316 already overlap + * that range and are told apart only by living in NIP-17 gift wraps + * instead, which is "an accident of routing rather than a decision, and + * the next family added should not rely on it." This is that next family. + * + * It is also the right neighbourhood on the merits. An archive is not a + * document kind; it is a statement about the record, which is what a key + * state is too. + */ + const val KIND: Kind = 30327 + + const val ALT_DESCRIPTION = "Group archive" + + /** + * The most JSON one page may carry. + * + * A byte cap rather than a count alone, because the payloads vary by two + * orders of magnitude -- a chunk is a paragraph, an artifact is a URL. + * Sized so the finished kind:445 clears the tightest relay limits with + * room for the MLS framing and the NIP-44 expansion that sit outside it; + * measure it against a real one rather than trusting the arithmetic. + */ + const val MAX_PAGE_BYTES = 64 * 1024 + + /** + * The most payloads one page may carry. + * + * The byte cap bounds the transport; this bounds the receiver's *work*, + * and they are not the same limit -- 64 KB of one-line dialects is a lot + * more signature verifies than 64 KB of chapters. + * + * **Sized under what the byte cap allows, or it would never fire.** An + * event carries a 64-character id, a 64-character pubkey and a + * 128-character signature before it says anything at all, so the floor is + * around 370 bytes and [MAX_PAGE_BYTES] cannot hold much past 170 of + * them. A count cap above that ceiling is a check that cannot fail -- + * which is how this was first written, and a test that asserts a page at + * exactly the cap decodes is what caught it. At 128 both caps bind + * something: the count stops a page of many small payloads, the bytes + * stop a page of few large ones. + * + * The two therefore have to be re-sized together. Raising [MAX_PAGE_BYTES] + * without raising this narrows the count cap's job; raising this without + * raising that retires it. + * + * Larger than `FrostSigningManager.MAX_BATCH_SIZE`, deliberately: a batch + * item costs nonce generation, a FROST session and a native sign, where + * an archive payload costs a signature verify and an upsert. + */ + const val MAX_PAGE_EVENTS = 128 + + /** + * The kinds an archive may carry, in the order they have to be applied. + * + * One list doing both jobs, because a separate allowlist is one more + * thing that can disagree with the order it is applied in. + * + * **The order is Room's, not nostr's.** Every one of these has a foreign + * key on the one before it, so an archive applied out of order is a + * constraint violation rather than a wrong answer. Kind order is not + * dependency order and never was -- a translation chapter (30308) hangs + * off a translation artifact version (30306) which hangs off an artifact + * version (30301) -- so this is a list rather than a `sortedBy { kind }`. + * + * **It is an allowlist first.** Verification admits an event to the apply + * path on the strength of the group's signature, which makes every kind + * the group has ever signed replayable by any member at any time. A + * `GroupKeyStateEvent` is group-signed and passes verification perfectly, + * so an archive carrying an old one is a validly signed statement about + * what the room signs with, replayed by whoever kept a copy. Nothing but + * this list stops it. + * + * Absent on purpose: the contributor-list kinds (30305, 30307, 30310). + * `ChatMessage.applyInnerEvent` has no arm that writes a row for any of + * them, so archiving them would cost bytes and restore nothing. They + * belong here on the day that changes, and the test that this list is a + * subset of what `applyInnerEvent` handles is what should catch it. + */ + private val APPLY_ORDER: List = listOf( + DialectEvent.KIND, + ArtifactEvent.KIND, + ArtifactVersionEvent.KIND, + ChapterEvent.KIND, + ChunkEvent.KIND, + TranslationArtifactVersionEvent.KIND, + TranslationChapterEvent.KIND, + TranslationChunkEvent.KIND, + TranslationEvent.KIND, + ) + + /** Every kind an archive may carry. */ + val ARCHIVABLE_KINDS: Set = APPLY_ORDER.toSet() + + /** + * Where [kind] sits in dependency order, or null if an archive may not + * carry it at all. + */ + fun applyRank(kind: Kind): Int? = APPLY_ORDER.indexOf(kind).takeIf { it >= 0 } + + fun isArchivable(kind: Kind): Boolean = kind in ARCHIVABLE_KINDS + + /** + * [payloads] in dependency order, so a page applies front to back. + * + * Stable within a rank: two chapters of one version have no order between + * them and keeping the caller's is one less thing that varies between two + * members assembling the same archive. + */ + fun inApplyOrder(payloads: List): List = + payloads.sortedBy { applyRank(it.kind) ?: Int.MAX_VALUE } + + /** + * One page's payloads as its content: a JSON array of whole events. + * + * Always an array, even for one. `FrostSigningEvents.encodeProposal` has + * a bare-object form only because a build predating batches has to be + * able to read a single proposal, and there is no such build here -- both + * kinds are new. Do not copy that shape. + */ + fun encodePage(events: List): String = + events.joinToString(separator = ",", prefix = "[", postfix = "]") { it.toJson() } + + /** + * The other half, with both caps enforced. + * + * **All-or-nothing, and that is not in tension with dropping forged + * payloads one at a time.** They are different questions. A page that + * will not parse has lost its framing, so nothing in it can be trusted to + * be what the sender wrote -- and a page silently shortened by one + * element would leave a receiver believing it holds a complete archive + * when the page count says so and the contents do not. A payload whose + * signature does not verify is a well-framed page containing one bad + * event, and costing its honest neighbours would let one forged payload + * deny an entire archive. + * + * Both caps are checked here rather than at the call site, because this + * is the boundary a remote party's page crosses. An archive is the second + * place in this protocol where somebody else decides how much work this + * device does; the first grew a cap on the way in for the same reason. + */ + fun decodePage(content: String): List? { + if (content.encodeToByteArray().size > MAX_PAGE_BYTES) return null + if (!content.trimStart().startsWith("[")) return null + + val elements = runCatching { CommonJson.parseToJsonElement(content).jsonArray } + .getOrNull() + ?.takeIf { it.isNotEmpty() && it.size <= MAX_PAGE_EVENTS } + ?: return null + + return elements.map { element -> Event.fromJsonOrNull(element.toString()) ?: return null } + } + + /** + * A page of [payloads], page [index] of [count] in archive [archiveId], + * for [recipient]. + * + * The caller sorts and pages; this only writes what it is handed. Both + * caps are checked anyway, because a page built over either of them is a + * page nobody can read back and it is better to fail where it was made. + */ + fun build( + payloads: List, + archiveId: String, + index: Int, + count: Int, + recipient: HexKey, + createdAt: Long = TimeUtils.now(), + initializer: TagArrayBuilder.() -> Unit = {}, + ): EventTemplate { + require(payloads.isNotEmpty()) { "An archive page carries at least one event" } + require(payloads.size <= MAX_PAGE_EVENTS) { + "An archive page carries at most $MAX_PAGE_EVENTS events, not ${payloads.size}" + } + payloads.forEach { + require(isArchivable(it.kind)) { "An archive may not carry kind ${it.kind}" } + } + + val content = encodePage(payloads) + require(content.encodeToByteArray().size <= MAX_PAGE_BYTES) { + "An archive page carries at most $MAX_PAGE_BYTES bytes" + } + + return eventTemplate(KIND, content, createdAt) { + alt(ALT_DESCRIPTION) + addUnique(ArchiveIdTag.assemble(archiveId)) + addUnique(ArchivePageTag.assemble(index, count)) + addUnique(ArchiveRecipientTag.assemble(recipient)) + initializer() + } + } + } +} diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/archive/ArchiveRequestEvent.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/archive/ArchiveRequestEvent.kt new file mode 100644 index 00000000..89ee815d --- /dev/null +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/archive/ArchiveRequestEvent.kt @@ -0,0 +1,70 @@ +package press.mantra.compose.nostr.archive + +import androidx.compose.runtime.Immutable +import com.vitorpamplona.quartz.nip01Core.core.Event +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.core.Kind +import com.vitorpamplona.quartz.nip01Core.core.TagArrayBuilder +import com.vitorpamplona.quartz.nip01Core.signers.EventTemplate +import com.vitorpamplona.quartz.nip01Core.signers.eventTemplate +import com.vitorpamplona.quartz.nip31Alts.alt +import com.vitorpamplona.quartz.utils.TimeUtils + +/** + * 30328 + * + * "I hold none of this group's work. Send it." + * + * The reliable half of handing a new member the group's history, and the reason + * [ArchiveEvent] is not simply pushed at them when they are invited. + * + * A push from the inviter is an application message in the epoch the add + * created. If it reaches the invitee before their Welcome does -- different + * transports, no ordering guarantee -- `MarmotInboundManager` drops it outright, + * because a message from an epoch ahead of the local one is dropped and not + * deferred. Nothing is retried and the inviter sees a success. That is the + * failure mode described in docs/marmot-membership.md, arriving here in a new + * costume. + * + * A request cannot lose that race, because sending one *is* the proof it was + * won: a device that can put an application message into the room has processed + * its Welcome and is at the group's epoch. + * + * It also covers three cases no invite-time push can reach, all of which look + * identical from inside the database and are therefore answered by one rule: + * + * - a reinstall, whose rows are gone and whose invite is long past; + * - a second device, which was never invited at all; + * - an archive that was sent and lost. + * + * Carries nothing. The room is the envelope, the asker is the MLS sender, and + * what they are missing is "all of it" -- a cursor would have to be a position + * in a dependency graph rather than a timestamp, and is deliberately left out of + * the first version. + */ +@Immutable +class ArchiveRequestEvent( + id: HexKey, + pubKey: HexKey, + createdAt: Long, + tags: Array>, + content: String, + sig: HexKey, +) : Event(id, pubKey, createdAt, KIND, tags, content, sig) { + + companion object { + /** Beside [ArchiveEvent] at 30327 -- see the note on its own kind. */ + const val KIND: Kind = 30328 + + const val ALT_DESCRIPTION = "Group archive request" + + fun build( + createdAt: Long = TimeUtils.now(), + initializer: TagArrayBuilder.() -> Unit = {}, + ): EventTemplate = + eventTemplate(KIND, "", createdAt) { + alt(ALT_DESCRIPTION) + initializer() + } + } +} diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/archive/tags/ArchiveIdTag.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/archive/tags/ArchiveIdTag.kt new file mode 100644 index 00000000..497d3fff --- /dev/null +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/archive/tags/ArchiveIdTag.kt @@ -0,0 +1,40 @@ +package press.mantra.compose.nostr.archive.tags + +import com.vitorpamplona.quartz.nip01Core.core.has +import com.vitorpamplona.quartz.utils.ensure + +/** + * Ties the pages of one archive together. + * + * An archive is more than one event and its pages arrive in no particular + * order, so a receiver has to be able to tell page 2 of this archive from page 2 + * of somebody else's. Two members answering the same request is the ordinary + * case rather than the odd one -- nothing stops them, and nothing should, since + * an incomplete answer is exactly what a second answer fixes -- and without an + * id their pages would interleave into one sequence that is neither. + * + * Fresh random bytes per archive, not derived from anything. Two members + * assembling the same rows must not collide on an id, because their page counts + * will differ whenever their databases do. + */ +class ArchiveIdTag( + val archiveId: String, +) { + fun toTagArray() = assemble(archiveId = archiveId) + + companion object { + const val TAG_NAME = "archiveId" + + fun parse(tag: Array): ArchiveIdTag? { + ensure(tag.has(1)) { return null } + ensure(tag[0] == TAG_NAME) { return null } + ensure(tag[1].isNotBlank()) { return null } + + return ArchiveIdTag(archiveId = tag[1]) + } + + fun assemble(archiveId: String): Array = arrayOf(TAG_NAME, archiveId) + + fun assemble(archiveIdTag: ArchiveIdTag) = assemble(archiveId = archiveIdTag.archiveId) + } +} diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/archive/tags/ArchivePageTag.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/archive/tags/ArchivePageTag.kt new file mode 100644 index 00000000..501c6ccf --- /dev/null +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/archive/tags/ArchivePageTag.kt @@ -0,0 +1,54 @@ +package press.mantra.compose.nostr.archive.tags + +import com.vitorpamplona.quartz.nip01Core.core.has +import com.vitorpamplona.quartz.utils.ensure + +/** + * Where this page sits in its archive, and how many there are. + * + * The count is what lets a receiver say whether it holds a whole archive, which + * is the only question it can answer about completeness on its own. It cannot + * tell whether the *sender* left anything out -- see the note on omission in + * docs/member-archive.md -- so this is a claim about the transfer, not about the + * group's record. + * + * The index is not an ordering instruction. Pages apply in whatever order they + * arrive and the sweep resolves what could not be applied yet, because relays + * give no ordering guarantee and a design that needed one would be wrong on the + * wire rather than merely slow. + */ +class ArchivePageTag( + val index: Int, + val count: Int, +) { + /** Whether this page claims to be the whole archive on its own. */ + fun isOnlyPage(): Boolean = count == 1 + + fun toTagArray() = assemble(index = index, count = count) + + companion object { + const val TAG_NAME = "archivePage" + + fun parse(tag: Array): ArchivePageTag? { + ensure(tag.has(2)) { return null } + ensure(tag[0] == TAG_NAME) { return null } + + val index = tag[1].toIntOrNull() ?: return null + val count = tag[2].toIntOrNull() ?: return null + + // A page outside its own archive is not a page. Refused rather than + // clamped: the pair is how a receiver decides it has everything, and + // a repaired one would let a truncated archive read as complete. + ensure(count >= 1) { return null } + ensure(index in 0 until count) { return null } + + return ArchivePageTag(index = index, count = count) + } + + fun assemble(index: Int, count: Int): Array = + arrayOf(TAG_NAME, index.toString(), count.toString()) + + fun assemble(archivePageTag: ArchivePageTag) = + assemble(index = archivePageTag.index, count = archivePageTag.count) + } +} diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/archive/tags/ArchiveRecipientTag.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/archive/tags/ArchiveRecipientTag.kt new file mode 100644 index 00000000..099a38b7 --- /dev/null +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/nostr/archive/tags/ArchiveRecipientTag.kt @@ -0,0 +1,43 @@ +package press.mantra.compose.nostr.archive.tags + +import com.vitorpamplona.quartz.nip01Core.core.HexKey +import com.vitorpamplona.quartz.nip01Core.core.has +import com.vitorpamplona.quartz.utils.ensure + +/** + * Who an archive is for. + * + * **A hint, not access control.** The page is an ordinary group message and + * every member can read it -- which is right, because it is their own history + * going back to them. What this decides is who *acts*: a device that is not + * named stores the page and applies nothing, since it already holds the work and + * re-applying would rewrite every one of its rows to point at an archive page + * rather than at the event that actually introduced it. + * + * So this is not a secret being kept from the group, and nothing downstream may + * treat it as one. Encrypting an archive to one member was considered and + * rejected in docs/member-archive.md: it protects nothing and costs a "sent a + * private message" line per page in everyone's transcript. + */ +class ArchiveRecipientTag( + val pubKey: HexKey, +) { + fun toTagArray() = assemble(pubKey = pubKey) + + companion object { + const val TAG_NAME = "p" + + fun parse(tag: Array): ArchiveRecipientTag? { + ensure(tag.has(1)) { return null } + ensure(tag[0] == TAG_NAME) { return null } + ensure(tag[1].length == 64) { return null } + + return ArchiveRecipientTag(pubKey = tag[1]) + } + + fun assemble(pubKey: HexKey): Array = arrayOf(TAG_NAME, pubKey) + + fun assemble(archiveRecipientTag: ArchiveRecipientTag) = + assemble(pubKey = archiveRecipientTag.pubKey) + } +} diff --git a/composeApp/src/commonTest/kotlin/press/mantra/compose/nostr/archive/ArchiveEventTest.kt b/composeApp/src/commonTest/kotlin/press/mantra/compose/nostr/archive/ArchiveEventTest.kt new file mode 100644 index 00000000..20515041 --- /dev/null +++ b/composeApp/src/commonTest/kotlin/press/mantra/compose/nostr/archive/ArchiveEventTest.kt @@ -0,0 +1,409 @@ +package press.mantra.compose.nostr.archive + +import com.vitorpamplona.quartz.nip01Core.core.Event +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue +import press.mantra.compose.nostr.archive.tags.ArchiveIdTag +import press.mantra.compose.nostr.archive.tags.ArchivePageTag +import press.mantra.compose.nostr.archive.tags.ArchiveRecipientTag +import press.mantra.compose.nostr.frost.FrostSigningEvents +import press.mantra.compose.nostr.frost.GroupKeyStateEvent +import press.mantra.compose.nostr.nip30303.ArtifactEvent +import press.mantra.compose.nostr.nip30303.ArtifactVersionEvent +import press.mantra.compose.nostr.nip30303.ChapterEvent +import press.mantra.compose.nostr.nip30303.ChunkEvent +import press.mantra.compose.nostr.nip30303.DialectEvent +import press.mantra.compose.nostr.nip30303.SubmissionEvent +import press.mantra.compose.nostr.nip30303.TranslationArtifactVersionContributorListEvent +import press.mantra.compose.nostr.nip30303.TranslationArtifactVersionEvent +import press.mantra.compose.nostr.nip30303.TranslationChapterContributorListEvent +import press.mantra.compose.nostr.nip30303.TranslationChapterEvent +import press.mantra.compose.nostr.nip30303.TranslationChunkEvent +import press.mantra.compose.nostr.nip30303.TranslationContributorListEvent +import press.mantra.compose.nostr.nip30303.TranslationEvent + +/** + * The envelope an archive travels in: what it carries, what it refuses, and the + * order it puts things in. + * + * Nothing here verifies a signature -- that is `GroupKeyStateTest`'s half, and + * `ArchiveManager`'s at apply time. What these cover is the framing, and the two + * bounds a page has to hold whoever sent it. + */ +class ArchiveEventTest { + private val recipient = "d".repeat(64) + private val archiveId = "e".repeat(64) + + private fun payload( + kind: Int, + content: String = "payload", + id: String = "a".repeat(63) + (kind % 10), + ) = Event( + id = id, + pubKey = "b".repeat(64), + createdAt = 1_700_000_000L, + kind = kind, + tags = arrayOf(arrayOf("alt", "test")), + content = content, + sig = "c".repeat(128) + ) + + // ---- The codec ------------------------------------------------------- + + @Test + fun `what is encoded is what comes back, whole`() { + val events = listOf( + payload(DialectEvent.KIND, "isiZulu"), + payload(ArtifactEvent.KIND, "an artifact"), + ) + + val decoded = ArchiveEvent.decodePage(ArchiveEvent.encodePage(events)) + + assertNotNull(decoded) + assertEquals(events.map { it.content }, decoded.map { it.content }) + // The point of carrying whole events rather than rebuilt ones: the + // signature is what the receiver checks, so losing it in transit would + // leave them nothing to check. + assertEquals(events.map { it.id }, decoded.map { it.id }) + assertEquals(events.map { it.pubKey }, decoded.map { it.pubKey }) + assertEquals(events.map { it.sig }, decoded.map { it.sig }) + } + + @Test + fun `a page of one is still an array`() { + // The contrast with FrostSigningEvents.encodeProposal, which keeps a bare + // object for a batch of one so that builds predating batching can read + // it. Both archive kinds are new, so there is no such build and no reason + // to carry the second shape. + val encoded = ArchiveEvent.encodePage(listOf(payload(DialectEvent.KIND))) + + assertTrue(encoded.startsWith("["), "a page is always an array") + assertTrue(encoded.endsWith("]")) + assertEquals(1, ArchiveEvent.decodePage(encoded)?.size) + } + + @Test + fun `a page that is not an array is not a page`() { + assertNull(ArchiveEvent.decodePage(payload(DialectEvent.KIND).toJson())) + assertNull(ArchiveEvent.decodePage("")) + assertNull(ArchiveEvent.decodePage("not json")) + assertNull(ArchiveEvent.decodePage("[unclosed")) + } + + @Test + fun `an empty page is refused rather than read as an empty archive`() { + assertNull(ArchiveEvent.decodePage("[]")) + } + + @Test + fun `one unreadable payload refuses the whole page`() { + // All-or-nothing here, and per-payload at verify time. They answer + // different questions: a page that will not parse has lost its framing, + // so a page quietly shortened by one element would report a complete + // archive on its page count while holding less than it says. A payload + // whose signature does not verify is a well-framed page with one bad + // event in it, and dropping its honest neighbours would let one forgery + // deny an entire archive. + val good = payload(DialectEvent.KIND) + + assertNull(ArchiveEvent.decodePage("[${good.toJson()},\"not an event\"]")) + assertNull(ArchiveEvent.decodePage("[\"not an event\",${good.toJson()}]")) + } + + // ---- The two caps ---------------------------------------------------- + + @Test + fun `a page over the event cap is refused on the way in`() { + val overCap = (0..ArchiveEvent.MAX_PAGE_EVENTS).map { + payload(DialectEvent.KIND, id = it.toString().padStart(64, '0')) + } + assertEquals(ArchiveEvent.MAX_PAGE_EVENTS + 1, overCap.size) + + assertNull(ArchiveEvent.decodePage(ArchiveEvent.encodePage(overCap))) + + // And exactly at the cap is fine, so the bound is not off by one. + assertNotNull(ArchiveEvent.decodePage(ArchiveEvent.encodePage(overCap.dropLast(1)))) + } + + @Test + fun `the event cap can actually be reached under the byte cap`() { + // The cap that cannot fire. An event is over 300 bytes before it says + // anything -- 64 characters of id, 64 of pubkey, 128 of signature -- so a + // count cap set above what MAX_PAGE_BYTES can hold is a check that never + // runs, and the page-over-the-cap test above passes for the wrong reason. + // Both caps have to be re-sized together; this is what says so. + val atCap = (0 until ArchiveEvent.MAX_PAGE_EVENTS).map { + payload(DialectEvent.KIND, id = it.toString().padStart(64, '0')) + } + val page = ArchiveEvent.encodePage(atCap) + + assertTrue( + page.encodeToByteArray().size <= ArchiveEvent.MAX_PAGE_BYTES, + "MAX_PAGE_EVENTS payloads take ${page.encodeToByteArray().size} bytes, " + + "over the ${ArchiveEvent.MAX_PAGE_BYTES}-byte cap: the event cap can never fire" + ) + assertEquals(ArchiveEvent.MAX_PAGE_EVENTS, ArchiveEvent.decodePage(page)?.size) + } + + @Test + fun `a page over the byte cap is refused on the way in`() { + // Checked before the JSON is parsed, because the cap exists to bound the + // work a remote party can ask this device to do. An archive is the second + // place in this protocol where somebody else sets that size. + val huge = ArchiveEvent.encodePage( + listOf(payload(DialectEvent.KIND, content = "x".repeat(ArchiveEvent.MAX_PAGE_BYTES))) + ) + assertTrue(huge.encodeToByteArray().size > ArchiveEvent.MAX_PAGE_BYTES) + + assertNull(ArchiveEvent.decodePage(huge)) + } + + @Test + fun `the byte cap counts bytes, not characters`() { + // A multi-byte payload that fits as characters and does not as bytes. If + // this were measured in characters the cap would let through up to four + // times what it says. + val wide = "δΈ–".repeat(ArchiveEvent.MAX_PAGE_BYTES / 2) + val page = ArchiveEvent.encodePage(listOf(payload(DialectEvent.KIND, content = wide))) + + assertTrue(page.length < ArchiveEvent.MAX_PAGE_BYTES) + assertTrue(page.encodeToByteArray().size > ArchiveEvent.MAX_PAGE_BYTES) + assertNull(ArchiveEvent.decodePage(page)) + } + + // ---- What an archive may carry --------------------------------------- + + @Test + fun `the allowlist is every kind an archive can restore, and only those`() { + listOf( + DialectEvent.KIND, + ArtifactEvent.KIND, + ArtifactVersionEvent.KIND, + ChapterEvent.KIND, + ChunkEvent.KIND, + TranslationArtifactVersionEvent.KIND, + TranslationChapterEvent.KIND, + TranslationChunkEvent.KIND, + TranslationEvent.KIND, + ).forEach { assertTrue(ArchiveEvent.isArchivable(it), "kind $it should be archivable") } + + assertEquals(9, ArchiveEvent.ARCHIVABLE_KINDS.size) + } + + @Test + fun `an archive may not carry a key state, however well signed`() { + // The attack the allowlist exists for, and the reason it is an allowlist + // rather than a denylist. A GroupKeyStateEvent is signed by the room and + // passes verification perfectly, so an archive carrying an old one is a + // validly signed statement about what the room signs with, replayed by + // whoever kept a copy. Nothing but this stops it. + assertFalse(ArchiveEvent.isArchivable(GroupKeyStateEvent.KIND)) + assertNull(ArchiveEvent.applyRank(GroupKeyStateEvent.KIND)) + } + + @Test + fun `an archive carries documents, not envelopes or protocol`() { + assertFalse(ArchiveEvent.isArchivable(SubmissionEvent.KIND)) + assertFalse(ArchiveEvent.isArchivable(ArchiveEvent.KIND)) + assertFalse(ArchiveEvent.isArchivable(ArchiveRequestEvent.KIND)) + FrostSigningEvents.ALL.forEach { + assertFalse(ArchiveEvent.isArchivable(it), "frost kind $it") + } + } + + @Test + fun `the contributor lists are left out while nothing applies them`() { + // Not an oversight. ChatMessage.applyInnerEvent has no arm that writes a + // row for any of these, so archiving them would cost bytes and restore + // nothing. They belong in the list on the day that changes. + assertFalse(ArchiveEvent.isArchivable(TranslationArtifactVersionContributorListEvent.KIND)) + assertFalse(ArchiveEvent.isArchivable(TranslationChapterContributorListEvent.KIND)) + assertFalse(ArchiveEvent.isArchivable(TranslationContributorListEvent.KIND)) + } + + // ---- Dependency order ------------------------------------------------ + + @Test + fun `the order is the foreign keys, not the kind numbers`() { + fun rank(kind: Int) = assertNotNull(ArchiveEvent.applyRank(kind), "kind $kind has no rank") + + // Every one of these is a foreign key in Room, so an archive applied the + // other way round is a constraint violation rather than a wrong answer. + assertTrue(rank(DialectEvent.KIND) < rank(ArtifactEvent.KIND)) + assertTrue(rank(ArtifactEvent.KIND) < rank(ArtifactVersionEvent.KIND)) + assertTrue(rank(ArtifactVersionEvent.KIND) < rank(ChapterEvent.KIND)) + assertTrue(rank(ChapterEvent.KIND) < rank(ChunkEvent.KIND)) + assertTrue(rank(ArtifactVersionEvent.KIND) < rank(TranslationArtifactVersionEvent.KIND)) + assertTrue(rank(DialectEvent.KIND) < rank(TranslationArtifactVersionEvent.KIND)) + assertTrue(rank(TranslationArtifactVersionEvent.KIND) < rank(TranslationChapterEvent.KIND)) + assertTrue(rank(ChapterEvent.KIND) < rank(TranslationChapterEvent.KIND)) + assertTrue(rank(ChunkEvent.KIND) < rank(TranslationChunkEvent.KIND)) + assertTrue(rank(TranslationChapterEvent.KIND) < rank(TranslationChunkEvent.KIND)) + assertTrue(rank(TranslationChunkEvent.KIND) < rank(TranslationEvent.KIND)) + assertTrue(rank(TranslationArtifactVersionEvent.KIND) < rank(TranslationEvent.KIND)) + + // And the numbers really do disagree with the order, which is why this is + // a list rather than a sortedBy { kind }. + assertTrue(DialectEvent.KIND > ArtifactEvent.KIND) + assertTrue(rank(DialectEvent.KIND) < rank(ArtifactEvent.KIND)) + } + + @Test + fun `sorting a page puts what is referenced before what refers to it`() { + val jumbled = listOf( + payload(TranslationEvent.KIND), + payload(ChunkEvent.KIND), + payload(DialectEvent.KIND), + payload(ChapterEvent.KIND), + payload(ArtifactEvent.KIND), + payload(ArtifactVersionEvent.KIND), + ) + + assertEquals( + listOf( + DialectEvent.KIND, + ArtifactEvent.KIND, + ArtifactVersionEvent.KIND, + ChapterEvent.KIND, + ChunkEvent.KIND, + TranslationEvent.KIND, + ), + ArchiveEvent.inApplyOrder(jumbled).map { it.kind } + ) + } + + @Test + fun `two of a kind keep the order they were handed in`() { + // Nothing orders two chapters of one version, and keeping the caller's + // order is one less thing that differs between two members assembling the + // same archive. + val chapters = listOf( + payload(ChapterEvent.KIND, "one", id = "1".repeat(64)), + payload(ChapterEvent.KIND, "two", id = "2".repeat(64)), + payload(ChapterEvent.KIND, "three", id = "3".repeat(64)), + ) + + assertEquals( + listOf("one", "two", "three"), + ArchiveEvent.inApplyOrder(chapters).map { it.content } + ) + } + + // ---- The tags -------------------------------------------------------- + + @Test + fun `a page says which archive it is, where in it, and who for`() { + val template = ArchiveEvent.build( + payloads = listOf(payload(DialectEvent.KIND)), + archiveId = archiveId, + index = 2, + count = 5, + recipient = recipient, + createdAt = 1_700_000_100L + ) + + val readBack = ArchiveEvent( + id = "f".repeat(64), + pubKey = "b".repeat(64), + createdAt = template.createdAt, + tags = template.tags, + content = template.content, + sig = "" + ) + + assertEquals(archiveId, readBack.archiveId()) + assertEquals(2, readBack.page()?.index) + assertEquals(5, readBack.page()?.count) + assertEquals(recipient, readBack.recipient()) + assertEquals(1, readBack.payloads()?.size) + } + + @Test + fun `a page outside its own archive is not a page`() { + // Refused rather than clamped: the pair is how a receiver decides it has + // everything, so a repaired one would let a truncated archive read as + // complete. + assertNull(ArchivePageTag.parse(arrayOf(ArchivePageTag.TAG_NAME, "5", "3"))) + assertNull(ArchivePageTag.parse(arrayOf(ArchivePageTag.TAG_NAME, "-1", "3"))) + assertNull(ArchivePageTag.parse(arrayOf(ArchivePageTag.TAG_NAME, "0", "0"))) + assertNull(ArchivePageTag.parse(arrayOf(ArchivePageTag.TAG_NAME, "0"))) + assertNull(ArchivePageTag.parse(arrayOf(ArchivePageTag.TAG_NAME, "one", "3"))) + + assertEquals(0, ArchivePageTag.parse(arrayOf(ArchivePageTag.TAG_NAME, "0", "1"))?.index) + assertTrue(ArchivePageTag.parse(arrayOf(ArchivePageTag.TAG_NAME, "0", "1"))!!.isOnlyPage()) + } + + @Test + fun `an archive id and a recipient have to be there to be read`() { + assertNull(ArchiveIdTag.parse(arrayOf(ArchiveIdTag.TAG_NAME))) + assertNull(ArchiveIdTag.parse(arrayOf(ArchiveIdTag.TAG_NAME, " "))) + assertNull(ArchiveIdTag.parse(arrayOf("other", archiveId))) + assertEquals(archiveId, ArchiveIdTag.parse(arrayOf(ArchiveIdTag.TAG_NAME, archiveId))?.archiveId) + + assertNull(ArchiveRecipientTag.parse(arrayOf(ArchiveRecipientTag.TAG_NAME, "short"))) + assertEquals( + recipient, + ArchiveRecipientTag.parse(arrayOf(ArchiveRecipientTag.TAG_NAME, recipient))?.pubKey + ) + } + + // ---- What build refuses to make -------------------------------------- + + @Test + fun `a page is not built over either cap, or out of what it may not carry`() { + // The outbound half of both checks. It is politeness rather than + // security -- the inbound one is the boundary -- but a page nobody can + // read back is better failed where it was made. + assertFailsWith { + ArchiveEvent.build(emptyList(), archiveId, 0, 1, recipient) + } + assertFailsWith { + ArchiveEvent.build( + payloads = (0..ArchiveEvent.MAX_PAGE_EVENTS).map { + payload(DialectEvent.KIND, id = it.toString().padStart(64, '0')) + }, + archiveId = archiveId, + index = 0, + count = 1, + recipient = recipient + ) + } + assertFailsWith { + ArchiveEvent.build( + payloads = listOf(payload(GroupKeyStateEvent.KIND)), + archiveId = archiveId, + index = 0, + count = 1, + recipient = recipient + ) + } + assertFailsWith { + ArchiveEvent.build( + payloads = listOf( + payload(DialectEvent.KIND, content = "x".repeat(ArchiveEvent.MAX_PAGE_BYTES)) + ), + archiveId = archiveId, + index = 0, + count = 1, + recipient = recipient + ) + } + } + + @Test + fun `a request carries nothing but what it is`() { + val template = ArchiveRequestEvent.build(createdAt = 1_700_000_100L) + + assertEquals(ArchiveRequestEvent.KIND, template.kind) + assertEquals("", template.content) + // The room is the envelope and the asker is the MLS sender, so there is + // nothing left for the event itself to say. + assertEquals(1, template.tags.size) + assertEquals("alt", template.tags[0][0]) + } +} diff --git a/docs/member-archive.md b/docs/member-archive.md index 58b3d25c..ba0ecedc 100644 --- a/docs/member-archive.md +++ b/docs/member-archive.md @@ -232,13 +232,23 @@ Tags, one value each, per the house convention: ```kotlin const val MAX_PAGE_BYTES = 64 * 1024 -const val MAX_PAGE_EVENTS = 256 +const val MAX_PAGE_EVENTS = 128 ``` A byte cap rather than a count alone, because the events vary by two orders of magnitude -- a chunk is a paragraph, an artifact is a URL. The count cap bounds the receiver's *work* where the byte cap bounds the *transport*. +**The count was 256 when this was written, and 256 can never fire.** An event +carries 64 characters of id, 64 of pubkey and 128 of signature before it says +anything, so the floor is about 370 bytes and 64 KB cannot hold much past 170 of +them -- the byte cap always binds first and the count cap is a check that never +runs. The two have to be sized against each other or one of them is decoration. +At 128 both bind something: the count stops a page of many small payloads, the +bytes stop a page of few large ones. The test that says so asserts a page at +exactly the cap still decodes, which is the assertion that fails when somebody +raises one number without the other. + Both are checked independently on the way in, for the reason the batch cap is: an archive is the second place in this protocol where a remote party decides how much work everyone else does. Measure the 64 KB against a finished kind:445