Implement marmot ingestion

This commit is contained in:
Kgothatso Ngako
2026-07-03 15:02:31 +02:00
parent 0a5a2194c4
commit bbe77936ed
14 changed files with 968 additions and 23 deletions

View File

@@ -2,7 +2,7 @@
"formatVersion": 1,
"database": {
"version": 1,
"identityHash": "8dd191b21e518a19f39b50a5f5b4a432",
"identityHash": "5c51783559255bdfff3112051ee22d05",
"entities": [
{
"tableName": "BroadcastNostrEventReceipt",
@@ -282,7 +282,7 @@
},
{
"tableName": "ChatMessage",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `senderPublicKey` TEXT NOT NULL, `isUserMessage` INTEGER NOT NULL, `giftWrapPayloadId` TEXT NOT NULL, `chatRoomId` TEXT NOT NULL, `replyToMessageId` INTEGER, `quotedMessageId` INTEGER, `content` TEXT NOT NULL, `createdAt` INTEGER NOT NULL, `updatedAt` INTEGER NOT NULL, `savedAt` INTEGER NOT NULL, `viewedAt` INTEGER, `deletedAt` INTEGER, FOREIGN KEY(`chatRoomId`) REFERENCES `ChatRoom`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE , FOREIGN KEY(`giftWrapPayloadId`) REFERENCES `GiftWrapPayload`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )",
"createSql": "CREATE TABLE IF NOT EXISTS `${TABLE_NAME}` (`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, `senderPublicKey` TEXT NOT NULL, `isUserMessage` INTEGER NOT NULL, `giftWrapPayloadId` TEXT, `groupEventId` TEXT, `chatRoomId` TEXT NOT NULL, `replyToMessageId` INTEGER, `quotedMessageId` INTEGER, `content` TEXT NOT NULL, `messageType` TEXT NOT NULL, `createdAt` INTEGER NOT NULL, `updatedAt` INTEGER NOT NULL, `savedAt` INTEGER NOT NULL, `viewedAt` INTEGER, `deletedAt` INTEGER, FOREIGN KEY(`chatRoomId`) REFERENCES `ChatRoom`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE , FOREIGN KEY(`giftWrapPayloadId`) REFERENCES `GiftWrapPayload`(`id`) ON UPDATE NO ACTION ON DELETE CASCADE )",
"fields": [
{
"fieldPath": "id",
@@ -305,8 +305,12 @@
{
"fieldPath": "giftWrapPayloadId",
"columnName": "giftWrapPayloadId",
"affinity": "TEXT",
"notNull": true
"affinity": "TEXT"
},
{
"fieldPath": "groupEventId",
"columnName": "groupEventId",
"affinity": "TEXT"
},
{
"fieldPath": "chatRoomId",
@@ -330,6 +334,12 @@
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "messageType",
"columnName": "messageType",
"affinity": "TEXT",
"notNull": true
},
{
"fieldPath": "createdAt",
"columnName": "createdAt",
@@ -3044,7 +3054,7 @@
],
"setupQueries": [
"CREATE TABLE IF NOT EXISTS room_master_table (id INTEGER PRIMARY KEY,identity_hash TEXT)",
"INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '8dd191b21e518a19f39b50a5f5b4a432')"
"INSERT OR REPLACE INTO room_master_table (id,identity_hash) VALUES(42, '5c51783559255bdfff3112051ee22d05')"
]
}
}

View File

@@ -14,22 +14,32 @@ import at.torch.compose.database.model.NostrEventRelay
import at.torch.compose.database.model.Participant
import at.torch.compose.database.model.Post
import at.torch.compose.database.model.Profile
import at.torch.compose.database.model.SynchronizeNostrEventRequest
import at.torch.compose.database.model.types.SynchronizationFilter
import at.torch.compose.exceptions.GiftWrapImpersonationException
import at.torch.compose.exceptions.GiftWrapSealDecryptionException
import at.torch.compose.exceptions.GiftWrapUnsealException
import at.torch.compose.exceptions.MarmotMissingChatGroupException
import at.torch.compose.exceptions.MarmotMissingKeyPackageBundleException
import at.torch.compose.exceptions.MarmotMissingNostrGroupDataExtension
import at.torch.compose.exceptions.MarmotNotMemberOfChatGroupException
import at.torch.compose.exceptions.MarmotUnsupportedWireFormatException
import at.torch.compose.exceptions.MarmotWelcomeEventMissingKeyPackageEventIdException
import at.torch.compose.extensions.exporterSecret
import at.torch.compose.extensions.toHex
import at.torch.compose.network.serialization.encodeToJsonString
import at.torch.compose.managers.MarmotInboundManager
import co.touchlab.kermit.Logger
import com.vitorpamplona.quartz.marmot.GroupEventResult
import com.vitorpamplona.quartz.marmot.mip00KeyPackages.KeyPackageEvent
import com.vitorpamplona.quartz.marmot.mip00KeyPackages.KeyPackageRelayListEvent
import com.vitorpamplona.quartz.marmot.mip02Welcome.WelcomeEvent
import com.vitorpamplona.quartz.marmot.mip03GroupMessages.GroupEventEncryption
import com.vitorpamplona.quartz.marmot.mls.codec.TlsReader
import com.vitorpamplona.quartz.marmot.mls.framing.ContentType
import com.vitorpamplona.quartz.marmot.mls.framing.MlsMessage
import com.vitorpamplona.quartz.marmot.mls.framing.PublicMessage
import com.vitorpamplona.quartz.marmot.mls.framing.WireFormat
import com.vitorpamplona.quartz.marmot.mls.group.MlsGroup
import com.vitorpamplona.quartz.marmot.mls.group.MlsGroupState
import com.vitorpamplona.quartz.marmot.mls.messages.KeyPackageBundle
import com.vitorpamplona.quartz.marmot.mls.messages.MlsKeyPackage
import com.vitorpamplona.quartz.marmot.mls.tree.Credential
@@ -44,7 +54,6 @@ import com.vitorpamplona.quartz.nip17Dm.messages.ChatMessageEvent
import com.vitorpamplona.quartz.nip17Dm.settings.ChatMessageRelayListEvent
import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent
import kotlin.io.encoding.Base64
import kotlin.math.log
import kotlin.time.Clock
import kotlin.time.Instant
@@ -328,6 +337,60 @@ abstract class NostrDao(
}
// TODO: Support MLSGroupEvent handling...
nostrEvent.toGroupEvent(
userPublicKey = activeKeyPair.pubKey.toHex()
)?.let { groupEvent ->
groupEvent.groupId()?.let { chatRoomId ->
val localChatRoom = database.chatRoomDao().findChatRoomById(chatRoomId)
if (localChatRoom != null) {
val mlsGroupStateByteArray = localChatRoom.chatRoom.mlsGroupState?.hexToByteArray()
if (mlsGroupStateByteArray != null) {
val mlsGroup = MlsGroup.restore(
MlsGroupState.decodeTls(
mlsGroupStateByteArray
)
)
val memberPubkeys = mlsGroup.members().mapNotNull { (leafIndex, leafNode) ->
val pubkey = when (val cred = leafNode.credential) {
is Credential.Basic -> cred.identity.toHexKey()
else -> null
}
pubkey
}
if (memberPubkeys.contains(activeKeyPair.pubKey.toHex())) {
// Ingest message...
val groupEventResult = MarmotInboundManager.processGroupEvent(
database = database,
activeKeyPair = activeKeyPair,
localChatRoom = localChatRoom,
mlsGroup = mlsGroup,
groupEvent = groupEvent
)?.let { chatMessage ->
database.chatMessageDao().upsert(
chatMessage
)
}
// Save groupEventResult as chatMessage...
} else {
throw MarmotNotMemberOfChatGroupException("We are not a member of the chat room ${localChatRoom.chatRoom.id}")
}
} else {
throw MarmotMissingNostrGroupDataExtension("Couldn't find chatRoom for $nostrEvent")
}
} else {
throw MarmotMissingChatGroupException("Couldn't find chatRoom for $nostrEvent")
}
}
}
nostrEvent.toGiftWrapMessageWithReceiverPTag()?.let { giftWrapMessage ->
logger.d("giftWrapMessage: $giftWrapMessage")
@@ -648,18 +711,15 @@ abstract class NostrDao(
)
)
// TODO: Add participants... by processing group.members()
val normalizedRelayUrl = NormalizedRelayUrl(relayURL)
val participants = group.members().mapNotNull { (leafIndex, leafNode) ->
val pubkey =
when (val cred = leafNode.credential) {
is Credential.Basic -> cred.identity.toHexKey()
else -> null
}
logger.d("Pubkey: $pubkey")
val pubkey = when (val cred = leafNode.credential) {
is Credential.Basic -> cred.identity.toHexKey()
else -> null
}
if (pubkey != null) {
Participant(
participantPublicKey = pubkey,

View File

@@ -7,7 +7,11 @@ import at.torch.compose.database.model.traits.LocalStoreEntity
import at.torch.compose.database.model.traits.SoftDeletableEntity
import at.torch.compose.database.model.traits.TimestampedEntity
import at.torch.compose.database.model.traits.UserViewableEntity
import at.torch.compose.extensions.toHex
import com.vitorpamplona.quartz.marmot.GroupEventResult
import com.vitorpamplona.quartz.marmot.mip03GroupMessages.GroupEvent
import com.vitorpamplona.quartz.nip01Core.core.HexKey
import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair
import kotlin.time.Clock
import kotlin.time.Instant
@@ -34,17 +38,118 @@ data class ChatMessage(
val senderPublicKey: String,
val isUserMessage: Boolean,
val giftWrapPayloadId: HexKey,
val giftWrapPayloadId: HexKey?,
val groupEventId: HexKey?,
val chatRoomId: String,
val replyToMessageId: Long? = null,
val quotedMessageId: Long? = null,
val content: String,
val messageType: String = "message",
override val createdAt: Instant = Clock.System.now(),
override val updatedAt: Instant = createdAt,
override val savedAt: Instant = Clock.System.now(),
override val viewedAt: Instant? = null,
override val deletedAt: Instant? = null, // Who this message should be seen by (used to derive conversationId)
): TimestampedEntity, LocalStoreEntity, UserViewableEntity, SoftDeletableEntity
): TimestampedEntity, LocalStoreEntity, UserViewableEntity, SoftDeletableEntity {
companion object {
fun fromGroupEventResult(
activeKeyPair: KeyPair,
groupEvent: GroupEvent,
groupEventResult: GroupEventResult
): ChatMessage? {
return when(groupEventResult) {
is GroupEventResult.ApplicationMessage -> {
ChatMessage(
giftWrapPayloadId = null,
messageType = "message",
groupEventId = groupEvent.id,
senderPublicKey = groupEvent.pubKey,
isUserMessage = activeKeyPair.pubKey.toHex() == groupEvent.pubKey,
chatRoomId = groupEventResult.groupId,
createdAt = Instant.fromEpochSeconds(groupEvent.createdAt),
content = groupEventResult.innerEventJson, // TODO: Figure out what to do here...
)
}
is GroupEventResult.CommitPending -> {
ChatMessage(
giftWrapPayloadId = null,
groupEventId = groupEvent.id,
messageType = "pendingCommit",
senderPublicKey = groupEvent.pubKey,
isUserMessage = activeKeyPair.pubKey.toHex() == groupEvent.pubKey,
chatRoomId = groupEventResult.groupId,
createdAt = Instant.fromEpochSeconds(groupEvent.createdAt),
content = "Pending Commit in epoch ${groupEventResult.epoch}", // TODO: Figure out what to do here...
)
}
is GroupEventResult.CommitProcessed -> {
ChatMessage(
giftWrapPayloadId = null,
groupEventId = groupEvent.id,
messageType = "processedCommit",
senderPublicKey = groupEvent.pubKey,
isUserMessage = activeKeyPair.pubKey.toHex() == groupEvent.pubKey,
chatRoomId = groupEventResult.groupId,
createdAt = Instant.fromEpochSeconds(groupEvent.createdAt),
content = "Processed Commit in epoch ${groupEventResult.newEpoch}", // TODO: Figure out what to do here...
)
}
is GroupEventResult.Duplicate -> {
// ChatMessage(
// giftWrapPayloadId = null,
// groupEventId = groupEvent.id,
// messageType = "duplicate",
// senderPublicKey = groupEvent.pubKey,
// isUserMessage = activeKeyPair.pubKey.toHex() == groupEvent.pubKey,
// chatRoomId = groupEventResult.groupId,
// createdAt = Instant.fromEpochSeconds(groupEvent.createdAt),
// content = groupEventResult.innerEventJson, // TODO: Figure out what to do here...
// )
null
}
is GroupEventResult.Error -> {
// ChatMessage(
// giftWrapPayloadId = null,
// groupEventId = groupEvent.id,
// messageType = "error",
// senderPublicKey = groupEvent.pubKey,
// isUserMessage = activeKeyPair.pubKey.toHex() == groupEvent.pubKey,
// chatRoomId = groupEventResult.groupId,
// createdAt = Instant.fromEpochSeconds(groupEvent.createdAt),
// content = groupEventResult.innerEventJson, // TODO: Figure out what to do here...
// )
null
}
is GroupEventResult.ProposalStaged -> {
ChatMessage(
giftWrapPayloadId = null,
groupEventId = groupEvent.id,
messageType = "proposalStaged",
senderPublicKey = groupEvent.pubKey,
isUserMessage = activeKeyPair.pubKey.toHex() == groupEvent.pubKey,
chatRoomId = groupEventResult.groupId,
createdAt = Instant.fromEpochSeconds(groupEvent.createdAt),
content = "Proposal staged: ${groupEventResult.senderLeafIndex}", // TODO: Figure out what to do here...
)
}
is GroupEventResult.UndecryptableOuterLayer -> {
ChatMessage(
giftWrapPayloadId = null,
groupEventId = groupEvent.id,
messageType = "undecryptableOuterLayer",
senderPublicKey = groupEvent.pubKey,
isUserMessage = activeKeyPair.pubKey.toHex() == groupEvent.pubKey,
chatRoomId = groupEventResult.groupId,
createdAt = Instant.fromEpochSeconds(groupEvent.createdAt),
content = "Undecryptable Message", // TODO: Figure out what to do here...
)
}
}
}
}
}

View File

@@ -52,7 +52,7 @@ data class ChatRoom(
*/
val subject: String?,
val mlsGroupState: String?, // TODO: Can this be nullable???
val mlsGroupState: String?,
/**
* Derived from kind14/15 tags

View File

@@ -14,6 +14,7 @@ import com.vitorpamplona.quartz.marmot.mip03GroupMessages.GroupEventEncryption
import com.vitorpamplona.quartz.marmot.mls.group.MlsGroup
import com.vitorpamplona.quartz.nip01Core.core.HexKey
import com.vitorpamplona.quartz.nip40Expiration.expiration
import com.vitorpamplona.quartz.utils.Hex
import com.vitorpamplona.quartz.utils.Log
import kotlin.time.Clock
import kotlin.time.Instant
@@ -32,6 +33,8 @@ data class MarmotGroupEvent(
*/
val userPublicKey: String,
val publicKey: HexKey,
val chatRoomId: String,
val encryptedContent: String,
@@ -43,7 +46,7 @@ data class MarmotGroupEvent(
override val nostrEventId: String,
override val createdAt: Instant = Clock.System.now(),
override val updatedAt: Instant = Clock.System.now(),
override val updatedAt: Instant = createdAt,
override val savedAt: Instant = Clock.System.now(),
override val deletedAt: Instant? = null,
override val broadcastedAt: Instant? = null,
@@ -64,6 +67,7 @@ data class MarmotGroupEvent(
createdAt = Instant.fromEpochSeconds(groupEvent.createdAt),
chatRoomId = chatRoomId,
userPublicKey = userPublicKey,
publicKey = groupEvent.pubKey,
expiresAt = groupEvent.expiration()?.let { Instant.fromEpochSeconds(it) }
)
}
@@ -71,7 +75,6 @@ data class MarmotGroupEvent(
}
fun toMarmotInnerEvent(group: MlsGroup): MarmotInnerEvent? {
val mlsBytes =
return null;
}

View File

@@ -5,6 +5,7 @@ import androidx.room3.Ignore
import androidx.room3.Index
import androidx.room3.PrimaryKey
import co.touchlab.kermit.Logger
import com.vitorpamplona.quartz.marmot.mip03GroupMessages.GroupEvent
import com.vitorpamplona.quartz.nip01Core.core.Event
import com.vitorpamplona.quartz.nip01Core.core.HexKey
import com.vitorpamplona.quartz.nip01Core.core.Kind
@@ -18,6 +19,7 @@ import com.vitorpamplona.quartz.nip18Reposts.quotes.QAddressableTag
import com.vitorpamplona.quartz.nip18Reposts.quotes.QEventTag
import com.vitorpamplona.quartz.nip18Reposts.quotes.firstTaggedQuote
import com.vitorpamplona.quartz.nip25Reactions.ReactionEvent
import com.vitorpamplona.quartz.nip40Expiration.expiration
import com.vitorpamplona.quartz.nip57Zaps.LnZapEvent
import com.vitorpamplona.quartz.nip59Giftwrap.wraps.GiftWrapEvent
import com.vitorpamplona.quartz.utils.EventFactory
@@ -216,6 +218,37 @@ data class NostrEvent(
return null
}
fun toGroupEvent(userPublicKey: HexKey): GroupEvent? = try {
if (kind == GroupEvent.KIND) {
val groupEvent = GroupEvent(
id = id,
pubKey = pubKey,
createdAt = createdAt.epochSeconds,
tags = tags,
content = content,
sig = sig
)
return groupEvent
// return groupEvent.groupId()?.let { nostrGroupId ->
// MarmotGroupEvent(
// id = groupEvent.id,
// createdAt = createdAt,
// userPublicKey = groupEvent.pubKey, // TODO: use the userPublicKey here
//// publicKey = groupEvent.pubKey,
// chatRoomId = nostrGroupId,
// encryptedContent = groupEvent.encryptedContent(),
// expiresAt = groupEvent.expiration()?.let { Instant.fromEpochSeconds(it) },
// nostrEventId = id,
// )
// }
}
return null
} catch (e: Throwable) {
logger.e("Couldn't cast GroupEvent", e)
return null
}
fun toReaction(): Reaction? = try {
if (kind == ReactionEvent.KIND) {
val reactionEvent = EventFactory.create<ReactionEvent>(
@@ -385,6 +418,7 @@ data class NostrEvent(
logger.e("Failed to return Repost: ", e)
return null
}
fun toZap(): Zap? = try {
if (kind == LnZapEvent.KIND) {
EventFactory.create<LnZapEvent>(

View File

@@ -135,7 +135,8 @@ class DatabaseChatRepository(
chatRoomId = localChatRoom.chatRoom.id,
senderPublicKey = localChatRoom.chatRoom.userPublicKey,
isUserMessage = true,
giftWrapPayloadId = giftWrapPayloadId
giftWrapPayloadId = giftWrapPayloadId,
groupEventId = null
)
)
}

View File

@@ -3,19 +3,36 @@ package at.torch.compose.database.repository
import at.torch.compose.database.TorchDatabase
import at.torch.compose.database.model.MarmotKeyPackageBundle
import at.torch.compose.database.model.UnsignedNostrEvent
import at.torch.compose.exceptions.MarmotMissingChatGroupException
import at.torch.compose.extensions.exporterSecret
import at.torch.compose.extensions.toHex
import at.torch.compose.nostr.Relays
import at.torch.compose.repository.MarmotRepository
import co.touchlab.kermit.Logger
import com.vitorpamplona.quartz.marmot.GroupEventResult
import com.vitorpamplona.quartz.marmot.mip00KeyPackages.KeyPackageEvent
import com.vitorpamplona.quartz.marmot.mip00KeyPackages.KeyPackageRotationManager.Companion.KEY_PACKAGE_LIFETIME_SECONDS
import com.vitorpamplona.quartz.marmot.mip00KeyPackages.KeyPackageUtils
import com.vitorpamplona.quartz.marmot.mip03GroupMessages.CommitOrdering
import com.vitorpamplona.quartz.marmot.mip03GroupMessages.GroupEvent
import com.vitorpamplona.quartz.marmot.mip03GroupMessages.GroupEventEncryption
import com.vitorpamplona.quartz.marmot.mls.codec.TlsReader
import com.vitorpamplona.quartz.marmot.mls.codec.TlsWriter
import com.vitorpamplona.quartz.marmot.mls.crypto.Ed25519
import com.vitorpamplona.quartz.marmot.mls.crypto.MlsCryptoProvider
import com.vitorpamplona.quartz.marmot.mls.crypto.X25519
import com.vitorpamplona.quartz.marmot.mls.framing.ContentType
import com.vitorpamplona.quartz.marmot.mls.framing.MlsMessage
import com.vitorpamplona.quartz.marmot.mls.framing.PrivateMessage
import com.vitorpamplona.quartz.marmot.mls.framing.PublicMessage
import com.vitorpamplona.quartz.marmot.mls.framing.WireFormat
import com.vitorpamplona.quartz.marmot.mls.group.DecryptedMessage
import com.vitorpamplona.quartz.marmot.mls.group.MlsGroup
import com.vitorpamplona.quartz.marmot.mls.group.RetainedEpochSecrets
import com.vitorpamplona.quartz.marmot.mls.messages.KeyPackageBundle
import com.vitorpamplona.quartz.marmot.mls.messages.MlsKeyPackage
import com.vitorpamplona.quartz.marmot.mls.schedule.KeySchedule
import com.vitorpamplona.quartz.marmot.mls.schedule.SecretTree
import com.vitorpamplona.quartz.marmot.mls.tree.Capabilities
import com.vitorpamplona.quartz.marmot.mls.tree.Credential
import com.vitorpamplona.quartz.marmot.mls.tree.Extension
@@ -30,6 +47,9 @@ import kotlinx.coroutines.CoroutineScope
import kotlin.io.encoding.Base64
import kotlin.time.Instant
/**
* Based on MlsGroupManager in quarts...
*/
class DatabaseMarmotRepository(
val database: TorchDatabase,
val scope: CoroutineScope

View File

@@ -0,0 +1,4 @@
package at.torch.compose.exceptions
class MarmotMissingChatGroupException(message: String? = null, cause: Throwable? = null) : Exception(message, cause) {
}

View File

@@ -0,0 +1,4 @@
package at.torch.compose.exceptions
class MarmotNotMemberOfChatGroupException(message: String? = null, cause: Throwable? = null) : Exception(message, cause) {
}

View File

@@ -0,0 +1,4 @@
package at.torch.compose.exceptions
class MarmotUnsupportedWireFormatException(message: String? = null, cause: Throwable? = null) : Exception(message, cause) {
}

View File

@@ -0,0 +1,9 @@
package at.torch.compose.extensions
import com.vitorpamplona.quartz.marmot.mls.group.MlsGroup
fun MlsGroup.exporterSecret() = this.exporterSecret(
"marmot",
"group-event".encodeToByteArray(),
32,
)

View File

@@ -0,0 +1,688 @@
package at.torch.compose.managers
import at.torch.compose.database.TorchDatabase
import at.torch.compose.database.model.ChatMessage
import at.torch.compose.database.model.intermdiate.LocalChatRoom
import at.torch.compose.exceptions.MarmotMissingChatGroupException
import at.torch.compose.exceptions.MarmotUnsupportedWireFormatException
import at.torch.compose.extensions.exporterSecret
import at.torch.compose.extensions.toHex
import co.touchlab.kermit.Logger
import com.vitorpamplona.quartz.marmot.GroupEventResult
import com.vitorpamplona.quartz.marmot.mip03GroupMessages.CommitOrdering
import com.vitorpamplona.quartz.marmot.mip03GroupMessages.GroupEvent
import com.vitorpamplona.quartz.marmot.mip03GroupMessages.GroupEventEncryption
import com.vitorpamplona.quartz.marmot.mls.codec.TlsReader
import com.vitorpamplona.quartz.marmot.mls.codec.TlsWriter
import com.vitorpamplona.quartz.marmot.mls.crypto.MlsCryptoProvider
import com.vitorpamplona.quartz.marmot.mls.framing.ContentType
import com.vitorpamplona.quartz.marmot.mls.framing.MlsMessage
import com.vitorpamplona.quartz.marmot.mls.framing.PrivateMessage
import com.vitorpamplona.quartz.marmot.mls.framing.PublicMessage
import com.vitorpamplona.quartz.marmot.mls.framing.WireFormat
import com.vitorpamplona.quartz.marmot.mls.group.DecryptedMessage
import com.vitorpamplona.quartz.marmot.mls.group.MlsGroup
import com.vitorpamplona.quartz.marmot.mls.group.RetainedEpochSecrets
import com.vitorpamplona.quartz.marmot.mls.schedule.KeySchedule
import com.vitorpamplona.quartz.marmot.mls.schedule.SecretTree
import com.vitorpamplona.quartz.nip01Core.core.HexKey
import com.vitorpamplona.quartz.nip01Core.crypto.KeyPair
import kotlin.time.Instant
object MarmotInboundManager {
private const val TAG = "MarmotInboundManager"
private val logger = Logger.withTag(TAG)
/**
* Number of past epochs to retain for late-arriving message decryption.
* Matches MDK's `DEFAULT_EPOCH_LOOKBACK` so a message encrypted under
* the prior N epochs' exporter secrets can still be decrypted after a
* Commit advances the group. Capped for forward-secrecy reasons.
*/
const val EPOCH_RETENTION_WINDOW = 5
/** Size of reuse_guard in PrivateMessage (RFC 9420 §6.3.1) */
private const val REUSE_GUARD_LENGTH = 4
private val retainedEpochs = mutableMapOf<HexKey, MutableList<RetainedEpochSecrets>>()
private val commitTracker = CommitOrdering.EpochCommitTracker()
suspend fun processGroupEvent(
database: TorchDatabase,
activeKeyPair: KeyPair,
localChatRoom: LocalChatRoom,
mlsGroup: MlsGroup,
groupEvent: GroupEvent
): ChatMessage? {
groupEvent.groupId()?.let { groupId ->
val result = try {
val mlsBytes = tryDecryptOuterLayer(
mlsGroup,
groupEvent.encryptedContent()
)
if (mlsBytes == null) {
// Expected when this kind:445 was encrypted with an epoch
// key that predates our join (classical MLS forward
// secrecy), or when the sender's epoch has drifted. Not
// an error — callers should log at DEBUG.
GroupEventResult.UndecryptableOuterLayer(
localChatRoom.chatRoom.id,
retainedEpochCount = retainedExporterSecrets(localChatRoom.chatRoom.id).size,
)
} else {
val mlsMessage = MlsMessage.decodeTls(TlsReader(mlsBytes))
when (mlsMessage.wireFormat) {
WireFormat.PRIVATE_MESSAGE -> {
processPrivateMessage(
database = database,
mlsGroup = mlsGroup,
mlsMessage = mlsMessage,
groupEvent = groupEvent
)
}
WireFormat.PUBLIC_MESSAGE -> {
processPublicMessage(
database = database,
mlsGroup = mlsGroup,
mlsMessage = mlsMessage,
groupEvent = groupEvent
)
}
else -> {
throw MarmotUnsupportedWireFormatException(
"Unexpected wire format: ${mlsMessage.wireFormat}"
)
}
}
}
} catch (e: Throwable) {
GroupEventResult.Error(groupId, "Failed to process GroupEvent: ${e.message}")
}
return ChatMessage.fromGroupEventResult(
activeKeyPair = activeKeyPair,
groupEvent = groupEvent,
groupEventResult = result,
)
}
return null
}
suspend fun processPrivateMessage(
database: TorchDatabase,
mlsGroup: MlsGroup,
mlsMessage: MlsMessage,
groupEvent: GroupEvent
): GroupEventResult {
val groupId = mlsGroup.currentMarmotData()?.nostrGroupId ?: mlsGroup.groupId.toHex()
// Peek at content type from the PrivateMessage header
val privMsg = PrivateMessage.decodeTls(TlsReader(mlsMessage.payload))
return when (privMsg.contentType) {
ContentType.APPLICATION -> {
// MLS decrypt to get the inner plaintext
val decrypted = decryptMessageBytes(
database,
mlsGroup,
mlsMessage.toTlsBytes()
)
val innerJson = decrypted.content.decodeToString()
// MIP-03: if the inner application payload is a Nostr event,
// its `pubkey` field MUST equal the MLS sender's credential
// identity. Reject any mismatch — otherwise a group member
// could mint events claiming a different author. Non-event
// payloads (raw bytes via buildGroupEventFromBytes) bypass
// this check since there is no author field to verify.
val innerEvent =
com.vitorpamplona.quartz.nip01Core.core.Event
.fromJsonOrNull(innerJson)
if (innerEvent != null) {
val senderIdentity = mlsGroup.memberIdentityHex(decrypted.senderLeafIndex)
if (senderIdentity == null || innerEvent.pubKey != senderIdentity) {
return GroupEventResult.Error(
groupId,
"MIP-03: inner event pubkey (${innerEvent.pubKey}) does not match MLS sender identity ($senderIdentity)",
)
}
}
GroupEventResult.ApplicationMessage(
groupId = groupId,
innerEventJson = innerJson,
senderLeafIndex = decrypted.senderLeafIndex,
epoch = decrypted.epoch,
)
}
ContentType.COMMIT -> {
handleCommitEvent(
database,
mlsGroup,
groupEvent
)
}
ContentType.PROPOSAL -> {
GroupEventResult.Error(groupId, "Standalone proposals not yet supported")
}
}
}
private suspend fun decryptMessageBytes(
database: TorchDatabase,
mlsGroup: MlsGroup,
messageBytes: ByteArray,
): DecryptedMessage {
val nostrGroupId = mlsGroup.currentMarmotData()?.nostrGroupId ?: mlsGroup.groupId.toHex()
// Try current epoch. If we hit an exception here we MUST surface
// it — commits that throw mid-processCommit leave the in-memory
// group half-mutated, and a retry via `group.decrypt(...)` will
// just report a stale "epoch mismatch" from the partial advance,
// hiding the real bug. Capture the original throwable, try
// retained epochs as a fallback, and re-raise the captured one
// if nothing decrypts.
val retainedBefore = mlsGroup.retainedSecrets()
val preEpoch = mlsGroup.epoch
val currentFailure: Throwable? =
try {
val result = mlsGroup.decrypt(messageBytes)
// PrivateMessage commits apply inline through
// `MlsGroup.decrypt` → `processCommit`; the epoch advances
// in memory but the CLI reopens a fresh Context on every
// command, so we MUST persist here or reloaded state
// silently reverts to the pre-commit extensions (including
// admin list).
if (result.contentType == ContentType.COMMIT && mlsGroup.epoch != preEpoch) {
pushRetainedEpoch(nostrGroupId, retainedBefore)
persistGroup(
database,
mlsGroup
)
}
return result
} catch (t: Throwable) {
t
}
val retained = retainedEpochs[nostrGroupId] ?: emptyList()
for (epochSecrets in retained) {
val result = tryDecryptWithRetainedEpoch(messageBytes, epochSecrets)
if (result != null) return result
}
throw currentFailure ?: IllegalStateException("Decrypt failed without captured cause")
}
public suspend fun processPublicMessage(
database: TorchDatabase,
mlsGroup: MlsGroup,
mlsMessage: MlsMessage,
groupEvent: GroupEvent
): GroupEventResult {
val groupId = mlsGroup.currentMarmotData()?.nostrGroupId ?: mlsGroup.groupId.toHex()
val pubMsg = PublicMessage.decodeTls(TlsReader(mlsMessage.payload))
return when (pubMsg.contentType) {
ContentType.COMMIT -> {
handleCommitEvent(
database,
mlsGroup,
groupEvent
)
}
ContentType.PROPOSAL -> {
// wn/openmls publishes SelfRemove as a standalone PublicMessage
// proposal — admins fold it into their next commit. Stage it
// locally so a subsequent commit's `ProposalRef` can resolve;
// without this every other member silently dropped the
// proposal and the admin's commit then failed with "Commit
// references unknown proposal" (marmot-interop test 15).
try {
mlsGroup.receivePublicMessageProposal(pubMsg)
GroupEventResult.ProposalStaged(groupId, pubMsg.sender.leafIndex)
} catch (e: Exception) {
GroupEventResult.Error(
groupId,
"Failed to stage standalone proposal: ${e.message}",
e,
)
}
}
ContentType.APPLICATION -> {
GroupEventResult.Error(groupId, "Application messages should use PrivateMessage")
}
}
}
private suspend fun handleCommitEvent(
database: TorchDatabase,
mlsGroup: MlsGroup,
groupEvent: GroupEvent
): GroupEventResult {
val groupId = mlsGroup.currentMarmotData()?.nostrGroupId ?: mlsGroup.groupId.toHex()
val currentEpoch = mlsGroup.epoch
commitTracker.addCommit(groupId, currentEpoch, groupEvent)
// If this is the only commit for this epoch, apply immediately
val pending = commitTracker.pendingForEpoch(groupId, currentEpoch)
return if (pending.size == 1) {
val result = applyCommit(
database,
mlsGroup,
groupEvent
)
commitTracker.clearEpoch(groupId, currentEpoch)
result
} else {
GroupEventResult.CommitPending(groupId, currentEpoch)
}
}
private suspend fun applyCommit(
database: TorchDatabase,
mlsGroup: MlsGroup,
commitEvent: GroupEvent,
): GroupEventResult =
try {
val groupId = mlsGroup.currentMarmotData()?.nostrGroupId ?: mlsGroup.groupId.toHex()
val mlsBytes =
tryDecryptOuterLayer(mlsGroup, commitEvent.encryptedContent())
?: return GroupEventResult.UndecryptableOuterLayer(
groupId,
retainedEpochCount = retainedExporterSecrets(groupId).size,
)
val mlsMessage = MlsMessage.decodeTls(TlsReader(mlsBytes))
when (mlsMessage.wireFormat) {
WireFormat.PRIVATE_MESSAGE -> {
// Sniff the PrivateMessage epoch without consuming any
// ratchet state. Past-epoch echoes and future-epoch
// arrivals must not advance the secret tree — otherwise
// the real handshake / application message gets rejected
// when it finally arrives.
val privPeek = PrivateMessage.decodeTls(TlsReader(mlsMessage.payload))
val currentEpoch = mlsGroup.epoch
when {
privPeek.epoch < currentEpoch -> {
GroupEventResult.Duplicate(groupId)
}
privPeek.epoch > currentEpoch -> {
GroupEventResult.Error(
groupId,
"PrivateMessage epoch ${privPeek.epoch} is ahead of local epoch $currentEpoch; ignoring",
)
}
else -> {
val decrypted = mlsGroup.decrypt(mlsMessage.toTlsBytes())
if (decrypted.contentType == ContentType.COMMIT) {
GroupEventResult.CommitProcessed(groupId, mlsGroup.epoch ?: 0)
} else {
GroupEventResult.Error(
groupId,
"Expected COMMIT but got ${decrypted.contentType}",
)
}
}
}
}
WireFormat.PUBLIC_MESSAGE -> {
val pubMsg = PublicMessage.decodeTls(TlsReader(mlsMessage.payload))
val tag = pubMsg.confirmationTag
val currentEpoch = mlsGroup.epoch
when {
tag == null -> {
GroupEventResult.Error(groupId, "PublicMessage commit missing confirmation_tag")
}
// Reject commits that are not for our current epoch.
// Happens most commonly when our own already-applied
// commit is echoed back from the relay after an app
// restart (the in-memory dedup set is cleared), and
// the outer layer decrypts via a retained epoch key.
// Calling `processCommit` on a past-epoch commit
// partially mutates tree / groupContext / epochSecrets
// before throwing on the confirmation-tag check,
// leaving the local state diverged from every other
// member's — they then can't decrypt anything we
// send next.
pubMsg.epoch < currentEpoch -> {
GroupEventResult.Duplicate(groupId)
}
pubMsg.epoch > currentEpoch -> {
GroupEventResult.Error(
groupId,
"Commit epoch ${pubMsg.epoch} is ahead of local epoch $currentEpoch; ignoring",
)
}
else -> {
// RFC 9420 §6.2 — reject PublicMessage commits
// whose membership_tag doesn't match what the
// current epoch's membership_key would produce.
// Without this an outsider with the outer
// exporter secret could inject arbitrary commit
// bytes and advance the group past them.
if (!mlsGroup.verifyPublicMessageCommitMembershipTag(pubMsg)) {
GroupEventResult.Error(
groupId,
"Invalid membership_tag on PublicMessage commit",
)
} else {
processCommit(
database,
mlsGroup,
commitBytes = pubMsg.content,
senderLeafIndex = pubMsg.sender.leafIndex,
confirmationTag = tag,
signature = pubMsg.signature,
)
GroupEventResult.CommitProcessed(groupId, mlsGroup.epoch)
}
}
}
}
else -> {
GroupEventResult.Error(groupId, "Unexpected wire format for commit")
}
}
} catch (e: Exception) {
GroupEventResult.Error(
groupId = mlsGroup.currentMarmotData()?.nostrGroupId ?: mlsGroup.groupId.toHex(),
"Failed to apply commit: ${e.message}",
e
)
}
/**
* Process a received Commit, advancing the epoch.
*
* @param mlsGroup MLS Group
* @param commitBytes TLS-serialized Commit
* @param senderLeafIndex sender's leaf index
* @param confirmationTag optional confirmation tag for verification
*/
suspend fun processCommit(
database: TorchDatabase,
mlsGroup: MlsGroup,
commitBytes: ByteArray,
senderLeafIndex: Int,
confirmationTag: ByteArray,
signature: ByteArray = ByteArray(0),
wireFormat: WireFormat = WireFormat.PUBLIC_MESSAGE,
) {
// Capture the outgoing epoch's secrets BEFORE advancing, but only
// commit them to the retention window once processCommit succeeds —
// otherwise a failed commit (e.g. "Duplicate encryption key" on an
// add-me relay echo) would pollute the window with a duplicate of
// the current epoch key, wasting the finite retention slots.
val retainedBefore = mlsGroup.retainedSecrets()
mlsGroup.processCommit(commitBytes, senderLeafIndex, confirmationTag, signature, wireFormat)
pushRetainedEpoch(
nostrGroupId = mlsGroup.currentMarmotData()?.nostrGroupId ?: mlsGroup.groupId.toHex(),
retainedBefore
)
persistGroup(
database,
mlsGroup
)
}
/**
* Decrypt the outer ChaCha20-Poly1305 layer, trying the current epoch's
* exporter key first and falling back to retained epoch exporter keys.
*
* After a commit advances the epoch, late-arriving messages encrypted
* with the previous epoch's exporter key would fail without this fallback.
*
* Returns null when neither the current epoch key nor any retained key
* decrypts. This happens normally for commits/application messages from
* epochs that predate our join (we never held those keys), so callers
* should treat null as an expected "nothing to do here" outcome and log
* at DEBUG, not as an error.
*/
private fun tryDecryptOuterLayer(
mlsGroup: MlsGroup,
encryptedContent: String,
): ByteArray? {
// Try current epoch key first
try {
val exporterKey = mlsGroup.exporterSecret()
return GroupEventEncryption.decrypt(encryptedContent, exporterKey)
} catch (_: Exception) {
// Current epoch key failed — try retained epoch keys
}
// Try retained epoch exporter keys (most recent first)
val retainedKeys = retainedExporterSecrets(
nostrGroupId = mlsGroup.currentMarmotData()?.nostrGroupId ?: mlsGroup.groupId.toHex()
)
for (retainedKey in retainedKeys) {
try {
return GroupEventEncryption.decrypt(encryptedContent, retainedKey)
} catch (_: Exception) {
// This retained key didn't work — try the next one
}
}
return null
}
private suspend fun persistGroup(
database: TorchDatabase,
mlsGroup: MlsGroup
) {
val nostrGroupId = mlsGroup.currentMarmotData()?.nostrGroupId ?: mlsGroup.groupId.toHex()
try {
val state = mlsGroup.saveState()
val encoded = state.encodeTls()
logger.d { "persistGroup($nostrGroupId): serialized ${encoded.size} bytes, calling store.save" }
val localChatRoom = database.chatRoomDao().findChatRoomById(nostrGroupId)
if (localChatRoom == null) {
throw MarmotMissingChatGroupException("chatRoom not found in database, skipping")
} else {
database.chatRoomDao().upsert(
localChatRoom.chatRoom.copy(
mlsGroupState = encoded.toHex()
)
)
}
// Also persist retained epochs
val retained = retainedEpochs[nostrGroupId]
if (retained != null) {
val retainedBytes =
retained.map { epoch ->
val writer = TlsWriter()
epoch.encodeTls(writer)
writer.toByteArray()
}
// TODO: store.saveRetainedEpochs(nostrGroupId, retainedBytes)
logger.d { "persistGroup($nostrGroupId): persisted ${retainedBytes.size} retained epochs" }
}
} catch (e: Exception) {
logger.e("persistGroup($nostrGroupId) FAILED: ${e.message}", e)
throw e
}
}
/**
* Push a previously-captured [RetainedEpochSecrets] into the bounded
* retention window. Call after the epoch advance has been applied
* successfully so that failed commits don't pollute the window with
* duplicate current-epoch keys.
*/
private fun pushRetainedEpoch(
nostrGroupId: HexKey,
retainedBefore: RetainedEpochSecrets,
) {
val retained = retainedEpochs.getOrPut(nostrGroupId) { mutableListOf() }
retained.add(retainedBefore)
// Trim to retention window (keep only the most recent N-1 epochs)
while (retained.size > EPOCH_RETENTION_WINDOW) {
retained.removeAt(0)
}
}
private fun tryDecryptWithRetainedEpoch(
messageBytes: ByteArray,
retained: RetainedEpochSecrets,
): DecryptedMessage? =
try {
val secretTree = SecretTree(retained.encryptionSecret, retained.leafCount)
val mlsMsg =
MlsMessage
.decodeTls(TlsReader(messageBytes))
if (mlsMsg.wireFormat != WireFormat.PRIVATE_MESSAGE) {
return null
}
val privMsg = PrivateMessage.decodeTls(
TlsReader(
mlsMsg.payload
)
)
if (privMsg.epoch != retained.epoch) return null
// Derive sender data key/nonce using ciphertext sample (RFC 9420 §6.3.1)
// RFC 9420 §6.3.2: ciphertext_sample is the first KDF.Nh bytes
// (32 for HKDF-SHA256), not AEAD.Nk (16). Using AEAD.Nk here made
// sender-data decryption silently fail for every retained-epoch
// message and turned the fallback path into a no-op — the
// symptom was interop Test 12 (offline catch-up): kind:9
// messages encrypted under epoch N arriving after a 1→N+1
// commit got rejected with "Message epoch X doesn't match
// current epoch Y" instead of being pulled through this path.
// Same fix already applied to MlsGroup.decrypt (line ~878);
// this branch was missed when that one was patched.
val ciphertextSample =
privMsg.ciphertext.copyOfRange(0, minOf(privMsg.ciphertext.size, MlsCryptoProvider.HASH_OUTPUT_LENGTH))
val senderDataKey =
MlsCryptoProvider.expandWithLabel(
retained.senderDataSecret,
"key",
ciphertextSample,
MlsCryptoProvider.AEAD_KEY_LENGTH,
)
val senderDataNonce =
MlsCryptoProvider.expandWithLabel(
retained.senderDataSecret,
"nonce",
ciphertextSample,
MlsCryptoProvider.AEAD_NONCE_LENGTH,
)
// Build SenderDataAAD
val senderDataAad = TlsWriter()
senderDataAad.putOpaqueVarInt(privMsg.groupId)
senderDataAad.putUint64(privMsg.epoch)
senderDataAad.putUint8(privMsg.contentType.value)
val senderDataPlain =
MlsCryptoProvider.aeadDecrypt(
senderDataKey,
senderDataNonce,
senderDataAad.toByteArray(),
privMsg.encryptedSenderData,
)
val senderReader = TlsReader(senderDataPlain)
val senderLeafIndex = senderReader.readUint32().toInt()
val generation = senderReader.readUint32().toInt()
val reuseGuard = senderReader.readBytes(REUSE_GUARD_LENGTH)
val kng = secretTree.applicationKeyNonceForGeneration(senderLeafIndex, generation)
// Apply reuse_guard XOR to nonce
val guardedNonce = kng.nonce.copyOf()
for (i in 0 until REUSE_GUARD_LENGTH) {
guardedNonce[i] = (guardedNonce[i].toInt() xor reuseGuard[i].toInt()).toByte()
}
// Build PrivateContentAAD
val contentAad = TlsWriter()
contentAad.putOpaqueVarInt(privMsg.groupId)
contentAad.putUint64(privMsg.epoch)
contentAad.putUint8(privMsg.contentType.value)
contentAad.putOpaqueVarInt(privMsg.authenticatedData)
val pmcBytes =
MlsCryptoProvider.aeadDecrypt(kng.key, guardedNonce, contentAad.toByteArray(), privMsg.ciphertext)
// AEAD plaintext is a PrivateMessageContent struct (RFC 9420
// §6.3.1): `applicationData<V> || signature<V> || padding`. The
// main decrypt path parses this and returns the inner
// applicationData; the retained-epoch branch was returning the
// raw struct, which made callers see a length-prefixed blob
// with signature + zero-padding glued onto the end. Extract the
// applicationData the same way.
val pmcReader = TlsReader(pmcBytes)
val applicationData = pmcReader.readOpaqueVarInt()
DecryptedMessage(
senderLeafIndex = senderLeafIndex,
contentType = privMsg.contentType,
content = applicationData,
epoch = privMsg.epoch,
)
} catch (_: Exception) {
null
}
/**
* Return exporter secrets from retained epochs for a group.
*
* Used by the inbound processor to attempt outer decryption with
* previous epoch keys when the current epoch's key fails (e.g.,
* after a commit has advanced the epoch but late-arriving messages
* still use the old exporter key).
*
* @param nostrGroupId hex-encoded Nostr group ID
* @return list of retained exporter secrets (most recent first), each
* derived via MLS-Exporter("marmot", "group-event", 32)
*/
fun retainedExporterSecrets(nostrGroupId: HexKey): List<ByteArray> {
val retained = retainedEpochs[nostrGroupId] ?: return emptyList()
return retained
.filter { it.exporterSecret.isNotEmpty() }
.sortedByDescending { it.epoch }
.map { epochSecrets ->
KeySchedule.mlsExporter(
epochSecrets.exporterSecret,
"marmot",
"group-event".encodeToByteArray(),
32,
)
}
}
}

View File

@@ -1,6 +1,10 @@
package at.torch.compose.repository
import at.torch.compose.database.model.MarmotKeyPackageBundle
import com.vitorpamplona.quartz.marmot.GroupEventResult
import com.vitorpamplona.quartz.marmot.mip03GroupMessages.GroupEvent
import com.vitorpamplona.quartz.marmot.mls.framing.MlsMessage
import com.vitorpamplona.quartz.marmot.mls.group.MlsGroup
import com.vitorpamplona.quartz.nip01Core.core.HexKey
interface MarmotRepository {
@@ -24,7 +28,6 @@ interface MarmotRepository {
override suspend fun getMarmotKeyPackageBundles(publicKey: HexKey): List<MarmotKeyPackageBundle> {
return emptyList()
}
}
}
}