diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/database/dao/NostrEventDao.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/database/dao/NostrEventDao.kt index 76a4c306..1066c814 100644 --- a/composeApp/src/commonMain/kotlin/press/mantra/compose/database/dao/NostrEventDao.kt +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/database/dao/NostrEventDao.kt @@ -65,6 +65,18 @@ interface NostrEventDao { query: RoomRawQuery ): Flow> + /** + * Events matching an arbitrary NIP-01 filter, built by + * [press.mantra.compose.database.query.NostrEventFilterQuery]. Raw because a nostr filter is + * a variable set of constraints over variable-length lists, which `@Query` cannot express + * without one method per shape — and the per-shape methods this replaced each dropped the + * constraints they had no parameter for. + */ + @RawQuery + fun getNostrEventsMatchingFilter( + query: RoomRawQuery + ): List + @Transaction @Query("SELECT * FROM NostrEvent WHERE kind in (:kinds) AND createdAt > :since ORDER BY createdAt ASC LIMIT :limit") fun getNostrEvents( diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/database/query/NostrEventFilterQuery.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/database/query/NostrEventFilterQuery.kt new file mode 100644 index 00000000..01b6c4d1 --- /dev/null +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/database/query/NostrEventFilterQuery.kt @@ -0,0 +1,166 @@ +package press.mantra.compose.database.query + +import androidx.room3.RoomRawQuery +import androidx.sqlite.SQLiteStatement +import press.mantra.compose.database.model.types.SynchronizationFilter +import kotlinx.serialization.json.Json + +/** + * Translates a [SynchronizationFilter] into the SQL that selects exactly the events a relay + * would return for the same NIP-01 filter. + * + * Negentropy compares two sets that are both defined by the *same* filter: ours, built here, + * and the relay's, built from the filter carried in NEG-OPEN. Any constraint we fail to apply + * locally makes our set a superset of the relay's, and every extra row comes back as an id the + * relay is "missing" — which this app then schedules as a broadcast. Any constraint we apply + * more tightly makes it a subset, and the difference comes back as ids to re-download that we + * already have. So the value here is not just tidiness: a filter clause that is silently + * dropped turns reconciliation into busywork in both directions. + * + * This replaced a chain of hand-written special cases that between them could not express + * `until` at all, ignored `tags` whenever `authors` was also set, and fell back to + * "kind 1 text notes" for any filter shape it did not recognise. + * + * Semantics mirror quartz's `FilterMatcher`, which is what the relays this app talks to + * implement: + * - `ids`/`authors`/`kinds`: membership; a present-but-empty list matches nothing. + * - `tags`: AND between tag names, OR between the values of one name. + * - `tagsAll`: AND between tag names, AND between the values of one name. + * - `since`/`until`: **inclusive** on both ends. + */ +object NostrEventFilterQuery { + + /** Sentinel for "no limit", matching SQLite's treatment of a negative LIMIT. */ + const val NO_LIMIT: Int = -1 + + fun build( + filter: SynchronizationFilter, + limit: Int = NO_LIMIT, + ): RoomRawQuery { + val clauses = mutableListOf() + val bindings = mutableListOf() + + filter.ids?.let { clauses += inClause("id", it.asList(), bindings) } + filter.authors?.let { clauses += inClause("pubKey", it.asList(), bindings) } + filter.kinds?.let { kinds -> + clauses += if (kinds.isEmpty()) { + MATCHES_NOTHING + } else { + bindings += kinds.map { Binding.Number(it.toLong()) } + "kind IN (${placeholders(kinds.size)})" + } + } + + // `since`/`until` are inclusive per NIP-01. The queries this replaced used a strict + // `createdAt > :since`, so an event stamped exactly on the boundary was in the relay's + // set and not in ours, and `until` was sent to the relay but never applied here at all. + filter.since?.let { + clauses += "createdAt >= ?" + bindings += Binding.Number(it.epochSeconds) + } + filter.until?.let { + clauses += "createdAt <= ?" + bindings += Binding.Number(it.epochSeconds) + } + + filter.search?.let { + clauses += "content LIKE '%' || ? || '%'" + bindings += Binding.Text(it) + } + + // AND between tag names, OR between the values of one name. + filter.tags?.forEach { (name, values) -> + clauses += if (values.isEmpty()) { + MATCHES_NOTHING + } else { + values.joinToString(separator = " OR ", prefix = "(", postfix = ")") { value -> + bindings += Binding.Text(tagPattern(name, value)) + TAG_LIKE + } + } + } + + // AND between tag names, AND between the values of one name. + filter.tagsAll?.forEach { (name, values) -> + values.forEach { value -> + bindings += Binding.Text(tagPattern(name, value)) + clauses += TAG_LIKE + } + } + + val where = if (clauses.isEmpty()) "" else " WHERE " + clauses.joinToString(" AND ") + + // Newest-first, which is the window a relay hands back when a filter carries a limit. + // The previous queries ordered ascending under the same limit and so returned the + // OLDEST rows instead. + val sql = buildString { + append("SELECT * FROM NostrEvent") + append(where) + append(" ORDER BY createdAt DESC, id DESC") + if (limit >= 0) { + append(" LIMIT ?") + } + } + if (limit >= 0) { + bindings += Binding.Number(limit.toLong()) + } + + val boundValues = bindings.toList() + return RoomRawQuery(sql) { statement -> + boundValues.forEachIndexed { index, binding -> + binding.bindTo(statement, index + 1) + } + } + } + + private const val MATCHES_NOTHING = "0" + + /** + * Tags are stored as the compact JSON of the tag array, e.g. `[["p",""],["e",""]]`, + * so a tag is matched by looking for the `["",""` fragment. Anchoring on both + * the name and the closing quote of the value is what keeps this from matching the same hex + * sitting in an unrelated tag position — the substring scan it replaced (`tags LIKE + * '%%'`) matched a pubkey anywhere in the row, including inside `e` tags. + */ + private const val TAG_LIKE = "tags LIKE ? ESCAPE '\\'" + + private fun tagPattern(name: String, value: String): String { + // Encode through the same serializer that wrote the column so any escaping matches, + // then drop the closing `]` so trailing tag elements (relay hint, marker) still match. + val fragment = Json.encodeToString(listOf(name, value)).dropLast(1) + return "%${escapeLike(fragment)}%" + } + + /** Keeps a `%`/`_` inside a tag value from silently widening the match. */ + private fun escapeLike(value: String): String = + buildString(value.length) { + value.forEach { char -> + if (char == '\\' || char == '%' || char == '_') append('\\') + append(char) + } + } + + private fun inClause( + column: String, + values: List, + bindings: MutableList, + ): String { + if (values.isEmpty()) return MATCHES_NOTHING + bindings += values.map { Binding.Text(it) } + return "$column IN (${placeholders(values.size)})" + } + + private fun placeholders(count: Int) = List(count) { "?" }.joinToString(", ") + + private sealed interface Binding { + fun bindTo(statement: SQLiteStatement, index: Int) + + data class Text(val value: String) : Binding { + override fun bindTo(statement: SQLiteStatement, index: Int) = statement.bindText(index, value) + } + + data class Number(val value: Long) : Binding { + override fun bindTo(statement: SQLiteStatement, index: Int) = statement.bindLong(index, value) + } + } +} diff --git a/composeApp/src/commonMain/kotlin/press/mantra/compose/database/repository/DatabaseNostrRepository.kt b/composeApp/src/commonMain/kotlin/press/mantra/compose/database/repository/DatabaseNostrRepository.kt index d819381d..27cfca37 100644 --- a/composeApp/src/commonMain/kotlin/press/mantra/compose/database/repository/DatabaseNostrRepository.kt +++ b/composeApp/src/commonMain/kotlin/press/mantra/compose/database/repository/DatabaseNostrRepository.kt @@ -8,6 +8,8 @@ import press.mantra.compose.database.model.UnsignedNostrEvent import press.mantra.compose.database.model.intermdiate.LocalAccount import press.mantra.compose.database.model.intermdiate.LocalBroadcastNostrEventRequest import press.mantra.compose.database.model.types.SynchronizationFilter +import press.mantra.compose.database.dao.NostrEventDao +import press.mantra.compose.database.query.NostrEventFilterQuery import press.mantra.compose.extensions.toHex import press.mantra.compose.nostr.Relays import co.touchlab.kermit.Logger @@ -60,6 +62,9 @@ class DatabaseNostrRepository( companion object { const val TAG = "DatabaseNostrRepository" + /** Falls back to the same window the per-shape queries this replaced hard-coded. */ + private const val DEFAULT_NEGENTROPY_LIMIT = 50 + private val storeNostrEventMutex = Mutex() } @@ -630,131 +635,82 @@ class DatabaseNostrRepository( applyLimits: Boolean ): List { logger.d("getNegentropicNostrFeedIds: ${synchronizationFilters.contentToString()}") - if (synchronizationFilters.size == 1) { - val synchronizationFilter = synchronizationFilters.first() - return when { - synchronizationFilter.kinds != null && synchronizationFilter.search != null -> { - logger.d("getFilteredNostrEvents($applyLimits): $synchronizationFilter") - database.nostrEventDao().getFilteredNostrEvents( - kinds = synchronizationFilter.kinds, - search = synchronizationFilter.search, - since = synchronizationFilter.since ?: GENESIS_AT, - limit = if (applyLimits) { - synchronizationFilter.limit ?: 50 - } else { - -1 - } - ) - } - synchronizationFilter.kinds != null && synchronizationFilter.ids != null -> { - logger.d("getFilteredNostrEvents($applyLimits): $synchronizationFilter") - database.nostrEventDao().getFilteredNostrEvents( - kinds = synchronizationFilter.kinds, - ids = synchronizationFilter.ids, - since = synchronizationFilter.since ?: GENESIS_AT, - limit = if (applyLimits) { - synchronizationFilter.limit ?: 50 - } else { - -1 - } - ) - } - synchronizationFilter.ids != null -> { - logger.d("getNostrEventsByIds($applyLimits): $synchronizationFilter") - database.nostrEventDao().getNostrEventsByIds( - ids = synchronizationFilter.ids, - since = synchronizationFilter.since ?: GENESIS_AT, - limit = if (applyLimits) { - synchronizationFilter.limit ?: 50 - } else { - -1 - } - ) - } - synchronizationFilter.kinds != null && synchronizationFilter.authors != null -> { - logger.d("getAuthoredNostrEvents($applyLimits): $synchronizationFilter") - database.nostrEventDao().getAuthoredNostrEvents( - kinds = synchronizationFilter.kinds, - authors = synchronizationFilter.authors, - since = synchronizationFilter.since ?: GENESIS_AT, - limit = if (applyLimits) { - synchronizationFilter.limit ?: 50 - } else { - -1 - } - ) - } - synchronizationFilter.kinds != null && synchronizationFilter.since != null && synchronizationFilter.tags?.contains("e") == true && synchronizationFilter.tags["e"]?.first()?.isNotEmpty() == true -> { - logger.d("getNostrEventReplies($applyLimits): $synchronizationFilter") - database.nostrEventDao().getNostrEventReplies( - kinds = synchronizationFilter.kinds, - eventId = synchronizationFilter.tags["e"]?.first()!!, - since = synchronizationFilter.since, - limit = if (applyLimits) { - synchronizationFilter.limit ?: 50 - } else { - -1 - } - ) - } - synchronizationFilter.kinds != null && synchronizationFilter.tags?.contains("p") == true && synchronizationFilter.tags["p"]?.first()?.isNotEmpty() == true -> { - logger.d("getPublicKeyMentionedNostrEvents($applyLimits): $synchronizationFilter") - database.nostrEventDao().getPublicKeyMentionedNostrEvents( - kinds = synchronizationFilter.kinds, - publicKey = synchronizationFilter.tags["p"]?.first()!!, - since = synchronizationFilter.since ?: GENESIS_AT, - limit = if (applyLimits) { - synchronizationFilter.limit ?: 50 - } else { - -1 - } - ) - } - synchronizationFilter.kinds?.isNotEmpty() == true && synchronizationFilter.tags?.contains("h") == true -> { - val chatRoomIds = synchronizationFilter.tags["h"]?.toTypedArray() ?: emptyArray() - logger.d("ChatRoomIds: ${chatRoomIds.contentToString()}") - logger.d("getMLSGroupEvents($applyLimits): $synchronizationFilter") - database.nostrEventDao().getMarmotGroupEvents( - chatRoomIds = chatRoomIds, - since = synchronizationFilter.since ?: GENESIS_AT, - until = synchronizationFilter.until ?: Instant.DISTANT_FUTURE, - limit = if (applyLimits) { - synchronizationFilter.limit ?: 50 - } else { - -1 - } - ).map { marmotGroupEvent -> - NostrEvent( - id = marmotGroupEvent.id, - content = marmotGroupEvent.encryptedContent, - kind = GroupEvent.KIND, - tags = emptyArray(), - createdAt = marmotGroupEvent.createdAt, - sig = "", // TODO: Use marmotGroupEvent.signature... - pubKey = marmotGroupEvent.publicKey - ) - } - } - else -> { - logger.d("getNostrEvents($applyLimits): $synchronizationFilter") - database.nostrEventDao().getNostrEvents( - kinds = arrayOf( - TextNoteEvent.KIND, - ), - since = synchronizationFilter.since ?: GENESIS_AT, - limit = if (applyLimits) { - synchronizationFilter.limit ?: 50 - } else { - -1 - } - ) - } + // A negentropy session reconciles against the union of its filters. Returning nothing + // for anything other than a single filter (as this used to) is the worst possible + // answer: an empty local set tells the relay we have none of these events, so it + // hands back its entire set as ids to download. + return synchronizationFilters + .flatMap { synchronizationFilter -> + matchNegentropicNostrEvents(synchronizationFilter, applyLimits) } + .distinctBy { it.id } + } + + private fun matchNegentropicNostrEvents( + synchronizationFilter: SynchronizationFilter, + applyLimits: Boolean, + ): List { + val limit = if (applyLimits) { + synchronizationFilter.limit ?: DEFAULT_NEGENTROPY_LIMIT } else { - logger.w("Multi synchronizationFilters not yet supported") - return emptyList() + NostrEventFilterQuery.NO_LIMIT } + + val chatRoomIds = synchronizationFilter.marmotGroupChatRoomIds() + if (chatRoomIds != null) { + // Group messages live in their own table, which carries the NIP-40 expiry a relay + // uses to decide whether it still serves the event, and an indexed chatRoomId + // rather than a scan of the tags JSON. + logger.d("getMarmotGroupEvents($applyLimits): $synchronizationFilter") + return database.nostrEventDao().getMarmotGroupEvents( + chatRoomIds = chatRoomIds, + since = synchronizationFilter.since ?: GENESIS_AT, + until = synchronizationFilter.until ?: Instant.DISTANT_FUTURE, + limit = limit, + ).map { marmotGroupEvent -> + // Only the id and createdAt reach the negentropy vector; the rest is filled in + // so callers keep a NostrEvent-shaped result. + NostrEvent( + id = marmotGroupEvent.id, + content = marmotGroupEvent.encryptedContent, + kind = GroupEvent.KIND, + tags = arrayOf(arrayOf("h", marmotGroupEvent.chatRoomId)), + createdAt = marmotGroupEvent.createdAt, + sig = marmotGroupEvent.signature, + pubKey = marmotGroupEvent.publicKey + ) + } + } + + logger.d("getNostrEventsMatchingFilter($applyLimits): $synchronizationFilter") + return database.nostrEventDao().getNostrEventsMatchingFilter( + NostrEventFilterQuery.build( + filter = synchronizationFilter, + limit = limit, + ) + ) + } + + /** + * The chat room ids of a filter that asks for nothing but Marmot group messages, or null if + * the filter says anything the [NostrEventDao.getMarmotGroupEvents] lookup cannot express. + * Kept deliberately narrow: the branch it guards answers from a different table, so a filter + * it only partly understands would silently drop the rest of the constraints — which is + * exactly what the shape-guessing chain this replaced did. + */ + private fun SynchronizationFilter.marmotGroupChatRoomIds(): Array? { + val tags = tags ?: return null + val rooms = tags["h"] ?: return null + + val onlyGroupEvents = kinds != null && kinds.size == 1 && kinds.first() == GroupEvent.KIND + val nothingElseAsked = ids == null && + authors == null && + search == null && + tagsAll.isNullOrEmpty() && + tags.keys == setOf("h") + + return if (onlyGroupEvents && nothingElseAsked) rooms.toTypedArray() else null } override suspend fun observeLocalNostrEventById(nostrEventId: String): Flow { diff --git a/composeApp/src/commonTest/kotlin/press/mantra/compose/database/query/NostrEventFilterQueryTest.kt b/composeApp/src/commonTest/kotlin/press/mantra/compose/database/query/NostrEventFilterQueryTest.kt new file mode 100644 index 00000000..f950561a --- /dev/null +++ b/composeApp/src/commonTest/kotlin/press/mantra/compose/database/query/NostrEventFilterQueryTest.kt @@ -0,0 +1,191 @@ +package press.mantra.compose.database.query + +import androidx.sqlite.SQLiteStatement +import press.mantra.compose.database.model.types.SynchronizationFilter +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.time.Instant + +/** + * Pins the SQL a [SynchronizationFilter] turns into. + * + * Negentropy is only as good as the agreement between the set we build from this query and the + * set the relay builds from the same filter sent in NEG-OPEN. A clause that is dropped or a + * bound that is off by one shows up not as an error but as reconciliation reporting differences + * that are not real — events re-downloaded, or events pushed at a relay that filtered them out + * on purpose. So the translation is asserted rather than eyeballed. + */ +class NostrEventFilterQueryTest { + + private val alice = "a".repeat(64) + private val bob = "b".repeat(64) + private val note = "c".repeat(64) + + @Test + fun `applies every clause a nostr filter can carry`() { + val query = NostrEventFilterQuery.build( + SynchronizationFilter( + ids = arrayOf(note), + authors = arrayOf(alice, bob), + kinds = arrayOf(1, 6), + since = Instant.fromEpochSeconds(1_000), + until = Instant.fromEpochSeconds(2_000), + ) + ) + + assertEquals( + "SELECT * FROM NostrEvent WHERE id IN (?) AND pubKey IN (?, ?) AND kind IN (?, ?) " + + "AND createdAt >= ? AND createdAt <= ? ORDER BY createdAt DESC, id DESC", + query.sql, + ) + assertEquals(listOf(note, alice, bob, 1L, 6L, 1_000L, 2_000L), query.boundValues()) + } + + /** + * `until` is the clause the per-shape queries this replaced could not express at all: it was + * sent to the relay and never applied locally, so every local event past the window came + * back as one the relay was "missing". + */ + @Test + fun `time bounds are inclusive on both ends`() { + val query = NostrEventFilterQuery.build( + SynchronizationFilter(since = Instant.fromEpochSeconds(7), until = Instant.fromEpochSeconds(9)) + ) + + assertEquals( + "SELECT * FROM NostrEvent WHERE createdAt >= ? AND createdAt <= ? ORDER BY createdAt DESC, id DESC", + query.sql, + ) + assertEquals(listOf(7L, 9L), query.boundValues()) + } + + /** AND between tag names, OR between the values of one name — as `FilterMatcher` does. */ + @Test + fun `tags are matched by name and value, not by substring`() { + val query = NostrEventFilterQuery.build( + SynchronizationFilter( + kinds = arrayOf(1059), + tags = mapOf("p" to listOf(alice, bob), "e" to listOf(note)), + ) + ) + + assertEquals( + "SELECT * FROM NostrEvent WHERE kind IN (?) " + + "AND (tags LIKE ? ESCAPE '\\' OR tags LIKE ? ESCAPE '\\') " + + "AND (tags LIKE ? ESCAPE '\\') ORDER BY createdAt DESC, id DESC", + query.sql, + ) + assertEquals( + listOf(1059L, "%[\"p\",\"$alice\"%", "%[\"p\",\"$bob\"%", "%[\"e\",\"$note\"%"), + query.boundValues(), + ) + } + + /** tagsAll is AND between values too, so each value gets its own clause. */ + @Test + fun `tagsAll requires every value`() { + val query = NostrEventFilterQuery.build( + SynchronizationFilter(tagsAll = mapOf("t" to listOf("nostr", "kmp"))) + ) + + assertEquals( + "SELECT * FROM NostrEvent WHERE tags LIKE ? ESCAPE '\\' AND tags LIKE ? ESCAPE '\\' " + + "ORDER BY createdAt DESC, id DESC", + query.sql, + ) + assertEquals(listOf("%[\"t\",\"nostr\"%", "%[\"t\",\"kmp\"%"), query.boundValues()) + } + + /** A wildcard in a tag value must not widen the match. */ + @Test + fun `like wildcards inside a tag value are escaped`() { + val query = NostrEventFilterQuery.build( + SynchronizationFilter(tags = mapOf("d" to listOf("100%_off"))) + ) + + assertEquals(listOf("%[\"d\",\"100\\%\\_off\"%"), query.boundValues()) + } + + /** A present-but-empty list matches nothing, the way a relay reads `{"kinds":[]}`. */ + @Test + fun `an empty list matches nothing`() { + assertEquals( + "SELECT * FROM NostrEvent WHERE 0 ORDER BY createdAt DESC, id DESC", + NostrEventFilterQuery.build(SynchronizationFilter(kinds = emptyArray())).sql, + ) + assertEquals( + "SELECT * FROM NostrEvent WHERE 0 ORDER BY createdAt DESC, id DESC", + NostrEventFilterQuery.build(SynchronizationFilter(tags = mapOf("h" to emptyList()))).sql, + ) + } + + @Test + fun `an unconstrained filter selects everything`() { + assertEquals( + "SELECT * FROM NostrEvent ORDER BY createdAt DESC, id DESC", + NostrEventFilterQuery.build(SynchronizationFilter()).sql, + ) + } + + /** A limited filter takes the newest rows, which is the window a relay returns. */ + @Test + fun `a limit is bound last and ordered newest first`() { + val query = NostrEventFilterQuery.build( + SynchronizationFilter(kinds = arrayOf(1)), + limit = 50, + ) + + assertEquals( + "SELECT * FROM NostrEvent WHERE kind IN (?) ORDER BY createdAt DESC, id DESC LIMIT ?", + query.sql, + ) + assertEquals(listOf(1L, 50L), query.boundValues()) + } + + @Test + fun `search still matches content`() { + val query = NostrEventFilterQuery.build(SynchronizationFilter(search = "mantra")) + + assertEquals( + "SELECT * FROM NostrEvent WHERE content LIKE '%' || ? || '%' ORDER BY createdAt DESC, id DESC", + query.sql, + ) + assertEquals(listOf("mantra"), query.boundValues()) + } + + private fun androidx.room3.RoomRawQuery.boundValues(): List = + RecordingStatement().also { getBindingFunction().invoke(it) }.bound + + /** Captures what the query binds, in order. Only the `bind*` calls are ever made. */ + private class RecordingStatement : SQLiteStatement { + val bound = mutableListOf() + + override fun bindText(index: Int, value: String) { + check(index == bound.size + 1) { "out of order bind at $index" } + bound += value + } + + override fun bindLong(index: Int, value: Long) { + check(index == bound.size + 1) { "out of order bind at $index" } + bound += value + } + + override fun bindBlob(index: Int, value: ByteArray) = unsupported() + override fun bindDouble(index: Int, value: Double) = unsupported() + override fun bindNull(index: Int) = unsupported() + override fun getBlob(index: Int): ByteArray = unsupported() + override fun getDouble(index: Int): Double = unsupported() + override fun getLong(index: Int): Long = unsupported() + override fun getText(index: Int): String = unsupported() + override fun isNull(index: Int): Boolean = unsupported() + override fun getColumnCount(): Int = unsupported() + override fun getColumnName(index: Int): String = unsupported() + override fun getColumnType(index: Int): Int = unsupported() + override fun step(): Boolean = unsupported() + override fun reset() = unsupported() + override fun clearBindings() = unsupported() + override fun close() = unsupported() + + private fun unsupported(): Nothing = error("only bind calls are expected") + } +}