diff --git a/src/main/kotlin/com/embabel/agent/rag/graph/GraphObjectManagerStore.kt b/src/main/kotlin/com/embabel/agent/rag/graph/GraphObjectManagerStore.kt index ae37baf..c970b1b 100644 --- a/src/main/kotlin/com/embabel/agent/rag/graph/GraphObjectManagerStore.kt +++ b/src/main/kotlin/com/embabel/agent/rag/graph/GraphObjectManagerStore.kt @@ -88,6 +88,7 @@ import org.drivine.query.dsl.instanceOf import org.drivine.query.dsl.query import org.drivine.schema.FullTextIndexSpec import org.drivine.schema.RangeIndexSpec +import org.drivine.schema.SchemaCatalog import org.drivine.schema.SimilarityFunction import org.drivine.schema.UniquenessConstraintSpec import org.drivine.schema.VectorIndexSpec @@ -107,6 +108,24 @@ private const val REEMBED_WINDOW_ATTEMPTS = 3 /** The label every document, section and chunk carries ([com.embabel.agent.rag.graph.model.ContentElementNode]). */ private const val CONTENT_ELEMENT_LABEL = "ContentElement" +/** + * The fragments every save, parent link and next-chunk link finds its nodes by. Each is a + * `MERGE (n: {id: …})` or a `MATCH` of the same shape: a document, a section and a chunk all + * carry `ContentElement` ([ContentElementFragment]), while the next-chunk link names `Chunk` alone + * ([ChunkPlaceFragment]). Each declares its id `@RangeIndex`, beside its `@NodeId`, so the model is the + * one place the index is stated. + * + * Without those indexes every one of those statements scans its label, so writing one document costs the + * size of the whole store, once per section, chunk and link — and on an engine that holds its write lock + * for a statement, every reader waits out each scan. A range index rather than a uniqueness constraint: + * it asks nothing of what a store already holds, so it cannot fail at startup. + */ +internal val CONTENT_ID_FRAGMENTS = listOf(ContentElementFragment::class, ChunkPlaceFragment::class) + +/** The range indexes [CONTENT_ID_FRAGMENTS] declare, read off their annotations. */ +internal fun contentIdIndexes(): List = + SchemaCatalog.fromFragments(*CONTENT_ID_FRAGMENTS.toTypedArray()).indexes.filterIsInstance() + /** * A threshold above this requires a raw full-text score above [FULL_TEXT_BM25_K] — already * a strong match. Empty results above it are far more likely a mis-set threshold than an empty corpus. @@ -224,18 +243,11 @@ class GraphObjectManagerStore( ) /* - * What every save, parent link and next-chunk link finds its nodes by. Each is a - * `MERGE (n: {id: …})` or a `MATCH` of the same shape: a document, a section and a chunk all - * carry `ContentElement`, while the next-chunk link names `Chunk` alone ([ChunkPlaceFragment]). - * Without these, every one of those statements scans its label, so writing one document costs the - * size of the whole store, once per section, chunk and link — and on an engine that holds its write - * lock for a statement, every reader waits out each scan. A range index rather than a uniqueness - * constraint: it asks nothing of what a store already holds, so it cannot fail at startup. + * What every save, parent link and next-chunk link finds its nodes by, declared on the model as + * `@NodeId @RangeIndex` on the fragments those statements name ([CONTENT_ID_FRAGMENTS]). Their labels + * are the fragments' own; the chunk one equals `properties.chunkNodeName`, which `init` pins to it. */ - private val contentElementIdIndexes = listOf( - RangeIndexSpec(CONTENT_ELEMENT_LABEL, listOf("id")), - RangeIndexSpec(properties.chunkNodeName, listOf("id")), - ) + private val contentElementIdIndexes = contentIdIndexes() private val provisioner = GraphProvisioner(persistenceManager) diff --git a/src/main/kotlin/com/embabel/agent/rag/graph/model/ChunkPlaceFragment.kt b/src/main/kotlin/com/embabel/agent/rag/graph/model/ChunkPlaceFragment.kt index e257bc6..199e3ad 100644 --- a/src/main/kotlin/com/embabel/agent/rag/graph/model/ChunkPlaceFragment.kt +++ b/src/main/kotlin/com/embabel/agent/rag/graph/model/ChunkPlaceFragment.kt @@ -18,14 +18,18 @@ package com.embabel.agent.rag.graph.model import org.drivine.annotation.GraphProperty import org.drivine.annotation.NodeFragment import org.drivine.annotation.NodeId +import org.drivine.annotation.RangeIndex /** * Where a chunk sits, and nothing else: enough to link it to its parent and to the chunk after it * without loading its text or its vector. + * + * Its `id` is range-indexed because the next-chunk link MATCHes both chunks by it under `Chunk` alone, + * which the `ContentElement` index does not serve. The store provisions this index from the annotation. */ @NodeFragment(labels = ["Chunk"]) data class ChunkPlaceFragment( - @NodeId val id: String, + @NodeId @RangeIndex val id: String, val parentId: String? = null, @GraphProperty("root_document_id") val rootDocumentId: String? = null, @GraphProperty("container_section_id") val containerSectionId: String? = null, diff --git a/src/main/kotlin/com/embabel/agent/rag/graph/model/ContentElementFragment.kt b/src/main/kotlin/com/embabel/agent/rag/graph/model/ContentElementFragment.kt index fd973be..f689ecc 100644 --- a/src/main/kotlin/com/embabel/agent/rag/graph/model/ContentElementFragment.kt +++ b/src/main/kotlin/com/embabel/agent/rag/graph/model/ContentElementFragment.kt @@ -17,15 +17,20 @@ package com.embabel.agent.rag.graph.model import org.drivine.annotation.NodeFragment import org.drivine.annotation.NodeId +import org.drivine.annotation.RangeIndex /** * A label-generic fragment matching **any** content element — the whole hierarchy shares the * `ContentElement` label, so a single fragment can stand in for a Document, Section, or Chunk when * we only care about tree structure and common properties. + * + * Its `id` is range-indexed because every save of a document, section or chunk MERGEs by it under + * `ContentElement`, and every parent link MATCHes both ends the same way: without the index, each of + * those statements scans the label. The store provisions this index from the annotation. */ @NodeFragment(labels = ["ContentElement"]) data class ContentElementFragment( - @NodeId val id: String, + @NodeId @RangeIndex val id: String, val text: String? = null, val title: String? = null, val uri: String? = null, diff --git a/src/test/kotlin/com/embabel/agent/rag/graph/ContentIdIndexesFollowTheModelTest.kt b/src/test/kotlin/com/embabel/agent/rag/graph/ContentIdIndexesFollowTheModelTest.kt new file mode 100644 index 0000000..6d99485 --- /dev/null +++ b/src/test/kotlin/com/embabel/agent/rag/graph/ContentIdIndexesFollowTheModelTest.kt @@ -0,0 +1,101 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.agent.rag.graph + +import com.embabel.agent.rag.graph.model.ContentElementFragment +import org.drivine.annotation.NodeFragment +import org.drivine.annotation.NodeId +import org.drivine.model.FragmentModel +import org.drivine.schema.RangeIndexSpec +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.DisplayName +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.AnnotatedBeanDefinition +import org.springframework.context.annotation.ClassPathScanningCandidateComponentProvider +import org.springframework.core.type.filter.AnnotationTypeFilter + +/** + * The content id indexes are declared on the model, as `@NodeId @RangeIndex`, and the store provisions what + * the model declares. These check that every content fragment's saves are served by one of them, so a new + * fragment, or a lost annotation, cannot leave the saves it MERGEs by id scanning their label. + */ +class ContentIdIndexesFollowTheModelTest { + + /** The labels whose nodes the store writes and finds by id. */ + private val contentLabels = setOf("ContentElement", GraphRagServiceProperties().chunkNodeName) + + /** Every `@NodeFragment` in the model package, interfaces included. */ + private fun modelFragments(): List> { + val scanner = object : ClassPathScanningCandidateComponentProvider(false) { + override fun isCandidateComponent(beanDefinition: AnnotatedBeanDefinition) = true + } + scanner.addIncludeFilter(AnnotationTypeFilter(NodeFragment::class.java)) + return scanner.findCandidateComponents(ContentElementFragment::class.java.packageName) + .map { Class.forName(it.beanClassName) } + } + + /** + * The content fragments in [fragments] whose `@NodeId` no index in [indexes] serves: a save of one is a + * MERGE by its id under all its labels, so an index on its id under any one of them is enough. + */ + private fun unindexed(fragments: List>, indexes: List): List = + fragments.mapNotNull { fragment -> + val labels = FragmentModel.labelsFor(fragment) + if (labels.none { it in contentLabels }) return@mapNotNull null + val nodeId = FragmentModel.from(fragment).nodeIdProperty ?: return@mapNotNull null + val served = indexes.any { it.label in labels && it.properties == listOf(nodeId) } + if (served) null else "${fragment.simpleName}($labels).$nodeId" + } + + @Test + @DisplayName("the indexes read off the model are the ones the store used to list by hand") + fun `derived indexes equal the old hand list`() { + assertEquals( + setOf( + RangeIndexSpec("ContentElement", listOf("id")), + RangeIndexSpec(GraphRagServiceProperties().chunkNodeName, listOf("id")), + ), + contentIdIndexes().toSet(), + ) + } + + @Test + @DisplayName("every content fragment's @NodeId is served by an index the model declares") + fun `every content fragment is indexed by its id`() { + val fragments = modelFragments() + assertTrue(fragments.size > CONTENT_ID_FRAGMENTS.size, "the scan should find the whole model, found $fragments") + assertEquals(emptyList(), unindexed(fragments, contentIdIndexes())) + } + + @NodeFragment(labels = ["Chunk"]) + data class ChunkByKey(@NodeId val key: String) + + @Test + @DisplayName("a content fragment whose @NodeId carries no @RangeIndex is caught") + fun `an unindexed node id is caught`() { + // Without the annotations, the two fragments the indexes are declared on are unserved. + assertEquals( + setOf("ContentElementFragment", "ChunkPlaceFragment"), + unindexed(CONTENT_ID_FRAGMENTS.map { it.java }, emptyList()).map { it.substringBefore('(') }.toSet(), + ) + // A fragment found by another property than the indexed `id` is caught too. + assertEquals( + listOf("ChunkByKey([Chunk]).key"), + unindexed(listOf(ChunkByKey::class.java), contentIdIndexes()), + ) + } +}