Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@ import org.drivine.query.dsl.instanceOf
import org.drivine.query.dsl.query
import org.drivine.schema.FullTextIndexSpec
import org.drivine.schema.RangeIndexSpec
import org.drivine.schema.SchemaCatalog
import org.drivine.schema.SimilarityFunction
import org.drivine.schema.UniquenessConstraintSpec
import org.drivine.schema.VectorIndexSpec
Expand All @@ -107,6 +108,24 @@ private const val REEMBED_WINDOW_ATTEMPTS = 3
/** The label every document, section and chunk carries ([com.embabel.agent.rag.graph.model.ContentElementNode]). */
private const val CONTENT_ELEMENT_LABEL = "ContentElement"

/**
* The fragments every save, parent link and next-chunk link finds its nodes by. Each is a
* `MERGE (n:<labels> {id: …})` or a `MATCH` of the same shape: a document, a section and a chunk all
* carry `ContentElement` ([ContentElementFragment]), while the next-chunk link names `Chunk` alone
* ([ChunkPlaceFragment]). Each declares its id `@RangeIndex`, beside its `@NodeId`, so the model is the
* one place the index is stated.
*
* Without those indexes every one of those statements scans its label, so writing one document costs the
* size of the whole store, once per section, chunk and link — and on an engine that holds its write lock
* for a statement, every reader waits out each scan. A range index rather than a uniqueness constraint:
* it asks nothing of what a store already holds, so it cannot fail at startup.
*/
internal val CONTENT_ID_FRAGMENTS = listOf(ContentElementFragment::class, ChunkPlaceFragment::class)

/** The range indexes [CONTENT_ID_FRAGMENTS] declare, read off their annotations. */
internal fun contentIdIndexes(): List<RangeIndexSpec> =
SchemaCatalog.fromFragments(*CONTENT_ID_FRAGMENTS.toTypedArray()).indexes.filterIsInstance<RangeIndexSpec>()

/**
* A threshold above this requires a raw full-text score above [FULL_TEXT_BM25_K] — already
* a strong match. Empty results above it are far more likely a mis-set threshold than an empty corpus.
Expand Down Expand Up @@ -224,18 +243,11 @@ class GraphObjectManagerStore(
)

/*
* What every save, parent link and next-chunk link finds its nodes by. Each is a
* `MERGE (n:<labels> {id: …})` or a `MATCH` of the same shape: a document, a section and a chunk all
* carry `ContentElement`, while the next-chunk link names `Chunk` alone ([ChunkPlaceFragment]).
* Without these, every one of those statements scans its label, so writing one document costs the
* size of the whole store, once per section, chunk and link — and on an engine that holds its write
* lock for a statement, every reader waits out each scan. A range index rather than a uniqueness
* constraint: it asks nothing of what a store already holds, so it cannot fail at startup.
* What every save, parent link and next-chunk link finds its nodes by, declared on the model as
* `@NodeId @RangeIndex` on the fragments those statements name ([CONTENT_ID_FRAGMENTS]). Their labels
* are the fragments' own; the chunk one equals `properties.chunkNodeName`, which `init` pins to it.
*/
private val contentElementIdIndexes = listOf(
RangeIndexSpec(CONTENT_ELEMENT_LABEL, listOf("id")),
RangeIndexSpec(properties.chunkNodeName, listOf("id")),
)
private val contentElementIdIndexes = contentIdIndexes()

private val provisioner = GraphProvisioner(persistenceManager)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,18 @@ package com.embabel.agent.rag.graph.model
import org.drivine.annotation.GraphProperty
import org.drivine.annotation.NodeFragment
import org.drivine.annotation.NodeId
import org.drivine.annotation.RangeIndex

/**
* Where a chunk sits, and nothing else: enough to link it to its parent and to the chunk after it
* without loading its text or its vector.
*
* Its `id` is range-indexed because the next-chunk link MATCHes both chunks by it under `Chunk` alone,
* which the `ContentElement` index does not serve. The store provisions this index from the annotation.
*/
@NodeFragment(labels = ["Chunk"])
data class ChunkPlaceFragment(
@NodeId val id: String,
@NodeId @RangeIndex val id: String,
val parentId: String? = null,
@GraphProperty("root_document_id") val rootDocumentId: String? = null,
@GraphProperty("container_section_id") val containerSectionId: String? = null,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,20 @@ package com.embabel.agent.rag.graph.model

import org.drivine.annotation.NodeFragment
import org.drivine.annotation.NodeId
import org.drivine.annotation.RangeIndex

/**
* A label-generic fragment matching **any** content element — the whole hierarchy shares the
* `ContentElement` label, so a single fragment can stand in for a Document, Section, or Chunk when
* we only care about tree structure and common properties.
*
* Its `id` is range-indexed because every save of a document, section or chunk MERGEs by it under
* `ContentElement`, and every parent link MATCHes both ends the same way: without the index, each of
* those statements scans the label. The store provisions this index from the annotation.
*/
@NodeFragment(labels = ["ContentElement"])
data class ContentElementFragment(
@NodeId val id: String,
@NodeId @RangeIndex val id: String,
val text: String? = null,
val title: String? = null,
val uri: String? = null,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
/*
* Copyright 2024-2026 Embabel Pty Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.embabel.agent.rag.graph

import com.embabel.agent.rag.graph.model.ContentElementFragment
import org.drivine.annotation.NodeFragment
import org.drivine.annotation.NodeId
import org.drivine.model.FragmentModel
import org.drivine.schema.RangeIndexSpec
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.DisplayName
import org.junit.jupiter.api.Test
import org.springframework.beans.factory.annotation.AnnotatedBeanDefinition
import org.springframework.context.annotation.ClassPathScanningCandidateComponentProvider
import org.springframework.core.type.filter.AnnotationTypeFilter

/**
* The content id indexes are declared on the model, as `@NodeId @RangeIndex`, and the store provisions what
* the model declares. These check that every content fragment's saves are served by one of them, so a new
* fragment, or a lost annotation, cannot leave the saves it MERGEs by id scanning their label.
*/
class ContentIdIndexesFollowTheModelTest {

/** The labels whose nodes the store writes and finds by id. */
private val contentLabels = setOf("ContentElement", GraphRagServiceProperties().chunkNodeName)

/** Every `@NodeFragment` in the model package, interfaces included. */
private fun modelFragments(): List<Class<*>> {
val scanner = object : ClassPathScanningCandidateComponentProvider(false) {
override fun isCandidateComponent(beanDefinition: AnnotatedBeanDefinition) = true
}
scanner.addIncludeFilter(AnnotationTypeFilter(NodeFragment::class.java))
return scanner.findCandidateComponents(ContentElementFragment::class.java.packageName)
.map { Class.forName(it.beanClassName) }
}

/**
* The content fragments in [fragments] whose `@NodeId` no index in [indexes] serves: a save of one is a
* MERGE by its id under all its labels, so an index on its id under any one of them is enough.
*/
private fun unindexed(fragments: List<Class<*>>, indexes: List<RangeIndexSpec>): List<String> =
fragments.mapNotNull { fragment ->
val labels = FragmentModel.labelsFor(fragment)
if (labels.none { it in contentLabels }) return@mapNotNull null
val nodeId = FragmentModel.from(fragment).nodeIdProperty ?: return@mapNotNull null
val served = indexes.any { it.label in labels && it.properties == listOf(nodeId) }
if (served) null else "${fragment.simpleName}($labels).$nodeId"
}

@Test
@DisplayName("the indexes read off the model are the ones the store used to list by hand")
fun `derived indexes equal the old hand list`() {
assertEquals(
setOf(
RangeIndexSpec("ContentElement", listOf("id")),
RangeIndexSpec(GraphRagServiceProperties().chunkNodeName, listOf("id")),
),
contentIdIndexes().toSet(),
)
}

@Test
@DisplayName("every content fragment's @NodeId is served by an index the model declares")
fun `every content fragment is indexed by its id`() {
val fragments = modelFragments()
assertTrue(fragments.size > CONTENT_ID_FRAGMENTS.size, "the scan should find the whole model, found $fragments")
assertEquals(emptyList<String>(), unindexed(fragments, contentIdIndexes()))
}

@NodeFragment(labels = ["Chunk"])
data class ChunkByKey(@NodeId val key: String)

@Test
@DisplayName("a content fragment whose @NodeId carries no @RangeIndex is caught")
fun `an unindexed node id is caught`() {
// Without the annotations, the two fragments the indexes are declared on are unserved.
assertEquals(
setOf("ContentElementFragment", "ChunkPlaceFragment"),
unindexed(CONTENT_ID_FRAGMENTS.map { it.java }, emptyList()).map { it.substringBefore('(') }.toSet(),
)
// A fragment found by another property than the indexed `id` is caught too.
assertEquals(
listOf("ChunkByKey([Chunk]).key"),
unindexed(listOf(ChunkByKey::class.java), contentIdIndexes()),
)
}
}
Loading