diff --git a/src/main/kotlin/com/embabel/agent/rag/graph/GraphObjectManagerStore.kt b/src/main/kotlin/com/embabel/agent/rag/graph/GraphObjectManagerStore.kt index 56bb9b6..ae37baf 100644 --- a/src/main/kotlin/com/embabel/agent/rag/graph/GraphObjectManagerStore.kt +++ b/src/main/kotlin/com/embabel/agent/rag/graph/GraphObjectManagerStore.kt @@ -104,6 +104,9 @@ private const val REEMBED_BATCH_SIZE = 256 /** How many times a window is loaded and re-embedded when a chunk in it changes meanwhile. */ private const val REEMBED_WINDOW_ATTEMPTS = 3 +/** The label every document, section and chunk carries ([com.embabel.agent.rag.graph.model.ContentElementNode]). */ +private const val CONTENT_ELEMENT_LABEL = "ContentElement" + /** * A threshold above this requires a raw full-text score above [FULL_TEXT_BM25_K] — already * a strong match. Empty results above it are far more likely a mis-set threshold than an empty corpus. @@ -212,7 +215,7 @@ class GraphObjectManagerStore( private val chunkFullTextIndex = FullTextIndexSpec(properties.chunkNodeName, listOf("text")) // Mirrors outline's `where` and `orderBy`, so a document's sections are read off the index in order. - private val sectionOrderIndex = RangeIndexSpec("ContentElement", listOf("root_document_id", "ordinal")) + private val sectionOrderIndex = RangeIndexSpec(CONTENT_ELEMENT_LABEL, listOf("root_document_id", "ordinal")) // What chunksOf looks a section's chunks up by: the leaf they were cut from, or the section holding them. private val chunkSectionIndexes = listOf( @@ -220,6 +223,20 @@ class GraphObjectManagerStore( RangeIndexSpec(properties.chunkNodeName, listOf("container_section_id")), ) + /* + * What every save, parent link and next-chunk link finds its nodes by. Each is a + * `MERGE (n: {id: …})` or a `MATCH` of the same shape: a document, a section and a chunk all + * carry `ContentElement`, while the next-chunk link names `Chunk` alone ([ChunkPlaceFragment]). + * Without these, every one of those statements scans its label, so writing one document costs the + * size of the whole store, once per section, chunk and link — and on an engine that holds its write + * lock for a statement, every reader waits out each scan. A range index rather than a uniqueness + * constraint: it asks nothing of what a store already holds, so it cannot fail at startup. + */ + private val contentElementIdIndexes = listOf( + RangeIndexSpec(CONTENT_ELEMENT_LABEL, listOf("id")), + RangeIndexSpec(properties.chunkNodeName, listOf("id")), + ) + private val provisioner = GraphProvisioner(persistenceManager) override fun provision() { @@ -228,7 +245,7 @@ class GraphObjectManagerStore( vectorIndexes = listOf(chunkVectorIndex), fullTextIndexes = listOf(chunkFullTextIndex), constraints = listOf(UniquenessConstraintSpec(properties.entityNodeName, "id")), - rangeIndexes = listOf(sectionOrderIndex) + chunkSectionIndexes, + rangeIndexes = contentElementIdIndexes + sectionOrderIndex + chunkSectionIndexes, ) logger.info("Provisioning complete") } diff --git a/src/main/kotlin/com/embabel/agent/rag/graph/GraphProvisioner.kt b/src/main/kotlin/com/embabel/agent/rag/graph/GraphProvisioner.kt index 275f756..f9b2bb7 100644 --- a/src/main/kotlin/com/embabel/agent/rag/graph/GraphProvisioner.kt +++ b/src/main/kotlin/com/embabel/agent/rag/graph/GraphProvisioner.kt @@ -79,7 +79,13 @@ class GraphProvisioner( private fun ensureUnderConventionName(spec: IndexSpec, rebuildAtAnotherWidth: Boolean) { when (val result = persistenceManager.indexes.ensure(spec)) { is EnsureResult.AlreadyMatching -> - if (result.info.name != spec.effectiveName) { + /* + * Only an index read by NAME needs the convention name: a vector or full-text search. A range + * index is chosen by the planner from its label and properties, so any index on them serves — + * and the one already there may belong to a uniqueness constraint (`Chunk(id)`), which cannot + * be dropped, so recreating it failed provisioning outright. + */ + if (spec !is RangeIndexSpec && result.info.name != spec.effectiveName) { logger.info( "Index on {}{} exists as '{}' but this store resolves '{}' — recreating under the convention name", spec.label, spec.properties, result.info.name, spec.effectiveName, diff --git a/src/test/kotlin/com/embabel/agent/rag/graph/Neo4jContentElementIdIndexTest.kt b/src/test/kotlin/com/embabel/agent/rag/graph/Neo4jContentElementIdIndexTest.kt new file mode 100644 index 0000000..2c7986e --- /dev/null +++ b/src/test/kotlin/com/embabel/agent/rag/graph/Neo4jContentElementIdIndexTest.kt @@ -0,0 +1,119 @@ +/* + * Copyright 2024-2026 Embabel Pty Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.embabel.agent.rag.graph + +import com.embabel.agent.rag.graph.test.DeterministicEmbeddingModel +import com.embabel.agent.rag.ingestion.ChunkTransformer +import com.embabel.agent.rag.ingestion.ContentChunker +import com.embabel.common.ai.model.EmbeddingService +import com.embabel.common.ai.model.SpringAiEmbeddingService +import org.drivine.autoconfigure.EnableDrivine +import org.drivine.autoconfigure.EnableDrivineTestConfig +import org.drivine.manager.GraphObjectManagerFactory +import org.drivine.manager.PersistenceManager +import org.drivine.manager.PersistenceManagerFactory +import org.drivine.query.QuerySpecification +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.beans.factory.annotation.Qualifier +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration +import org.springframework.context.annotation.EnableAspectJAutoProxy +import org.springframework.context.annotation.Profile +import org.springframework.test.context.ActiveProfiles + +/** + * Every node a document's ingest writes is found by `id` — each save is `MERGE (n:…:ContentElement {id: …})`, + * each parent link and next-chunk link MATCHes both ends the same way — so the store indexes `id` under the + * labels those statements name. Without the index each statement scanned its label: one document's ingest cost + * the size of the whole store, once per section, chunk and link. + */ +@SpringBootTest(classes = [Neo4jContentElementIdIndexTest.Config::class]) +@ActiveProfiles("neo4j") +class Neo4jContentElementIdIndexTest { + + @Configuration + @Profile("neo4j") + @EnableDrivine + @EnableDrivineTestConfig + @EnableAspectJAutoProxy(proxyTargetClass = true) + @EnableConfigurationProperties(GraphRagServiceProperties::class) + class Config { + @Bean("graph") + fun persistenceManager(factory: PersistenceManagerFactory): PersistenceManager = factory.get("graph") + + @Bean + fun embeddingService(): EmbeddingService = SpringAiEmbeddingService("fake", "embabel", DeterministicEmbeddingModel()) + + @Bean + fun gomStore( + factory: GraphObjectManagerFactory, + pm: PersistenceManager, + properties: GraphRagServiceProperties, + embeddingService: EmbeddingService, + ): GraphObjectManagerStore = GraphObjectManagerStore( + gom = factory.stateless("graph"), + persistenceManager = pm, + properties = properties, + chunkerConfig = ContentChunker.Config(), + chunkTransformer = ChunkTransformer.NO_OP, + embeddingService = embeddingService, + ) + } + + @Autowired lateinit var store: GraphObjectManagerStore + @Autowired @Qualifier("graph") lateinit var pm: PersistenceManager + + @BeforeEach + fun setUp() = store.provision() + + private fun indexedIds(): List = pm.query( + QuerySpecification + .withStatement( + "SHOW INDEXES YIELD type, labelsOrTypes, properties " + + "WHERE type = 'RANGE' AND properties = ['id'] RETURN labelsOrTypes[0] AS label", + ) + .transform(String::class.java), + ) + + @Test + fun `provisioning indexes id under ContentElement and Chunk`() { + val labels = indexedIds() + assertTrue("ContentElement" in labels, "ContentElement(id) range index, got $labels") + assertTrue("Chunk" in labels, "Chunk(id) range index, got $labels") + } + + @Test + fun `a store whose ids are already uniquely constrained provisions without touching the constraint`() { + pm.execute(QuerySpecification.withStatement("DROP INDEX Chunk_id_range IF EXISTS")) + pm.execute( + QuerySpecification.withStatement("CREATE CONSTRAINT chunk_id_unique IF NOT EXISTS FOR (c:Chunk) REQUIRE c.id IS UNIQUE"), + ) + try { + store.provision() + val constraints = pm.query( + QuerySpecification.withStatement("SHOW CONSTRAINTS YIELD name RETURN name").transform(String::class.java), + ) + assertTrue("chunk_id_unique" in constraints, "the constraint survives provisioning, got $constraints") + } finally { + pm.execute(QuerySpecification.withStatement("DROP CONSTRAINT chunk_id_unique IF EXISTS")) + } + } +}