Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,9 @@ private const val REEMBED_BATCH_SIZE = 256
/** How many times a window is loaded and re-embedded when a chunk in it changes meanwhile. */
private const val REEMBED_WINDOW_ATTEMPTS = 3

/** The label every document, section and chunk carries ([com.embabel.agent.rag.graph.model.ContentElementNode]). */
private const val CONTENT_ELEMENT_LABEL = "ContentElement"

/**
* A threshold above this requires a raw full-text score above [FULL_TEXT_BM25_K] — already
* a strong match. Empty results above it are far more likely a mis-set threshold than an empty corpus.
Expand Down Expand Up @@ -212,14 +215,28 @@ class GraphObjectManagerStore(
private val chunkFullTextIndex = FullTextIndexSpec(properties.chunkNodeName, listOf("text"))

// Mirrors outline's `where` and `orderBy`, so a document's sections are read off the index in order.
private val sectionOrderIndex = RangeIndexSpec("ContentElement", listOf("root_document_id", "ordinal"))
private val sectionOrderIndex = RangeIndexSpec(CONTENT_ELEMENT_LABEL, listOf("root_document_id", "ordinal"))

// What chunksOf looks a section's chunks up by: the leaf they were cut from, or the section holding them.
private val chunkSectionIndexes = listOf(
RangeIndexSpec(properties.chunkNodeName, listOf("leaf_section_id")),
RangeIndexSpec(properties.chunkNodeName, listOf("container_section_id")),
)

/*
* What every save, parent link and next-chunk link finds its nodes by. Each is a
* `MERGE (n:<labels> {id: …})` or a `MATCH` of the same shape: a document, a section and a chunk all
* carry `ContentElement`, while the next-chunk link names `Chunk` alone ([ChunkPlaceFragment]).
* Without these, every one of those statements scans its label, so writing one document costs the
* size of the whole store, once per section, chunk and link — and on an engine that holds its write
* lock for a statement, every reader waits out each scan. A range index rather than a uniqueness
* constraint: it asks nothing of what a store already holds, so it cannot fail at startup.
*/
private val contentElementIdIndexes = listOf(
RangeIndexSpec(CONTENT_ELEMENT_LABEL, listOf("id")),
RangeIndexSpec(properties.chunkNodeName, listOf("id")),
)

private val provisioner = GraphProvisioner(persistenceManager)

override fun provision() {
Expand All @@ -228,7 +245,7 @@ class GraphObjectManagerStore(
vectorIndexes = listOf(chunkVectorIndex),
fullTextIndexes = listOf(chunkFullTextIndex),
constraints = listOf(UniquenessConstraintSpec(properties.entityNodeName, "id")),
rangeIndexes = listOf(sectionOrderIndex) + chunkSectionIndexes,
rangeIndexes = contentElementIdIndexes + sectionOrderIndex + chunkSectionIndexes,
)
logger.info("Provisioning complete")
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,13 @@ class GraphProvisioner(
private fun ensureUnderConventionName(spec: IndexSpec, rebuildAtAnotherWidth: Boolean) {
when (val result = persistenceManager.indexes.ensure(spec)) {
is EnsureResult.AlreadyMatching ->
if (result.info.name != spec.effectiveName) {
/*
* Only an index read by NAME needs the convention name: a vector or full-text search. A range
* index is chosen by the planner from its label and properties, so any index on them serves —
* and the one already there may belong to a uniqueness constraint (`Chunk(id)`), which cannot
* be dropped, so recreating it failed provisioning outright.
*/
if (spec !is RangeIndexSpec && result.info.name != spec.effectiveName) {
logger.info(
"Index on {}{} exists as '{}' but this store resolves '{}' — recreating under the convention name",
spec.label, spec.properties, result.info.name, spec.effectiveName,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
/*
* Copyright 2024-2026 Embabel Pty Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.embabel.agent.rag.graph

import com.embabel.agent.rag.graph.test.DeterministicEmbeddingModel
import com.embabel.agent.rag.ingestion.ChunkTransformer
import com.embabel.agent.rag.ingestion.ContentChunker
import com.embabel.common.ai.model.EmbeddingService
import com.embabel.common.ai.model.SpringAiEmbeddingService
import org.drivine.autoconfigure.EnableDrivine
import org.drivine.autoconfigure.EnableDrivineTestConfig
import org.drivine.manager.GraphObjectManagerFactory
import org.drivine.manager.PersistenceManager
import org.drivine.manager.PersistenceManagerFactory
import org.drivine.query.QuerySpecification
import org.junit.jupiter.api.Assertions.assertTrue
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Test
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.boot.context.properties.EnableConfigurationProperties
import org.springframework.boot.test.context.SpringBootTest
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
import org.springframework.context.annotation.EnableAspectJAutoProxy
import org.springframework.context.annotation.Profile
import org.springframework.test.context.ActiveProfiles

/**
* Every node a document's ingest writes is found by `id` — each save is `MERGE (n:…:ContentElement {id: …})`,
* each parent link and next-chunk link MATCHes both ends the same way — so the store indexes `id` under the
* labels those statements name. Without the index each statement scanned its label: one document's ingest cost
* the size of the whole store, once per section, chunk and link.
*/
@SpringBootTest(classes = [Neo4jContentElementIdIndexTest.Config::class])
@ActiveProfiles("neo4j")
class Neo4jContentElementIdIndexTest {

@Configuration
@Profile("neo4j")
@EnableDrivine
@EnableDrivineTestConfig
@EnableAspectJAutoProxy(proxyTargetClass = true)
@EnableConfigurationProperties(GraphRagServiceProperties::class)
class Config {
@Bean("graph")
fun persistenceManager(factory: PersistenceManagerFactory): PersistenceManager = factory.get("graph")

@Bean
fun embeddingService(): EmbeddingService = SpringAiEmbeddingService("fake", "embabel", DeterministicEmbeddingModel())

@Bean
fun gomStore(
factory: GraphObjectManagerFactory,
pm: PersistenceManager,
properties: GraphRagServiceProperties,
embeddingService: EmbeddingService,
): GraphObjectManagerStore = GraphObjectManagerStore(
gom = factory.stateless("graph"),
persistenceManager = pm,
properties = properties,
chunkerConfig = ContentChunker.Config(),
chunkTransformer = ChunkTransformer.NO_OP,
embeddingService = embeddingService,
)
}

@Autowired lateinit var store: GraphObjectManagerStore
@Autowired @Qualifier("graph") lateinit var pm: PersistenceManager

@BeforeEach
fun setUp() = store.provision()

private fun indexedIds(): List<String> = pm.query(
QuerySpecification
.withStatement(
"SHOW INDEXES YIELD type, labelsOrTypes, properties " +
"WHERE type = 'RANGE' AND properties = ['id'] RETURN labelsOrTypes[0] AS label",
)
.transform(String::class.java),
)

@Test
fun `provisioning indexes id under ContentElement and Chunk`() {
val labels = indexedIds()
assertTrue("ContentElement" in labels, "ContentElement(id) range index, got $labels")
assertTrue("Chunk" in labels, "Chunk(id) range index, got $labels")
}

@Test
fun `a store whose ids are already uniquely constrained provisions without touching the constraint`() {
pm.execute(QuerySpecification.withStatement("DROP INDEX Chunk_id_range IF EXISTS"))
pm.execute(
QuerySpecification.withStatement("CREATE CONSTRAINT chunk_id_unique IF NOT EXISTS FOR (c:Chunk) REQUIRE c.id IS UNIQUE"),
)
try {
store.provision()
val constraints = pm.query(
QuerySpecification.withStatement("SHOW CONSTRAINTS YIELD name RETURN name").transform(String::class.java),
)
assertTrue("chunk_id_unique" in constraints, "the constraint survives provisioning, got $constraints")
} finally {
pm.execute(QuerySpecification.withStatement("DROP CONSTRAINT chunk_id_unique IF EXISTS"))
}
}
}
Loading