From a968c3228d8352360a5d767390e555ff33483b31 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 10 Aug 2026 11:04:51 -0600 Subject: [PATCH 01/36] feat(rag): add driver/attached fields, validation floors and force-reingest Phase 1 of the RAG driver abstraction (design doc sections 5.1-5.5a). Data model: - Add `driver: String` (serde default "yaml" via RagData::default_driver) and `attached: bool` as the first two fields of RagData, so driver metadata sits at the top of each RAG YAML. Old files without them load unchanged. - Add `#[serde(default)]` to the non-Option fields so a minimal attached-RAG YAML deserializes, and add `skip_serializing_if` to `vectors` so an empty map renders no `vectors:` key. - Add a hand-written `impl Default for RagData` delegating to `RagData::new()`. It is deliberately not derived: a derived impl yields `driver: ""`, which is not a valid driver string. Validation (the price of the new serde defaults): - Add `RagData::validate()`, called from `Rag::load()` after deserialization. It enforces the (driver, attached) matrix and, critically, numeric floors that the new defaults would otherwise mask: `top_k >= 1` unconditionally (a 0 makes every query return nothing, silently), and `chunk_size >= 1` plus `chunk_overlap < chunk_size` when not attached (a 0 chunk_size is a real divide-by-zero panic while sizing embedding batches). - Reject `.set rag_top_k 0` at the setter, before the set/update fork. Without this, the new load-time floor turns one keystroke into an unloadable RAG: the setter saves immediately and no dot-command can reach the file again. Rebuild actually re-embeds now: - `.rebuild rag` and `--rebuild-rag` previously re-scanned paths and re-embedded nothing, because the content-hash skip fired regardless of the refresh flag. Extract that decision into a module-level `find_hash_skip()` free function and thread a `force_reingest` flag through `sync_documents()` and `refresh_document_paths()`, set true only from `rebuild_rag()`. `.edit rag-docs` stays incremental. Re-embedding costs time and API spend, so `rebuild_rag()` now prints a one-line file-count warning first (no prompt: the path is reachable from a non-interactive CLI flag). Attached-RAG guards: - Block `.rebuild rag` / `--rebuild-rag` and `.edit rag-docs` on attached RAGs, which Coyote did not index and whose source documents it does not own. - Add `Rag::driver()`, `Rag::is_attached()` and `Rag::file_count()`, and surface driver/attached through `Rag::export()` so `.info rag` shows them. Adds 12 unit tests (1299 -> 1311), including the two gate tests pinning that a forced re-ingest does not hash-skip while an ordinary refresh still does. --- src/config/request_context.rs | 38 +++- src/rag/mod.rs | 332 +++++++++++++++++++++++++++++++++- 2 files changed, 357 insertions(+), 13 deletions(-) diff --git a/src/config/request_context.rs b/src/config/request_context.rs index 7c07c4a..f01850d 100644 --- a/src/config/request_context.rs +++ b/src/config/request_context.rs @@ -2773,7 +2773,12 @@ impl RequestContext { } } "rag_top_k" => { - let value = value.parse().with_context(|| "Invalid value")?; + let value: usize = value.parse().with_context(|| "Invalid value")?; + if value == 0 { + bail!( + "rag_top_k must be >= 1; a top_k of 0 makes every query return no results." + ); + } if !self.set_rag_top_k(value)? { self.update_app_config(|app| app.rag_top_k = value); } @@ -4125,6 +4130,12 @@ impl RequestContext { None => bail!("No RAG"), }; + if rag.is_attached() { + bail!( + "Cannot edit documents on an attached RAG — Coyote does not own its source documents." + ); + } + let document_paths = rag.document_paths(); let temp_file = temp_file(&format!("-rag-{}", rag.name()), ".txt"); tokio::fs::write(&temp_file, &document_paths.join("\n")) @@ -4157,8 +4168,14 @@ impl RequestContext { }; self.rag_cache().invalidate(&key); - rag.refresh_document_paths(&new_document_paths, false, &self.app.config, abort_signal) - .await?; + rag.refresh_document_paths( + &new_document_paths, + false, + false, + &self.app.config, + abort_signal, + ) + .await?; self.rag = Some(Arc::new(rag)); Ok(()) } @@ -4169,6 +4186,14 @@ impl RequestContext { None => bail!("No RAG"), }; + if rag.is_attached() { + bail!( + "Cannot rebuild an attached RAG — Coyote does not own its source documents. \ + Re-index from the system that originally created '{}'.", + rag.name() + ); + } + let key = if self.agent.is_some() { RagKey::Agent(rag.name().to_string()) } else { @@ -4177,7 +4202,12 @@ impl RequestContext { self.rag_cache().invalidate(&key); let document_paths = rag.document_paths().to_vec(); - rag.refresh_document_paths(&document_paths, true, &self.app.config, abort_signal) + println!( + "Rebuilding re-embeds every document ({} files). \ + This will call the embedding API and may take a while.", + rag.file_count() + ); + rag.refresh_document_paths(&document_paths, true, true, &self.app.config, abort_signal) .await?; self.rag = Some(Arc::new(rag)); Ok(()) diff --git a/src/rag/mod.rs b/src/rag/mod.rs index b06928d..1c91f68 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -132,7 +132,7 @@ impl Rag { let loaders = app.document_loaders.clone(); let (spinner, spinner_rx) = Spinner::create(""); abortable_run_with_spinner_rx( - rag.sync_documents(doc_paths, true, loaders, Some(spinner)), + rag.sync_documents(doc_paths, true, false, loaders, Some(spinner)), spinner_rx, abort_signal, ) @@ -276,7 +276,7 @@ impl Rag { let loaders = app.document_loaders.clone(); let (spinner, spinner_rx) = Spinner::create(""); abortable_run_with_spinner_rx( - rag.sync_documents(&paths, true, loaders, Some(spinner)), + rag.sync_documents(&paths, true, false, loaders, Some(spinner)), spinner_rx, abort_signal, ) @@ -291,6 +291,7 @@ impl Rag { let err = || format!("Failed to load rag '{name}' at '{}'", path.display()); let content = fs::read_to_string(path).with_context(err)?; let data: RagData = serde_yaml::from_str(&content).with_context(err)?; + data.validate().with_context(err)?; Self::create(app, name, path, data) } @@ -322,13 +323,20 @@ impl Rag { &mut self, document_paths: &[String], refresh: bool, + force_reingest: bool, app: &AppConfig, abort_signal: AbortSignal, ) -> Result<()> { let loaders = app.document_loaders.clone(); let (spinner, spinner_rx) = Spinner::create(""); abortable_run_with_spinner_rx( - self.sync_documents(document_paths, refresh, loaders, Some(spinner)), + self.sync_documents( + document_paths, + refresh, + force_reingest, + loaders, + Some(spinner), + ), spinner_rx, abort_signal, ) @@ -455,6 +463,8 @@ impl Rag { .collect(); let data = json!({ "path": self.path, + "driver": self.driver(), + "attached": self.is_attached(), "embedding_model": self.embedding_model.id(), "chunk_size": self.data.chunk_size, "chunk_overlap": self.data.chunk_overlap, @@ -476,6 +486,18 @@ impl Rag { &self.name } + pub fn is_attached(&self) -> bool { + self.data.attached + } + + pub fn driver(&self) -> &str { + &self.data.driver + } + + pub fn file_count(&self) -> usize { + self.data.files.len() + } + pub fn is_temp(&self) -> bool { self.name == TEMP_RAG_NAME } @@ -565,9 +587,15 @@ impl Rag { &mut self, paths: &[String], refresh: bool, + force_reingest: bool, loaders: HashMap, spinner: Option, ) -> Result<()> { + debug_assert!( + !force_reingest || refresh, + "force_reingest requires refresh" + ); + let refresh = refresh || force_reingest; if let Some(spinner) = &spinner { let _ = spinner.set_message(String::new()); } @@ -685,11 +713,9 @@ impl Rag { } in loaded_documents { let hash = sha256(&contents); - if let Some(file_ids) = to_deleted.get_mut(&hash) - && let Some((i, _)) = file_ids - .iter() - .enumerate() - .find(|(_, v)| self.data.files[*v].path == path) + if let Some((i, _)) = + find_hash_skip(force_reingest, &to_deleted, &self.data.files, &hash, &path) + && let Some(file_ids) = to_deleted.get_mut(&hash) { if file_ids.len() == 1 { to_deleted.swap_remove(&hash); @@ -1084,16 +1110,31 @@ impl Rag { #[derive(Clone, Serialize, Deserialize)] pub struct RagData { + #[serde(default = "RagData::default_driver")] + pub driver: String, + #[serde(default)] + pub attached: bool, + pub embedding_model: String, + #[serde(default)] pub chunk_size: usize, + #[serde(default)] pub chunk_overlap: usize, pub reranker_model: Option, + #[serde(default)] pub top_k: usize, pub batch_size: Option, + #[serde(default)] pub next_file_id: FileId, + #[serde(default)] pub document_paths: Vec, + #[serde(default)] pub files: IndexMap, - #[serde(with = "serde_vectors")] + #[serde( + default, + with = "serde_vectors", + skip_serializing_if = "IndexMap::is_empty" + )] pub vectors: IndexMap>, #[serde(default)] pub extractor_model: Option, @@ -1108,6 +1149,8 @@ pub struct RagData { impl Debug for RagData { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("RagData") + .field("driver", &self.driver) + .field("attached", &self.attached) .field("embedding_model", &self.embedding_model) .field("chunk_size", &self.chunk_size) .field("chunk_overlap", &self.chunk_overlap) @@ -1135,6 +1178,8 @@ impl RagData { graph: GraphRagConfig, ) -> Self { Self { + driver: "yaml".to_string(), + attached: false, embedding_model, chunk_size, chunk_overlap, @@ -1152,6 +1197,52 @@ impl RagData { } } + fn default_driver() -> String { + "yaml".to_string() + } + + pub fn validate(&self) -> Result<()> { + if self.top_k == 0 { + bail!( + "top_k must be >= 1 (got 0). A top_k of 0 makes every query return \ + no results with no error. Set `top_k:` in the RAG YAML." + ); + } + if !self.attached { + if self.chunk_size == 0 { + bail!( + "chunk_size must be >= 1 (got 0) for a non-attached RAG. A \ + chunk_size of 0 panics with a divide-by-zero while sizing \ + embedding batches. Set `chunk_size:` in the RAG YAML." + ); + } + if self.chunk_overlap >= self.chunk_size { + bail!( + "chunk_overlap ({}) must be strictly less than chunk_size ({}).", + self.chunk_overlap, + self.chunk_size + ); + } + } + match (self.driver.as_str(), self.attached) { + ("yaml", false) => Ok(()), + ("duckdb", false) => Ok(()), + ("qdrant", true) => Ok(()), + ("qdrant", false) => Ok(()), + ("yaml", true) => bail!( + "driver 'yaml' cannot be attached (attached: true). \ + Attached RAGs require an external driver (qdrant)." + ), + ("duckdb", true) => bail!( + "driver 'duckdb' cannot be attached (attached: true). \ + DuckDB is a local-only driver; use 'qdrant' for external collections." + ), + (other, _) => { + bail!("Unknown RAG driver '{other}'. Valid drivers: yaml, duckdb, qdrant.") + } + } + } + pub fn get(&self, id: DocumentId) -> Option<&RagDocument> { let (file_index, document_index) = id.split(); let file = self.files.get(&file_index)?; @@ -1208,6 +1299,20 @@ impl RagData { } } +impl Default for RagData { + fn default() -> Self { + RagData::new( + String::new(), + 0, + 0, + None, + 5, + None, + GraphRagConfig::default(), + ) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RagFile { hash: String, @@ -1451,6 +1556,30 @@ fn progress(spinner: &Option, message: String) { } } +/// Decide whether a just-loaded document may skip re-chunking and re-embedding. +/// +/// Returns the position of the matching `FileId` within `to_deleted[hash]`, together with +/// that `FileId`. The caller needs the position to un-mark the file for deletion. `None` +/// means "ingest this document": either a full re-ingest was requested, or no +/// already-indexed file has both this content hash and this path. +fn find_hash_skip( + force_reingest: bool, + to_deleted: &IndexMap>, + files: &IndexMap, + hash: &str, + path: &str, +) -> Option<(usize, FileId)> { + if force_reingest { + return None; + } + let file_ids = to_deleted.get(hash)?; + file_ids + .iter() + .enumerate() + .find(|(_, v)| files[*v].path == path) + .map(|(i, v)| (i, *v)) +} + fn reciprocal_rank_fusion( list_of_document_ids: Vec>, list_of_weights: Vec, @@ -1868,4 +1997,189 @@ mod tests { "higher-weight signal's top doc should rank first" ); } + + fn hash_skip_fixture() -> (IndexMap, IndexMap>) { + let mut files: IndexMap = Default::default(); + files.insert( + 7, + RagFile { + hash: "abc".into(), + path: "test.txt".into(), + documents: vec![RagDocument::new("unchanged")], + }, + ); + let mut to_deleted: IndexMap> = Default::default(); + to_deleted.insert("abc".into(), vec![7]); + (files, to_deleted) + } + + #[test] + fn force_reingest_re_embeds_hash_identical_files() { + let (files, to_deleted) = hash_skip_fixture(); + assert_eq!( + find_hash_skip(true, &to_deleted, &files, "abc", "test.txt"), + None, + "a forced re-ingest must not skip an unchanged file" + ); + } + + #[test] + fn refresh_without_force_still_hash_skips() { + let (files, to_deleted) = hash_skip_fixture(); + assert_eq!( + find_hash_skip(false, &to_deleted, &files, "abc", "test.txt"), + Some((0, 7)), + "an unchanged file should be skipped and un-marked for deletion" + ); + } + + #[test] + fn find_hash_skip_returns_none_on_path_change() { + let (files, to_deleted) = hash_skip_fixture(); + assert_eq!( + find_hash_skip(false, &to_deleted, &files, "abc", "moved.txt"), + None + ); + assert_eq!( + find_hash_skip(true, &to_deleted, &files, "abc", "moved.txt"), + None + ); + } + + #[test] + fn ragdata_new_has_yaml_driver_and_not_attached() { + let data = RagData::new( + "text-embedding-3-small".to_string(), + 1024, + 50, + None, + 5, + None, + GraphRagConfig::default(), + ); + assert_eq!(data.driver, "yaml"); + assert!(!data.attached); + } + + #[test] + fn ragdata_deserializes_without_driver_field() { + let yaml = " +embedding_model: text-embedding-3-small +chunk_size: 1024 +chunk_overlap: 50 +top_k: 5 +next_file_id: 0 +document_paths: [] +files: {} +vectors: {} +"; + let data: RagData = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(data.driver, "yaml"); + assert!(!data.attached); + } + + #[test] + fn ragdata_round_trips_driver_and_attached() { + let mut data = RagData::new( + "text-embedding-3-small".to_string(), + 1024, + 50, + None, + 5, + None, + GraphRagConfig::default(), + ); + data.driver = "qdrant".to_string(); + data.attached = true; + + let yaml = serde_yaml::to_string(&data).unwrap(); + let restored: RagData = serde_yaml::from_str(&yaml).unwrap(); + assert_eq!(restored.driver, "qdrant"); + assert!(restored.attached); + } + + #[test] + fn ragdata_validate_rejects_yaml_attached() { + let mut data = RagData::new( + "m".into(), + 1024, + 50, + None, + 5, + None, + GraphRagConfig::default(), + ); + data.attached = true; + let err = data.validate().unwrap_err().to_string(); + assert!(err.contains("cannot be attached"), "got: {err}"); + } + + #[test] + fn ragdata_validate_accepts_qdrant_attached() { + let mut data = RagData::new( + "m".into(), + 1024, + 50, + None, + 5, + None, + GraphRagConfig::default(), + ); + data.driver = "qdrant".to_string(); + data.attached = true; + assert!(data.validate().is_ok()); + } + + #[test] + fn ragdata_validate_rejects_zero_top_k_from_a_truncated_yaml() { + let yaml = " +embedding_model: text-embedding-3-small +chunk_size: 1024 +chunk_overlap: 50 +"; + let data: RagData = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(data.top_k, 0, "a missing top_k must default to 0"); + let err = data.validate().unwrap_err().to_string(); + assert!(err.contains("top_k must be >= 1"), "got: {err}"); + } + + #[test] + fn ragdata_validate_rejects_zero_chunk_size_when_not_attached() { + let yaml = " +embedding_model: text-embedding-3-small +top_k: 5 +"; + let data: RagData = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(data.chunk_size, 0); + let err = data.validate().unwrap_err().to_string(); + assert!(err.contains("chunk_size must be >= 1"), "got: {err}"); + } + + #[test] + fn ragdata_validate_allows_zero_chunk_size_when_attached() { + let yaml = " +driver: qdrant +attached: true +embedding_model: text-embedding-3-small +top_k: 5 +"; + let data: RagData = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(data.chunk_size, 0); + assert!(data.validate().is_ok()); + } + + #[test] + fn ragdata_validate_rejects_overlap_not_less_than_chunk_size() { + let data = RagData::new( + "m".into(), + 100, + 100, + None, + 5, + None, + GraphRagConfig::default(), + ); + let err = data.validate().unwrap_err().to_string(); + assert!(err.contains("chunk_overlap"), "got: {err}"); + } } From 5049143fcc6e4684ea29cb62b6c78e47ddc4893e Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 10 Aug 2026 11:41:37 -0600 Subject: [PATCH 02/36] refactor(rag): extract RagProvider trait and add YamlProvider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduce a narrow `RagProvider` trait covering vector search and content retrieval, and make `Rag` delegate to a boxed provider instead of owning an HNSW index directly. `YamlProvider` is the sole implementation for now. The trait deliberately stays narrow: embeddings, chunking, BM25 keyword search, graph RAG, entity extraction, RRF merging and persistence all remain on `Rag`/`RagData`, so a new storage backend does not have to reimplement Coyote's indexing logic. Notable points: - `fetch_content`'s ordering contract is part of the trait, not an accident. Implementations must return results in input-`ids` order; `hybrid_search` passes an RRF-ranked list straight to the prompt builder, so a provider returning storage order would silently discard the ranking. - The content store is keyed on `data.files`, never `data.vectors`. Both the content map and BM25 now route through the new `RagData::iter_documents()` so the two key spaces match by construction. `RagData::add` zips document ids with embeddings and truncates silently, so ids in `files \ vectors` are genuinely reachable. - A provider keyword-search failure degrades to an empty ranker with a warning rather than failing the whole query; it is one of three RRF inputs. It deliberately does not fall back to the local BM25, which would be a silent ranking-algorithm swap once a provider with native FTS exists. - The rerank path builds its text and id vectors from a single `fetch_content` result in one pass, so the reranker's positional indices cannot desync. This is not a bit-for-bit no-op. `vector_search` now dedups by best score and sorts globally instead of concatenating per-chunk hit lists. Single-chunk queries (the common case) are unaffected. Multi-chunk queries get corrected rank assignment and no longer let a document that matched several query chunks accumulate multiple RRF contributions. There is no overall cap on the merged pool — truncation remains `reciprocal_rank_fusion`'s job. `RagData::get()` is removed: its only two callers were the content lookups replaced here, and an unused private-module method fails the build under `--deny warnings`. Its three tests were rewritten against `iter_documents()`, one of which now guards the files-vs-vectors keying directly. Implements Phase 2 of the RAG driver abstraction design (§6). --- src/rag/mod.rs | 265 +++++++++++++++++++++++++++----------- src/rag/provider.rs | 87 +++++++++++++ src/rag/providers/mod.rs | 6 + src/rag/providers/yaml.rs | 244 +++++++++++++++++++++++++++++++++++ 4 files changed, 526 insertions(+), 76 deletions(-) create mode 100644 src/rag/provider.rs create mode 100644 src/rag/providers/mod.rs create mode 100644 src/rag/providers/yaml.rs diff --git a/src/rag/mod.rs b/src/rag/mod.rs index 1c91f68..9011296 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -5,10 +5,14 @@ use crate::config::*; use crate::utils::*; mod graph; +mod provider; +mod providers; mod serde_vectors; mod splitter; use self::graph::{KnowledgeGraph, extract_entities}; +use self::provider::RagProvider; +use self::providers::YamlProvider; use anyhow::{Context, Result, anyhow, bail}; use bm25::{Language, SearchEngine, SearchEngineBuilder}; @@ -56,8 +60,11 @@ pub struct Rag { name: String, path: String, embedding_model: Model, - hnsw: Hnsw<'static, f32, DistCosine>, + // Local BM25: keyword search + graph seeding. Always built from `data.files` + // regardless of driver, and kept on `Rag` so the sync `graph_search` can use it. bm25: SearchEngine, + // Vector storage + content retrieval. + provider: Box, data: RagData, last_sources: RwLock>, node_to_docs: IndexMap>, @@ -74,6 +81,19 @@ impl Debug for Rag { } } +// CLONING A `Rag` DOES NOT SNAPSHOT ITS BACKING STORE. +// +// `provider.duplicate(&self.data)` is a true snapshot for YamlProvider only. +// DuckDbProvider Arc-clones one shared `Mutex` over one file, and +// QdrantProvider addresses the same remote collection. So for those drivers the +// clone and the original are two views of ONE store. +// +// INVARIANT: after calling `rebuild_indexes` on a cloned `Rag`, the pre-clone +// instance MUST be discarded immediately and MUST NOT serve further queries. +// Cloning to READ is always fine; cloning to REBUILD makes the original a +// half-truth (pre-rebuild `data`, post-rebuild store). Note that reassigning +// `RequestContext.rag` drops only one holder of the old `Arc` — forked +// request contexts, agents, captured inputs and the RAG cache keep theirs. impl Clone for Rag { fn clone(&self) -> Self { Self { @@ -81,8 +101,8 @@ impl Clone for Rag { name: self.name.clone(), path: self.path.clone(), embedding_model: self.embedding_model.clone(), - hnsw: self.data.build_hnsw(), bm25: self.data.build_bm25(), + provider: self.provider.duplicate(&self.data), node_to_docs: self.data.knowledge_graph.build_node_to_docs(), data: self.data.clone(), last_sources: RwLock::new(None), @@ -296,8 +316,8 @@ impl Rag { } pub fn create(app: &AppConfig, name: &str, path: &Path, data: RagData) -> Result { - let hnsw = data.build_hnsw(); let bm25 = data.build_bm25(); + let provider: Box = Box::new(YamlProvider::from_data(&data)); let node_to_docs = data.knowledge_graph.build_node_to_docs(); let embedding_model = Model::retrieve_model(app, &data.embedding_model, ModelType::Embedding)?; @@ -307,8 +327,8 @@ impl Rag { path: path.display().to_string(), data, embedding_model, - hnsw, bm25, + provider, node_to_docs, last_sources: RwLock::new(None), }; @@ -821,7 +841,11 @@ impl Rag { } progress(&spinner, "Building store".into()); - self.hnsw = self.data.build_hnsw(); + // `refresh` is true for a full re-index (.rebuild rag / --rebuild-rag / + // initial build) and false for an incremental .edit rag-docs change. + // Passing it through is what stops a remote provider from wiping its + // collection on a one-file add. + self.provider.rebuild_indexes(&self.data, refresh).await?; self.bm25 = self.data.build_bm25(); self.node_to_docs = self.data.knowledge_graph.build_node_to_docs(); @@ -834,17 +858,30 @@ impl Rag { top_k: usize, rerank_model: Option<&str>, ) -> Result> { - let (vector_search_results, keyword_search_results) = tokio::join!( - self.vector_search(query, top_k, 0.0), - self.keyword_search(query, top_k, 0.0), - ); - - let vector_search_results = vector_search_results?; + let vector_search_results = self.vector_search(query, top_k, 0.0).await?; debug!("vector_search_results: {vector_search_results:?}",); let vector_search_ids: Vec = vector_search_results.into_iter().map(|(v, _)| v).collect(); - let keyword_search_results = keyword_search_results?; + let keyword_search_results: Vec<(DocumentId, f32)> = + if self.provider.has_native_keyword_search() { + // Keyword is ONE of three RRF rankers (vector + keyword + graph); + // its absence is survivable and produces a slightly worse ranking, + // whereas a `?` here turns a provider FTS fault into TOTAL query + // failure — the user gets an error instead of the results the + // vector and graph rankers already retrieved. Degrade, do not + // propagate, and do not silently swap in the local BM25 either: + // that would change the ranking algorithm mid-query. + match self.provider.keyword_search(query, top_k).await { + Ok(v) => v, + Err(e) => { + warn!("native keyword search failed, dropping the keyword ranker: {e}"); + Vec::new() + } + } + } else { + self.keyword_search(query, top_k, 0.0) + }; debug!("keyword_search_results: {keyword_search_results:?}",); let keyword_search_ids: Vec = keyword_search_results.into_iter().map(|(v, _)| v).collect(); @@ -857,13 +894,20 @@ impl Rag { .concat() .into_iter() .collect(); - let mut documents = vec![]; - let mut documents_ids = vec![]; - for id in ids { - if let Some(document) = self.data.get(id) { - documents_ids.push(id); - documents.push(document.page_content.to_string()); - } + // `ids` is an `IndexSet` here, not the `Vec` of the RRF branch below, + // and `&IndexSet<_>` does not coerce to `&[DocumentId]`. + let ids: Vec = ids.into_iter().collect(); + let fetched = self.provider.fetch_content(&ids).await?; + // Build both vectors from the SAME source in the SAME iteration — + // never zip two independently-built lists. The reranker returns + // positional indices into `documents`, so any drift between the two + // resolves reranked hits to the wrong document's text. A partial + // fetch simply yields a shorter pair, and both shrink together. + let mut documents_ids = Vec::with_capacity(fetched.len()); + let mut documents = Vec::with_capacity(fetched.len()); + for (id, text) in fetched { + documents_ids.push(id); + documents.push(text); } let data = RerankData::new(query.to_string(), documents, top_k); let list = client.rerank(&data).await.context("Failed to rerank")?; @@ -895,13 +939,9 @@ impl Rag { ids } }; - let output = ids - .into_iter() - .filter_map(|id| { - let document = self.data.get(id)?; - Some((id, document.page_content.clone())) - }) - .collect(); + // `ids` is the ranked list; `fetch_content` preserves that order per the + // trait's ordering contract, so the result is returned as-is. + let output = self.provider.fetch_content(&ids).await?; Ok(output) } @@ -918,35 +958,24 @@ impl Rag { ); let texts = splitter.split_text(query); let embeddings_data = EmbeddingsData::new(texts, true); - let embeddings = self.create_embeddings(embeddings_data, None).await?; - let output = self - .hnsw - .parallel_search(&embeddings, top_k, 30) - .into_iter() - .flat_map(|list| { - list.into_iter() - .filter_map(|v| { - let score = 1.0 - v.distance; - if score > min_score { - Some((DocumentId(v.d_id), score)) - } else { - None - } - }) - .collect::>() - }) - .collect(); - Ok(output) + let query_embeddings = self.create_embeddings(embeddings_data, None).await?; + + let mut results: Vec<(DocumentId, f32)> = vec![]; + for embedding in &query_embeddings { + let batch = self + .provider + .vector_search(embedding, top_k, min_score) + .await?; + results.extend(batch); + } + Ok(merge_vector_results(results)) } - async fn keyword_search( - &self, - query: &str, - top_k: usize, - min_score: f32, - ) -> Result> { + /// Local in-memory BM25 over `data.files` — empty for attached RAGs, which is + /// correct: they have no local text. + fn keyword_search(&self, query: &str, top_k: usize, min_score: f32) -> Vec<(DocumentId, f32)> { let results = self.bm25.search(query, top_k); - let output: Vec<(DocumentId, f32)> = results + results .into_iter() .filter_map(|v| { let score = v.score; @@ -956,8 +985,7 @@ impl Rag { None } }) - .collect(); - Ok(output) + .collect() } fn graph_search(&self, query: &str, top_k: usize) -> Vec { @@ -1243,11 +1271,21 @@ impl RagData { } } - pub fn get(&self, id: DocumentId) -> Option<&RagDocument> { - let (file_index, document_index) = id.split(); - let file = self.files.get(&file_index)?; - let document = file.documents.get(document_index)?; - Some(document) + /// Every (DocumentId, &RagDocument) in the corpus, in `files` order. + /// + /// This — NOT `vectors` — is the authoritative document id space. BM25, the + /// knowledge graph and content lookup all key off it; `vectors` is a subset, + /// since `add`'s zip truncates whenever fewer embeddings come back than + /// document ids were sent. + pub fn iter_documents(&self) -> impl Iterator { + self.files.iter().flat_map(|(file_index, file)| { + file.documents + .iter() + .enumerate() + .map(move |(document_index, document)| { + (DocumentId::new(*file_index, document_index), document) + }) + }) } pub fn del(&mut self, file_ids: Vec) { @@ -1285,13 +1323,12 @@ impl RagData { } pub fn build_bm25(&self) -> SearchEngine { - let mut documents = vec![]; - for (file_index, file) in self.files.iter() { - for (document_index, document) in file.documents.iter().enumerate() { - let id = DocumentId::new(*file_index, document_index); - documents.push(bm25::Document::new(id, &document.page_content)) - } - } + // Shares `iter_documents` with the providers' content maps so the BM25 key + // space and the content key space are identical by construction. + let documents: Vec<_> = self + .iter_documents() + .map(|(id, doc)| bm25::Document::new(id, &doc.page_content)) + .collect(); SearchEngineBuilder::::with_documents(Language::English, documents) .k1(1.5) .b(0.75) @@ -1580,6 +1617,27 @@ fn find_hash_skip( .map(|(i, v)| (i, *v)) } +/// Global score sort + dedup keeping the best score per document. +/// +/// NO overall cap: each `provider.vector_search` call already returns <= top_k, +/// so the pool is bounded by `top_k * query_chunks`, and `reciprocal_rank_fusion` +/// truncates to `top_k` itself. Capping here would let whichever query chunk has +/// the strongest absolute scores crowd out every other chunk's hits. +/// +/// Free function (not a method) so it is unit-testable without an embeddings client. +fn merge_vector_results(mut results: Vec<(DocumentId, f32)>) -> Vec<(DocumentId, f32)> { + debug_assert!( + results.iter().all(|(_, score)| score.is_finite()), + "provider returned a non-finite score; NaN silently degrades sort order" + ); + results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(Ordering::Equal)); + let mut seen = IndexSet::new(); + results + .into_iter() + .filter(|(id, _)| seen.insert(*id)) + .collect() +} + fn reciprocal_rank_fusion( list_of_document_ids: Vec>, list_of_weights: Vec, @@ -1695,7 +1753,7 @@ mod tests { } #[test] - fn rag_data_get_returns_document() { + fn rag_data_iter_documents_yields_all_documents_in_file_order() { let mut data = RagData::new( "m".into(), 100, @@ -1712,15 +1770,21 @@ mod tests { }; data.files.insert(0, file); - let doc = data.get(DocumentId::new(0, 0)).unwrap(); - assert_eq!(doc.page_content, "first"); - - let doc = data.get(DocumentId::new(0, 1)).unwrap(); - assert_eq!(doc.page_content, "second"); + let documents: Vec<_> = data + .iter_documents() + .map(|(id, doc)| (id, doc.page_content.as_str())) + .collect(); + assert_eq!( + documents, + vec![ + (DocumentId::new(0, 0), "first"), + (DocumentId::new(0, 1), "second"), + ] + ); } #[test] - fn rag_data_get_returns_none_for_missing_file() { + fn rag_data_iter_documents_is_empty_without_files() { let data = RagData::new( "m".into(), 100, @@ -1730,11 +1794,14 @@ mod tests { None, GraphRagConfig::default(), ); - assert!(data.get(DocumentId::new(99, 0)).is_none()); + assert_eq!(data.iter_documents().count(), 0); } + /// The document id space is `files`, never `vectors`: `add`'s zip truncates + /// silently, so a vector may exist for an id no file provides. Content lookup + /// and BM25 both key off this iterator and must agree. #[test] - fn rag_data_get_returns_none_for_missing_document() { + fn rag_data_iter_documents_ignores_vector_only_ids() { let mut data = RagData::new( "m".into(), 100, @@ -1750,7 +1817,10 @@ mod tests { documents: vec![RagDocument::new("only one")], }; data.files.insert(0, file); - assert!(data.get(DocumentId::new(0, 5)).is_none()); + data.vectors.insert(DocumentId::new(0, 5), vec![1.0]); + + let ids: Vec<_> = data.iter_documents().map(|(id, _)| id).collect(); + assert_eq!(ids, vec![DocumentId::new(0, 0)]); } #[test] @@ -1998,6 +2068,49 @@ mod tests { ); } + #[test] + fn merge_vector_results_empty_input() { + let result = super::merge_vector_results(vec![]); + assert!(result.is_empty(), "empty input should produce empty output"); + } + + #[test] + fn merge_vector_results_keeps_best_score_per_document() { + let doc = DocumentId::new(0, 0); + let result = super::merge_vector_results(vec![(doc, 0.2), (doc, 0.9)]); + assert_eq!(result.len(), 1, "a document must not be double-counted"); + assert_eq!(result[0].0, doc); + assert_eq!( + result[0].1, 0.9, + "dedup must keep the highest score, not the first seen" + ); + } + + #[test] + fn merge_vector_results_sorts_globally_by_descending_score() { + let doc_a = DocumentId::new(0, 0); + let doc_b = DocumentId::new(1, 0); + let doc_c = DocumentId::new(2, 0); + // Interleaved as two per-chunk hit lists would arrive: concatenating them + // would yield a, c, b — only a global sort produces c, a, b. + let result = super::merge_vector_results(vec![(doc_a, 0.5), (doc_c, 0.9), (doc_b, 0.1)]); + let ids: Vec = result.iter().map(|(id, _)| *id).collect(); + assert_eq!(ids, vec![doc_c, doc_a, doc_b]); + } + + #[test] + fn merge_vector_results_does_not_truncate() { + let input: Vec<(DocumentId, f32)> = (0..10) + .map(|i| (DocumentId::new(i, 0), i as f32 / 10.0)) + .collect(); + let result = super::merge_vector_results(input); + assert_eq!( + result.len(), + 10, + "merging must not cap the pool; truncation belongs to reciprocal_rank_fusion" + ); + } + fn hash_skip_fixture() -> (IndexMap, IndexMap>) { let mut files: IndexMap = Default::default(); files.insert( diff --git a/src/rag/provider.rs b/src/rag/provider.rs new file mode 100644 index 0000000..6889e81 --- /dev/null +++ b/src/rag/provider.rs @@ -0,0 +1,87 @@ +use super::{DocumentId, RagData}; +use anyhow::Result; +use async_trait::async_trait; + +/// Abstracts where RAG vector data is stored and queried. +/// +/// Implementors: +/// - YamlProvider: HNSW in-memory, state derived from RagData.vectors/files +/// - DuckDbProvider: DuckDB on-disk vector index + document store +/// - QdrantProvider: remote Qdrant collection +/// +/// The Rag orchestrator owns: embeddings, chunking, BM25 keyword search, graph RAG, +/// entity extraction, RRF merging. Providers own: vector storage and content retrieval. +#[async_trait] +pub trait RagProvider: Send + Sync { + /// Vector similarity search. Returns (DocumentId, score) sorted by score desc. + /// `embedding` is a single query vector from Coyote's embedding model. + async fn vector_search( + &self, + embedding: &[f32], + top_k: usize, + min_score: f32, + ) -> Result>; + + /// Resolve document IDs to their page content. + /// + /// **Ordering contract:** implementations MUST return results in the same + /// relative order as the input `ids` slice. `hybrid_search` passes an + /// RRF-ranked list and feeds the result straight to the LLM — a provider + /// that returns rows in storage order (e.g. Qdrant `get_points`, DuckDB + /// `WHERE id IN (...)`) would silently discard the ranking. Implementations + /// that query an unordered backend must re-sort by input position before + /// returning. + /// + /// Returns only IDs that were found; callers must handle partial returns + /// (a missing ID is skipped, not an error). + /// YamlProvider: reads from an in-memory content map built from data.files. + /// DuckDbProvider: queries the documents table by id. + /// QdrantProvider: fetches payload from the remote collection. + async fn fetch_content(&self, ids: &[DocumentId]) -> Result>; + + /// Rebuild internal indexes from freshly updated RagData. + /// Called once at the end of every sync_documents pass. + /// + /// `full_rebuild` mirrors `sync_documents`' `refresh` parameter: + /// - `true` — a full re-index (`.rebuild rag`, `--rebuild-rag`, initial build). + /// Destructive strategies (wipe-then-reindex) are permitted. + /// - `false` — an incremental change (`.edit rag-docs` adding/removing a file). + /// Implementations MUST NOT wipe existing state; upsert only. + /// + /// The parameter is part of the signature from the outset so it is fixed + /// while there is exactly one implementor. Yaml/DuckDb ignore it — + /// rebuilding their local state wholesale is fast and always correct. + /// Only a remote provider is destructive enough to care. + /// + /// YamlProvider: rebuilds HNSW + content map from data.vectors/files. + /// DuckDbProvider: writes new rows to DuckDB, deletes removed rows. + /// QdrantProvider: no-op while attach-only — remote data is unchanged. + async fn rebuild_indexes(&mut self, data: &RagData, full_rebuild: bool) -> Result<()>; + + /// Keyword / full-text search. Returns (DocumentId, BM25-style score) sorted desc. + /// + /// Default impl returns `Ok(vec![])` — callers fall back to `Rag.bm25` (local in-memory + /// BM25 built from `data.files`). DuckDbProvider overrides this with a native FTS query + /// (DuckDB's `fts` extension, installed once at schema-creation time). + /// + /// Callers check `has_native_keyword_search()` before deciding which path to take: + /// - true → call this method; skip `Rag.bm25` + /// - false → call `Rag.keyword_search()` which uses `Rag.bm25` (sync, infallible) + /// + /// YamlProvider and QdrantProvider do NOT override this (return empty). + async fn keyword_search(&self, query: &str, top_k: usize) -> Result> { + let _ = (query, top_k); + Ok(vec![]) + } + + /// Returns true if this provider implements a native keyword-search index. + /// When false, `Rag.hybrid_search` uses the local `Rag.bm25` field instead. + fn has_native_keyword_search(&self) -> bool { + false + } + + /// Deep-clone the provider with fresh indexes derived from `data`. + /// Required because Box is not Clone. + /// Called by Rag's Clone impl (which clones before mutating in rebuild_rag/edit_rag_docs). + fn duplicate(&self, data: &RagData) -> Box; +} diff --git a/src/rag/providers/mod.rs b/src/rag/providers/mod.rs new file mode 100644 index 0000000..509f02c --- /dev/null +++ b/src/rag/providers/mod.rs @@ -0,0 +1,6 @@ +mod yaml; +// Use `self::` on every re-export in this file. Once a `mod duckdb;` sits here +// alongside a dependency on the `duckdb` CRATE, a bare `pub use duckdb::...` +// is ambiguous (E0659) — `use` paths resolve against both this module's items +// and the extern prelude, and `use` declarations may not shadow. +pub use self::yaml::YamlProvider; diff --git a/src/rag/providers/yaml.rs b/src/rag/providers/yaml.rs new file mode 100644 index 0000000..df1f6f5 --- /dev/null +++ b/src/rag/providers/yaml.rs @@ -0,0 +1,244 @@ +use crate::rag::provider::RagProvider; +use crate::rag::{DocumentId, RagData}; + +use anyhow::Result; +use async_trait::async_trait; +use hnsw_rs::prelude::*; +use indexmap::IndexMap; + +pub struct YamlProvider { + hnsw: Hnsw<'static, f32, DistCosine>, + content_map: IndexMap, +} + +impl YamlProvider { + pub fn from_data(data: &RagData) -> Self { + Self { + hnsw: data.build_hnsw(), + content_map: Self::build_content_map(data), + } + } + + fn build_content_map(data: &RagData) -> IndexMap { + // Keyed on `files`, NOT `vectors`: this is the exact replacement for the + // per-id document lookup it supersedes, and it must resolve every id that + // BM25 or graph_search can produce — both of which enumerate `files`. + data.iter_documents() + .map(|(id, doc)| (id, doc.page_content.clone())) + .collect() + } +} + +#[async_trait] +impl RagProvider for YamlProvider { + async fn vector_search( + &self, + embedding: &[f32], + top_k: usize, + min_score: f32, + ) -> Result> { + let results = self + .hnsw + .parallel_search(&[embedding.to_vec()], top_k, 30) + .into_iter() + .flat_map(|list| { + list.into_iter().filter_map(|v| { + let score = 1.0 - v.distance; + if score > min_score { + Some((DocumentId(v.d_id), score)) + } else { + None + } + }) + }) + .collect(); + Ok(results) + } + + async fn fetch_content(&self, ids: &[DocumentId]) -> Result> { + // Iterating `ids` (not `content_map`) satisfies the trait's ordering + // contract for free — output order mirrors input order. + Ok(ids + .iter() + .filter_map(|id| self.content_map.get(id).map(|text| (*id, text.clone()))) + .collect()) + } + + async fn rebuild_indexes(&mut self, data: &RagData, _full_rebuild: bool) -> Result<()> { + // Local in-memory state — a wholesale rebuild is fast and always correct, + // so the incremental/full distinction is irrelevant here. + self.hnsw = data.build_hnsw(); + self.content_map = Self::build_content_map(data); + Ok(()) + } + + fn duplicate(&self, data: &RagData) -> Box { + Box::new(YamlProvider::from_data(data)) + } +} + +#[cfg(test)] +mod provider_tests { + use super::*; + // `RagFile` and `RagDocument` are not used by the impl above, so they are + // imported here rather than at module scope. Both have private fields, which + // is why these tests must live in-crate rather than under `tests/`. + use crate::rag::{RagDocument, RagFile}; + + fn minimal_rag_data() -> RagData { + // `..Default::default()` rather than an exhaustive struct literal so that + // later additions to `RagData` do not break this helper. + RagData { + embedding_model: "text-embedding-3-small".to_string(), + chunk_size: 1024, + chunk_overlap: 50, + top_k: 5, + driver: "yaml".to_string(), + attached: false, + ..Default::default() + } + } + + /// Two files, one chunk each, with vectors — the minimum needed to exercise + /// `build_content_map` and the `fetch_content` ordering contract. + /// `DocumentId::new(f, d)` packs (file_index, document_index); `RagData::add` + /// is the real insertion path but a direct literal is sufficient and avoids + /// the embedding pipeline. + fn populated_rag_data() -> RagData { + let mut data = minimal_rag_data(); + // `files` must be populated: build_content_map iterates data.iter_documents(), + // which enumerates `files`. Populating `vectors` alone would produce an EMPTY + // content map, and every assertion below would vacuously pass on a broken impl. + // The vectors inserted at the end are for the HNSW side only. + data.files.insert( + 0, + RagFile { + hash: "h0".to_string(), + path: "/tmp/a.md".to_string(), + documents: vec![RagDocument { + page_content: "alpha".to_string(), + metadata: Default::default(), + }], + }, + ); + data.files.insert( + 1, + RagFile { + hash: "h1".to_string(), + path: "/tmp/b.md".to_string(), + documents: vec![RagDocument { + page_content: "beta".to_string(), + metadata: Default::default(), + }], + }, + ); + data.vectors + .insert(DocumentId::new(0, 0), vec![1.0, 0.0, 0.0]); + data.vectors + .insert(DocumentId::new(1, 0), vec![0.0, 1.0, 0.0]); + data + } + + #[tokio::test] + async fn yaml_provider_empty_data_returns_nothing() { + let data = minimal_rag_data(); + let provider = YamlProvider::from_data(&data); + let results = provider.fetch_content(&[]).await.unwrap(); + assert!(results.is_empty()); + } + + /// `fetch_content` MUST return results in the same relative order as the input + /// ids. The reversed-input case is the one that fails if an implementation ever + /// iterates its own map instead of `ids`. + #[tokio::test] + async fn yaml_provider_fetch_content_preserves_input_order() { + let data = populated_rag_data(); + let provider = YamlProvider::from_data(&data); + + let a = DocumentId::new(0, 0); + let b = DocumentId::new(1, 0); + + let forward = provider.fetch_content(&[a, b]).await.unwrap(); + assert_eq!(forward.len(), 2, "both documents must resolve"); + assert_eq!(forward[0].1, "alpha"); + assert_eq!(forward[1].1, "beta"); + + // Reversed input must produce reversed output — NOT storage order. + let reversed = provider.fetch_content(&[b, a]).await.unwrap(); + assert_eq!( + reversed[0].1, "beta", + "fetch_content must honor input order" + ); + assert_eq!(reversed[1].1, "alpha"); + } + + /// A missing id is skipped, not an error, and does not disturb the order of + /// the ids that DO resolve. + #[tokio::test] + async fn yaml_provider_fetch_content_skips_missing_ids() { + let data = populated_rag_data(); + let provider = YamlProvider::from_data(&data); + + let a = DocumentId::new(0, 0); + let missing = DocumentId::new(99, 0); + let b = DocumentId::new(1, 0); + + let out = provider.fetch_content(&[a, missing, b]).await.unwrap(); + assert_eq!(out.len(), 2, "missing id is skipped, not an error"); + assert_eq!(out[0].1, "alpha"); + assert_eq!(out[1].1, "beta"); + } + + /// `YamlProvider::duplicate()` rebuilds from `data`, so the clone is a genuine + /// independent snapshot. Providers backed by a shared store deliberately are not. + #[tokio::test] + async fn yaml_provider_duplicate_returns_equivalent_content() { + // MUST be populated_rag_data(): on minimal_rag_data() both providers hold an + // EMPTY content map, so `assert_eq!(r1, r2)` compares two empty vectors and + // passes against a duplicate() that returns nothing at all. + let data = populated_rag_data(); + let provider = YamlProvider::from_data(&data); + let dup = provider.duplicate(&data); + let ids = [DocumentId::new(0, 0), DocumentId::new(1, 0)]; + // Query with REAL ids, not `&[]` — an empty slice is answered without ever + // touching the content map, so it would pass against a broken duplicate(). + let r1 = provider.fetch_content(&ids).await.unwrap(); + let r2 = dup.fetch_content(&ids).await.unwrap(); + // Guard against the vacuous case: if both sides resolved nothing, the equality + // below proves nothing. Assert the fixture actually produced content first. + assert_eq!(r1.len(), 2, "fixture must resolve both documents"); + assert_eq!( + r1, r2, + "duplicate must resolve the same content as the original" + ); + } + + /// The content store is keyed on `files`, never on `vectors`. A vector may exist + /// for an id with no backing file (a stale entry, or a file dropped mid-sync); + /// keying on `vectors` would surface such an id with empty text instead of + /// dropping it. The shared fixture only ever inserts vectors for ids that also + /// have files, so this case has to be constructed here. + #[tokio::test] + async fn yaml_provider_content_is_keyed_on_files_not_vectors() { + let mut data = populated_rag_data(); + let orphan = DocumentId::new(9, 0); + data.vectors.insert(orphan, vec![0.0, 0.0, 1.0]); + + let provider = YamlProvider::from_data(&data); + + let out = provider.fetch_content(&[orphan]).await.unwrap(); + assert!( + out.is_empty(), + "an id present only in `vectors` must not resolve to content" + ); + + // The file-backed ids still resolve, so the assertion above is not vacuous. + let real = provider + .fetch_content(&[DocumentId::new(0, 0), DocumentId::new(1, 0)]) + .await + .unwrap(); + assert_eq!(real.len(), 2, "file-backed documents must still resolve"); + assert_eq!(real[0].1, "alpha"); + assert_eq!(real[1].1, "beta"); + } +} From d734276927a9b60985d1e5d8f192477b68ea15c3 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 10 Aug 2026 11:59:57 -0600 Subject: [PATCH 03/36] build(rag): add duckdb dependency and pin comfy-table to 7.1.4 Adds the duckdb crate with the bundled feature ahead of any provider code, so the dependency and build surface can be proven on every CI target on its own. duckdb constrains comfy-table to ~7.1, so comfy-table moves from 7.2.2 to 7.1.4 while keeping custom_styling. That feature swaps measure_text_width for an ANSI-stripping implementation that render_table's pre-styled cells depend on; dropping it still compiles and still passes every other test, and only corrupts column widths. A regression test now renders a styled table at a fixed wrap width and asserts every line has an equal ANSI-stripped display width. comfy-table 7.1.4 pins crossterm 0.28 while coyote pins 0.29, so both now build side by side. comfy_table::Color, Attribute and Cell are consequently crossterm 0.28 types and must not be used; table styling stays ANSI-string based. Caches the DuckDB extension directory in CI, keyed on the runner OS and the DuckDB version, so vss and fts survive an upstream outage. Also switches merge_vector_results to f32::total_cmp. The previous partial_cmp().unwrap_or(Equal) comparator is not total under NaN, which sort_by is permitted to answer with a panic in release builds. --- .github/workflows/ci.yaml | 6 + Cargo.lock | 443 ++++++++++++++++++++++++++++++++++++-- Cargo.toml | 3 +- src/rag/mod.rs | 2 +- src/render/markdown.rs | 91 ++++++++ 5 files changed, 522 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1f06695..a7bc97a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -36,6 +36,12 @@ jobs: - uses: Swatinem/rust-cache@v2 + - name: Cache DuckDB Extensions + uses: actions/cache@v4 + with: + path: ~/.duckdb/extensions + key: duckdb-ext-${{ matrix.os }}-v1.5.5 + - name: Test run: cargo test --all diff --git a/Cargo.lock b/Cargo.lock index 5ce821f..e42c605 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,6 +34,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "const-random", + "getrandom 0.3.4", "once_cell", "version_check", "zerocopy", @@ -244,6 +246,169 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +[[package]] +name = "arrow" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-data" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ord" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "arrow-select" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + [[package]] name = "async-compression" version = "0.4.43" @@ -289,6 +454,15 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -1096,6 +1270,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ade8366b8bd5ba243f0a58f036cc0ca8a2f069cff1a2351ef1cac6b083e16fc0" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.4.0" @@ -1314,13 +1494,13 @@ dependencies = [ [[package]] name = "comfy-table" -version = "7.2.2" +version = "7.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +checksum = "4a65ebfec4fb190b6f90e944a817d60499ee0744e582530e2c9900a22e591d9a" dependencies = [ "ansi-str", - "console", - "crossterm", + "console 0.15.11", + "crossterm 0.28.1", "unicode-segmentation", "unicode-width", ] @@ -1355,6 +1535,19 @@ dependencies = [ "thiserror 2.0.19", ] +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.59.0", +] + [[package]] name = "console" version = "0.16.4" @@ -1379,6 +1572,26 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + [[package]] name = "convert_case" version = "0.10.0" @@ -1465,8 +1678,9 @@ dependencies = [ "clap_complete_nushell", "colored", "comfy-table", - "crossterm", + "crossterm 0.29.0", "dirs", + "duckdb", "duct", "dunce", "eventsource-stream", @@ -1595,6 +1809,19 @@ version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags 2.13.1", + "crossterm_winapi", + "parking_lot", + "rustix 0.38.44", + "winapi", +] + [[package]] name = "crossterm" version = "0.29.0" @@ -1608,7 +1835,7 @@ dependencies = [ "filedescriptor", "mio", "parking_lot", - "rustix", + "rustix 1.1.4", "serde", "signal-hook", "signal-hook-mio", @@ -1624,6 +1851,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -1886,7 +2119,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25f104b501bf2364e78d0d3974cbc774f738f5865306ed128e1e0d7499c0ad96" dependencies = [ - "console", + "console 0.16.4", "shell-words", "tempfile", "zeroize", @@ -1993,6 +2226,23 @@ dependencies = [ "dtoa", ] +[[package]] +name = "duckdb" +version = "1.10505.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "970e05eedd3f55c435194d9104f90a9b4a79a80d6e73251bc9ff43e178130c4e" +dependencies = [ + "arrow", + "cast", + "comfy-table", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libduckdb-sys", + "num-integer", + "strum", +] + [[package]] name = "duct" version = "1.1.1" @@ -2168,6 +2418,18 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fancy-regex" version = "0.14.0" @@ -2192,7 +2454,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", - "rustix", + "rustix 1.1.4", "windows-sys 0.59.0", ] @@ -2448,7 +2710,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" dependencies = [ - "rustix", + "rustix 1.1.4", "windows-link", ] @@ -2526,7 +2788,7 @@ dependencies = [ "clap_complete", "confy", "crc32c", - "crossterm", + "crossterm 0.29.0", "dialoguer", "dirs", "futures", @@ -2590,6 +2852,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -2613,6 +2887,15 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "heck" version = "0.5.0" @@ -3122,7 +3405,7 @@ version = "0.18.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c" dependencies = [ - "console", + "console 0.16.4", "portable-atomic", "unicode-width", "unit-prefix", @@ -3154,7 +3437,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6654738b8024300cf062d04a1c13c10c8e2cea598ec1c47dc9b6641159429756" dependencies = [ "bitflags 2.13.1", - "crossterm", + "crossterm 0.29.0", "dyn-clone", "fuzzy-matcher", "unicode-segmentation", @@ -3387,12 +3670,86 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + [[package]] name = "libc" version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libduckdb-sys" +version = "1.10505.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb514dab5e271e849235c1cb98bd65a2ae107fbd619a6740219319c54a71d95" +dependencies = [ + "cc", + "flate2", + "pkg-config", + "serde", + "serde_json", + "tar", + "ureq", + "vcpkg", + "zip", +] + [[package]] name = "libloading" version = "0.8.9" @@ -3403,6 +3760,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "libredox" version = "0.1.19" @@ -3412,6 +3775,12 @@ dependencies = [ "libc", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -3744,6 +4113,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -3766,6 +4144,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -4717,7 +5096,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2066729dce9fecd28d1c6850a159ee68719130f149b22467c362353e16994e90" dependencies = [ "chrono", - "crossterm", + "crossterm 0.29.0", "fd-lock", "itertools 0.13.0", "nu-ansi-term", @@ -5032,6 +5411,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.1", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + [[package]] name = "rustix" version = "1.1.4" @@ -5041,7 +5433,7 @@ dependencies = [ "bitflags 2.13.1", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.12.1", "windows-sys 0.61.2", ] @@ -5952,7 +6344,7 @@ dependencies = [ "fastrand", "getrandom 0.4.3", "once_cell", - "rustix", + "rustix 1.1.4", "windows-sys 0.61.2", ] @@ -5999,7 +6391,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ - "rustix", + "rustix 1.1.4", "windows-sys 0.61.2", ] @@ -6114,6 +6506,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -6845,7 +7246,7 @@ checksum = "016ccf01d1c58b6f8999612813e17c9b2390f7d70671428869913310f83f54b8" dependencies = [ "cc", "downcast-rs", - "rustix", + "rustix 1.1.4", "smallvec", "wayland-sys", ] @@ -6857,7 +7258,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c36a0f861ad76d0901f2800b46321410d9f73f2ea88aac0650d86c32688073" dependencies = [ "bitflags 2.13.1", - "rustix", + "rustix 1.1.4", "wayland-backend", "wayland-scanner", ] @@ -7347,7 +7748,7 @@ dependencies = [ "libc", "log", "os_pipe", - "rustix", + "rustix 1.1.4", "thiserror 2.0.19", "tree_magic_mini", "wayland-backend", @@ -7369,7 +7770,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" dependencies = [ "gethostname", - "rustix", + "rustix 1.1.4", "x11rb-protocol", ] @@ -7386,7 +7787,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", - "rustix", + "rustix 1.1.4", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 32c56cb..d51421c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,8 +17,9 @@ exclude = [".github", "CONTRIBUTING.md"] anyhow = "1.0.69" bytes = "1.4.0" clap = { version = "4.5.40", features = ["cargo", "derive", "wrap_help"] } -comfy-table = { version = "7.2.2", features = ["custom_styling"] } +comfy-table = { version = "7.1.4", features = ["custom_styling"] } dirs = "6.0.0" +duckdb = { version = "1.10505.0", features = ["bundled"] } dunce = "1.0.5" futures-util = "0.3.29" inquire = "0.9.4" diff --git a/src/rag/mod.rs b/src/rag/mod.rs index 9011296..dd2ba91 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -1630,7 +1630,7 @@ fn merge_vector_results(mut results: Vec<(DocumentId, f32)>) -> Vec<(DocumentId, results.iter().all(|(_, score)| score.is_finite()), "provider returned a non-finite score; NaN silently degrades sort order" ); - results.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(Ordering::Equal)); + results.sort_by(|a, b| b.1.total_cmp(&a.1)); let mut seen = IndexSet::new(); results .into_iter() diff --git a/src/render/markdown.rs b/src/render/markdown.rs index 5a1a975..cc93c2c 100644 --- a/src/render/markdown.rs +++ b/src/render/markdown.rs @@ -1749,6 +1749,97 @@ std::error::Error>> { ); } + /// Removes CSI escape sequences so only printable content is measured. + /// + /// Deliberately tolerant of malformed input: a sequence that was sliced + /// mid-escape swallows the following characters, which is precisely the + /// corruption `render_table_pads_columns_by_display_width` exists to catch. + fn strip_ansi(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut chars = text.chars(); + while let Some(c) = chars.next() { + if c != '\u{1b}' { + out.push(c); + continue; + } + if chars.next() == Some('[') { + for c in chars.by_ref() { + if ('\u{40}'..='\u{7e}').contains(&c) { + break; + } + } + } + } + out + } + + #[test] + fn strip_ansi_removes_sgr_and_keeps_text() { + assert_eq!(strip_ansi("\x1b[1mbold\x1b[0m"), "bold"); + assert_eq!(strip_ansi("\x1b[38;5;120mx\x1b[39m"), "x"); + assert_eq!(strip_ansi("plain"), "plain"); + } + + /// `render_table` hands comfy-table pre-styled cells that already contain + /// ANSI escapes, and `colorize_box_chars` adds more afterwards. Column + /// widths are therefore only correct if the escapes are excluded from the + /// width calculation. When they are not, the table still renders and every + /// other assertion in this file still passes -- only the alignment silently + /// degrades -- so this is the sole guard over that behaviour. + #[test] + fn render_table_pads_columns_by_display_width() { + use unicode_width::UnicodeWidthStr; + + const WRAP_WIDTH: u16 = 80; + + let options = RenderOptions::default(); + let mut render = MarkdownRender::init(options).unwrap(); + render.wrap_width = Some(WRAP_WIDTH); + + let header = vec![ + "**Setting**".into(), + "*Default*".into(), + "`Description`".into(), + ]; + let alignments = vec![ + CellAlignment::Left, + CellAlignment::Right, + CellAlignment::Center, + ]; + let rows = vec![ + vec![ + "**temperature**".into(), + "`0.7`".into(), + "Controls how *random* the sampled reply is allowed to be".into(), + ], + vec![ + "**top_p**".into(), + "`1.0`".into(), + "Nucleus sampling cutoff, applied **after** temperature".into(), + ], + ]; + + let output = render.render_table(header, alignments, rows); + + assert!( + output.contains('\u{1b}'), + "fixture must actually contain ANSI escapes: {output:?}", + ); + + let widths: Vec = output + .lines() + .map(|line| strip_ansi(line).width()) + .collect(); + assert!(!widths.is_empty(), "table rendered no lines"); + + for (index, width) in widths.iter().enumerate() { + assert_eq!( + *width, WRAP_WIDTH as usize, + "line {index} display width; all widths were {widths:?} in output:\n{output}", + ); + } + } + #[test] fn state_machine_renders_full_table_and_flushes_on_paragraph() { let options = RenderOptions::default(); From 98d3ba4a835ab139e1e0e2e09f1c3e8071282d05 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 10 Aug 2026 12:51:56 -0600 Subject: [PATCH 04/36] feat(rag): add DuckDB provider behind the RAG driver abstraction Phase 3 of the RAG driver abstraction. Adds a `DuckDbProvider` that keeps vectors and document content in a `.duckdb` sidecar next to the existing YAML metadata, selected by the `driver: duckdb` field. - `src/rag/providers/duckdb.rs` (new): vector search via the vss extension and keyword search via fts, an all-or-nothing hydration path (a partial read is an error, never a shorter map), and an anti-wipe guard that refuses the destructive `CREATE OR REPLACE TABLE` when `data.vectors` is empty while `data.files` is not and the store still holds rows. - `src/rag/mod.rs`: `sync_documents` now refreshes `bm25`/`node_to_docs` BEFORE the fallible `provider.rebuild_indexes`. `self.data` is already mutated by that point, so propagating a provider error afterwards would leave the derived in-memory state describing the previous corpus while `data` describes the new one. Both rebuilds are pure functions of `self.data` and cannot fail, so running them first is always safe. - `src/config/paths.rs`: sidecar path helpers. - `src/rag/providers/mod.rs`, `src/config/agent.rs`: driver dispatch and RAG cache keying. Also keeps `RequestContext::rag_key` in lockstep with `rag` at the two sites that were still missing it, so that a cache insert and its matching invalidate are structurally incapable of disagreeing: - `use_agent` assigned `self.rag` from the agent but never set `rag_key`. This one was live. Agent RAGs are inserted under `RagKey::Agent()`, so with `rag_key == None` the invalidation guards in `rebuild_rag` and `edit_rag_docs` matched nothing and `.rebuild rag` left the stale cache entry in place. Worse, a preceding `.rag ` left a stale `Named()` key attached to the agent's RAG, pointing the invalidation at an unrelated RAG's cache entry. Now mirrors the insert key exactly, yielding `None` when the agent has no RAG. - `exit_agent` cleared `self.rag` but left `rag_key` behind. Latent rather than live, since `rebuild_rag`/`edit_rag_docs` both bail on `rag.is_none()` before reaching the invalidate guards, but the guards that make it unobservable are not the kind of thing to depend on. Covered by `use_agent_does_not_carry_stale_rag_key`, and by a new assertion in `exit_agent_clears_all_agent_state`. --- src/config/agent.rs | 14 +- src/config/paths.rs | 116 +++- src/config/request_context.rs | 155 +++++- src/rag/mod.rs | 138 ++++- src/rag/providers/duckdb.rs | 964 ++++++++++++++++++++++++++++++++++ src/rag/providers/mod.rs | 7 + 6 files changed, 1364 insertions(+), 30 deletions(-) create mode 100644 src/rag/providers/duckdb.rs diff --git a/src/config/agent.rs b/src/config/agent.rs index d9d95ea..16081ff 100644 --- a/src/config/agent.rs +++ b/src/config/agent.rs @@ -171,7 +171,15 @@ impl Agent { let rag = app_state .rag_cache .load_with(key, || async move { - Rag::init(&app_clone, "rag", &rag_path_clone, &document_paths, abort).await + Rag::init( + &app_clone, + "rag", + &rag_path_clone, + &document_paths, + abort, + false, + ) + .await }) .await?; Some(rag) @@ -983,6 +991,10 @@ async fn init_graph_rags( extractor_model: rag_node.extractor_model.clone(), extractor_prompt: rag_node.extractor_prompt.clone(), graph_hops: rag_node.graph_hops, + // Graph-node RAGs are yaml-only: `RagNode` has no `driver` field, so + // there is nothing to forward. The rest-pattern also keeps this literal + // from breaking on future `RagInitConfig` additions. + ..Default::default() }; let fully_specified = config.embedding_model.is_some() && config.chunk_size.is_some() diff --git a/src/config/paths.rs b/src/config/paths.rs index 49f2135..268e497 100644 --- a/src/config/paths.rs +++ b/src/config/paths.rs @@ -16,7 +16,7 @@ use anyhow::{Context, Result, anyhow, bail}; use log::LevelFilter; use std::collections::HashSet; use std::env; -use std::fs::{read_dir, read_to_string}; +use std::fs::{read_dir, read_to_string, remove_file}; use std::path::{Path, PathBuf}; pub fn config_dir() -> PathBuf { @@ -414,6 +414,11 @@ pub fn list_rags() -> Vec { for entry in rd.flatten() { let name = entry.file_name(); if let Some(name) = name.to_string_lossy().strip_suffix(".yaml") { + // Sidecars are not RAGs. `.duckdb` files are already excluded by + // the `.yaml` suffix check above; this rejects `.sbx-mixin`. + if is_rag_sidecar_name(name) { + continue; + } names.push(name.to_string()); } } @@ -424,6 +429,43 @@ pub fn list_rags() -> Vec { } } +/// True for the sidecar YAML files that must never be listed or deleted as RAGs. +/// `name` is the already-stripped stem (i.e. after `strip_suffix(".yaml")`). +/// Uses `ends_with`, not `contains('.')`, so a RAG legitimately named "v2.docs" is +/// not rejected. +pub(crate) fn is_rag_sidecar_name(name: &str) -> bool { + name.ends_with(".sbx-mixin") +} + +/// Remove every sidecar belonging to RAG `name` in `dir`. Missing files are NOT an +/// error. A failure to remove an EXISTING mixin IS an error and must propagate — a +/// silently-orphaned mixin keeps a sandbox network permission alive after the user +/// believes it is gone. The `.duckdb` orphan is only wasted disk, so its removal +/// failure is ignorable; the asymmetry is deliberate. +/// +/// Callers must run this BEFORE unlinking the primary `.yaml`. If the YAML goes first +/// and this then fails, the RAG disappears from `list_rags()` — so the user can no +/// longer select it to retry — while its `allowedDomains` entry keeps being injected +/// into every sandbox launch. +pub(crate) fn remove_rag_sidecars(dir: &Path, name: &str) -> Result<()> { + let duckdb_path = dir.join(format!("{name}.duckdb")); + if duckdb_path.exists() { + let _ = remove_file(&duckdb_path); + } + let mixin_path = dir.join(format!("{name}.sbx-mixin.yaml")); + if mixin_path.exists() { + remove_file(&mixin_path).with_context(|| { + format!( + "Failed to remove the sandbox mixin for RAG '{name}' at '{}'. \ + The RAG was NOT deleted so you can retry; this host remains \ + whitelisted in the sandbox until the file is removed.", + mixin_path.display() + ) + })?; + } + Ok(()) +} + pub fn list_macros() -> Vec { list_file_names(macros_dir(), ".yaml") } @@ -846,4 +888,76 @@ mod tests { } let _ = fs::remove_dir_all(&root); } + + /// Unique temp dir for the sidecar helper tests. These take `dir: &Path` directly, + /// so no env-var mutation and therefore no `#[serial]` is needed. + fn sidecar_temp_dir(label: &str) -> PathBuf { + let unique = time::SystemTime::now() + .duration_since(time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = env::temp_dir().join(format!("coyote-{label}-test-{unique}")); + fs::create_dir_all(&root).unwrap(); + root + } + + #[test] + fn is_rag_sidecar_name_accepts_dotted_rag_names() { + // A RAG legitimately named "v2.docs" must not be mistaken for a sidecar. + assert!(!is_rag_sidecar_name("v2.docs")); + assert!(!is_rag_sidecar_name("myrag")); + assert!(is_rag_sidecar_name("myrag.sbx-mixin")); + assert!(is_rag_sidecar_name("v2.docs.sbx-mixin")); + } + + #[test] + fn remove_rag_sidecars_removes_both() { + let root = sidecar_temp_dir("rag-sidecars-both"); + let duckdb = root.join("docs.duckdb"); + let mixin = root.join("docs.sbx-mixin.yaml"); + fs::write(&duckdb, "db").unwrap(); + fs::write(&mixin, "mixin").unwrap(); + + remove_rag_sidecars(&root, "docs").unwrap(); + + assert!(!duckdb.exists(), "the .duckdb sidecar must be removed"); + assert!( + !mixin.exists(), + "the .sbx-mixin.yaml sidecar must be removed" + ); + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn remove_rag_sidecars_is_ok_when_absent() { + let root = sidecar_temp_dir("rag-sidecars-absent"); + assert!(remove_rag_sidecars(&root, "docs").is_ok()); + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn remove_rag_sidecars_runs_before_yaml_unlink() { + let root = sidecar_temp_dir("rag-sidecars-order"); + let yaml = root.join("docs.yaml"); + fs::write(&yaml, "rag").unwrap(); + // A non-empty DIRECTORY at the mixin path makes remove_file fail, standing in + // for any real removal failure (permissions, a busy mount). + let mixin = root.join("docs.sbx-mixin.yaml"); + fs::create_dir_all(&mixin).unwrap(); + fs::write(mixin.join("blocker"), "x").unwrap(); + + let err = remove_rag_sidecars(&root, "docs").unwrap_err(); + assert!( + err.to_string() + .contains("Failed to remove the sandbox mixin"), + "got: {err}" + ); + // The whole point of removing sidecars first: the RAG is still on disk, still + // listed, and the deletion is retryable. + assert!( + yaml.exists(), + "the .yaml must survive a sidecar-removal failure so the delete is retryable" + ); + let _ = fs::remove_dir_all(&root); + } } diff --git a/src/config/request_context.rs b/src/config/request_context.rs index f01850d..e2f3883 100644 --- a/src/config/request_context.rs +++ b/src/config/request_context.rs @@ -142,6 +142,12 @@ pub struct RequestContext { pub role: Option, pub session: Option, pub rag: Option>, + /// The cache key `self.rag` was actually inserted under, carried rather than + /// reconstructed. Reconstruction was the bug: the invalidation sites do not have + /// the information needed to rebuild the key (agent RAGs are inserted under the + /// AGENT's name but `rag.name()` is the constant "rag"), so insert and invalidate + /// silently disagreed. `None` for the temp RAG, which bypasses the cache entirely. + pub rag_key: Option, pub agent: Option, pub last_message: Option, @@ -176,6 +182,7 @@ impl RequestContext { role: None, session: None, rag: None, + rag_key: None, agent: None, last_message: None, tool_scope: ToolScope::default(), @@ -229,6 +236,7 @@ impl RequestContext { role: None, session: None, rag: None, + rag_key: None, agent: None, last_message: None, tool_scope: ToolScope { @@ -277,6 +285,7 @@ impl RequestContext { role: self.role.clone(), session: self.session.clone(), rag: self.rag.clone(), + rag_key: self.rag_key.clone(), agent: self.agent.clone(), last_message: self.last_message.clone(), tool_scope: self.tool_scope.clone(), @@ -315,6 +324,7 @@ impl RequestContext { role: None, session: None, rag: None, + rag_key: None, agent: None, last_message: None, tool_scope: ToolScope { @@ -2554,6 +2564,14 @@ impl RequestContext { match file_ext { Some(file_ext) => { if let Some(name) = name.to_string_lossy().strip_suffix(file_ext) { + // Sidecars are not independently deletable assets. + // Guarded on `kind == "rag"` because this scan is shared + // by all six kinds, and `session`/`macro` also use + // `.yaml`. The helper lives in paths.rs beside + // list_rags() so both filters cannot drift apart. + if kind == "rag" && paths::is_rag_sidecar_name(name) { + continue; + } names.push(name.to_string()); } } @@ -2590,6 +2608,13 @@ impl RequestContext { match file_ext { Some(ext) => { let path = dir.join(format!("{name}{ext}")); + // Sidecars FIRST. If this fails, the .yaml is still on disk, the + // RAG is still listed, and the user can retry. Unlinking the .yaml + // first would make the deletion unretryable while leaving an + // orphaned mixin whitelisting a host in every sandbox launch. + if kind == "rag" { + paths::remove_rag_sidecars(&dir, &name)?; + } remove_file(&path).with_context(|| { format!("Failed to delete {kind} at '{}'", path.display()) })?; @@ -3729,6 +3754,14 @@ impl RequestContext { .then(|| Arc::new(RwLock::new(Supervisor::new(max_concurrent, max_depth)))); self.rag = agent.rag(); + // Keep `rag_key` in lockstep with `rag`. Agent RAGs are cached under + // `RagKey::Agent()` (see `Agent::init`), so mirror that key exactly; + // leaving the previous key in place would let `.rebuild rag` invalidate an + // unrelated RAG's cache entry, and leaving it `None` would invalidate nothing. + self.rag_key = self + .rag + .is_some() + .then(|| RagKey::Agent(agent.name().to_string())); self.agent = Some(agent); self.supervisor = supervisor; self.inbox = None; @@ -3777,6 +3810,11 @@ impl RequestContext { self.pending_agents_guardrail_count = 0; self.todo_list = TodoList::default(); self.rag.take(); + // Cleared alongside `rag` so the pair never disagrees: an agent RAG is + // cached under `RagKey::Agent()`, and leaving that key behind + // would outlive the RAG it names. Latent rather than live today only + // because `rebuild_rag`/`edit_rag_docs` bail on `rag.is_none()` first. + self.rag_key = None; self.discontinuous_last_message(); } Ok(()) @@ -4087,7 +4125,9 @@ impl RequestContext { let rag_cache = self.rag_cache(); let working_mode = self.working_mode; - let rag: Arc = match rag { + // The key is returned alongside the Rag rather than assigned inside the match: + // `rag_cache` borrows `self`, so writing `self.rag_key` there is E0506. + let (rag, rag_key): (Arc, Option) = match rag { None => { let rag_path = self.rag_file(super::TEMP_RAG_NAME); if rag_path.exists() { @@ -4095,14 +4135,28 @@ impl RequestContext { format!("Failed to cleanup previous '{}' rag", super::TEMP_RAG_NAME) })?; } - Arc::new(Rag::init(&app, super::TEMP_RAG_NAME, &rag_path, &[], abort_signal).await?) + // The temp RAG is never inserted into the cache, so it has no key. + ( + Arc::new( + Rag::init( + &app, + super::TEMP_RAG_NAME, + &rag_path, + &[], + abort_signal, + false, + ) + .await?, + ), + None, + ) } Some(name) => { let rag_path = self.rag_file(name); let key = RagKey::Named(name.to_string()); - rag_cache - .load_with(key, || { + let loaded = rag_cache + .load_with(key.clone(), || { let app = app.clone(); let rag_path = rag_path.clone(); let abort_signal = abort_signal.clone(); @@ -4111,16 +4165,19 @@ impl RequestContext { if working_mode.is_cmd() { bail!("Unknown RAG '{name}'"); } - Rag::init(&app, name, &rag_path, &[], abort_signal.clone()).await + Rag::init(&app, name, &rag_path, &[], abort_signal.clone(), true) + .await } else { Rag::load(&app, name, &rag_path) } } }) - .await? + .await?; + (loaded, Some(key)) } }; self.rag = Some(rag); + self.rag_key = rag_key; Ok(()) } @@ -4161,12 +4218,9 @@ impl RequestContext { bail!("No changes") } - let key = if self.agent.is_some() { - RagKey::Agent(rag.name().to_string()) - } else { - RagKey::Named(rag.name().to_string()) - }; - self.rag_cache().invalidate(&key); + if let Some(key) = self.rag_key.clone() { + self.rag_cache().invalidate(&key); + } rag.refresh_document_paths( &new_document_paths, @@ -4194,12 +4248,9 @@ impl RequestContext { ); } - let key = if self.agent.is_some() { - RagKey::Agent(rag.name().to_string()) - } else { - RagKey::Named(rag.name().to_string()) - }; - self.rag_cache().invalidate(&key); + if let Some(key) = self.rag_key.clone() { + self.rag_cache().invalidate(&key); + } let document_paths = rag.document_paths().to_vec(); println!( @@ -4613,6 +4664,49 @@ mod tests { assert!(ctx.agent.is_none()); assert!(ctx.rag.is_none()); + assert_eq!(ctx.rag_key, None); + } + + #[test] + #[serial] + fn use_agent_does_not_carry_stale_rag_key() { + let _guard = TestConfigDirGuard::new(); + let mut ctx = create_test_ctx(); + let app = ctx.app.config.clone(); + let agent_name = format!( + "test_agent_{}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + ); + let agent_dir = paths::agent_data_dir(&agent_name); + create_dir_all(&agent_dir).unwrap(); + write( + agent_dir.join("config.yaml"), + format!("name: {agent_name}\ninstructions: hi\n"), + ) + .unwrap(); + + // Stand in for the state `.rag docs` leaves behind: `use_rag` sets `rag` and + // `rag_key` together, so a named key is live when the agent is entered. + ctx.rag_key = Some(RagKey::Named("docs".to_string())); + + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { + ctx.use_agent(&app, &agent_name, None, utils::create_abort_signal()) + .await + .unwrap(); + }); + + // This agent has no RAG, so `rag` is None and `rag_key` must be None as well. + // Carrying `Named("docs")` across the transition would point `.rebuild rag` + // at an unrelated RAG's cache entry. + assert!(ctx.rag.is_none()); + assert_eq!(ctx.rag_key, None); } #[test] @@ -6000,6 +6094,31 @@ mod tests { assert!(paths::list_rags().is_empty()); } + /// A `.sbx-mixin.yaml` sidecar must not appear as a phantom RAG in TAB + /// completion or `.list rag`. A RAG whose name legitimately contains a dot must + /// still be listed — the filter uses `ends_with`, not `contains('.')`. + #[test] + #[serial] + fn list_rags_skips_sbx_mixin_sidecars() { + let _guard = TestConfigDirGuard::new(); + let rags_dir = paths::rags_dir(); + create_dir_all(&rags_dir).unwrap(); + write(rags_dir.join("docs.yaml"), "embedding_model: test").unwrap(); + write(rags_dir.join("docs.sbx-mixin.yaml"), "kind: mixin").unwrap(); + write(rags_dir.join("v2.docs.yaml"), "embedding_model: test").unwrap(); + + let names = paths::list_rags(); + assert!(names.contains(&"docs".to_string())); + assert!( + names.contains(&"v2.docs".to_string()), + "a dotted RAG name must still be listed: {names:?}" + ); + assert!( + !names.contains(&"docs.sbx-mixin".to_string()), + "the sandbox mixin sidecar must not appear as a RAG: {names:?}" + ); + } + #[test] #[serial] fn use_agent_errors_when_already_in_session() { diff --git a/src/rag/mod.rs b/src/rag/mod.rs index dd2ba91..6c8d41a 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -12,7 +12,10 @@ mod splitter; use self::graph::{KnowledgeGraph, extract_entities}; use self::provider::RagProvider; -use self::providers::YamlProvider; +// `providers::duckdb_path_from_yaml(path)` is called through the module path in +// `create()`, so `providers` itself must stay in scope — do not collapse it into +// the `use` below. +use self::providers::{DuckDbProvider, YamlProvider}; use anyhow::{Context, Result, anyhow, bail}; use bm25::{Language, SearchEngine, SearchEngineBuilder}; @@ -121,6 +124,9 @@ pub struct RagInitConfig { pub extractor_model: Option, pub extractor_prompt: Option, pub graph_hops: Option, + /// `None` -> "yaml". No serde attribute: this struct derives only + /// `Debug, Clone, Default` and is built in Rust, never deserialized. + pub driver: Option, } #[derive(Debug, Clone, Default)] @@ -147,7 +153,8 @@ impl Rag { bail!("Cannot build RAG knowledge base '{name}' with no documents"); } println!("⚙ Initializing RAG..."); - let data = Self::resolve_init_data(app, config)?; + let mut data = Self::resolve_init_data(app, config)?; + data.driver = config.driver.clone().unwrap_or_else(|| "yaml".to_string()); let mut rag = Self::create(app, name, save_path, data)?; let loaders = app.document_loaders.clone(); let (spinner, spinner_rx) = Spinner::create(""); @@ -257,12 +264,37 @@ impl Rag { save_path: &Path, doc_paths: &[String], abort_signal: AbortSignal, + prompt_for_driver: bool, ) -> Result { if !*IS_STDOUT_TERMINAL { bail!("Failed to init rag in non-interactive mode"); } println!("⚙ Initializing RAG..."); let (embedding_model, chunk_size, chunk_overlap) = Self::create_config(app)?; + // Only interactive named-RAG creation offers a driver choice. Temp RAGs and + // agent startup pass `false`; an explicit flag is used rather than inferring + // from the name because the agent path passes the literal name "rag", which is + // indistinguishable from a user creating a RAG genuinely named `rag`. + let driver = if prompt_for_driver { + let options = vec![ + "yaml — portable, in-memory HNSW; usable from several Coyote processes at once (default)", + "duckdb — persistent on-disk store; vectors and content survive restarts; HNSW approximate search. Can only be open in ONE Coyote process at a time, and its driver cannot be changed later without recreating the RAG", + ]; + let sel = Select::new("RAG storage driver:", options) + .with_starting_cursor(0) + .prompt()?; + if sel.starts_with("duckdb") { + println!( + "Note: a duckdb RAG can only be open in one Coyote process at a time, \ + and changing its driver later means deleting and recreating the RAG." + ); + "duckdb" + } else { + "yaml" + } + } else { + "yaml" + }; let reranker_model = app.rag_reranker_model.clone(); let top_k = app.rag_top_k; let extractor_model = match app.rag_extractor_model.clone() { @@ -275,7 +307,7 @@ impl Rag { app.rag_graph_hops }; let extractor_prompt = app.rag_extractor_prompt.clone(); - let data = RagData::new( + let mut data = RagData::new( embedding_model.id(), chunk_size, chunk_overlap, @@ -288,6 +320,7 @@ impl Rag { graph_hops: Some(graph_hops), }, ); + data.driver = driver.to_string(); let mut rag = Self::create(app, name, save_path, data)?; let mut paths = doc_paths.to_vec(); if paths.is_empty() { @@ -315,9 +348,50 @@ impl Rag { Self::create(app, name, path, data) } - pub fn create(app: &AppConfig, name: &str, path: &Path, data: RagData) -> Result { - let bm25 = data.build_bm25(); - let provider: Box = Box::new(YamlProvider::from_data(&data)); + /// `mut data` — the duckdb arm rehydrates `data.vectors` from the sidecar. + pub fn create(app: &AppConfig, name: &str, path: &Path, mut data: RagData) -> Result { + // Deliberately does NOT call rebuild_indexes: both callers construct the Rag + // before any documents are added, so rebuilding empty data would be a no-op. + // Actual population happens later via sync_documents. + let (provider, bm25): (Box, _) = match data.driver.as_str() { + "duckdb" => { + let db_path = providers::duckdb_path_from_yaml(path); + let dim = embedding_dim_for_model(&data.embedding_model); + let duck = DuckDbProvider::open(&db_path, dim)?; + // HYDRATE — mandatory, not an optimization. The YAML file for a duckdb + // RAG deliberately omits `vectors`, so `data.vectors` arrives empty from + // disk. Refilling it from the sidecar is what makes the NEXT incremental + // sync non-destructive: rebuild_indexes does CREATE OR REPLACE TABLE and + // writes exactly what data.vectors holds. Skip this and the first + // `.edit rag-docs` after a restart wipes every previously indexed vector. + // + // Guarded on is_empty() so a caller that already has vectors in memory + // is never overwritten by an empty table. + // + // 🔴 `?`, NOT `unwrap_or_default()`. A hydration failure must propagate. + // Degrading to an empty map here loads a RAG that looks healthy, answers + // every query with nothing, and then loses the store permanently on the + // first `.edit rag-docs`. The legitimate "nothing indexed yet" case is + // already Ok(empty) — open() runs CREATE TABLE IF NOT EXISTS — so `?` + // costs a new RAG nothing. + if data.vectors.is_empty() { + data.vectors = duck.read_all_vectors()?; + } + // data.files is always populated for duckdb, so build_bm25() is the only + // path; there is no from-DuckDB fallback. + let bm25 = data.build_bm25(); + (Box::new(duck), bm25) + } + "qdrant" => bail!( + "Qdrant RAGs cannot be constructed via Rag::create(); \ + use Rag::attach() or Rag::load_async() instead" + ), + _ => { + // "yaml" and any unknown driver — in-memory HNSW. + let bm25 = data.build_bm25(); + (Box::new(YamlProvider::from_data(&data)), bm25) + } + }; let node_to_docs = data.knowledge_graph.build_node_to_docs(); let embedding_model = Model::retrieve_model(app, &data.embedding_model, ModelType::Embedding)?; @@ -460,8 +534,18 @@ impl Rag { let path = Path::new(&self.path); ensure_parent_exists(path)?; - let content = serde_yaml::to_string(&self.data) - .with_context(|| format!("Failed to serde rag '{}'", self.name))?; + let content = if self.data.driver == "duckdb" { + // Embeddings live in the .duckdb sidecar; keep them out of the YAML file. + // Clone-and-empty rather than mutating self.data — the live map must stay + // complete for the next incremental sync, and save() takes &self, so any + // clear-then-restore would leave the object corrupted on an early return. + let mut on_disk = self.data.clone(); + on_disk.vectors.clear(); + serde_yaml::to_string(&on_disk) + } else { + serde_yaml::to_string(&self.data) + } + .with_context(|| format!("Failed to serde rag '{}'", self.name))?; fs::write(path, content).with_context(|| { format!("Failed to save rag '{}' to '{}'", self.name, path.display()) })?; @@ -841,13 +925,18 @@ impl Rag { } progress(&spinner, "Building store".into()); + // Derived in-memory state is refreshed BEFORE the fallible provider rebuild. + // `self.data` has already been mutated at this point, so returning early on a + // provider error while `bm25`/`node_to_docs` still describe the previous corpus + // would leave this Rag internally inconsistent. Both are pure functions of + // `self.data` and cannot fail, so doing them first is always safe. + self.bm25 = self.data.build_bm25(); + self.node_to_docs = self.data.knowledge_graph.build_node_to_docs(); // `refresh` is true for a full re-index (.rebuild rag / --rebuild-rag / // initial build) and false for an incremental .edit rag-docs change. // Passing it through is what stops a remote provider from wiping its // collection on a one-file add. self.provider.rebuild_indexes(&self.data, refresh).await?; - self.bm25 = self.data.build_bm25(); - self.node_to_docs = self.data.knowledge_graph.build_node_to_docs(); Ok(()) } @@ -1660,10 +1749,39 @@ fn reciprocal_rank_fusion( .collect() } +/// Map an embedding model id to its vector dimension. +/// +/// The DuckDB `FLOAT[N]` column type and its HNSW index are fixed at schema-creation +/// time, so this value must be decided before the first insert. An unrecognized model +/// falls back to 1536; if that is wrong, DuckDB raises a dimension-mismatch error on +/// the first insert rather than silently corrupting the schema, and the recovery is to +/// delete the sidecar and re-ingest from source. +fn embedding_dim_for_model(model_id: &str) -> usize { + match model_id { + m if m.contains("3-large") => 3072, + m if m.contains("3-small") || m.contains("ada-002") => 1536, + m if m.contains("nomic-embed-text") || m.contains("all-minilm") => 768, + m if m.contains("jina-embeddings-v2") => 1024, + _ => 1536, + } +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn embedding_dim_for_model_maps_known_models() { + assert_eq!(embedding_dim_for_model("text-embedding-3-large"), 3072); + assert_eq!(embedding_dim_for_model("text-embedding-3-small"), 1536); + assert_eq!(embedding_dim_for_model("text-embedding-ada-002"), 1536); + assert_eq!(embedding_dim_for_model("nomic-embed-text"), 768); + assert_eq!(embedding_dim_for_model("all-minilm"), 768); + assert_eq!(embedding_dim_for_model("jina-embeddings-v2-base-en"), 1024); + // Unknown models fall back to the OpenAI-compatible default. + assert_eq!(embedding_dim_for_model("some-unknown-model"), 1536); + } + #[test] fn document_id_round_trip() { let id = DocumentId::new(5, 17); diff --git a/src/rag/providers/duckdb.rs b/src/rag/providers/duckdb.rs new file mode 100644 index 0000000..5c877fe --- /dev/null +++ b/src/rag/providers/duckdb.rs @@ -0,0 +1,964 @@ +use crate::rag::provider::RagProvider; +use crate::rag::{DocumentId, RagData}; + +use anyhow::{Context, Result, anyhow, bail}; +use async_trait::async_trait; +use duckdb::Connection; +use indexmap::IndexMap; +use log::warn; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; + +/// Derive the DuckDB sidecar path from a RAG's YAML path: `docs.yaml` -> `docs.duckdb`. +pub(crate) fn duckdb_path_from_yaml(yaml_path: &Path) -> PathBuf { + yaml_path.with_extension("duckdb") +} + +pub struct DuckDbProvider { + path: PathBuf, + conn: Arc>, + /// Embedding dimension; fixed at open time because the `FLOAT[N]` column depends on it. + dim: usize, + /// True once an FTS index has been built on `documents`. Until then + /// `fts_main_documents.match_bm25` does not exist and any keyword query would + /// fail with a DuckDB catalog error. Backs `has_native_keyword_search`. + fts_ready: AtomicBool, +} + +impl DuckDbProvider { + /// Open (or create) the DuckDB file. `dim` is the embedding vector dimension, + /// supplied by the caller who knows the model. + pub fn open(db_path: &Path, dim: usize) -> Result { + let conn = Connection::open(db_path).with_context(|| { + format!( + "Failed to open the DuckDB store at '{}'. If another Coyote process (or \ + another window) has this RAG open, close it and retry — a duckdb RAG can \ + only be open in ONE process at a time. Unlike the yaml driver, its data \ + lives in a single file with an exclusive lock.", + db_path.display() + ) + })?; + // Statement order is load-bearing. `hnsw_enable_experimental_persistence` is + // registered BY the vss extension, so setting it before `LOAD vss` fails with + // "Setting with name ... is not in the catalog, but it exists in the vss + // extension". Omitting it entirely makes CREATE INDEX ... USING HNSW on a + // file-backed database fail with "HNSW index persistence is not yet supported + // by default". LOAD vss -> LOAD fts -> SET -> CREATE INDEX. + conn.execute_batch(&format!( + "LOAD vss; + LOAD fts; + SET hnsw_enable_experimental_persistence = true; + CREATE TABLE IF NOT EXISTS vectors ( + doc_id UBIGINT PRIMARY KEY, + embedding FLOAT[{dim}] + ); + CREATE INDEX IF NOT EXISTS hnsw_idx + ON vectors USING HNSW (embedding) + WITH (metric = 'cosine'); + CREATE TABLE IF NOT EXISTS documents ( + doc_id UBIGINT PRIMARY KEY, + page_content TEXT NOT NULL + );" + )) + .context("Failed to initialize DuckDB schema")?; + // A reopened file may already carry a live FTS index from a previous session, + // in which case keyword search works immediately. + let fts_exists = Self::probe_fts_index(&conn); + Ok(Self { + path: db_path.to_path_buf(), + conn: Arc::new(Mutex::new(conn)), + dim, + fts_ready: AtomicBool::new(fts_exists), + }) + } + + /// Read all `(doc_id, embedding)` pairs so `create()` can hydrate `data.vectors` + /// from disk. This is what makes the next incremental sync non-destructive, and is + /// mandatory rather than an optimization. + /// + /// 🔴 THIS PATTERN IS DUCKDB-ONLY. NEVER write the qdrant equivalent. Qdrant Cosine + /// collections L2-normalize stored vectors on write, so hydrating `data.vectors` + /// from a Qdrant read fills it with NORMALIZED vectors and the next save() destroys + /// the originals permanently. The symmetry with this function is exactly why that + /// bug is easy to introduce. + /// + /// ALL-OR-NOTHING. An empty `vectors` table is `Ok(empty)`; ANY decode failure or + /// non-finite embedding is an `Err`. A partially-hydrated map is worse than no map: + /// `rebuild_indexes` does `CREATE OR REPLACE TABLE` and writes exactly what it is + /// given, so a thinned map is committed as the new truth on the next sync. + pub(crate) fn read_all_vectors(&self) -> Result>> { + let conn = self.lock_conn()?; + let mut stmt = conn.prepare("SELECT doc_id, embedding FROM vectors")?; + // Collect into a Result, NOT a filter_map. `.filter_map(|r| r.ok())` here would + // turn a systematic decode failure (e.g. a schema written by a different duckdb + // version) into a silently short map, indistinguishable from an empty store. + let raw: Vec<(u64, Vec)> = stmt + .query_map([], |row| { + let id: u64 = row.get(0)?; + // The duckdb crate does not implement `FromSql` for `Vec`, so the + // column is read as a `Value` and destructured. A FLOAT[N] column yields + // `Value::Array`, a FLOAT[] column yields `Value::List`; match both so + // the reader survives a file written under either schema. + let embedding: Vec = match row.get::<_, duckdb::types::Value>(1)? { + duckdb::types::Value::Array(vals) | duckdb::types::Value::List(vals) => vals + .into_iter() + .map(|v| match v { + duckdb::types::Value::Float(f) => f, + duckdb::types::Value::Double(d) => d as f32, + _ => f32::NAN, + }) + .collect(), + _ => Vec::new(), + }; + Ok((id, embedding)) + })? + .collect::>>() + .with_context(|| { + format!( + "Failed to decode the `vectors` table in '{}'. The DuckDB sidecar is \ + unreadable; delete it and run `.rebuild rag` to re-ingest from source.", + self.path.display() + ) + })?; + + // Validate OUTSIDE the closure so the error can name the offending row. A + // non-finite embedding is NOT droppable: dropping it thins the map, and the + // thinned map is what the next CREATE OR REPLACE commits. + let mut out = IndexMap::with_capacity(raw.len()); + for (id, embedding) in raw { + if embedding.is_empty() || embedding.iter().any(|f| !f.is_finite()) { + bail!( + "Vector for doc_id {id} in '{}' is empty or contains a non-finite \ + value. Refusing to hydrate a partial vector map — that would erase \ + the remaining vectors on the next sync. Delete the sidecar and run \ + `.rebuild rag` to re-ingest from source.", + self.path.display() + ); + } + out.insert(DocumentId(id as usize), embedding); + } + Ok(out) + } + + /// Does a LIVE FTS index exist on `documents`? Probed at open time so a reopened + /// database reports native keyword search accurately. + /// + /// A schema-existence check alone is NOT sufficient. After + /// `CREATE OR REPLACE TABLE documents`, the `fts_main_documents` schema still + /// exists and `match_bm25` still SUCCEEDS — but returns zero rows for every term. + /// The index is silently dead. The probe therefore asserts that a KNOWN row comes + /// back rather than merely that the call did not error. + fn probe_fts_index(conn: &Connection) -> bool { + // 1. Structural check — cheap, and short-circuits a never-built index. + let schema_exists = conn + .query_row( + "SELECT COUNT(*) FROM duckdb_schemas() WHERE schema_name = 'fts_main_documents'", + [], + |row| row.get::<_, i64>(0), + ) + .map(|n| n > 0) + .unwrap_or(false); + if !schema_exists { + return false; + } + // 2. Liveness check — pull a real token out of a real row and confirm the index + // scores that same row. A live index returns >= 1; a stale one returns 0 + // without erroring. An empty `documents` table cannot be probed, and + // reporting false is correct: there is nothing to keyword-search, and the + // next rebuild_indexes sets the flag directly. + conn.query_row( + "SELECT COUNT(*) FROM documents d + WHERE fts_main_documents.match_bm25( + d.doc_id, + (SELECT string_split(trim(page_content), ' ')[1] + FROM documents WHERE length(trim(page_content)) > 0 LIMIT 1) + ) IS NOT NULL", + [], + |row| row.get::<_, i64>(0), + ) + .map(|n| n > 0) + .unwrap_or(false) + } + + /// Lock the shared connection, converting mutex poisoning into an `anyhow` error. + /// + /// Never `.lock().unwrap()` here: a panic anywhere inside a locked scope poisons the + /// mutex permanently, and an unwrap would then turn every subsequent RAG query into + /// a panic for the remaining life of the process. + fn lock_conn(&self) -> Result> { + self.conn + .lock() + .map_err(|e| anyhow!("DuckDB connection mutex was poisoned: {e}")) + } +} + +#[async_trait] +impl RagProvider for DuckDbProvider { + // NOTE: the MutexGuard is held across the body of these `async fn`s. That is sound + // ONLY because no `.await` appears inside a locked scope. Adding one would make the + // generated future non-`Send` and break the `RagProvider: Send + Sync` bound. + async fn vector_search( + &self, + embedding: &[f32], + top_k: usize, + min_score: f32, + ) -> Result> { + // Validate before building the SQL literal — a NaN would produce malformed SQL. + if embedding.iter().any(|f| !f.is_finite()) { + bail!("Query embedding contains a non-finite value (NaN or infinity)"); + } + let vals: String = embedding + .iter() + .map(|f| f.to_string()) + .collect::>() + .join(", "); + let dim = self.dim; + let conn = self.lock_conn()?; + // array_cosine_distance requires a FLOAT[N] ARRAY, not the LIST type FLOAT[]. + // ORDER BY distance ASC is required for the planner to use hnsw_idx; the + // similarity form (DESC) does NOT trigger the ANN index. Distance is converted + // back to a similarity score on return. + let sql = format!( + "SELECT doc_id, \ + array_cosine_distance(embedding, [{vals}]::FLOAT[{dim}]) AS distance \ + FROM vectors ORDER BY distance ASC LIMIT {top_k}" + ); + let mut stmt = conn.prepare(&sql)?; + let results = stmt + .query_map([], |row| { + let id: u64 = row.get(0)?; + // `array_cosine_distance` on a FLOAT[N] column returns FLOAT (f32), NOT + // DOUBLE. Reading it as f64 raises InvalidColumnType INSIDE the closure, + // which a bare `.filter_map(|r| r.ok())` would silently discard — + // yielding ZERO results with no error and no log line. + let distance: f32 = match row.get::<_, duckdb::types::Value>(1)? { + duckdb::types::Value::Float(f) => f, + duckdb::types::Value::Double(d) => d as f32, + other => { + warn!("unexpected distance type from DuckDB: {other:?}"); + return Err(duckdb::Error::InvalidQuery); + } + }; + Ok((DocumentId(id as usize), 1.0_f32 - distance)) + })? + // Log-and-drop rather than a bare `.ok()`: a systematic decode failure here + // is otherwise indistinguishable from "no matches". + .filter_map(|r| match r { + Ok(v) => Some(v), + Err(e) => { + warn!("vector_search row decode failed: {e}"); + None + } + }) + .filter(|(_, score)| *score > min_score) + .collect(); + Ok(results) + } + + async fn fetch_content(&self, ids: &[DocumentId]) -> Result> { + if ids.is_empty() { + return Ok(vec![]); + } + let conn = self.lock_conn()?; + let placeholders = ids.iter().map(|_| "?").collect::>().join(", "); + let sql = + format!("SELECT doc_id, page_content FROM documents WHERE doc_id IN ({placeholders})"); + let params: Vec = ids + .iter() + .map(|id| duckdb::types::Value::UBigInt(id.0 as u64)) + .collect(); + let mut stmt = conn.prepare(&sql)?; + let mut rows: Vec<(DocumentId, String)> = stmt + .query_map(duckdb::params_from_iter(params.iter()), |row| { + let id: u64 = row.get(0)?; + let text: String = row.get(1)?; + Ok((DocumentId(id as usize), text)) + })? + .filter_map(|r| match r { + Ok(v) => Some(v), + Err(e) => { + warn!("fetch_content row decode failed: {e}"); + None + } + }) + .collect(); + // Ordering contract: `WHERE doc_id IN (...)` returns rows in storage order, NOT + // in the order of `ids`. Since `ids` is the RRF-ranked list, returning storage + // order would silently discard the ranking. Re-sort by input position. + let position: std::collections::HashMap = + ids.iter().enumerate().map(|(i, id)| (*id, i)).collect(); + rows.sort_by_key(|(id, _)| position.get(id).copied().unwrap_or(usize::MAX)); + Ok(rows) + } + + async fn rebuild_indexes(&mut self, data: &RagData, _full_rebuild: bool) -> Result<()> { + // Local on-disk state — a wholesale table reset plus re-INSERT is always + // correct, so the incremental/full distinction is ignored. + // + // 🔴 ANTI-WIPE GUARD. The CREATE OR REPLACE TABLE below writes exactly what + // `data.vectors` holds. An empty map on a RAG that HAS indexed files, against a + // store that ALREADY holds vectors, is always a bug — a failed/skipped + // hydration, or a caller that emptied the live map. Refuse rather than commit + // the loss. Every legitimate empty-vector rebuild also has an empty `files` + // (a fresh RAG; the zero-document tests), so this cannot fire on a correct call. + // The `existing > 0` conjunct is load-bearing, not belt-and-braces: a populated + // fixture rebuilt against a fresh temp database has count 0 and must proceed. + if data.vectors.is_empty() && !data.files.is_empty() { + let existing: i64 = { + // Scoped: the guard MUST be dropped before `lock_conn()` is taken again + // below. `Mutex` is not reentrant — holding both self-deadlocks at + // runtime, with no compile error. + let conn = self.lock_conn()?; + conn.query_row("SELECT count(*) FROM vectors", [], |r| r.get(0)) + .context("Failed to count existing vectors before rebuild")? + }; + if existing > 0 { + bail!( + "Refusing to rebuild the DuckDB store at '{}': the in-memory vector \ + map is empty, but this RAG has {} indexed file(s) and the store \ + already holds {existing} vector(s). Rebuilding would erase them. \ + Vector hydration failed, or `data.vectors` was cleared on a live \ + Rag. Nothing was written; the store is intact.", + self.path.display(), + data.files.len() + ); + } + } + // Validate BEFORE opening the transaction so a bad embedding aborts before any + // write, not mid-write with the guard held. + for (doc_id, embedding) in &data.vectors { + if embedding.iter().any(|f| !f.is_finite()) { + bail!( + "Embedding for document {} contains a non-finite value", + doc_id.0 + ); + } + } + let dim = self.dim; + // `Connection::transaction()` takes `&mut self`, so this binding must be `mut`. + let mut conn = self.lock_conn()?; + let tx = conn + .transaction() + .context("Failed to begin DuckDB transaction")?; + + // Use `CREATE OR REPLACE TABLE`, not `DELETE FROM`. Deleting rows and + // re-inserting the SAME primary keys inside ONE transaction violates DuckDB's PK + // constraint: the index holds deleted keys until commit, so the second rebuild + // fails with a duplicate-key error. `CREATE OR REPLACE` drops the table and its + // indexes atomically, leaving no stale keys. It also drops the HNSW index, which + // is recreated after commit. + tx.execute_batch(&format!( + "CREATE OR REPLACE TABLE vectors ( + doc_id UBIGINT PRIMARY KEY, + embedding FLOAT[{dim}] + ); + CREATE OR REPLACE TABLE documents ( + doc_id UBIGINT PRIMARY KEY, + page_content TEXT NOT NULL + );" + )) + .context("Failed to reset DuckDB tables")?; + + { + // Plain INSERT: tables are empty after CREATE OR REPLACE, so no PK conflict. + // The embedding is bound as TEXT and cast in SQL — the duckdb crate's + // `bind_parameter` has no arm for List/Array, so binding a vector directly + // fails at runtime with "binding List parameters is not yet supported". + let mut vstmt = tx.prepare(&format!( + "INSERT INTO vectors (doc_id, embedding) VALUES (?, CAST(? AS FLOAT[{dim}]))" + ))?; + let mut dstmt = + tx.prepare("INSERT INTO documents (doc_id, page_content) VALUES (?, ?)")?; + + // 🔴 TWO INDEPENDENT LOOPS. `documents` is keyed on `files`, `vectors` on + // `vectors` — they are DIFFERENT key sets and neither is a subset of the + // other. Do not merge these into one loop over `&data.vectors` gated on a + // lookup into `files`: that produces keys(documents) ⊆ keys(vectors), and + // every id in `files \ vectors` (which `RagData::add`'s zip truncation + // really does produce) becomes a live, rankable id from BM25 and + // graph_search that resolves to nothing — no error, no log line. + for (id, doc) in data.iter_documents() { + dstmt.execute(duckdb::params![id.0 as u64, doc.page_content.as_str()])?; + } + + for (doc_id, embedding) in &data.vectors { + // Serialize as a DuckDB array literal: "[0.1,0.2,...]". Finiteness was + // validated above, so `to_string()` cannot emit NaN/inf here. + let embedding_text = { + let mut s = String::with_capacity(embedding.len() * 12 + 2); + s.push('['); + for (i, f) in embedding.iter().enumerate() { + if i > 0 { + s.push(','); + } + s.push_str(&f.to_string()); + } + s.push(']'); + s + }; + vstmt.execute(duckdb::params![doc_id.0 as u64, embedding_text.as_str()])?; + } + } // drop prepared statements before commit + + tx.commit().context("Failed to commit DuckDB transaction")?; + + // Recreate the HNSW index, dropped by CREATE OR REPLACE TABLE above. When + // CREATE INDEX USING HNSW fails, the preceding COMMIT still returns Ok, so the + // failure lands here; swallowing it would leave the database populated but + // UNINDEXED, with every vector_search silently falling back to a full scan. + conn.execute_batch( + "CREATE INDEX IF NOT EXISTS hnsw_idx \ + ON vectors USING HNSW (embedding) WITH (metric = 'cosine');", + ) + .context("Failed to recreate HNSW index")?; + + // Rebuild the FTS index (must be outside the transaction). This rebuild is + // MANDATORY on every pass, not an optimization: CREATE OR REPLACE TABLE above + // leaves the old fts_main_documents schema in place but DEAD — match_bm25 keeps + // succeeding while returning zero rows for every term. + // GUARDED on a non-empty table: FTS cannot index an empty table, and + // rebuild_indexes is legitimately called with zero documents (a fresh RAG, and + // three of the tests below). + let doc_count: i64 = conn + .query_row("SELECT count(*) FROM documents", [], |r| r.get(0)) + .context("Failed to count documents before FTS rebuild")?; + + if doc_count > 0 { + conn.execute_batch( + "DROP FUNCTION IF EXISTS fts_main_documents_match_bm25; + PRAGMA create_fts_index('documents', 'doc_id', 'page_content', overwrite=1);", + ) + .context("Failed to rebuild DuckDB FTS index")?; + } + + // Live only if an index was actually built. With zero documents there is no FTS + // index, so `has_native_keyword_search()` must stay false and hybrid_search must + // fall back to local BM25 — which is also empty, and therefore correct. + self.fts_ready.store(doc_count > 0, Ordering::Relaxed); + + Ok(()) + } + + async fn keyword_search(&self, query: &str, top_k: usize) -> Result> { + let conn = self.lock_conn()?; + // match_bm25 returns NULL for non-matching rows; WHERE filters them out. + let mut stmt = conn.prepare( + "SELECT doc_id, fts_main_documents.match_bm25(doc_id, ?) AS score + FROM documents + WHERE score IS NOT NULL + ORDER BY score DESC + LIMIT ?", + )?; + let results = stmt + .query_map(duckdb::params![query, top_k as u64], |row| { + let id: u64 = row.get(0)?; + // Same hazard as vector_search's distance column: guessing the width + // wrong raises InvalidColumnType INSIDE the closure, which a bare + // `.filter_map(|r| r.ok())` silently discards — yielding ZERO keyword + // hits, indistinguishable from "the query matched nothing". + let score: f32 = match row.get::<_, duckdb::types::Value>(1)? { + duckdb::types::Value::Double(d) => d as f32, + duckdb::types::Value::Float(f) => f, + other => { + warn!("unexpected match_bm25 score type from DuckDB: {other:?}"); + return Err(duckdb::Error::InvalidQuery); + } + }; + Ok((DocumentId(id as usize), score)) + })? + .filter_map(|r| match r { + Ok(v) => Some(v), + Err(e) => { + warn!("keyword_search row decode failed: {e}"); + None + } + }) + .collect(); + Ok(results) + } + + fn has_native_keyword_search(&self) -> bool { + // NOT an unconditional `true`. The FTS schema only exists after + // rebuild_indexes has run the pragma (or after reopening a database where a + // previous session did). `create()` deliberately does NOT call rebuild_indexes, + // so a freshly created RAG reaches this point with no FTS index at all. + // Returning true there would route every query into keyword_search, whose `?` + // would propagate a DuckDB catalog error and fail the WHOLE search. + self.fts_ready.load(Ordering::Relaxed) + } + + fn duplicate(&self, _data: &RagData) -> Box { + // Do NOT call DuckDbProvider::open() here. `Connection::open()` instantiates a + // NEW DuckDB *database* handle on the same file; the first handle still holds + // the file lock, so the second open fails with a locking error. Cloning the Arc + // shares the already-open connection, serialized by the Mutex. + // + // This means DuckDbProvider clones are NOT independent snapshots: state lives on + // disk and a rebuild through one handle is immediately visible to the other. + // That is unavoidable for any on-disk store and is handled by the discipline + // documented on `Rag`'s Clone impl — the pre-clone instance must be discarded. + Box::new(DuckDbProvider { + path: self.path.clone(), + conn: Arc::clone(&self.conn), + dim: self.dim, + fts_ready: AtomicBool::new(self.fts_ready.load(Ordering::Relaxed)), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + // Trait methods are only callable with the trait in scope. + use crate::rag::provider::RagProvider; + // `RagFile` / `RagDocument` have private fields; struct-literal construction + // compiles only because this module is a descendant of `crate::rag`. They are + // deliberately not in the non-test `use` block — unused there, and `--deny warnings` + // rejects that. + use crate::rag::{RagDocument, RagFile}; + + /// Unique temp path per test. `tempfile` is not a dev-dependency; this mirrors the + /// house pattern used elsewhere in the tree. + struct TempDb { + path: PathBuf, + } + + impl TempDb { + fn new(tag: &str) -> Self { + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = std::env::temp_dir().join(format!("coyote-duckdb-{tag}-{unique}.duckdb")); + Self { path } + } + } + + impl Drop for TempDb { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.path); + // DuckDB writes a `.wal` sidecar; remove it too or /tmp accumulates + // one per test run. + let _ = std::fs::remove_file(self.path.with_extension("duckdb.wal")); + } + } + + /// ⚠️ NOTE THE EMPTY `files`. This fixture describes a RAG with ZERO documents. + /// Since `rebuild_indexes` populates the `documents` table from + /// `data.iter_documents()` (keyed on `files`), any test built on this helper alone + /// exercises the `vectors` table and NOTHING ELSE — no `documents` rows, no FTS + /// index. That is correct for the rebuild tests below, and is exactly why the + /// FTS-flag test and the `fetch_content` tests use `populated_rag_data()` instead. + /// Do not "simplify" them back onto this helper. + fn minimal_rag_data() -> RagData { + RagData { + embedding_model: "text-embedding-3-small".to_string(), + chunk_size: 1024, + chunk_overlap: 50, + top_k: 5, + driver: "duckdb".to_string(), + attached: false, + ..Default::default() + } + } + + /// Two files, one document each — the minimum fixture that produces `documents` + /// rows. + fn populated_rag_data() -> RagData { + let mut data = minimal_rag_data(); + debug_assert_eq!(DocumentId::new(0, 0), DocumentId(0)); + data.files.insert( + 0, + RagFile { + hash: "h0".to_string(), + path: "/tmp/a.md".to_string(), + documents: vec![RagDocument { + page_content: "alpha keyword".to_string(), + metadata: Default::default(), + }], + }, + ); + data.files.insert( + 1, + RagFile { + hash: "h1".to_string(), + path: "/tmp/b.md".to_string(), + documents: vec![RagDocument { + page_content: "beta keyword".to_string(), + metadata: Default::default(), + }], + }, + ); + data + } + + #[tokio::test] + async fn open_creates_schema() { + let db = TempDb::new("schema"); + let provider = DuckDbProvider::open(&db.path, 3).unwrap(); + let conn = provider.conn.lock().unwrap(); + let v: i64 = conn + .query_row("SELECT COUNT(*) FROM vectors", [], |r| r.get(0)) + .unwrap(); + let d: i64 = conn + .query_row("SELECT COUNT(*) FROM documents", [], |r| r.get(0)) + .unwrap(); + assert_eq!(v, 0); + assert_eq!(d, 0); + } + + #[tokio::test] + async fn vector_search_returns_top_result() { + let db = TempDb::new("vsearch"); + let provider = DuckDbProvider::open(&db.path, 3).unwrap(); + { + let conn = provider.conn.lock().unwrap(); + // The ::FLOAT[3] cast is REQUIRED: a bare [0.1, 0.2, 0.3] literal infers + // DOUBLE[], which does not match the FLOAT[N] ARRAY column type. + conn.execute( + "INSERT INTO vectors (doc_id, embedding) VALUES (0, [0.1, 0.2, 0.3]::FLOAT[3])", + [], + ) + .unwrap(); + } + let results = provider + .vector_search(&[0.1, 0.2, 0.3], 5, 0.0) + .await + .unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0].0.0, 0); + assert!(results[0].1 > 0.99); + } + + #[tokio::test] + async fn fetch_content_returns_stored_text() { + let db = TempDb::new("fetch"); + let provider = DuckDbProvider::open(&db.path, 3).unwrap(); + { + let conn = provider.conn.lock().unwrap(); + conn.execute( + "INSERT INTO documents (doc_id, page_content) VALUES (42, 'hello world')", + [], + ) + .unwrap(); + } + let results = provider.fetch_content(&[DocumentId(42)]).await.unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0].1, "hello world"); + } + + #[tokio::test] + async fn rebuild_indexes_then_vector_search() { + let db = TempDb::new("rebuild"); + let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); + let data = minimal_rag_data(); + provider.rebuild_indexes(&data, true).await.unwrap(); + let results = provider + .vector_search(&[0.1, 0.2, 0.3], 5, 0.0) + .await + .unwrap(); + assert!(results.is_empty()); // no vectors in empty data + } + + /// Rebuilding TWICE must succeed. + /// + /// The first rebuild always passes because the tables start empty. The bug this + /// guards against — `DELETE FROM` plus re-INSERT of the same PKs inside one + /// transaction — only fires on the SECOND rebuild, with "Duplicate key ... violates + /// primary key constraint". A single-rebuild test cannot catch it. + #[tokio::test] + async fn rebuild_indexes_is_idempotent_across_two_passes() { + let db = TempDb::new("idempotent"); + let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); + let mut data = minimal_rag_data(); + data.vectors.insert(DocumentId(0), vec![0.1, 0.2, 0.3]); + data.vectors.insert(DocumentId(1), vec![0.4, 0.5, 0.6]); + + provider.rebuild_indexes(&data, true).await.unwrap(); + // Second pass over the SAME doc_ids — this is the assertion that matters. + provider + .rebuild_indexes(&data, true) + .await + .expect("second rebuild must not violate the primary key constraint"); + + let conn = provider.conn.lock().unwrap(); + let count: i64 = conn + .query_row("SELECT COUNT(*) FROM vectors", [], |r| r.get(0)) + .unwrap(); + assert_eq!(count, 2, "rebuild must replace rows, not duplicate them"); + } + + /// Guards the incremental data-loss bug: `sync_documents` hash-skips unchanged + /// files, so on an incremental pass `data.vectors` holds ONLY the newly embedded + /// chunks. If the live map is ever emptied, the `CREATE OR REPLACE TABLE` in + /// rebuild_indexes writes only those, destroying every previously indexed vector. + /// Hydration via `read_all_vectors()` is what prevents it. + /// + /// This simulates the real restart-then-edit sequence: build, drop the in-memory + /// map, rehydrate from disk, add one vector, rebuild incrementally. + #[tokio::test] + async fn rebuild_indexes_preserves_prior_vectors_on_incremental_pass() { + let db = TempDb::new("incremental"); + let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); + + // Pass 1 — initial full build with two vectors. + let mut data = minimal_rag_data(); + data.vectors.insert(DocumentId(0), vec![0.1, 0.2, 0.3]); + data.vectors.insert(DocumentId(1), vec![0.4, 0.5, 0.6]); + provider.rebuild_indexes(&data, true).await.unwrap(); + + // Simulate a restart: the YAML on disk carries NO vectors, so a fresh RagData + // starts empty. create() hydrates it back from the sidecar. + let mut reloaded = minimal_rag_data(); + assert!( + reloaded.vectors.is_empty(), + "YAML-loaded data starts with no vectors" + ); + reloaded.vectors = provider.read_all_vectors().unwrap(); + assert_eq!( + reloaded.vectors.len(), + 2, + "hydration must restore both vectors" + ); + + // Pass 2 — incremental add of ONE new chunk. + reloaded.vectors.insert(DocumentId(2), vec![0.7, 0.8, 0.9]); + provider.rebuild_indexes(&reloaded, false).await.unwrap(); + + let conn = provider.conn.lock().unwrap(); + let count: i64 = conn + .query_row("SELECT COUNT(*) FROM vectors", [], |r| r.get(0)) + .unwrap(); + // Without hydration this is 1 — the two originals silently vanish. + assert_eq!( + count, 3, + "an incremental pass must not destroy previously indexed vectors" + ); + } + + /// `has_native_keyword_search()` must be false before any FTS index exists, + /// otherwise hybrid_search routes into keyword_search and the `?` turns a DuckDB + /// catalog error into a total query failure. + /// + /// 🔴 THE FIXTURE MUST CARRY DOCUMENTS. `rebuild_indexes` builds the FTS index only + /// when the `documents` table is non-empty, and that table is filled from + /// `data.iter_documents()` (keyed on `files`), never from `vectors`. With `files` + /// empty the pragma is skipped and `fts_ready` stores `false`. + #[tokio::test] + async fn keyword_search_is_not_advertised_before_first_rebuild() { + let db = TempDb::new("ftsflag"); + let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); + assert!( + !provider.has_native_keyword_search(), + "a freshly opened DB has no FTS index yet" + ); + + // 2 files × 1 document → 2 `documents` rows → doc_count > 0 → pragma runs. + let mut data = populated_rag_data(); + data.vectors.insert(DocumentId(0), vec![0.1, 0.2, 0.3]); + provider.rebuild_indexes(&data, true).await.unwrap(); + { + // Pin the precondition explicitly. If this ever reads 0 the assertion below + // is vacuous and the FTS hazard is unguarded again. + let conn = provider.conn.lock().unwrap(); + let docs: i64 = conn + .query_row("SELECT COUNT(*) FROM documents", [], |r| r.get(0)) + .unwrap(); + assert_eq!(docs, 2, "the fixture must produce documents rows"); + } + assert!( + provider.has_native_keyword_search(), + "rebuild_indexes creates the FTS index and must flip the flag" + ); + } + + /// The `documents` table is keyed on `files`, NOT on `vectors`. + /// + /// `fetch_content` is the sink for BOTH the BM25 and graph_search paths, and both + /// enumerate `files` via `iter_documents()`. If `rebuild_indexes` only inserts a + /// documents row for ids that also appear in `data.vectors`, every id in + /// `files \ vectors` resolves to nothing: healthy BM25 scores, healthy graph hits, + /// zero results, no error. + /// + /// The incremental test CANNOT catch this — its fixture has an empty `files`, so the + /// documents table is empty in every assertion either way. + #[tokio::test] + async fn fetch_content_resolves_documents_without_vectors() { + let db = TempDb::new("docsnovec"); + let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); + + // Two files, one document each — but a vector for ONLY THE FIRST. This is the + // `RagData::add` zip-truncation shape. + let mut data = populated_rag_data(); + let id_a = DocumentId::new(0, 0); + let id_b = DocumentId::new(1, 0); + data.vectors.insert(id_a, vec![0.1, 0.2, 0.3]); + assert!( + !data.vectors.contains_key(&id_b), + "precondition: b has no vector" + ); + + provider.rebuild_indexes(&data, true).await.unwrap(); + + let got = provider.fetch_content(&[id_a, id_b]).await.unwrap(); + assert_eq!( + got.len(), + 2, + "fetch_content must resolve BOTH documents; the one without a vector is \ + still reachable by keyword and graph search" + ); + assert_eq!(got[0].1, "alpha keyword"); + assert_eq!(got[1].1, "beta keyword"); + } + + /// `fetch_content` must return rows in INPUT order, not storage order. + /// + /// `YamlProvider` satisfies this contract structurally — it maps over `ids` — so the + /// existing yaml test proves nothing about DuckDB. DuckDB's `WHERE doc_id IN (...)` + /// returns storage order and relies on an explicit positional re-sort, which is what + /// can actually regress. `ids` is the RRF-ranked list, so losing the order silently + /// discards the ranking while still returning the right documents. + #[tokio::test] + async fn duckdb_fetch_content_preserves_input_order() { + let db = TempDb::new("order"); + let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); + let data = populated_rag_data(); + provider.rebuild_indexes(&data, true).await.unwrap(); + + let id_a = DocumentId::new(0, 0); // inserted first → storage order 0 + let id_b = DocumentId::new(1, 0); // inserted second → storage order 1 + + // REVERSED relative to storage order. Without the positional re-sort this + // returns ["alpha keyword", "beta keyword"] and the assertion fails. + let got = provider.fetch_content(&[id_b, id_a]).await.unwrap(); + assert_eq!(got.len(), 2); + assert_eq!(got[0].0, id_b, "input order must win over storage order"); + assert_eq!(got[0].1, "beta keyword"); + assert_eq!(got[1].0, id_a); + assert_eq!(got[1].1, "alpha keyword"); + } + + /// The anti-wipe guard. + /// + /// `rebuild_indexes` does `CREATE OR REPLACE TABLE` and writes exactly what + /// `data.vectors` holds. An empty map on a RAG that HAS indexed files, against a + /// store that already holds vectors, is always a bug (failed hydration, or a caller + /// that emptied the live map) and must be refused rather than committed. The loss + /// would otherwise be permanent: nothing re-embeds it back. + #[tokio::test] + async fn rebuild_indexes_refuses_to_wipe_when_vectors_are_empty_but_files_are_not() { + let db = TempDb::new("nowipe"); + let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); + + // A healthy store: two files with documents, two vectors. + let mut data = populated_rag_data(); + data.vectors + .insert(DocumentId::new(0, 0), vec![0.1, 0.2, 0.3]); + data.vectors + .insert(DocumentId::new(1, 0), vec![0.4, 0.5, 0.6]); + provider.rebuild_indexes(&data, true).await.unwrap(); + + // Now the failure state: vectors lost in memory, files intact. + let broken = populated_rag_data(); // same files, NO vectors + assert!( + broken.vectors.is_empty() && !broken.files.is_empty(), + "precondition" + ); + + let err = provider.rebuild_indexes(&broken, true).await.unwrap_err(); + assert!( + err.to_string().contains("Refusing to rebuild"), + "got: {err}" + ); + + // The store must be untouched — this is the whole point. + let conn = provider.conn.lock().unwrap(); + let count: i64 = conn + .query_row("SELECT COUNT(*) FROM vectors", [], |r| r.get(0)) + .unwrap(); + assert_eq!( + count, 2, + "the guard must abort BEFORE the CREATE OR REPLACE" + ); + } + + /// The guard must NOT fire on a legitimately empty RAG — otherwise a fresh + /// `Rag::init()` cannot complete. Empty `vectors` AND empty `files` is fine. + #[tokio::test] + async fn rebuild_indexes_allows_empty_vectors_when_files_are_also_empty() { + let db = TempDb::new("emptyok"); + let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); + let data = minimal_rag_data(); // no files, no vectors + provider + .rebuild_indexes(&data, true) + .await + .expect("a fresh RAG with nothing indexed must rebuild cleanly"); + } + + /// `duplicate()` must clone the Arc, NOT call `open()` again. + /// + /// This asserts SHARED state, which is the actual contract. It deliberately does NOT + /// use `fetch_content(&[])` — that early-returns before ever touching the + /// connection, so it would pass even against a broken double-opening implementation. + #[tokio::test] + async fn duplicate_shares_the_same_connection() { + let db = TempDb::new("dup"); + let provider = DuckDbProvider::open(&db.path, 3).unwrap(); + let dup = provider.duplicate(&minimal_rag_data()); + + // Write through the ORIGINAL... + { + let conn = provider.conn.lock().unwrap(); + conn.execute( + "INSERT INTO documents (doc_id, page_content) VALUES (7, 'shared row')", + [], + ) + .unwrap(); + } + // ...and read it back through the DUPLICATE. Only possible if they share state. + let via_dup = dup.fetch_content(&[DocumentId(7)]).await.unwrap(); + assert_eq!( + via_dup.len(), + 1, + "duplicate() must see writes made via the original" + ); + assert_eq!(via_dup[0].1, "shared row"); + } + + /// A decode failure must abort hydration rather than yield a thinned map: a short + /// map is what the next CREATE OR REPLACE commits as the new truth. + #[tokio::test] + async fn read_all_vectors_rejects_non_finite_embeddings() { + let db = TempDb::new("nonfinite"); + let provider = DuckDbProvider::open(&db.path, 3).unwrap(); + { + let conn = provider.conn.lock().unwrap(); + conn.execute( + "INSERT INTO vectors (doc_id, embedding) VALUES (0, [0.1, 0.2, 0.3]::FLOAT[3])", + [], + ) + .unwrap(); + conn.execute( + // Each element is cast individually: a bare ['nan', 0.2, 0.3] literal + // mixes text and numerics, so DuckDB infers DECIMAL and the conversion + // fails before the row is ever stored. + "INSERT INTO vectors (doc_id, embedding) VALUES \ + (1, [CAST('nan' AS FLOAT), CAST(0.2 AS FLOAT), CAST(0.3 AS FLOAT)]::FLOAT[3])", + [], + ) + .unwrap(); + } + let err = provider.read_all_vectors().unwrap_err(); + assert!( + err.to_string().contains("non-finite"), + "hydration must abort rather than silently drop the row; got: {err}" + ); + } + + #[test] + fn duckdb_path_from_yaml_swaps_extension() { + let p = duckdb_path_from_yaml(Path::new("/tmp/rags/docs.yaml")); + assert_eq!(p, PathBuf::from("/tmp/rags/docs.duckdb")); + } +} diff --git a/src/rag/providers/mod.rs b/src/rag/providers/mod.rs index 509f02c..19d9c9a 100644 --- a/src/rag/providers/mod.rs +++ b/src/rag/providers/mod.rs @@ -4,3 +4,10 @@ mod yaml; // is ambiguous (E0659) — `use` paths resolve against both this module's items // and the extern prelude, and `use` declarations may not shadow. pub use self::yaml::YamlProvider; + +mod duckdb; +pub use self::duckdb::DuckDbProvider; +// `create()` in rag/mod.rs derives the sidecar path through this. It is `pub(crate)` +// in providers/duckdb.rs, and the re-export must be `pub(crate)` too — a `pub use` of +// a `pub(crate)` item is E0364/E0365. +pub(crate) use self::duckdb::duckdb_path_from_yaml; From 7a732436aa5b59c05f241467fec06d0693cc3089 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 10 Aug 2026 13:40:22 -0600 Subject: [PATCH 05/36] feat(rag): add attach-only Qdrant provider, attach wizard and sandbox wiring Adds QdrantProvider as a read-only driver for pre-existing remote Qdrant collections, an interactive '.rag attach' wizard, and the sandbox credential and domain-whitelisting wiring that lets an attached RAG work inside a sandbox. Attach-only by design: rebuild_indexes bails for both the attached and the unattached case rather than silently succeeding. Coyote never writes to Qdrant in this change. Vectors are never hydrated back from Qdrant. Cosine collections L2-normalize stored vectors on write, so reading them back returns unit-length copies of the originals; the YAML vector copy is authoritative and the serialization guard stays scoped to the duckdb driver alone. Collections keyed by string or UUID point IDs are rejected at attach time. The read path parses point ids as u64 inside a filter_map, so such a collection would otherwise yield zero results with no error. The three response-shape parsers are pure functions over an already-parsed JSON body, unit-tested against captured fixtures, with the async wrappers delegating to them rather than duplicating the logic. '.rag' now splits its first argument, so '.rag attach ' no longer tries to load a RAG literally named 'attach '. --- src/config/agent.rs | 12 +- src/config/request_context.rs | 30 +- src/rag/mod.rs | 598 +++++++++++++++++++++++++++++++++- src/rag/providers/mod.rs | 3 + src/rag/providers/qdrant.rs | 491 ++++++++++++++++++++++++++++ src/repl/mod.rs | 26 +- src/sandbox/mixins.rs | 82 +++++ src/sandbox/mod.rs | 61 ++++ 8 files changed, 1295 insertions(+), 8 deletions(-) create mode 100644 src/rag/providers/qdrant.rs diff --git a/src/config/agent.rs b/src/config/agent.rs index 16081ff..a6c051a 100644 --- a/src/config/agent.rs +++ b/src/config/agent.rs @@ -146,11 +146,18 @@ impl Agent { let rag = if rag_path.exists() { let key = RagKey::Agent(name.to_string()); let app_clone = app.clone(); + let vault_clone = app_state.vault.clone(); let rag_path_clone = rag_path.clone(); let rag = app_state .rag_cache .load_with(key, || async move { - Rag::load(&app_clone, DEFAULT_AGENT_NAME, &rag_path_clone) + Rag::load_async( + &app_clone, + &vault_clone, + DEFAULT_AGENT_NAME, + &rag_path_clone, + ) + .await }) .await?; Some(rag) @@ -972,12 +979,13 @@ async fn init_graph_rags( }; let rag = if rag_path.exists() { let app_clone = app.clone(); + let vault_clone = app_state.vault.clone(); let path_clone = rag_path.clone(); let name_clone = node_id.clone(); app_state .rag_cache .load_with(key, || async move { - Rag::load(&app_clone, &name_clone, &path_clone) + Rag::load_async(&app_clone, &vault_clone, &name_clone, &path_clone).await }) .await? } else { diff --git a/src/config/request_context.rs b/src/config/request_context.rs index e2f3883..5260518 100644 --- a/src/config/request_context.rs +++ b/src/config/request_context.rs @@ -4122,6 +4122,9 @@ impl RequestContext { } let app = self.app.config.clone(); + // Hoisted: `rag_cache` below borrows `self`, so the loader closure cannot + // reach through `self` for the vault. `GlobalVault` is an Arc, so this is cheap. + let vault = self.app.vault.clone(); let rag_cache = self.rag_cache(); let working_mode = self.working_mode; @@ -4158,6 +4161,7 @@ impl RequestContext { let loaded = rag_cache .load_with(key.clone(), || { let app = app.clone(); + let vault = vault.clone(); let rag_path = rag_path.clone(); let abort_signal = abort_signal.clone(); async move { @@ -4168,7 +4172,7 @@ impl RequestContext { Rag::init(&app, name, &rag_path, &[], abort_signal.clone(), true) .await } else { - Rag::load(&app, name, &rag_path) + Rag::load_async(&app, &vault, name, &rag_path).await } } }) @@ -4181,6 +4185,30 @@ impl RequestContext { Ok(()) } + pub async fn attach_rag(&mut self, name: &str) -> Result<()> { + let rag_path = self.rag_file(name); + if rag_path.exists() { + bail!( + "RAG '{name}' already exists at '{}'. \ + Use a different name, or delete the existing file first.", + rag_path.display() + ); + } + let app = self.app.config.as_ref(); + let vault = self.app.vault.clone(); + let rag = Rag::attach(app, &vault, name, &rag_path).await?; + let rag = Arc::new(rag); + // Populate the cache so a later `.rag ` reuses this instance rather + // than re-running the network preflight. Attach is always a global RAG. + let key = RagKey::Named(name.to_string()); + self.rag_cache().insert(key.clone(), &rag); + self.rag = Some(rag); + // Carried so invalidation in rebuild_rag()/edit_rag_docs() can find this + // entry; without it a stale Arc would linger in the cache all session. + self.rag_key = Some(key); + Ok(()) + } + pub async fn edit_rag_docs(&mut self, abort_signal: AbortSignal) -> Result<()> { let mut rag = match self.rag.clone() { Some(v) => v.as_ref().clone(), diff --git a/src/rag/mod.rs b/src/rag/mod.rs index 6c8d41a..abe5319 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -15,7 +15,8 @@ use self::provider::RagProvider; // `providers::duckdb_path_from_yaml(path)` is called through the module path in // `create()`, so `providers` itself must stay in scope — do not collapse it into // the `use` below. -use self::providers::{DuckDbProvider, YamlProvider}; +use self::providers::{DuckDbProvider, QdrantProvider, YamlProvider}; +use crate::vault::{Vault, interpolate_secrets}; use anyhow::{Context, Result, anyhow, bail}; use bm25::{Language, SearchEngine, SearchEngineBuilder}; @@ -348,6 +349,232 @@ impl Rag { Self::create(app, name, path, data) } + /// Loads a RAG from a YAML file. External drivers need an async constructor + /// because building their provider performs a network preflight. + pub async fn load_async( + app: &AppConfig, + vault: &Vault, + name: &str, + path: &Path, + ) -> Result { + let err = || format!("Failed to load rag '{name}' at '{}'", path.display()); + let raw_content = fs::read_to_string(path).with_context(err)?; + + // Parsed WITHOUT secret interpolation, so `driver_config` keeps its + // `{{...}}` placeholders. Interpolating here would bake the resolved API + // key into `self.data`, which `save()` then writes back to disk in + // plaintext. + let data: RagData = serde_yaml::from_str(&raw_content).with_context(err)?; + + // Validated before the match so the rule applies to every driver, including + // the sync fallthrough below. + data.validate().with_context(err)?; + + match data.driver.as_str() { + "qdrant" => { + let host = data + .driver_config + .get("host") + .context("qdrant driver requires 'host' in driver_config")? + .clone(); + let collection = data + .driver_config + .get("collection") + .context("qdrant driver requires 'collection' in driver_config")? + .clone(); + + // Resolved out of band and kept in a local; it never enters `data`. + let api_key: Option = match data.driver_config.get("api_key") { + Some(placeholder) => { + let (resolved, _) = + interpolate_secrets(placeholder, vault).with_context(|| { + format!("Failed to resolve api_key secret for RAG '{name}'") + })?; + Some(resolved) + } + None => None, + }; + + let provider = QdrantProvider::new(&host, &collection, api_key.as_deref()).await?; + let embedding_model = + Model::retrieve_model(app, &data.embedding_model, ModelType::Embedding)?; + Ok(Rag { + app_config: Arc::new(app.clone()), + name: name.to_string(), + path: path.display().to_string(), + embedding_model, + bm25: data.build_bm25(), + provider: Box::new(provider), + node_to_docs: data.knowledge_graph.build_node_to_docs(), + data, + last_sources: RwLock::new(None), + }) + } + // yaml/duckdb take the sync path. It re-reads and re-parses the file; + // that cost is accepted to keep every existing caller untouched. + _ => Self::load(app, name, path), + } + } + + /// Connects to a pre-existing external collection. Coyote is a query-only + /// client here: it never indexes documents into it. + pub async fn attach( + app: &AppConfig, + vault: &Vault, + name: &str, + save_path: &Path, + ) -> Result { + if !*IS_STDOUT_TERMINAL { + bail!("Cannot run attach wizard in non-interactive mode"); + } + println!("⚙ Attaching to external RAG..."); + + let driver = Select::new("Select driver:", vec!["qdrant"]).prompt()?; + + let host = Text::new("Host (e.g. qdrant.company.com:6333):") + .with_validator(required!("This field is required")) + .with_validator(|input: &str| { + // Bracketed IPv6 literals would produce a malformed sandbox + // allowedDomains entry, so refuse them at the prompt. + Ok(if input.contains('[') || input.contains(']') { + Validation::Invalid( + "Bracketed IPv6 literals are not supported; use a hostname.".into(), + ) + } else { + Validation::Valid + }) + }) + .prompt()?; + + let api_key_entry: Option<(String, String)> = { + let needs_key = Confirm::new("Does this instance require an API key?") + .with_default(true) + .prompt()?; + if needs_key { + let secret_name = Text::new("Vault secret name for API key:") + .with_default("QDRANT_API_KEY") + .with_validator(required!("This field is required")) + .prompt()?; + let resolved = vault.get_secret(&secret_name, false).with_context(|| { + format!( + "Secret '{secret_name}' not found in vault. \ + Run `coyote --add-secret {secret_name}` first." + ) + })?; + Some((secret_name, resolved)) + } else { + None + } + }; + + println!("⚙ Connecting to {host}..."); + let api_key = api_key_entry.as_ref().map(|(_, v)| v.as_str()); + let collections = QdrantProvider::list_collections(&host, api_key) + .await + .with_context(|| format!("Failed to connect to {host}. Check host and API key."))?; + + if collections.is_empty() { + bail!("No collections found in this Qdrant instance"); + } + println!( + "✓ Connected. {} collection(s) available.", + collections.len() + ); + + let collection = Select::new("Select collection:", collections).prompt()?; + + // Point IDs are read with `as_u64()`, which yields None for a JSON string. + // A UUID-keyed collection would therefore return zero hits with no error, + // so refuse it here instead of attaching something silently broken. + if let Some(raw_id) = QdrantProvider::sample_point_id(&host, &collection, api_key).await? + && raw_id.starts_with('"') + { + bail!( + "Collection '{collection}' uses string (UUID) point IDs. \ + Coyote requires integer point IDs. Rebuild the collection with integer IDs \ + (e.g. LangChain: pass ids=list(range(len(docs))) to add_documents())." + ); + } + println!("ℹ This collection must store document text in a 'page_content' payload field."); + + let dim = QdrantProvider::get_vector_dimension(&host, &collection, api_key) + .await + .unwrap_or(0); + // Queries send a single unnamed vector, which a named/multi-vector + // collection rejects with HTTP 400 every time. Checked separately from + // `dim` because `dim == 0` also means "the request failed". + if QdrantProvider::is_multi_vector(&host, &collection, api_key).await? { + bail!( + "Collection '{collection}' uses named (multi-vector) configuration. \ + Coyote queries with a single unnamed vector and would fail with HTTP 400 \ + on every request. Attach a single-vector collection instead." + ); + } + if dim > 0 { + let candidates = embedding_model_candidates_for_dimension(dim); + if !candidates.is_empty() { + println!( + "Collection uses {dim}-dim vectors. Likely models: {}", + candidates.join(", ") + ); + } + } + println!( + "⚠️ If the embedding model doesn't match what built this collection, \ + queries will return bad results." + ); + let models = list_models(app, ModelType::Embedding); + if models.is_empty() { + bail!("No available embedding model"); + } + let embedding_model_id = select_embedding_model(&models)?; + + let mut driver_config = IndexMap::new(); + driver_config.insert("host".to_string(), host.clone()); + driver_config.insert("collection".to_string(), collection.clone()); + if let Some((secret_name, _)) = &api_key_entry { + driver_config.insert("api_key".to_string(), format!("{{{{{secret_name}}}}}")); + } + + let data = RagData { + driver: driver.to_string(), + attached: true, + driver_config, + embedding_model: embedding_model_id, + chunk_size: app.rag_chunk_size.unwrap_or(1024), + chunk_overlap: app.rag_chunk_overlap.unwrap_or(50), + // A top_k of 0 makes every query return nothing. + top_k: app.rag_top_k.max(1), + ..RagData::default() + }; + data.validate()?; + + let embedding_model = + Model::retrieve_model(app, &data.embedding_model, ModelType::Embedding)?; + let provider = QdrantProvider::new(&host, &collection, api_key).await?; + let rag = Rag { + app_config: Arc::new(app.clone()), + name: name.to_string(), + path: save_path.display().to_string(), + embedding_model, + // Both empty: an attached RAG holds no local text and no local graph. + bm25: data.build_bm25(), + node_to_docs: IndexMap::new(), + provider: Box::new(provider), + data, + last_sources: RwLock::new(None), + }; + + rag.save()?; + println!("✓ Attached '{name}' → collection '{collection}' on {host}."); + + let env_var = rag_env_var_name(name); + let (header_name, value_format) = driver_auth_header(driver); + generate_rag_sbx_mixin(save_path, &host, name, &env_var, header_name, value_format)?; + + Ok(rag) + } + /// `mut data` — the duckdb arm rehydrates `data.vectors` from the sidecar. pub fn create(app: &AppConfig, name: &str, path: &Path, mut data: RagData) -> Result { // Deliberately does NOT call rebuild_indexes: both callers construct the Rag @@ -491,6 +718,13 @@ impl Rag { } pub fn set_last_sources(&self, ids: &[DocumentId]) { + if self.data.attached { + // `data.files` is empty for an attached RAG; the local index is not the + // source of truth. A static label is honest, an empty list is not. + *self.last_sources.write() = + Some("[attached RAG — source list unavailable]".to_string()); + return; + } let mut sources: IndexMap> = IndexMap::new(); for id in ids { let (file_index, _) = id.split(); @@ -665,6 +899,9 @@ impl Rag { } fn resolve_source(&self, id: &DocumentId) -> String { + if self.data.attached { + return self.data.attached_source_label(); + } let (file_index, _) = id.split(); self.data .files @@ -674,6 +911,9 @@ impl Rag { } fn format_sources(&self, ids: &[DocumentId]) -> String { + if self.data.attached { + return format!("- {}", self.data.attached_source_label()); + } let mut seen = IndexSet::new(); for id in ids { let (file_index, _) = id.split(); @@ -1231,6 +1471,11 @@ pub struct RagData { pub driver: String, #[serde(default)] pub attached: bool, + /// Driver-specific connection parameters (qdrant: `host`, `collection`, `api_key`). + /// Secret-bearing values are stored as `{{SECRET_NAME}}` placeholders and resolved + /// out of band at load time, so a resolved credential never reaches disk. + #[serde(default, skip_serializing_if = "IndexMap::is_empty")] + pub driver_config: IndexMap, pub embedding_model: String, #[serde(default)] @@ -1268,6 +1513,7 @@ impl Debug for RagData { f.debug_struct("RagData") .field("driver", &self.driver) .field("attached", &self.attached) + .field("driver_config", &self.driver_config) .field("embedding_model", &self.embedding_model) .field("chunk_size", &self.chunk_size) .field("chunk_overlap", &self.chunk_overlap) @@ -1297,6 +1543,7 @@ impl RagData { Self { driver: "yaml".to_string(), attached: false, + driver_config: Default::default(), embedding_model, chunk_size, chunk_overlap, @@ -1318,6 +1565,15 @@ impl RagData { "yaml".to_string() } + /// Citation label for an attached RAG. Its documents live in a remote + /// collection, so there is no local file path to cite. + fn attached_source_label(&self) -> String { + match self.driver_config.get("collection") { + Some(collection) => format!("[external collection: {collection}]"), + None => "[external collection]".to_string(), + } + } + pub fn validate(&self) -> Result<()> { if self.top_k == 0 { bail!( @@ -1497,6 +1753,131 @@ impl DocumentId { } } +/// Writes the per-RAG sandbox sidecar that whitelists the external host and tells +/// the sbx proxy which header to rewrite with the stored credential. +/// +/// Two details are load-bearing and fail silently if guessed: +/// 1. The schema envelope is mandatory. `wrap_mixin_as_kit` copies this file +/// byte-for-byte to `spec.yaml` inside a kit dir handed to `sbx create --kit`, +/// with no `kind` rewrite — so an envelope-less file can break the launch +/// itself, not merely this RAG's traffic. +/// 2. `allowedDomains` entries carry a port; `serviceDomains` keys are bare +/// hostnames. The asymmetry is deliberate. +fn generate_rag_sbx_mixin( + rag_yaml_path: &Path, + host: &str, + service_name: &str, + env_var: &str, + header_name: &str, + value_format: &str, +) -> Result<()> { + let (bare_host, allowed_domain) = sbx_domain_forms(host); + let mixin_path = rag_yaml_path.with_extension("sbx-mixin.yaml"); + let content = format!( + r#"schemaVersion: "1" +kind: mixin +name: rag-{service_name} +description: > + Auto-generated by the Coyote attach wizard for RAG '{service_name}'. Allows + outbound traffic to its external vector store and tells the sbx proxy which + header to rewrite with the stored credential. Do not edit manually. + +network: + allowedDomains: + - "{allowed_domain}" + serviceDomains: + {bare_host}: {service_name} + serviceAuth: + {service_name}: + headerName: {header_name} + valueFormat: "{value_format}" + +credentials: + sources: + {service_name}: + env: + - {env_var} + +environment: + proxyManaged: + - {env_var} +"# + ); + fs::write(&mixin_path, &content).with_context(|| { + format!( + "Failed to write sandbox mixin to '{}'", + mixin_path.display() + ) + })?; + println!("✓ Sandbox mixin: '{}'.", mixin_path.display()); + Ok(()) +} + +/// Splits a user-supplied host into the two forms sbx needs: +/// `(bare_host, allowed_domain)`. +/// +/// `bare_host` is the `serviceDomains` KEY — hostname only, no scheme, no port. +/// `allowed_domain` is an `allowedDomains` ENTRY — always `host:port`. +/// +/// The rule: emit the host verbatim when it already carries a numeric port, +/// otherwise append the default for the scheme. Stripping the port fails +/// silently — the sandbox just denies the connection, with no compile or test +/// signal. Defaults match `normalize_base_url`, which assumes `http://` when no +/// scheme is given: plain host → 6333, explicit `https://` → 443. +fn sbx_domain_forms(host: &str) -> (String, String) { + let is_https = host.starts_with("https://"); + let hostport = host + .strip_prefix("https://") + .or_else(|| host.strip_prefix("http://")) + .unwrap_or(host) + .trim_end_matches('/') + // A trailing ':' with no digits is a typo, not a port. Trim it so the + // default-port arm cannot emit "host::6333". + .trim_end_matches(':'); + match hostport.rsplit_once(':') { + Some((h, p)) if !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit()) => { + (h.to_string(), hostport.to_string()) + } + _ => { + let port = if is_https { 443 } else { 6333 }; + (hostport.to_string(), format!("{hostport}:{port}")) + } + } +} + +/// Derives a deterministic env var name from a RAG name: +/// `company-docs` → `COMPANY_DOCS_API_KEY`. +fn rag_env_var_name(rag_name: &str) -> String { + format!( + "{}_API_KEY", + rag_name.to_uppercase().replace(['-', ' '], "_") + ) +} + +/// How a driver authenticates its HTTP requests. Qdrant uses a bare `api-key` +/// header rather than `Authorization: Bearer`. +fn driver_auth_header(driver: &str) -> (&'static str, &'static str) { + match driver { + "qdrant" => ("api-key", "%s"), + _ => ("Authorization", "Bearer %s"), + } +} + +/// Embedding models known to produce a given vector dimension, used to hint the +/// user toward a model compatible with the collection they just picked. +fn embedding_model_candidates_for_dimension(dim: u64) -> Vec<&'static str> { + match dim { + 1536 => vec!["text-embedding-3-small", "text-embedding-ada-002"], + 3072 => vec!["text-embedding-3-large"], + 768 => vec!["nomic-embed-text", "all-minilm-l6-v2"], + 1024 => vec![ + "text-embedding-3-small (matryoshka-1024)", + "jina-embeddings-v2-base", + ], + _ => vec![], + } +} + fn select_embedding_model(models: &[&Model]) -> Result { let max_width = models.iter().map(|v| v.id().len()).max().unwrap_or(0); let models: Vec<_> = models @@ -1770,6 +2151,221 @@ fn embedding_dim_for_model(model_id: &str) -> usize { mod tests { use super::*; + /// Scratch directory for tests that must write a real file. + struct TempDir { + path: std::path::PathBuf, + } + + impl TempDir { + fn new(tag: &str) -> Self { + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = env::temp_dir().join(format!("coyote-rag-{tag}-{unique}")); + fs::create_dir_all(&path).unwrap(); + Self { path } + } + } + + impl Drop for TempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } + } + + #[test] + fn attached_ragdata_serializes_driver_and_attached() { + let mut data = RagData { + driver: "qdrant".to_string(), + attached: true, + embedding_model: "text-embedding-3-small".to_string(), + top_k: 5, + ..Default::default() + }; + data.driver_config + .insert("host".into(), "localhost:6333".into()); + data.driver_config.insert("collection".into(), "c".into()); + data.driver_config + .insert("api_key".into(), "{{QDRANT_API_KEY}}".into()); + data.validate().unwrap(); + let yaml = serde_yaml::to_string(&data).unwrap(); + assert!(yaml.contains("attached: true")); + assert!(yaml.contains("driver: qdrant")); + // The placeholder is what reaches disk — never a resolved secret. + assert!(yaml.contains("{{QDRANT_API_KEY}}")); + } + + /// A qdrant RAG's vectors MUST survive serialization. + /// + /// `save()` omits vectors only for `driver == "duckdb"`. Qdrant must not join + /// that guard: Cosine collections L2-normalize on write, so the YAML copy is + /// the only place the unnormalized originals survive. This fails loudly the + /// day someone "tidies" the guard into `matches!(driver, "duckdb" | "qdrant")`. + #[test] + fn save_round_trips_qdrant_vectors_intact() { + let mut data = RagData { + driver: "qdrant".to_string(), + attached: true, + embedding_model: "text-embedding-3-small".to_string(), + top_k: 5, + ..Default::default() + }; + // Deliberately NOT unit-length: magnitude 5, so any normalization is visible. + data.vectors.insert(DocumentId(0), vec![3.0, 0.0, 0.0, 4.0]); + let yaml = serde_yaml::to_string(&data).unwrap(); + assert!( + yaml.contains("vectors:"), + "qdrant vectors must be serialized, not omitted" + ); + let back: RagData = serde_yaml::from_str(&yaml).unwrap(); + assert_eq!( + back.vectors.get(&DocumentId(0)), + Some(&vec![3.0, 0.0, 0.0, 4.0]), + "magnitudes must survive — Qdrant normalizes, the YAML copy must not" + ); + } + + /// `allowedDomains` entries are host:PORT; `serviceDomains` keys are bare + /// hostnames. Stripping the port breaks sandbox whitelisting silently — no + /// compile error, no runtime error on the host, just a denied connection + /// inside the sandbox. This test is the only signal. + #[test] + fn sbx_domain_forms_keeps_explicit_ports_and_defaults_the_rest() { + assert_eq!( + sbx_domain_forms("rag.example.com:6333"), + ("rag.example.com".into(), "rag.example.com:6333".into()) + ); + // A non-default explicit port must survive. + assert_eq!( + sbx_domain_forms("rag.example.com:7777").1, + "rag.example.com:7777" + ); + // Bare host → Qdrant's REST default, matching normalize_base_url. + assert_eq!( + sbx_domain_forms("rag.example.com"), + ("rag.example.com".into(), "rag.example.com:6333".into()) + ); + // The scheme is stripped; http keeps 6333. + assert_eq!( + sbx_domain_forms("http://localhost:6333").1, + "localhost:6333" + ); + // https with no port → 443 (the Qdrant Cloud shape). + assert_eq!( + sbx_domain_forms("https://xyz.cloud.qdrant.io"), + ( + "xyz.cloud.qdrant.io".into(), + "xyz.cloud.qdrant.io:443".into() + ) + ); + // A dangling colon is a typo, not a port: trimmed, then defaulted. + assert_eq!( + sbx_domain_forms("rag.example.com:").1, + "rag.example.com:6333" + ); + } + + /// The generated mixin must carry the schema envelope: `wrap_mixin_as_kit` + /// copies it verbatim to `spec.yaml` for `sbx create --kit`, with no `kind` + /// rewrite, so an envelope-less file can break the launch itself. + #[test] + fn generated_sbx_mixin_carries_the_schema_envelope() { + let dir = TempDir::new("mixin"); + let yaml_path = dir.path.join("company-docs.yaml"); + generate_rag_sbx_mixin( + &yaml_path, + "rag.example.com", + "company-docs", + "COMPANY_DOCS_API_KEY", + "api-key", + "%s", + ) + .unwrap(); + let text = fs::read_to_string(dir.path.join("company-docs.sbx-mixin.yaml")).unwrap(); + + assert!( + text.starts_with("schemaVersion:"), + "envelope must come first:\n{text}" + ); + assert!( + text.contains("kind: mixin"), + "kind must be `mixin`, not `sandbox`" + ); + assert!(text.contains("name: rag-company-docs")); + assert!(text.contains("description:")); + + // It must parse as YAML at all — a broken format! escape is invisible otherwise. + let parsed: serde_yaml::Value = serde_yaml::from_str(&text).unwrap(); + assert_eq!(parsed["kind"].as_str(), Some("mixin")); + // The list entry carries the port; the map key does not. + assert_eq!( + parsed["network"]["allowedDomains"][0].as_str(), + Some("rag.example.com:6333") + ); + assert_eq!( + parsed["network"]["serviceDomains"]["rag.example.com"].as_str(), + Some("company-docs") + ); + // The proxy rewrites this header with the credential it holds. + assert_eq!( + parsed["network"]["serviceAuth"]["company-docs"]["headerName"].as_str(), + Some("api-key") + ); + assert_eq!( + parsed["credentials"]["sources"]["company-docs"]["env"][0].as_str(), + Some("COMPANY_DOCS_API_KEY") + ); + assert_eq!( + parsed["environment"]["proxyManaged"][0].as_str(), + Some("COMPANY_DOCS_API_KEY") + ); + } + + #[test] + fn rag_env_var_name_uppercases_and_underscores() { + assert_eq!(rag_env_var_name("company-docs"), "COMPANY_DOCS_API_KEY"); + assert_eq!(rag_env_var_name("my rag"), "MY_RAG_API_KEY"); + assert_eq!(rag_env_var_name("docs"), "DOCS_API_KEY"); + } + + #[test] + fn driver_auth_header_uses_a_bare_api_key_for_qdrant() { + // Qdrant's REST API reads `api-key`, NOT `Authorization: Bearer`. + assert_eq!(driver_auth_header("qdrant"), ("api-key", "%s")); + assert_eq!( + driver_auth_header("something-else"), + ("Authorization", "Bearer %s") + ); + } + + /// An attached RAG has no local `files`, so the citation helpers would + /// otherwise emit "unknown" and an empty source list for every result. + #[test] + fn attached_rag_citation_helpers_do_not_fall_back_to_the_empty_file_index() { + let mut data = RagData { + driver: "qdrant".to_string(), + attached: true, + embedding_model: "text-embedding-3-small".to_string(), + top_k: 5, + ..Default::default() + }; + data.driver_config + .insert("collection".into(), "company-kb".into()); + assert!(data.files.is_empty()); + + // The real helper — `resolve_source`/`format_sources` both delegate here. + assert_eq!( + data.attached_source_label(), + "[external collection: company-kb]" + ); + + // Degrades to a generic label rather than "unknown" when the collection + // name is absent. + data.driver_config.shift_remove("collection"); + assert_eq!(data.attached_source_label(), "[external collection]"); + } + #[test] fn embedding_dim_for_model_maps_known_models() { assert_eq!(embedding_dim_for_model("text-embedding-3-large"), 3072); diff --git a/src/rag/providers/mod.rs b/src/rag/providers/mod.rs index 19d9c9a..6fd4472 100644 --- a/src/rag/providers/mod.rs +++ b/src/rag/providers/mod.rs @@ -11,3 +11,6 @@ pub use self::duckdb::DuckDbProvider; // in providers/duckdb.rs, and the re-export must be `pub(crate)` too — a `pub use` of // a `pub(crate)` item is E0364/E0365. pub(crate) use self::duckdb::duckdb_path_from_yaml; + +mod qdrant; +pub use self::qdrant::QdrantProvider; diff --git a/src/rag/providers/qdrant.rs b/src/rag/providers/qdrant.rs new file mode 100644 index 0000000..ffff33a --- /dev/null +++ b/src/rag/providers/qdrant.rs @@ -0,0 +1,491 @@ +use crate::rag::provider::RagProvider; +use crate::rag::{DocumentId, RagData}; + +use anyhow::{Context, Result, bail}; +use async_trait::async_trait; +use std::collections::HashMap; + +/// Render Qdrant's error envelope into a human-readable message. +/// +/// `body` is the raw response text. Two shapes have to be tolerated: +/// * application-level errors carry `{"status": {"error": "..."}, "time": 0.0}`, +/// while successful responses carry a bare string `{"status": "ok", ...}` — so +/// `status` is string-or-object and a struct with `status: String` fails to +/// parse every error body; +/// * routing-level 404s (a wrong HTTP verb) return an EMPTY body with no JSON at +/// all, which without the length check surfaces as "EOF while parsing a value" +/// instead of the actual 404. +fn format_error_body(status: reqwest::StatusCode, body: &str) -> String { + if body.is_empty() { + return format!("HTTP {status} (empty body — check the HTTP verb and path)"); + } + serde_json::from_str::(body) + .ok() + .and_then(|v| v["status"]["error"].as_str().map(str::to_string)) + .unwrap_or_else(|| format!("HTTP {status}: {body}")) +} + +/// Read the vector dimension out of a parsed `GET /collections/{name}` response. +/// +/// Unnamed collections put `size` directly under `vectors`; named ones nest it +/// under the vector's name. Both shapes occur in the wild, so try the flat one +/// first and fall back to the first named entry. +fn vector_dimension_from_collection(body: &serde_json::Value) -> Result { + let params = &body["result"]["config"]["params"]; + params["vectors"]["size"] + .as_u64() + .or_else(|| { + params["vectors"] + .as_object() + .and_then(|m| m.values().next()) + .and_then(|v| v["size"].as_u64()) + }) + .context("Could not determine vector dimension from collection config") +} + +/// True if a parsed `GET /collections/{name}` response describes a NAMED +/// (multi-vector) collection. +/// +/// `vector_search` posts an unnamed vector, which a named-vector collection +/// rejects with HTTP 400 on every query — so attaching one yields a RAG that is +/// silently 100% broken. A named collection holding a SINGLE vector is +/// structurally a map, identical in kind to the multi-named case, and rejects +/// the same way; testing for a numeric `size` directly under `vectors` catches +/// it, whereas counting keys (`len() > 1`) would wrongly accept it. +fn is_multi_vector_config(body: &serde_json::Value) -> bool { + body["result"]["config"]["params"]["vectors"]["size"] + .as_u64() + .is_none() +} + +/// Query-only client for an external Qdrant collection. +/// +/// Attach-only: this provider never writes to the remote collection. Coyote does +/// not own the data, and `rebuild_indexes` refuses rather than pretending to. +pub struct QdrantProvider { + /// `reqwest::Client` is Arc-backed, so `clone()` is O(1) and shares both the + /// connection pool and the `api-key` default header injected at build time. + client: reqwest::Client, + /// Includes the scheme, e.g. `http://qdrant.example.com:6333`. + base_url: String, + collection: String, +} + +impl QdrantProvider { + /// The resolved API key is injected as a default header here and is + /// deliberately NOT stored on the struct: the plaintext value stays a local + /// of the caller and never outlives it. + fn make_client(api_key: Option<&str>) -> Result { + let mut headers = reqwest::header::HeaderMap::new(); + if let Some(key) = api_key { + let mut value = reqwest::header::HeaderValue::from_str(key) + .context("api-key header value is not valid ASCII")?; + value.set_sensitive(true); + headers.insert("api-key", value); + } + reqwest::Client::builder() + .default_headers(headers) + .build() + .context("Failed to build reqwest client") + } + + fn normalize_base_url(host: &str) -> String { + if host.starts_with("http://") || host.starts_with("https://") { + host.to_string() + } else { + format!("http://{host}") + } + } + + async fn error_message(resp: reqwest::Response) -> String { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + format_error_body(status, &body) + } + + /// Shared `GET /collections/{name}` fetch. Both the dimension and the + /// multi-vector probe discriminate on this same response. + async fn fetch_collection( + host: &str, + collection: &str, + api_key: Option<&str>, + ) -> Result { + let base_url = Self::normalize_base_url(host); + let client = Self::make_client(api_key)?; + let resp = client + .get(format!("{base_url}/collections/{collection}")) + .send() + .await + .with_context(|| format!("Failed to connect to {host}"))?; + if !resp.status().is_success() { + bail!( + "Failed to read collection '{collection}': {}", + Self::error_message(resp).await + ); + } + Ok(resp.json().await?) + } + + pub async fn new(host: &str, collection: &str, api_key: Option<&str>) -> Result { + let base_url = Self::normalize_base_url(host); + let client = Self::make_client(api_key)?; + // Preflight: confirm the collection exists and we may read it. + let resp = client + .get(format!("{base_url}/collections/{collection}")) + .send() + .await + .with_context(|| format!("Failed to connect to {host}"))?; + if !resp.status().is_success() { + bail!( + "Collection '{collection}' not accessible at {host}: {}", + Self::error_message(resp).await + ); + } + Ok(Self { + client, + base_url, + collection: collection.to_string(), + }) + } + + pub async fn list_collections(host: &str, api_key: Option<&str>) -> Result> { + let base_url = Self::normalize_base_url(host); + let client = Self::make_client(api_key)?; + let resp = client + .get(format!("{base_url}/collections")) + .send() + .await + .with_context(|| format!("Failed to connect to {host}"))?; + if !resp.status().is_success() { + bail!( + "Failed to list collections: {}", + Self::error_message(resp).await + ); + } + let body: serde_json::Value = resp.json().await?; + let names = body["result"]["collections"] + .as_array() + .context("Unexpected /collections response shape")? + .iter() + .filter_map(|v| v["name"].as_str().map(str::to_string)) + .collect(); + Ok(names) + } + + pub async fn get_vector_dimension( + host: &str, + collection: &str, + api_key: Option<&str>, + ) -> Result { + let body = Self::fetch_collection(host, collection, api_key).await?; + vector_dimension_from_collection(&body) + } + + pub async fn is_multi_vector( + host: &str, + collection: &str, + api_key: Option<&str>, + ) -> Result { + let body = Self::fetch_collection(host, collection, api_key).await?; + Ok(is_multi_vector_config(&body)) + } + + /// Peek at one point to learn how its ID is typed. Returns the raw JSON + /// rendering, so a string ID comes back quoted and an integer one bare. + pub async fn sample_point_id( + host: &str, + collection: &str, + api_key: Option<&str>, + ) -> Result> { + let base_url = Self::normalize_base_url(host); + let client = Self::make_client(api_key)?; + let url = format!("{base_url}/collections/{collection}/points/scroll"); + let body = serde_json::json!({ "limit": 1, "with_payload": false }); + let resp = client + .post(&url) + .json(&body) + .send() + .await + .with_context(|| format!("Failed to connect to {host}"))?; + if !resp.status().is_success() { + bail!( + "Failed to sample a point from '{collection}': {}", + Self::error_message(resp).await + ); + } + let data: serde_json::Value = resp.json().await?; + let id_val = data["result"]["points"] + .as_array() + .and_then(|pts| pts.first()) + .map(|pt| pt["id"].to_string()); + Ok(id_val) + } +} + +#[async_trait] +impl RagProvider for QdrantProvider { + async fn vector_search( + &self, + embedding: &[f32], + top_k: usize, + min_score: f32, + ) -> Result> { + let url = format!( + "{}/collections/{}/points/search", + self.base_url, self.collection + ); + // `score_threshold` is deliberately NOT sent. It is metric-aware: on Cosine + // collections 0.0 means "no floor" as expected, but Euclid collections score + // by negative distance, where 0.0 filters everything out. The attach wizard + // does not pin the distance metric, so filter locally instead. + let body = serde_json::json!({ + "vector": embedding, + "limit": top_k, + "with_payload": false, + }); + let resp = self.client.post(&url).json(&body).send().await?; + if !resp.status().is_success() { + bail!( + "Qdrant search on '{}' failed: {}", + self.collection, + Self::error_message(resp).await + ); + } + let data: serde_json::Value = resp.json().await?; + let results = data["result"] + .as_array() + .context("Unexpected /points/search response shape")? + .iter() + .filter_map(|pt| { + // String (UUID) IDs yield None here and are dropped. The attach + // wizard rejects such collections up front so this cannot silently + // become "zero results, no error". + let id = pt["id"].as_u64()? as usize; + let score = pt["score"].as_f64()? as f32; + Some((DocumentId(id), score)) + }) + .filter(|(_, score)| *score > min_score) + .collect(); + Ok(results) + } + + async fn fetch_content(&self, ids: &[DocumentId]) -> Result> { + if ids.is_empty() { + return Ok(vec![]); + } + let url = format!("{}/collections/{}/points", self.base_url, self.collection); + let id_list: Vec = ids.iter().map(|d| d.0 as u64).collect(); + let body = serde_json::json!({ + "ids": id_list, + "with_payload": true, + }); + let resp = self.client.post(&url).json(&body).send().await?; + if !resp.status().is_success() { + bail!( + "Qdrant point fetch on '{}' failed: {}", + self.collection, + Self::error_message(resp).await + ); + } + let data: serde_json::Value = resp.json().await?; + let mut rows: Vec<(DocumentId, String)> = data["result"] + .as_array() + .context("Unexpected /points response shape")? + .iter() + .filter_map(|pt| { + let id = pt["id"].as_u64()? as usize; + let text = pt["payload"]["page_content"].as_str()?.to_string(); + Some((DocumentId(id), text)) + }) + .collect(); + // `/points` does not guarantee response order matches request order, and the + // caller's RRF ranking is carried by that order. Restore it. + let position: HashMap = + ids.iter().enumerate().map(|(i, id)| (*id, i)).collect(); + rows.sort_by_key(|(id, _)| position.get(id).copied().unwrap_or(usize::MAX)); + Ok(rows) + } + + async fn rebuild_indexes(&mut self, data: &RagData, _full_rebuild: bool) -> Result<()> { + // Both arms refuse. A silent `Ok(())` would make `.rebuild rag` and + // `.edit rag-docs` look like they worked while writing nothing to the + // remote — leaving the user believing the collection was updated. + if data.attached { + bail!( + "This RAG is attached to an external Qdrant collection. Coyote does not own \ + its documents and cannot rebuild it. Manage the collection directly, or \ + create a Coyote-owned RAG with `.rag `." + ); + } + bail!("Writing to Qdrant is not supported yet (attach-only)."); + } + + fn duplicate(&self, _data: &RagData) -> Box { + // Cloning the client shares the connection pool and the injected api-key + // header. Sharing is correct: both handles address the same remote + // collection, and neither of them writes to it. + Box::new(Self { + client: self.client.clone(), + base_url: self.base_url.clone(), + collection: self.collection.clone(), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn error_message_reads_the_object_status_envelope() { + let body = + r#"{"status": {"error": "Wrong input: Not existing vector name error:"}, "time": 0.0}"#; + let msg = format_error_body(reqwest::StatusCode::BAD_REQUEST, body); + assert!(msg.contains("Not existing vector name"), "got: {msg}"); + assert!( + !msg.contains("EOF"), + "must not fall through to a parse error" + ); + } + + #[test] + fn error_message_survives_the_string_status_and_the_empty_body() { + // Success envelope: `status` is a bare string, so the object lookup misses + // and we must fall back rather than panic or invent an error text. + let ok = format_error_body(reqwest::StatusCode::OK, r#"{"status": "ok", "time": 0.0}"#); + assert!( + ok.contains("200"), + "no `status.error` present → fall back to status+body: {ok}" + ); + // Routing-level 404 from a wrong HTTP verb: empty body, no JSON at all. + let empty = format_error_body(reqwest::StatusCode::NOT_FOUND, ""); + assert!(empty.contains("empty body"), "got: {empty}"); + assert!( + empty.contains("verb"), + "the message must point at the likely cause: {empty}" + ); + } + + #[test] + fn vector_dimension_handles_both_collection_shapes() { + let unnamed = serde_json::json!({ + "result": {"config": {"params": {"vectors": {"size": 1536, "distance": "Cosine"}}}} + }); + assert_eq!(vector_dimension_from_collection(&unnamed).unwrap(), 1536); + + let named = serde_json::json!({ + "result": {"config": {"params": {"vectors": {"text": {"size": 768, "distance": "Cosine"}}}}} + }); + assert_eq!(vector_dimension_from_collection(&named).unwrap(), 768); + + let junk = serde_json::json!({"result": {"config": {"params": {}}}}); + assert!(vector_dimension_from_collection(&junk).is_err()); + } + + #[test] + fn is_multi_vector_rejects_the_named_single_collection() { + // The only supported shape: a single unnamed vector. + let unnamed = serde_json::json!({ + "result": {"config": {"params": {"vectors": {"size": 1536, "distance": "Cosine"}}}} + }); + assert!(!is_multi_vector_config(&unnamed)); + + // Named but SINGLE — structurally a map, and writes to it fail with + // `400 "Wrong input: Not existing vector name error:"`. A `len() > 1` check + // would wrongly accept this one; that is the bug this case exists to catch. + let named_single = serde_json::json!({ + "result": {"config": {"params": {"vectors": {"text": {"size": 1536}}}}} + }); + assert!( + is_multi_vector_config(&named_single), + "named-single must be rejected too" + ); + + let named_multi = serde_json::json!({ + "result": {"config": {"params": {"vectors": {"text": {"size": 1536}, "image": {"size": 512}}}}} + }); + assert!(is_multi_vector_config(&named_multi)); + } + + #[test] + fn normalize_base_url_only_adds_a_scheme_when_missing() { + assert_eq!( + QdrantProvider::normalize_base_url("qdrant.example.com:6333"), + "http://qdrant.example.com:6333" + ); + assert_eq!( + QdrantProvider::normalize_base_url("https://xyz.cloud.qdrant.io"), + "https://xyz.cloud.qdrant.io" + ); + assert_eq!( + QdrantProvider::normalize_base_url("http://localhost:6333"), + "http://localhost:6333" + ); + } + + #[tokio::test] + async fn rebuild_indexes_refuses_for_attached_and_unattached_alike() { + let mut provider = QdrantProvider { + client: reqwest::Client::new(), + base_url: "http://localhost:6333".to_string(), + collection: "c".to_string(), + }; + + let attached = RagData { + driver: "qdrant".to_string(), + attached: true, + ..Default::default() + }; + let err = provider + .rebuild_indexes(&attached, true) + .await + .expect_err("an attached qdrant RAG must never report a successful rebuild"); + assert!(err.to_string().contains("cannot rebuild"), "got: {err}"); + + // `attached: false` is reserved for the (unimplemented) write path. It must + // also refuse: silently succeeding would run a full paid embedding pass and + // then discard every vector. + let owned = RagData { + driver: "qdrant".to_string(), + attached: false, + ..Default::default() + }; + let err = provider + .rebuild_indexes(&owned, true) + .await + .expect_err("writing to qdrant is unimplemented and must fail loudly"); + assert!(err.to_string().contains("not supported yet"), "got: {err}"); + } + + #[tokio::test] + async fn fetch_content_short_circuits_on_an_empty_id_list() { + // No network is touched: the early return happens before any request, which + // is why this can assert against an unreachable host. + let provider = QdrantProvider { + client: reqwest::Client::new(), + base_url: "http://127.0.0.1:1".to_string(), + collection: "c".to_string(), + }; + assert!(provider.fetch_content(&[]).await.unwrap().is_empty()); + } + + #[tokio::test] + #[ignore] + async fn qdrant_list_collections_requires_running_instance() { + let collections = QdrantProvider::list_collections("http://localhost:6333", None) + .await + .unwrap(); + assert!(!collections.is_empty()); + } + + #[tokio::test] + #[ignore] + async fn qdrant_vector_search_returns_results() { + let provider = QdrantProvider::new("http://localhost:6333", "test-collection", None) + .await + .unwrap(); + let embedding = vec![0.0f32; 1536]; + let results = provider.vector_search(&embedding, 5, 0.0).await.unwrap(); + assert!(results.len() <= 5); + } +} diff --git a/src/repl/mod.rs b/src/repl/mod.rs index c33d067..68a871b 100644 --- a/src/repl/mod.rs +++ b/src/repl/mod.rs @@ -53,7 +53,7 @@ pub const DEFAULT_CONTINUATION_PROMPT: &str = indoc! {" 4. Continue with the next pending item now. Call tools immediately." }; -static REPL_COMMANDS: LazyLock<[ReplCommand; 59]> = LazyLock::new(|| { +static REPL_COMMANDS: LazyLock<[ReplCommand; 60]> = LazyLock::new(|| { [ ReplCommand::new(".help", "Show this help guide", AssertState::pass()), ReplCommand::new(".info", "Show system info", AssertState::pass()), @@ -217,6 +217,11 @@ static REPL_COMMANDS: LazyLock<[ReplCommand; 59]> = LazyLock::new(|| { "Initialize or access RAG", AssertState::False(StateFlags::AGENT), ), + ReplCommand::new( + ".rag attach", + "Attach to a pre-existing external RAG (Qdrant)", + AssertState::False(StateFlags::AGENT), + ), ReplCommand::new( ".edit rag-docs", "Add or remove documents from an existing RAG", @@ -885,7 +890,20 @@ pub async fn run_repl_command( task::spawn_blocking(move || config::run_self_update(version, false)).await??; } ".rag" => { - ctx.use_rag(args, abort_signal.clone()).await?; + // `split_first_arg` rather than `starts_with("attach ")`: the latter + // misses a bare `.rag attach`, which would silently create a RAG + // literally named "attach". + match split_first_arg(args) { + Some(("attach", rest)) => match rest { + Some(name) if !name.trim().is_empty() => { + ctx.attach_rag(name.trim()).await?; + } + _ => println!("Usage: .rag attach "), + }, + _ => { + ctx.use_rag(args, abort_signal.clone()).await?; + } + } } ".agent" => match split_first_arg(args) { Some((agent_name, args)) => { @@ -1711,8 +1729,8 @@ mod tests { } #[test] - fn repl_commands_has_59_entries() { - assert_eq!(REPL_COMMANDS.len(), 59); + fn repl_commands_has_60_entries() { + assert_eq!(REPL_COMMANDS.len(), 60); } #[test] diff --git a/src/sandbox/mixins.rs b/src/sandbox/mixins.rs index 9a5d8f2..e9574a9 100644 --- a/src/sandbox/mixins.rs +++ b/src/sandbox/mixins.rs @@ -10,6 +10,7 @@ use sha2::{Digest, Sha256}; use crate::config::paths; const SBX_MIXIN_FILE_NAME: &str = "sbx-mixin.yaml"; +const SBX_MIXIN_FILE_SUFFIX: &str = ".sbx-mixin.yaml"; const KIT_SPEC_FILE_NAME: &str = "spec.yaml"; #[derive(Debug, Clone)] @@ -72,6 +73,13 @@ pub fn discover() -> Result> { for path in collect_subdir_mixins(&paths::agents_data_dir()) { out.push(read_mixin(path)?); } + // RAG sidecars are FLAT files named `.sbx-mixin.yaml` inside rags/, not + // the `/sbx-mixin.yaml` shape the two scans above walk. Loaded + // unconditionally, mirroring agents/*: a RAG mixin only adds an outbound + // allowlist entry for that RAG's host and opens no inbound rules. + for path in collect_flat_mixins(&paths::rags_dir()) { + out.push(read_mixin(path)?); + } if let Ok(cwd) = env::current_dir() && let Some(path) = paths::find_workspace_sbx_mixin(&cwd) @@ -173,6 +181,30 @@ fn collect_subdir_mixins(dir: &Path) -> Vec { result } +/// Mixins stored as flat `.sbx-mixin.yaml` files directly inside `dir`, +/// matched by suffix rather than by exact filename. +fn collect_flat_mixins(dir: &Path) -> Vec { + let mut result = Vec::new(); + let Ok(rd) = read_dir(dir) else { return result }; + + let mut entries: Vec<_> = rd + .flatten() + .filter(|e| e.file_type().map(|t| t.is_file()).unwrap_or(false)) + .filter(|e| { + e.file_name() + .to_str() + .is_some_and(|n| n.ends_with(SBX_MIXIN_FILE_SUFFIX)) + }) + .collect(); + entries.sort_by_key(|e| e.file_name()); + + for entry in entries { + result.push(entry.path()); + } + + result +} + #[cfg(test)] mod tests { use super::*; @@ -439,4 +471,54 @@ network: ); } } + + /// RAG sidecars are flat `.sbx-mixin.yaml` files, matched by SUFFIX. + #[test] + fn collect_flat_mixins_matches_rag_sidecars_by_suffix() { + let root = unique_root("flat-mixins"); + fs::write(root.join("company-docs.sbx-mixin.yaml"), "kind: mixin\n").unwrap(); + fs::write(root.join("alpha.sbx-mixin.yaml"), "kind: mixin\n").unwrap(); + // The RAGs themselves must not be picked up, only their sidecars. + fs::write(root.join("company-docs.yaml"), "driver: qdrant\n").unwrap(); + fs::write(root.join("notes.yaml"), "driver: yaml\n").unwrap(); + // A directory whose name ends in the suffix is not a mixin file. + fs::create_dir_all(root.join("decoy.sbx-mixin.yaml")).unwrap(); + + let found = collect_flat_mixins(&root); + let names: Vec<_> = found + .iter() + .map(|p| p.file_name().unwrap().to_str().unwrap()) + .collect(); + // Sorted by file name, so the order is deterministic. + assert_eq!( + names, + vec!["alpha.sbx-mixin.yaml", "company-docs.sbx-mixin.yaml"] + ); + + let _ = fs::remove_dir_all(&root); + } + + /// Why `collect_flat_mixins` had to be written: the existing collector walks + /// SUBDIRECTORIES for a file named exactly `sbx-mixin.yaml`, so it cannot see + /// a flat sidecar. If this ever starts finding them, the new collector is + /// redundant — but until then, removing it silently drops every RAG mixin. + #[test] + fn collect_subdir_mixins_cannot_see_flat_rag_sidecars() { + let root = unique_root("flat-vs-subdir"); + fs::write(root.join("company-docs.sbx-mixin.yaml"), "kind: mixin\n").unwrap(); + + assert!(collect_subdir_mixins(&root).is_empty()); + assert_eq!(collect_flat_mixins(&root).len(), 1); + + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn collect_flat_mixins_tolerates_a_missing_directory() { + let root = unique_root("flat-missing"); + let absent = root.join("nope"); + assert!(collect_flat_mixins(&absent).is_empty()); + + let _ = fs::remove_dir_all(&root); + } } diff --git a/src/sandbox/mod.rs b/src/sandbox/mod.rs index 19a9fc4..91fc33b 100644 --- a/src/sandbox/mod.rs +++ b/src/sandbox/mod.rs @@ -16,6 +16,7 @@ use crate::config::AppConfig; use crate::config::Config; use crate::config::VAULT_DATA_FILE_NAME; use crate::config::paths; +use crate::rag::RagData; use crate::sandbox::mixins::DiscoveredMixin; use crate::utils::run_command_with_output; use crate::vault::SECRET_RE; @@ -51,6 +52,7 @@ pub fn launch(name: Option, fresh: bool) -> Result<()> { inject_llm_secret(&config_content, &vault, ®istered)?; if !fresh { inject_mcp_secrets(&vault, ®istered)?; + inject_rag_secrets(&vault, ®istered)?; } let discovered = mixins::discover()?; @@ -301,6 +303,65 @@ fn inject_mcp_secrets(vault: &Vault, registered: &HashSet) -> Result<()> Ok(()) } +/// Registers the API key of every attached RAG with the sbx proxy. +/// +/// `launch()` has no notion of an active RAG — that is runtime state set by +/// `--rag` / `.rag` and never persisted — so every attached RAG is scanned +/// unconditionally, exactly as `inject_mcp_secrets` does for MCP servers. +fn inject_rag_secrets(vault: &Vault, registered: &HashSet) -> Result<()> { + let rags_dir = paths::rags_dir(); + if !rags_dir.exists() { + return Ok(()); + } + for entry in fs::read_dir(&rags_dir)?.flatten() { + let path = entry.path(); + if path.extension().and_then(|e| e.to_str()) != Some("yaml") { + continue; + } + let stem = match path.file_stem().and_then(|s| s.to_str()) { + // Skip sidecars ("myrag.sbx-mixin.yaml" has stem "myrag.sbx-mixin"). + Some(s) if !paths::is_rag_sidecar_name(s) => s.to_string(), + _ => continue, + }; + let Ok(raw) = fs::read_to_string(&path) else { + continue; + }; + let Ok(data) = serde_yaml::from_str::(&raw) else { + continue; + }; + if !data.attached { + continue; + } + let Some(placeholder) = data.driver_config.get("api_key") else { + continue; + }; + if registered.contains(&stem) { + continue; + } + let secret_name = placeholder + .trim_start_matches("{{") + .trim_end_matches("}}") + .trim(); + // Degrade rather than abort: one stale RAG key must not block the whole + // sandbox launch. Queries to that RAG fail with a 401 at runtime, which + // is recoverable without a restart. + match vault.get_secret(secret_name, false) { + Ok(secret_value) => { + sbx_secret_set(&stem, &secret_value) + .context("Failed to register RAG secret with sbx")?; + } + Err(e) => { + eprintln!( + "Warning: could not load secret '{secret_name}' for RAG '{stem}': {e}. \ + Queries to this RAG will fail inside the sandbox. \ + Run `coyote --add-secret {secret_name}` to fix." + ); + } + } + } + Ok(()) +} + fn provider_to_sbx_service(provider_type: &str, client_name: Option<&str>) -> String { match provider_type { "claude" => "anthropic".to_string(), From f68937611e422a7df01f343b84679bc43b1ef571 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 10 Aug 2026 15:47:15 -0600 Subject: [PATCH 06/36] fix(rag): install DuckDB vss and fts extensions when they are missing The DuckDB schema init loaded the vss and fts extensions but nothing ever installed them, so any machine without them already present failed with 'IO Error: Extension "vss.duckdb_extension" not found'. This surfaced as 13 failing tests in CI while passing locally, because local runs had the extensions installed already. Loading is attempted first so an extension that is already present costs nothing and never touches the network; INSTALL is reached only once, on a machine seeing the extension for the first time, and reports an actionable message if it cannot download. CI cached the extension directory but nothing populated it, so the cache saved an empty directory forever. The cache key now derives from Cargo.lock rather than a hardcoded DuckDB version, and a step on cache miss installs the extensions so the post-job save has something to store. --- .github/workflows/ci.yaml | 15 ++++++++++++++- src/rag/providers/duckdb.rs | 32 ++++++++++++++++++++++++++++---- 2 files changed, 42 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a7bc97a..ead1292 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -36,11 +36,24 @@ jobs: - uses: Swatinem/rust-cache@v2 + # The extension directory is DuckDB-version-specific (v//), so the + # key is derived from Cargo.lock, which pins the duckdb crate. A hardcoded version in + # the key would keep hitting after a crate bump and, because an exact hit skips the + # save, the new extensions would be re-downloaded on every run and never cached. - name: Cache DuckDB Extensions + id: duckdb-extensions uses: actions/cache@v4 with: path: ~/.duckdb/extensions - key: duckdb-ext-${{ matrix.os }}-v1.5.5 + key: duckdb-ext-${{ matrix.os }}-${{ hashFiles('Cargo.lock') }} + + # Populates the cache on a miss: opening a DuckDB store installs vss and fts when + # they are absent, and the post-job save then has something to store. Runs the + # DuckDB tests only, so a download failure is reported here rather than as a wall of + # unrelated-looking test failures. + - name: Install DuckDB Extensions + if: steps.duckdb-extensions.outputs.cache-hit != 'true' + run: cargo test --all duckdb - name: Test run: cargo test --all diff --git a/src/rag/providers/duckdb.rs b/src/rag/providers/duckdb.rs index 5c877fe..64b3eee 100644 --- a/src/rag/providers/duckdb.rs +++ b/src/rag/providers/duckdb.rs @@ -44,11 +44,12 @@ impl DuckDbProvider { // "Setting with name ... is not in the catalog, but it exists in the vss // extension". Omitting it entirely makes CREATE INDEX ... USING HNSW on a // file-backed database fail with "HNSW index persistence is not yet supported - // by default". LOAD vss -> LOAD fts -> SET -> CREATE INDEX. + // by default". ensure vss (installing it if missing) -> ensure fts -> SET -> + // CREATE INDEX. + Self::ensure_extension(&conn, "vss")?; + Self::ensure_extension(&conn, "fts")?; conn.execute_batch(&format!( - "LOAD vss; - LOAD fts; - SET hnsw_enable_experimental_persistence = true; + "SET hnsw_enable_experimental_persistence = true; CREATE TABLE IF NOT EXISTS vectors ( doc_id UBIGINT PRIMARY KEY, embedding FLOAT[{dim}] @@ -73,6 +74,29 @@ impl DuckDbProvider { }) } + /// Make a DuckDB extension available on `conn`, installing it if this machine does + /// not have it yet. `LOAD` is attempted first so an extension that is already + /// installed costs nothing and never touches the network; `INSTALL` is only reached + /// once, on a machine seeing the extension for the first time. + fn ensure_extension(conn: &Connection, name: &str) -> Result<()> { + if conn.execute_batch(&format!("LOAD {name};")).is_ok() { + return Ok(()); + } + conn.execute_batch(&format!("INSTALL {name};")) + .with_context(|| { + format!( + "Failed to install the DuckDB `{name}` extension. The duckdb RAG driver \ + needs it, and downloading it needs network access the first time. If this \ + machine is offline, connect once and retry, or run `INSTALL {name};` \ + yourself from a DuckDB shell." + ) + })?; + conn.execute_batch(&format!("LOAD {name};")) + .with_context(|| { + format!("Failed to load the DuckDB `{name}` extension after installing it.") + }) + } + /// Read all `(doc_id, embedding)` pairs so `create()` can hydrate `data.vectors` /// from disk. This is what makes the next incremental sync non-destructive, and is /// mandatory rather than an optimization. From 3abc30d6336f6eee765f6cec9ba7caa266fd7336 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 10 Aug 2026 15:58:40 -0600 Subject: [PATCH 07/36] fix(rag): emit an sbx kit v2 mixin and declare RAG credentials to the proxy The RAG attach sidecar was written against the sbx kit v1 spec and still emitted schemaVersion "1" with network.allowedDomains, network.serviceDomains, network.serviceAuth, credentials.sources..env and environment.proxyManaged. Every one of those keys was removed in kit v2. Coyote does not validate mixins, it copies them byte-for-byte into spec.yaml, so the invalid document surfaced only as an opaque sbx failure with no indication of which mixin caused it. generate_rag_sbx_mixin now builds the document from the shared serializer structs instead of a format! string, which is how the envelope drifted unnoticed in the first place. render_mixin_yaml and the RAG sidecar both go through a new render_mixin_document, giving one definition of the envelope and one enforcement point for the rule that every inject domain must also appear in permissions.network.allow. Fix an auth bug the port exposed: inject_rag_secrets bound the API key with sbx secret set, but nothing ever emitted a matching credentials entry, so the proxy held a value with no inject rule and never rewrote the auth header. An attached RAG credential silently did not work inside the sandbox. The sidecar now declares that credential; a RAG with no API key declares none while still receiving egress. Fix the service id: the bind passed the raw file stem instead of routing it through secret_service_id, so a RAG named My_Docs produced an illegal id. The bind and the generated credentials service now share that derivation and cannot disagree. Retire sbx_domain_forms in favour of allow_entry_for_url, now pub(crate). It emitted both a bare host and host:port because v1 serviceDomains needed a bare key; v2 has no such need, so the extra entry is simply wrong. It also defaulted a schemeless host to port 6333 while normalize_base_url resolves it to http and port 80, meaning the allow entry named a port the client never dialled. --- src/config/paths.rs | 2 +- src/rag/mod.rs | 376 ++++++++++++++++++++------------- src/rag/providers/qdrant.rs | 2 +- src/sandbox/mcp_credentials.rs | 102 +++++---- src/sandbox/mod.rs | 11 +- 5 files changed, 308 insertions(+), 185 deletions(-) diff --git a/src/config/paths.rs b/src/config/paths.rs index 268e497..177864d 100644 --- a/src/config/paths.rs +++ b/src/config/paths.rs @@ -445,7 +445,7 @@ pub(crate) fn is_rag_sidecar_name(name: &str) -> bool { /// /// Callers must run this BEFORE unlinking the primary `.yaml`. If the YAML goes first /// and this then fails, the RAG disappears from `list_rags()` — so the user can no -/// longer select it to retry — while its `allowedDomains` entry keeps being injected +/// longer select it to retry — while its network allow entry keeps being injected /// into every sandbox launch. pub(crate) fn remove_rag_sidecars(dir: &Path, name: &str) -> Result<()> { let duckdb_path = dir.join(format!("{name}.duckdb")); diff --git a/src/rag/mod.rs b/src/rag/mod.rs index abe5319..29f298a 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -16,6 +16,7 @@ use self::provider::RagProvider; // `create()`, so `providers` itself must stay in scope — do not collapse it into // the `use` below. use self::providers::{DuckDbProvider, QdrantProvider, YamlProvider}; +use crate::sandbox::mcp_credentials; use crate::vault::{Vault, interpolate_secrets}; use anyhow::{Context, Result, anyhow, bail}; @@ -435,7 +436,7 @@ impl Rag { .with_validator(required!("This field is required")) .with_validator(|input: &str| { // Bracketed IPv6 literals would produce a malformed sandbox - // allowedDomains entry, so refuse them at the prompt. + // network allow entry, so refuse them at the prompt. Ok(if input.contains('[') || input.contains(']') { Validation::Invalid( "Bracketed IPv6 literals are not supported; use a hostname.".into(), @@ -568,9 +569,16 @@ impl Rag { rag.save()?; println!("✓ Attached '{name}' → collection '{collection}' on {host}."); - let env_var = rag_env_var_name(name); + let env_var = api_key_entry.as_ref().map(|_| rag_env_var_name(name)); let (header_name, value_format) = driver_auth_header(driver); - generate_rag_sbx_mixin(save_path, &host, name, &env_var, header_name, value_format)?; + generate_rag_sbx_mixin( + save_path, + &host, + name, + env_var.as_deref(), + header_name, + value_format, + )?; Ok(rag) } @@ -1761,48 +1769,57 @@ impl DocumentId { /// byte-for-byte to `spec.yaml` inside a kit dir handed to `sbx create --kit`, /// with no `kind` rewrite — so an envelope-less file can break the launch /// itself, not merely this RAG's traffic. -/// 2. `allowedDomains` entries carry a port; `serviceDomains` keys are bare -/// hostnames. The asymmetry is deliberate. +/// 2. The `service` declared here must be the id the host binds the value +/// under with `sbx secret set`; both derive from the RAG name through +/// `secret_service_id`, so they cannot spell it differently. +/// +/// `api_key_env` is `None` for a store that needs no credential: the host is +/// still allowed, but no binding is declared, because nothing binds a value. fn generate_rag_sbx_mixin( rag_yaml_path: &Path, host: &str, service_name: &str, - env_var: &str, + api_key_env: Option<&str>, header_name: &str, value_format: &str, ) -> Result<()> { - let (bare_host, allowed_domain) = sbx_domain_forms(host); + // The client reaches the store through `normalize_base_url`, so deriving the + // allow entry from that same URL keeps the whitelist and the actual dialled + // port from drifting apart. + let base_url = QdrantProvider::normalize_base_url(host); + let Some(allow_entry) = mcp_credentials::allow_entry_for_url(&base_url) else { + eprintln!( + "Warning: host '{host}' has no representation in the sbx network allow \ + grammar, so no sandbox mixin was written for RAG '{service_name}'. \ + Queries to this RAG will be blocked inside the sandbox." + ); + return Ok(()); + }; + + let credentials = api_key_env + .map(|env_var| mcp_credentials::CredentialEntry { + service: mcp_credentials::secret_service_id(service_name), + description: format!("API key for the attached RAG '{service_name}'"), + api_key: mcp_credentials::ApiKey { + name: env_var.to_string(), + proxy_managed: true, + inject: vec![rag_inject_rule(&allow_entry, header_name, value_format)], + }, + }) + .into_iter() + .collect(); + let mixin_path = rag_yaml_path.with_extension("sbx-mixin.yaml"); - let content = format!( - r#"schemaVersion: "1" -kind: mixin -name: rag-{service_name} -description: > - Auto-generated by the Coyote attach wizard for RAG '{service_name}'. Allows - outbound traffic to its external vector store and tells the sbx proxy which - header to rewrite with the stored credential. Do not edit manually. - -network: - allowedDomains: - - "{allowed_domain}" - serviceDomains: - {bare_host}: {service_name} - serviceAuth: - {service_name}: - headerName: {header_name} - valueFormat: "{value_format}" - -credentials: - sources: - {service_name}: - env: - - {env_var} - -environment: - proxyManaged: - - {env_var} -"# - ); + let content = mcp_credentials::render_mixin_document( + &format!("rag-{service_name}"), + &format!( + "Auto-generated by the Coyote attach wizard for RAG '{service_name}'. Allows \ + outbound traffic to its external vector store and declares the credential the \ + sbx proxy injects into each request. Do not edit manually." + ), + credentials, + &[allow_entry], + )?; fs::write(&mixin_path, &content).with_context(|| { format!( "Failed to write sandbox mixin to '{}'", @@ -1813,34 +1830,28 @@ environment: Ok(()) } -/// Splits a user-supplied host into the two forms sbx needs: -/// `(bare_host, allowed_domain)`. -/// -/// `bare_host` is the `serviceDomains` KEY — hostname only, no scheme, no port. -/// `allowed_domain` is an `allowedDomains` ENTRY — always `host:port`. -/// -/// The rule: emit the host verbatim when it already carries a numeric port, -/// otherwise append the default for the scheme. Stripping the port fails -/// silently — the sandbox just denies the connection, with no compile or test -/// signal. Defaults match `normalize_base_url`, which assumes `http://` when no -/// scheme is given: plain host → 6333, explicit `https://` → 443. -fn sbx_domain_forms(host: &str) -> (String, String) { - let is_https = host.starts_with("https://"); - let hostport = host - .strip_prefix("https://") - .or_else(|| host.strip_prefix("http://")) - .unwrap_or(host) - .trim_end_matches('/') - // A trailing ':' with no digits is a typo, not a port. Trim it so the - // default-port arm cannot emit "host::6333". - .trim_end_matches(':'); - match hostport.rsplit_once(':') { - Some((h, p)) if !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit()) => { - (h.to_string(), hostport.to_string()) +/// Bearer credentials are spelled as a `scheme`, everything else as an explicit +/// `header`; the two are mutually exclusive in the inject grammar. +fn rag_inject_rule( + domain: &str, + header_name: &str, + value_format: &str, +) -> mcp_credentials::InjectRule { + if header_name.eq_ignore_ascii_case("authorization") + && value_format.eq_ignore_ascii_case("Bearer %s") + { + mcp_credentials::InjectRule { + domain: domain.to_string(), + header: None, + format: None, + scheme: Some("bearer".to_string()), } - _ => { - let port = if is_https { 443 } else { 6333 }; - (hostport.to_string(), format!("{hostport}:{port}")) + } else { + mcp_credentials::InjectRule { + domain: domain.to_string(), + header: Some(header_name.to_string()), + format: Some(value_format.to_string()), + scheme: None, } } } @@ -2226,99 +2237,180 @@ mod tests { ); } - /// `allowedDomains` entries are host:PORT; `serviceDomains` keys are bare - /// hostnames. Stripping the port breaks sandbox whitelisting silently — no - /// compile error, no runtime error on the host, just a denied connection - /// inside the sandbox. This test is the only signal. - #[test] - fn sbx_domain_forms_keeps_explicit_ports_and_defaults_the_rest() { - assert_eq!( - sbx_domain_forms("rag.example.com:6333"), - ("rag.example.com".into(), "rag.example.com:6333".into()) - ); - // A non-default explicit port must survive. - assert_eq!( - sbx_domain_forms("rag.example.com:7777").1, - "rag.example.com:7777" - ); - // Bare host → Qdrant's REST default, matching normalize_base_url. - assert_eq!( - sbx_domain_forms("rag.example.com"), - ("rag.example.com".into(), "rag.example.com:6333".into()) - ); - // The scheme is stripped; http keeps 6333. - assert_eq!( - sbx_domain_forms("http://localhost:6333").1, - "localhost:6333" - ); - // https with no port → 443 (the Qdrant Cloud shape). - assert_eq!( - sbx_domain_forms("https://xyz.cloud.qdrant.io"), - ( - "xyz.cloud.qdrant.io".into(), - "xyz.cloud.qdrant.io:443".into() - ) - ); - // A dangling colon is a typo, not a port: trimmed, then defaulted. - assert_eq!( - sbx_domain_forms("rag.example.com:").1, - "rag.example.com:6333" - ); - } - - /// The generated mixin must carry the schema envelope: `wrap_mixin_as_kit` - /// copies it verbatim to `spec.yaml` for `sbx create --kit`, with no `kind` - /// rewrite, so an envelope-less file can break the launch itself. - #[test] - fn generated_sbx_mixin_carries_the_schema_envelope() { + /// Renders a sidecar into a scratch dir and returns its text plus the + /// parsed document. Parsing is not optional: a malformed document is + /// otherwise copied verbatim into `spec.yaml` and only rejected by sbx. + fn render_rag_mixin( + host: &str, + name: &str, + api_key_env: Option<&str>, + header_name: &str, + value_format: &str, + ) -> (String, serde_yaml::Value) { let dir = TempDir::new("mixin"); - let yaml_path = dir.path.join("company-docs.yaml"); + let yaml_path = dir.path.join(format!("{name}.yaml")); generate_rag_sbx_mixin( &yaml_path, - "rag.example.com", - "company-docs", - "COMPANY_DOCS_API_KEY", - "api-key", - "%s", + host, + name, + api_key_env, + header_name, + value_format, ) .unwrap(); - let text = fs::read_to_string(dir.path.join("company-docs.sbx-mixin.yaml")).unwrap(); + let text = fs::read_to_string(dir.path.join(format!("{name}.sbx-mixin.yaml"))).unwrap(); + let parsed = serde_yaml::from_str(&text).unwrap(); + (text, parsed) + } + + fn allow_list(parsed: &serde_yaml::Value) -> Vec { + parsed["permissions"]["network"]["allow"] + .as_sequence() + .expect("a mixin without an allow list whitelists nothing") + .iter() + .map(|v| v.as_str().unwrap().to_string()) + .collect() + } + + /// The generated mixin must carry the kit v2 schema envelope: + /// `wrap_mixin_as_kit` copies it verbatim to `spec.yaml` for + /// `sbx create --kit`, with no `kind` rewrite and no validation on Coyote's + /// side, so a stale envelope breaks the launch itself. + #[test] + fn generated_sbx_mixin_carries_the_schema_envelope() { + let (text, parsed) = render_rag_mixin( + "rag.example.com:6333", + "company-docs", + Some("COMPANY_DOCS_API_KEY"), + "api-key", + "%s", + ); assert!( text.starts_with("schemaVersion:"), "envelope must come first:\n{text}" ); - assert!( - text.contains("kind: mixin"), - "kind must be `mixin`, not `sandbox`" - ); - assert!(text.contains("name: rag-company-docs")); - assert!(text.contains("description:")); - - // It must parse as YAML at all — a broken format! escape is invisible otherwise. - let parsed: serde_yaml::Value = serde_yaml::from_str(&text).unwrap(); + // Quoted string, never the numeric 2 — the kit rejects an int here. + assert_eq!(parsed["schemaVersion"].as_str(), Some("2")); assert_eq!(parsed["kind"].as_str(), Some("mixin")); - // The list entry carries the port; the map key does not. + assert_eq!(parsed["name"].as_str(), Some("rag-company-docs")); + assert!(parsed["description"].as_str().is_some()); + + // https/443 is the only bare-host case; 6333 must carry its port, and + // the bare host must NOT also be listed. + assert_eq!(allow_list(&parsed), vec!["rag.example.com:6333"]); + + let credential = &parsed["credentials"][0]; + assert_eq!(credential["service"].as_str(), Some("company-docs")); assert_eq!( - parsed["network"]["allowedDomains"][0].as_str(), - Some("rag.example.com:6333") - ); - assert_eq!( - parsed["network"]["serviceDomains"]["rag.example.com"].as_str(), - Some("company-docs") - ); - // The proxy rewrites this header with the credential it holds. - assert_eq!( - parsed["network"]["serviceAuth"]["company-docs"]["headerName"].as_str(), - Some("api-key") - ); - assert_eq!( - parsed["credentials"]["sources"]["company-docs"]["env"][0].as_str(), + credential["apiKey"]["name"].as_str(), Some("COMPANY_DOCS_API_KEY") ); + assert_eq!(credential["apiKey"]["proxyManaged"].as_bool(), Some(true)); + + let inject = &credential["apiKey"]["inject"][0]; + assert_eq!(inject["domain"].as_str(), Some("rag.example.com:6333")); + assert_eq!(inject["header"].as_str(), Some("api-key")); + assert_eq!(inject["format"].as_str(), Some("%s")); + + // sbx does not derive allow entries from inject rules; an inject + // domain that is not allowed is a dead rule. + assert!( + allow_list(&parsed).contains(&inject["domain"].as_str().unwrap().to_string()), + "every inject domain must also appear in allow:\n{text}" + ); + + // The v1 vocabulary is gone, not merely unused. + for dead in ["allowedDomains", "serviceDomains", "serviceAuth"] { + assert!(!text.contains(dead), "v1 key '{dead}' survived:\n{text}"); + } + assert!( + parsed["network"].is_null(), + "v1 top-level `network` survived:\n{text}" + ); + assert!( + parsed["environment"].is_null(), + "v1 `environment.proxyManaged` survived:\n{text}" + ); + } + + /// The allow entry has to name the port the client actually dials, which is + /// whatever `normalize_base_url` resolves to — not a Qdrant-specific guess. + #[test] + fn generated_sbx_mixin_allows_the_port_the_client_dials() { + let cases = [ + // No scheme means http, and http means port 80 — normalize_base_url + // does not silently append Qdrant's 6333. + ("rag.example.com", "rag.example.com:80"), + ("rag.example.com:7777", "rag.example.com:7777"), + ("http://localhost:6333", "localhost:6333"), + // https on the default port is the one bare-host case. + ("https://xyz.cloud.qdrant.io", "xyz.cloud.qdrant.io"), + ( + "https://xyz.cloud.qdrant.io:6333", + "xyz.cloud.qdrant.io:6333", + ), + ]; + for (host, expected) in cases { + let (_, parsed) = render_rag_mixin(host, "docs", Some("DOCS_API_KEY"), "api-key", "%s"); + assert_eq!( + allow_list(&parsed), + vec![expected.to_string()], + "host {host}" + ); + } + } + + /// `Authorization: Bearer` is spelled as a scheme; `header` and `scheme` are + /// mutually exclusive in the inject grammar. + #[test] + fn generated_sbx_mixin_spells_bearer_as_a_scheme() { + let (_, parsed) = render_rag_mixin( + "https://store.example.com", + "docs", + Some("DOCS_API_KEY"), + "Authorization", + "Bearer %s", + ); + let inject = &parsed["credentials"][0]["apiKey"]["inject"][0]; + assert_eq!(inject["scheme"].as_str(), Some("bearer")); + assert!(inject["header"].is_null()); + assert!(inject["format"].is_null()); + } + + /// The bind in `inject_rag_secrets` and the `service` declared here both run + /// the RAG name through `secret_service_id`. If they disagreed, the proxy + /// would hold a value under one id and an inject rule under another, and the + /// header would never be rewritten. + #[test] + fn generated_sbx_mixin_service_id_matches_the_host_side_bind() { + let (_, parsed) = render_rag_mixin( + "https://store.example.com", + "My_Docs", + Some("MY_DOCS_API_KEY"), + "api-key", + "%s", + ); assert_eq!( - parsed["environment"]["proxyManaged"][0].as_str(), - Some("COMPANY_DOCS_API_KEY") + parsed["credentials"][0]["service"].as_str(), + Some(crate::sandbox::mcp_credentials::secret_service_id("My_Docs").as_str()) + ); + assert_eq!( + parsed["credentials"][0]["service"].as_str(), + Some("my-docs") + ); + } + + /// A store with no API key still needs egress, but declaring a credential + /// nothing ever binds would leave sbx waiting on a binding that never comes. + #[test] + fn generated_sbx_mixin_omits_credentials_when_there_is_no_api_key() { + let (text, parsed) = + render_rag_mixin("https://store.example.com", "docs", None, "api-key", "%s"); + assert_eq!(allow_list(&parsed), vec!["store.example.com"]); + assert!( + parsed["credentials"].is_null(), + "no key means no credential declaration:\n{text}" ); } diff --git a/src/rag/providers/qdrant.rs b/src/rag/providers/qdrant.rs index ffff33a..a2d2c34 100644 --- a/src/rag/providers/qdrant.rs +++ b/src/rag/providers/qdrant.rs @@ -89,7 +89,7 @@ impl QdrantProvider { .context("Failed to build reqwest client") } - fn normalize_base_url(host: &str) -> String { + pub(crate) fn normalize_base_url(host: &str) -> String { if host.starts_with("http://") || host.starts_with("https://") { host.to_string() } else { diff --git a/src/sandbox/mcp_credentials.rs b/src/sandbox/mcp_credentials.rs index 0132e21..a16d8e6 100644 --- a/src/sandbox/mcp_credentials.rs +++ b/src/sandbox/mcp_credentials.rs @@ -387,7 +387,11 @@ pub(crate) fn collect_server_allow_entries( out.into_iter().collect() } -fn allow_entry_for_url(raw: &str) -> Option { +/// The single definition of the sbx kit v2 allow-list entry grammar: https on +/// the default port yields a bare host, anything else is spelled `host:port`. +/// Bracketed IPv6 hosts and non-http(s) schemes have no representation in the +/// grammar and yield `None`. +pub(crate) fn allow_entry_for_url(raw: &str) -> Option { let url = Url::parse(raw).ok()?; let scheme = url.scheme(); if scheme != "https" && scheme != "http" { @@ -426,8 +430,8 @@ fn placeholders(text: &str) -> Result> { struct CredentialsMixin { schema_version: &'static str, kind: &'static str, - name: &'static str, - description: &'static str, + name: String, + description: String, #[serde(skip_serializing_if = "Vec::is_empty")] credentials: Vec, #[serde(skip_serializing_if = "Option::is_none")] @@ -435,20 +439,20 @@ struct CredentialsMixin { } #[derive(Serialize)] -struct CredentialEntry { - service: String, - description: String, +pub(crate) struct CredentialEntry { + pub service: String, + pub description: String, #[serde(rename = "apiKey")] - api_key: ApiKey, + pub api_key: ApiKey, } #[derive(Serialize)] #[serde(rename_all = "camelCase")] -struct ApiKey { - name: String, - proxy_managed: bool, +pub(crate) struct ApiKey { + pub name: String, + pub proxy_managed: bool, #[serde(skip_serializing_if = "Vec::is_empty")] - inject: Vec, + pub inject: Vec, } #[derive(Serialize)] @@ -461,40 +465,30 @@ struct Network { allow: Vec, } -pub(crate) fn render_mixin_yaml( - credentials: &[CredentialSpec], - server_allow_entries: &[String], +/// Serializes one sbx kit v2 mixin document. +/// +/// Every `inject[].domain` is unioned into `permissions.network.allow`: sbx +/// does not derive allow entries from inject rules, so a rule whose domain is +/// not allowed would be dead. Enforcing it here keeps the invariant in one +/// place for every mixin Coyote generates. +pub(crate) fn render_mixin_document( + name: &str, + description: &str, + credentials: Vec, + extra_allow_entries: &[String], ) -> Result { let mut allow: BTreeSet = credentials .iter() - .flat_map(|c| c.inject.iter().map(|r| r.domain.clone())) + .flat_map(|c| c.api_key.inject.iter().map(|r| r.domain.clone())) .collect(); - allow.extend(server_allow_entries.iter().cloned()); + allow.extend(extra_allow_entries.iter().cloned()); let mixin = CredentialsMixin { schema_version: "2", kind: "mixin", - name: MCP_MIXIN_NAME, - description: "Auto-generated by Coyote at launch: allows network egress to the user's \ - remote MCP servers and declares their credentials so Docker Sandboxes \ - binds them (bindings are approved on first interactive run). Values are \ - pre-seeded from Coyote's vault via `sbx secret set`.", - credentials: credentials - .iter() - .map(|c| CredentialEntry { - service: c.service_id.clone(), - description: format!( - "Coyote vault secret '{}', used by MCP server(s) {}", - c.secret_name, - quoted_list(&c.servers) - ), - api_key: ApiKey { - name: c.env_var.clone(), - proxy_managed: c.proxy_managed, - inject: c.inject.clone(), - }, - }) - .collect(), + name: name.to_string(), + description: description.to_string(), + credentials, permissions: (!allow.is_empty()).then(|| Permissions { network: Network { allow: allow.into_iter().collect(), @@ -502,7 +496,39 @@ pub(crate) fn render_mixin_yaml( }), }; - serde_yaml::to_string(&mixin).context("Failed to serialize generated MCP credentials mixin") + serde_yaml::to_string(&mixin).context("Failed to serialize generated sandbox mixin") +} + +pub(crate) fn render_mixin_yaml( + credentials: &[CredentialSpec], + server_allow_entries: &[String], +) -> Result { + let entries = credentials + .iter() + .map(|c| CredentialEntry { + service: c.service_id.clone(), + description: format!( + "Coyote vault secret '{}', used by MCP server(s) {}", + c.secret_name, + quoted_list(&c.servers) + ), + api_key: ApiKey { + name: c.env_var.clone(), + proxy_managed: c.proxy_managed, + inject: c.inject.clone(), + }, + }) + .collect(); + + render_mixin_document( + MCP_MIXIN_NAME, + "Auto-generated by Coyote at launch: allows network egress to the user's remote MCP \ + servers and declares their credentials so Docker Sandboxes binds them (bindings are \ + approved on first interactive run). Values are pre-seeded from Coyote's vault via \ + `sbx secret set`.", + entries, + server_allow_entries, + ) } #[cfg(test)] diff --git a/src/sandbox/mod.rs b/src/sandbox/mod.rs index 8ac1736..4fb9e08 100644 --- a/src/sandbox/mod.rs +++ b/src/sandbox/mod.rs @@ -10,7 +10,7 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use which::which; -mod mcp_credentials; +pub(crate) mod mcp_credentials; mod mixins; pub(crate) use mcp_credentials::sandbox_secret_env_var; @@ -346,7 +346,12 @@ fn inject_rag_secrets(vault: &Vault, registered: &HashSet) -> Result<()> let Some(placeholder) = data.driver_config.get("api_key") else { continue; }; - if registered.contains(&stem) { + // The sidecar mixin declares `credentials[].service` under the same + // derivation, so the bound value and the inject rule that consumes it + // always name the same service. Passing the raw stem here would produce + // an id sbx rejects for any RAG whose name is not already a valid id. + let service_id = mcp_credentials::secret_service_id(&stem); + if service_id.is_empty() || registered.contains(&service_id) { continue; } let secret_name = placeholder @@ -358,7 +363,7 @@ fn inject_rag_secrets(vault: &Vault, registered: &HashSet) -> Result<()> // is recoverable without a restart. match vault.get_secret(secret_name, false) { Ok(secret_value) => { - sbx_secret_set(&stem, &secret_value) + sbx_secret_set(&service_id, &secret_value) .context("Failed to register RAG secret with sbx")?; } Err(e) => { From 3e598065f8086dc46daf4f0b5e600e9ac9f7a1f9 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 10 Aug 2026 16:13:36 -0600 Subject: [PATCH 08/36] fix(rag): serialize DuckDB extension installs to stop a Windows race `ensure_extension` fell back to `INSTALL` whenever `LOAD` failed. With a cold extension cache every thread's `LOAD` fails at once, so every thread ran `INSTALL` concurrently for the same extension. DuckDB installs by downloading to a temp file and then MOVING it into `~/.duckdb/extensions/...`; POSIX allows replacing a file other handles hold open, so Linux and macOS survived, but Windows rejects that move with "Access is denied" and the losing threads failed. Guard the install step with a process-global mutex and re-check `LOAD` after acquiring it. The re-check is what bounds the work to a single install: without it every thread queued behind the winner would still run a redundant `INSTALL` and repeat the same move over a file that is now open. `LOAD` is per-connection, so it still runs on every connection; only `INSTALL` is serialized. An already-installed extension takes the pre-lock fast path and costs neither a lock nor network. The lock is never held across the connection mutex, so it cannot invert lock order. Traced with strace on a cold cache under default test parallelism: before, 17 threads moved files into the store (13 racing on vss alone); after, exactly one rename per extension. --- src/rag/providers/duckdb.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/rag/providers/duckdb.rs b/src/rag/providers/duckdb.rs index 64b3eee..f5e9f07 100644 --- a/src/rag/providers/duckdb.rs +++ b/src/rag/providers/duckdb.rs @@ -10,6 +10,15 @@ use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, MutexGuard}; +/// Serializes `INSTALL` across every thread in this process. DuckDB installs an +/// extension by downloading it to a temp file and then MOVING that file into +/// `~/.duckdb/extensions/...`. Two threads installing the same extension at once +/// both perform that move; on Windows the loser's move targets a file the winner +/// already holds open and fails with "Access is denied", where POSIX would let the +/// replacement through. Guards nothing but the install step, so it is never held +/// across a `DuckDbProvider::conn` guard and cannot invert lock order. +static INSTALL_LOCK: Mutex<()> = Mutex::new(()); + /// Derive the DuckDB sidecar path from a RAG's YAML path: `docs.yaml` -> `docs.duckdb`. pub(crate) fn duckdb_path_from_yaml(yaml_path: &Path) -> PathBuf { yaml_path.with_extension("duckdb") @@ -78,7 +87,20 @@ impl DuckDbProvider { /// not have it yet. `LOAD` is attempted first so an extension that is already /// installed costs nothing and never touches the network; `INSTALL` is only reached /// once, on a machine seeing the extension for the first time. + /// + /// `LOAD` is per-connection and so runs on every connection; only `INSTALL` is + /// serialized, and the second `LOAD` under the lock is what keeps it to one + /// install. Without that re-check, every thread that queued behind the winner + /// would still run a redundant `INSTALL` and re-trigger the same file move. fn ensure_extension(conn: &Connection, name: &str) -> Result<()> { + if conn.execute_batch(&format!("LOAD {name};")).is_ok() { + return Ok(()); + } + // A poisoned lock means some other thread panicked mid-install; the lock owns + // no state to corrupt, so recover rather than failing every later open. + let _install_guard = INSTALL_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + // Re-check now that we hold the lock: whoever held it before us may already + // have installed the extension, in which case this `LOAD` finds it on disk. if conn.execute_batch(&format!("LOAD {name};")).is_ok() { return Ok(()); } From ecda258d3a47a989b2ca96f5a14795de4c48e3da Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 11 Aug 2026 13:04:27 -0600 Subject: [PATCH 09/36] style: Cleaned up some minor styling issues --- .github/workflows/ci.yaml | 8 - Cargo.lock | 264 ++++++++++++++++++--------------- src/config/paths.rs | 25 +--- src/config/request_context.rs | 26 +--- src/rag/mod.rs | 148 +++++++----------- src/rag/provider.rs | 25 +--- src/rag/providers/duckdb.rs | 190 +++++++----------------- src/rag/providers/mod.rs | 7 - src/rag/providers/qdrant.rs | 91 ++++++------ src/rag/providers/yaml.rs | 43 ++---- src/render/markdown.rs | 11 -- src/repl/mod.rs | 25 ++-- src/sandbox/mcp_credentials.rs | 10 -- src/sandbox/mixins.rs | 10 -- src/sandbox/mod.rs | 15 +- 15 files changed, 331 insertions(+), 567 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ead1292..fc98eba 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -36,10 +36,6 @@ jobs: - uses: Swatinem/rust-cache@v2 - # The extension directory is DuckDB-version-specific (v//), so the - # key is derived from Cargo.lock, which pins the duckdb crate. A hardcoded version in - # the key would keep hitting after a crate bump and, because an exact hit skips the - # save, the new extensions would be re-downloaded on every run and never cached. - name: Cache DuckDB Extensions id: duckdb-extensions uses: actions/cache@v4 @@ -47,10 +43,6 @@ jobs: path: ~/.duckdb/extensions key: duckdb-ext-${{ matrix.os }}-${{ hashFiles('Cargo.lock') }} - # Populates the cache on a miss: opening a DuckDB store installs vss and fts when - # they are absent, and the post-job save then has something to store. Runs the - # DuckDB tests only, so a download failure is reported here rather than as a wall of - # unrelated-looking test failures. - name: Install DuckDB Extensions if: steps.duckdb-extensions.outputs.cache-hit != 'true' run: cargo test --all duckdb diff --git a/Cargo.lock b/Cargo.lock index e42c605..a71758f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -58,9 +58,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android_system_properties" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" dependencies = [ "libc", ] @@ -209,7 +209,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17060e608fbc0809d62a996a65cdee9e7c441a979f40f2d1d2fbdce9eef60dad" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "convert_case 0.11.0", "dirs", "either", @@ -321,7 +321,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64", + "base64 0.22.1", "chrono", "comfy-table", "half", @@ -445,9 +445,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.91" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", @@ -520,11 +520,11 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.17.3" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" +checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" dependencies = [ - "aws-lc-sys 0.43.0", + "aws-lc-sys 0.44.0", "zeroize", ] @@ -543,9 +543,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.43.0" +version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" +checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483" dependencies = [ "cc", "cmake", @@ -1052,6 +1052,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1248,7 +1254,7 @@ dependencies = [ "cached_proc_macro_types", "hashbrown 0.15.5", "once_cell", - "thiserror 2.0.19", + "thiserror 2.0.20", "web-time", ] @@ -1278,9 +1284,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -1382,9 +1388,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.5" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" dependencies = [ "clap_builder", "clap_derive", @@ -1392,9 +1398,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.5" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" dependencies = [ "anstream", "anstyle", @@ -1405,9 +1411,9 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.6.8" +version = "4.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f84a88507dbd05c695f2cb5e8558e747179134005e9893882dec964190ed89" +checksum = "3be2ad0423bdbbb0e25bc89add796f3559706d4a95e1bc98e4d9662a957b6a19" dependencies = [ "clap", "clap_lex", @@ -1417,9 +1423,9 @@ dependencies = [ [[package]] name = "clap_complete_nushell" -version = "4.6.1" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "933b05d5d83ff65fd7eaf5d106c792f2264908790a2642aca57429767b762ce2" +checksum = "ffb66bc82eb9c92b1727310ae2c5868df22ae7cf46185bc5c544a4fa71955e49" dependencies = [ "clap", "clap_complete", @@ -1532,7 +1538,7 @@ dependencies = [ "lazy_static", "serde", "serde_yaml", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -1612,9 +1618,9 @@ dependencies = [ [[package]] name = "cookie" -version = "0.18.1" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87" dependencies = [ "percent-encoding", "time", @@ -1667,7 +1673,7 @@ dependencies = [ "async-trait", "aws-smithy-eventstream", "aws-smithy-types", - "base64", + "base64 0.22.1", "bincode 2.0.1", "bitflags 2.13.1", "bm25", @@ -2033,7 +2039,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" dependencies = [ - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -2487,9 +2493,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "fixedbitset" @@ -2562,9 +2568,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -2577,9 +2583,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -2587,15 +2593,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -2604,38 +2610,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-macro" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] name = "futures-sink" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -2781,7 +2787,7 @@ dependencies = [ "azure_identity", "azure_security_keyvault_secrets", "backtrace", - "base64", + "base64 0.22.1", "chacha20poly1305", "chrono", "clap", @@ -2807,7 +2813,7 @@ dependencies = [ "serde_with", "serde_yaml", "tempfile", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "validator", "which", @@ -3214,7 +3220,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -3528,10 +3534,12 @@ dependencies = [ "defmt", "jiff-core", "jiff-static", + "jiff-tzdb-platform", "log", "portable-atomic", "portable-atomic-util", "serde_core", + "windows-link", ] [[package]] @@ -3555,6 +3563,21 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "jiff-tzdb" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jni" version = "0.22.4" @@ -3567,7 +3590,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.19", + "thiserror 2.0.20", "walkdir", "windows-link", ] @@ -3616,9 +3639,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.103" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" dependencies = [ "cfg-if", "futures-util", @@ -3634,7 +3657,7 @@ dependencies = [ "jsonptr", "serde", "serde_json", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -3653,7 +3676,7 @@ version = "10.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" dependencies = [ - "base64", + "base64 0.22.1", "getrandom 0.2.17", "js-sys", "pem", @@ -3845,7 +3868,7 @@ dependencies = [ "serde-value", "serde_json", "serde_yaml", - "thiserror 2.0.19", + "thiserror 2.0.20", "thread-id", "typemap-ors", "unicode-segmentation", @@ -3992,7 +4015,7 @@ dependencies = [ "mach2", "nix 0.30.1", "sysctl", - "thiserror 2.0.19", + "thiserror 2.0.20", "widestring", "windows 0.48.0", ] @@ -4389,9 +4412,9 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "open" -version = "5.4.0" +version = "5.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0b3d059e795d52b8a72fef45658620edd4d9c359b338564aa14391ffa511ed5" +checksum = "f9cfef937e9c486488c7e3d949ae31c0f1d06bdacd75b99c086cb35356e30408" dependencies = [ "is-wsl", "libc", @@ -4566,7 +4589,7 @@ version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64", + "base64 0.22.1", "serde_core", ] @@ -4705,7 +4728,7 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da1d65da6dd5d1e44199ac0f58712d241c0f439f80adea8924d832384087f85" dependencies = [ - "base64", + "base64 0.22.1", "indexmap 2.14.0", "quick-xml 0.41.0", "serde", @@ -4725,9 +4748,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "portable-atomic-util" @@ -4912,7 +4935,7 @@ dependencies = [ "rustc-hash", "rustls 0.23.43", "socket2 0.6.5", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tracing", "web-time", @@ -4935,7 +4958,7 @@ dependencies = [ "rustls 0.23.43", "rustls-pki-types", "slab", - "thiserror 2.0.19", + "thiserror 2.0.20", "tinyvec", "tracing", "web-time", @@ -5086,7 +5109,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -5103,7 +5126,7 @@ dependencies = [ "serde", "strip-ansi-escapes", "strum", - "thiserror 2.0.19", + "thiserror 2.0.20", "unicase", "unicode-segmentation", "unicode-width", @@ -5143,9 +5166,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.16" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -5170,7 +5193,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "futures-util", @@ -5214,7 +5237,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-core", @@ -5285,7 +5308,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d1f571c72940a19d9532fe52dbea8bc9912bf1d766c2970bb824056b86f3f59" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "chrono", "futures", "http 1.5.0", @@ -5298,7 +5321,7 @@ dependencies = [ "serde", "serde_json", "sse-stream", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tokio-stream", "tokio-util", @@ -5460,7 +5483,7 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.13", + "rustls-webpki 0.103.14", "subtle", "zeroize", ] @@ -5501,7 +5524,7 @@ dependencies = [ "rustls 0.23.43", "rustls-native-certs", "rustls-platform-verifier-android", - "rustls-webpki 0.103.13", + "rustls-webpki 0.103.14", "security-framework", "security-framework-sys", "webpki-root-certs", @@ -5526,9 +5549,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a" dependencies = [ "aws-lc-rs", "ring", @@ -5828,16 +5851,17 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.21.0" +version = "3.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a" dependencies = [ - "base64", + "base64 0.22.1", "bs58", "chrono", "hex", "indexmap 1.9.3", "indexmap 2.14.0", + "jiff", "schemars 0.9.0", "schemars 1.2.2", "serde_core", @@ -5848,9 +5872,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.21.0" +version = "3.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -6055,7 +6079,7 @@ checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" dependencies = [ "num-bigint", "num-traits", - "thiserror 2.0.19", + "thiserror 2.0.20", "time", ] @@ -6283,7 +6307,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "thiserror 2.0.19", + "thiserror 2.0.20", "walkdir", ] @@ -6417,11 +6441,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.19", + "thiserror-impl 2.0.20", ] [[package]] @@ -6437,9 +6461,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", @@ -6658,7 +6682,7 @@ checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "axum", - "base64", + "base64 0.22.1", "bytes", "h2 0.4.15", "http 1.5.0", @@ -6779,9 +6803,9 @@ dependencies = [ [[package]] name = "tree-sitter" -version = "0.26.11" +version = "0.26.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af1c71c1c4cc0920b20d6b0f6572e7682cd07a6a2faec71067a31fa394c586df" +checksum = "83c567a8e18ae93f20982c90370b16fd24023aeaf52f6052b96957ab253a0fec" dependencies = [ "cc", "regex", @@ -6855,7 +6879,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dd1eb4a538c1ab3d5c05437129bc16891296146b23c9b0bb3f5df99f5b3a18d" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures", "serde", @@ -6870,7 +6894,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e632235c99ae896a3c451d1ead00cea11a2219aeda1b35a74027fe99ea3f3b72" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "dyn-clone", "futures", "getrandom 0.3.4", @@ -6981,11 +7005,11 @@ checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" [[package]] name = "ureq" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" +checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d" dependencies = [ - "base64", + "base64 0.23.1", "cookie_store", "encoding_rs", "flate2", @@ -7003,11 +7027,11 @@ dependencies = [ [[package]] name = "ureq-proto" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +checksum = "da5f78b09e6941e1a0f2e30e695e4b120377b54d5e0aec11b594bb57b3971613" dependencies = [ - "base64", + "base64 0.23.1", "http 1.5.0", "httparse", "log", @@ -7159,9 +7183,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" dependencies = [ "cfg-if", "once_cell", @@ -7172,9 +7196,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.76" +version = "0.4.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" dependencies = [ "js-sys", "wasm-bindgen", @@ -7182,9 +7206,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -7192,9 +7216,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" dependencies = [ "bumpalo", "proc-macro2", @@ -7205,9 +7229,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" dependencies = [ "unicode-ident", ] @@ -7310,9 +7334,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.103" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" dependencies = [ "js-sys", "wasm-bindgen", @@ -7749,7 +7773,7 @@ dependencies = [ "log", "os_pipe", "rustix 1.1.4", - "thiserror 2.0.19", + "thiserror 2.0.20", "tree_magic_mini", "wayland-backend", "wayland-client", @@ -7844,18 +7868,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" dependencies = [ "proc-macro2", "quote", @@ -7957,9 +7981,9 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dba6063ff82cdbd9a765add16d369abe81e520f836054e997c2db217ceca40c0" dependencies = [ - "base64", + "base64 0.22.1", "ed25519-dalek", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] diff --git a/src/config/paths.rs b/src/config/paths.rs index 177864d..f49f688 100644 --- a/src/config/paths.rs +++ b/src/config/paths.rs @@ -414,11 +414,10 @@ pub fn list_rags() -> Vec { for entry in rd.flatten() { let name = entry.file_name(); if let Some(name) = name.to_string_lossy().strip_suffix(".yaml") { - // Sidecars are not RAGs. `.duckdb` files are already excluded by - // the `.yaml` suffix check above; this rejects `.sbx-mixin`. if is_rag_sidecar_name(name) { continue; } + names.push(name.to_string()); } } @@ -429,24 +428,10 @@ pub fn list_rags() -> Vec { } } -/// True for the sidecar YAML files that must never be listed or deleted as RAGs. -/// `name` is the already-stripped stem (i.e. after `strip_suffix(".yaml")`). -/// Uses `ends_with`, not `contains('.')`, so a RAG legitimately named "v2.docs" is -/// not rejected. pub(crate) fn is_rag_sidecar_name(name: &str) -> bool { name.ends_with(".sbx-mixin") } -/// Remove every sidecar belonging to RAG `name` in `dir`. Missing files are NOT an -/// error. A failure to remove an EXISTING mixin IS an error and must propagate — a -/// silently-orphaned mixin keeps a sandbox network permission alive after the user -/// believes it is gone. The `.duckdb` orphan is only wasted disk, so its removal -/// failure is ignorable; the asymmetry is deliberate. -/// -/// Callers must run this BEFORE unlinking the primary `.yaml`. If the YAML goes first -/// and this then fails, the RAG disappears from `list_rags()` — so the user can no -/// longer select it to retry — while its network allow entry keeps being injected -/// into every sandbox launch. pub(crate) fn remove_rag_sidecars(dir: &Path, name: &str) -> Result<()> { let duckdb_path = dir.join(format!("{name}.duckdb")); if duckdb_path.exists() { @@ -463,6 +448,7 @@ pub(crate) fn remove_rag_sidecars(dir: &Path, name: &str) -> Result<()> { ) })?; } + Ok(()) } @@ -889,8 +875,6 @@ mod tests { let _ = fs::remove_dir_all(&root); } - /// Unique temp dir for the sidecar helper tests. These take `dir: &Path` directly, - /// so no env-var mutation and therefore no `#[serial]` is needed. fn sidecar_temp_dir(label: &str) -> PathBuf { let unique = time::SystemTime::now() .duration_since(time::UNIX_EPOCH) @@ -903,7 +887,6 @@ mod tests { #[test] fn is_rag_sidecar_name_accepts_dotted_rag_names() { - // A RAG legitimately named "v2.docs" must not be mistaken for a sidecar. assert!(!is_rag_sidecar_name("v2.docs")); assert!(!is_rag_sidecar_name("myrag")); assert!(is_rag_sidecar_name("myrag.sbx-mixin")); @@ -940,8 +923,6 @@ mod tests { let root = sidecar_temp_dir("rag-sidecars-order"); let yaml = root.join("docs.yaml"); fs::write(&yaml, "rag").unwrap(); - // A non-empty DIRECTORY at the mixin path makes remove_file fail, standing in - // for any real removal failure (permissions, a busy mount). let mixin = root.join("docs.sbx-mixin.yaml"); fs::create_dir_all(&mixin).unwrap(); fs::write(mixin.join("blocker"), "x").unwrap(); @@ -952,8 +933,6 @@ mod tests { .contains("Failed to remove the sandbox mixin"), "got: {err}" ); - // The whole point of removing sidecars first: the RAG is still on disk, still - // listed, and the deletion is retryable. assert!( yaml.exists(), "the .yaml must survive a sidecar-removal failure so the delete is retryable" diff --git a/src/config/request_context.rs b/src/config/request_context.rs index 5260518..c0eea43 100644 --- a/src/config/request_context.rs +++ b/src/config/request_context.rs @@ -142,11 +142,6 @@ pub struct RequestContext { pub role: Option, pub session: Option, pub rag: Option>, - /// The cache key `self.rag` was actually inserted under, carried rather than - /// reconstructed. Reconstruction was the bug: the invalidation sites do not have - /// the information needed to rebuild the key (agent RAGs are inserted under the - /// AGENT's name but `rag.name()` is the constant "rag"), so insert and invalidate - /// silently disagreed. `None` for the temp RAG, which bypasses the cache entirely. pub rag_key: Option, pub agent: Option, @@ -4122,14 +4117,10 @@ impl RequestContext { } let app = self.app.config.clone(); - // Hoisted: `rag_cache` below borrows `self`, so the loader closure cannot - // reach through `self` for the vault. `GlobalVault` is an Arc, so this is cheap. let vault = self.app.vault.clone(); let rag_cache = self.rag_cache(); let working_mode = self.working_mode; - // The key is returned alongside the Rag rather than assigned inside the match: - // `rag_cache` borrows `self`, so writing `self.rag_key` there is E0506. let (rag, rag_key): (Arc, Option) = match rag { None => { let rag_path = self.rag_file(super::TEMP_RAG_NAME); @@ -4138,7 +4129,6 @@ impl RequestContext { format!("Failed to cleanup previous '{}' rag", super::TEMP_RAG_NAME) })?; } - // The temp RAG is never inserted into the cache, so it has no key. ( Arc::new( Rag::init( @@ -4198,13 +4188,9 @@ impl RequestContext { let vault = self.app.vault.clone(); let rag = Rag::attach(app, &vault, name, &rag_path).await?; let rag = Arc::new(rag); - // Populate the cache so a later `.rag ` reuses this instance rather - // than re-running the network preflight. Attach is always a global RAG. let key = RagKey::Named(name.to_string()); self.rag_cache().insert(key.clone(), &rag); self.rag = Some(rag); - // Carried so invalidation in rebuild_rag()/edit_rag_docs() can find this - // entry; without it a stale Arc would linger in the cache all session. self.rag_key = Some(key); Ok(()) } @@ -4217,7 +4203,7 @@ impl RequestContext { if rag.is_attached() { bail!( - "Cannot edit documents on an attached RAG — Coyote does not own its source documents." + "Cannot edit documents on an attached RAG; Coyote does not own its source documents." ); } @@ -4270,7 +4256,7 @@ impl RequestContext { if rag.is_attached() { bail!( - "Cannot rebuild an attached RAG — Coyote does not own its source documents. \ + "Cannot rebuild an attached RAG; Coyote does not own its source documents. \ Re-index from the system that originally created '{}'.", rag.name() ); @@ -4716,8 +4702,6 @@ mod tests { ) .unwrap(); - // Stand in for the state `.rag docs` leaves behind: `use_rag` sets `rag` and - // `rag_key` together, so a named key is live when the agent is entered. ctx.rag_key = Some(RagKey::Named("docs".to_string())); tokio::runtime::Builder::new_current_thread() @@ -4730,9 +4714,6 @@ mod tests { .unwrap(); }); - // This agent has no RAG, so `rag` is None and `rag_key` must be None as well. - // Carrying `Named("docs")` across the transition would point `.rebuild rag` - // at an unrelated RAG's cache entry. assert!(ctx.rag.is_none()); assert_eq!(ctx.rag_key, None); } @@ -6122,9 +6103,6 @@ mod tests { assert!(paths::list_rags().is_empty()); } - /// A `.sbx-mixin.yaml` sidecar must not appear as a phantom RAG in TAB - /// completion or `.list rag`. A RAG whose name legitimately contains a dot must - /// still be listed — the filter uses `ends_with`, not `contains('.')`. #[test] #[serial] fn list_rags_skips_sbx_mixin_sidecars() { diff --git a/src/rag/mod.rs b/src/rag/mod.rs index 29f298a..dc20877 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -12,9 +12,6 @@ mod splitter; use self::graph::{KnowledgeGraph, extract_entities}; use self::provider::RagProvider; -// `providers::duckdb_path_from_yaml(path)` is called through the module path in -// `create()`, so `providers` itself must stay in scope — do not collapse it into -// the `use` below. use self::providers::{DuckDbProvider, QdrantProvider, YamlProvider}; use crate::sandbox::mcp_credentials; use crate::vault::{Vault, interpolate_secrets}; @@ -65,10 +62,7 @@ pub struct Rag { name: String, path: String, embedding_model: Model, - // Local BM25: keyword search + graph seeding. Always built from `data.files` - // regardless of driver, and kept on `Rag` so the sync `graph_search` can use it. bm25: SearchEngine, - // Vector storage + content retrieval. provider: Box, data: RagData, last_sources: RwLock>, @@ -126,8 +120,7 @@ pub struct RagInitConfig { pub extractor_model: Option, pub extractor_prompt: Option, pub graph_hops: Option, - /// `None` -> "yaml". No serde attribute: this struct derives only - /// `Debug, Clone, Default` and is built in Rust, never deserialized. + /// `None` -> "yaml" pub driver: Option, } @@ -367,8 +360,6 @@ impl Rag { // plaintext. let data: RagData = serde_yaml::from_str(&raw_content).with_context(err)?; - // Validated before the match so the rule applies to every driver, including - // the sync fallthrough below. data.validate().with_context(err)?; match data.driver.as_str() { @@ -384,7 +375,6 @@ impl Rag { .context("qdrant driver requires 'collection' in driver_config")? .clone(); - // Resolved out of band and kept in a local; it never enters `data`. let api_key: Option = match data.driver_config.get("api_key") { Some(placeholder) => { let (resolved, _) = @@ -411,14 +401,10 @@ impl Rag { last_sources: RwLock::new(None), }) } - // yaml/duckdb take the sync path. It re-reads and re-parses the file; - // that cost is accepted to keep every existing caller untouched. _ => Self::load(app, name, path), } } - /// Connects to a pre-existing external collection. Coyote is a query-only - /// client here: it never indexes documents into it. pub async fn attach( app: &AppConfig, vault: &Vault, @@ -435,8 +421,6 @@ impl Rag { let host = Text::new("Host (e.g. qdrant.company.com:6333):") .with_validator(required!("This field is required")) .with_validator(|input: &str| { - // Bracketed IPv6 literals would produce a malformed sandbox - // network allow entry, so refuse them at the prompt. Ok(if input.contains('[') || input.contains(']') { Validation::Invalid( "Bracketed IPv6 literals are not supported; use a hostname.".into(), @@ -583,7 +567,6 @@ impl Rag { Ok(rag) } - /// `mut data` — the duckdb arm rehydrates `data.vectors` from the sidecar. pub fn create(app: &AppConfig, name: &str, path: &Path, mut data: RagData) -> Result { // Deliberately does NOT call rebuild_indexes: both callers construct the Rag // before any documents are added, so rebuilding empty data would be a no-op. @@ -603,11 +586,11 @@ impl Rag { // Guarded on is_empty() so a caller that already has vectors in memory // is never overwritten by an empty table. // - // 🔴 `?`, NOT `unwrap_or_default()`. A hydration failure must propagate. - // Degrading to an empty map here loads a RAG that looks healthy, answers - // every query with nothing, and then loses the store permanently on the - // first `.edit rag-docs`. The legitimate "nothing indexed yet" case is - // already Ok(empty) — open() runs CREATE TABLE IF NOT EXISTS — so `?` + // WARNING: `?`, NOT `unwrap_or_default()`. A hydration failure must + // propagate. Degrading to an empty map here loads a RAG that looks healthy, + // answers every query with nothing, and then loses the store permanently + // on the first `.edit rag-docs`. The legitimate "nothing indexed yet" case + // is already Ok(empty) (open() runs CREATE TABLE IF NOT EXISTS) so `?` // costs a new RAG nothing. if data.vectors.is_empty() { data.vectors = duck.read_all_vectors()?; @@ -622,7 +605,6 @@ impl Rag { use Rag::attach() or Rag::load_async() instead" ), _ => { - // "yaml" and any unknown driver — in-memory HNSW. let bm25 = data.build_bm25(); (Box::new(YamlProvider::from_data(&data)), bm25) } @@ -730,7 +712,7 @@ impl Rag { // `data.files` is empty for an attached RAG; the local index is not the // source of truth. A static label is honest, an empty list is not. *self.last_sources.write() = - Some("[attached RAG — source list unavailable]".to_string()); + Some("[Using attached RAG. Source list unavailable]".to_string()); return; } let mut sources: IndexMap> = IndexMap::new(); @@ -922,6 +904,7 @@ impl Rag { if self.data.attached { return format!("- {}", self.data.attached_source_label()); } + let mut seen = IndexSet::new(); for id in ids { let (file_index, _) = id.split(); @@ -1202,20 +1185,13 @@ impl Rag { let keyword_search_results: Vec<(DocumentId, f32)> = if self.provider.has_native_keyword_search() { - // Keyword is ONE of three RRF rankers (vector + keyword + graph); - // its absence is survivable and produces a slightly worse ranking, - // whereas a `?` here turns a provider FTS fault into TOTAL query - // failure — the user gets an error instead of the results the - // vector and graph rankers already retrieved. Degrade, do not - // propagate, and do not silently swap in the local BM25 either: - // that would change the ranking algorithm mid-query. - match self.provider.keyword_search(query, top_k).await { - Ok(v) => v, - Err(e) => { + self.provider + .keyword_search(query, top_k) + .await + .unwrap_or_else(|e| { warn!("native keyword search failed, dropping the keyword ranker: {e}"); Vec::new() - } - } + }) } else { self.keyword_search(query, top_k, 0.0) }; @@ -1231,8 +1207,6 @@ impl Rag { .concat() .into_iter() .collect(); - // `ids` is an `IndexSet` here, not the `Vec` of the RRF branch below, - // and `&IndexSet<_>` does not coerce to `&[DocumentId]`. let ids: Vec = ids.into_iter().collect(); let fetched = self.provider.fetch_content(&ids).await?; // Build both vectors from the SAME source in the SAME iteration — @@ -1276,8 +1250,6 @@ impl Rag { ids } }; - // `ids` is the ranked list; `fetch_content` preserves that order per the - // trait's ordering contract, so the result is returned as-is. let output = self.provider.fetch_content(&ids).await?; Ok(output) } @@ -1308,7 +1280,7 @@ impl Rag { Ok(merge_vector_results(results)) } - /// Local in-memory BM25 over `data.files` — empty for attached RAGs, which is + /// Local in-memory BM25 over `data.files`. This is empty for attached RAGs, which is /// correct: they have no local text. fn keyword_search(&self, query: &str, top_k: usize, min_score: f32) -> Vec<(DocumentId, f32)> { let results = self.bm25.search(query, top_k); @@ -1479,9 +1451,6 @@ pub struct RagData { pub driver: String, #[serde(default)] pub attached: bool, - /// Driver-specific connection parameters (qdrant: `host`, `collection`, `api_key`). - /// Secret-bearing values are stored as `{{SECRET_NAME}}` placeholders and resolved - /// out of band at load time, so a resolved credential never reaches disk. #[serde(default, skip_serializing_if = "IndexMap::is_empty")] pub driver_config: IndexMap, @@ -1589,6 +1558,7 @@ impl RagData { no results with no error. Set `top_k:` in the RAG YAML." ); } + if !self.attached { if self.chunk_size == 0 { bail!( @@ -1597,6 +1567,7 @@ impl RagData { embedding batches. Set `chunk_size:` in the RAG YAML." ); } + if self.chunk_overlap >= self.chunk_size { bail!( "chunk_overlap ({}) must be strictly less than chunk_size ({}).", @@ -1605,6 +1576,7 @@ impl RagData { ); } } + match (self.driver.as_str(), self.attached) { ("yaml", false) => Ok(()), ("duckdb", false) => Ok(()), @@ -1626,7 +1598,7 @@ impl RagData { /// Every (DocumentId, &RagDocument) in the corpus, in `files` order. /// - /// This — NOT `vectors` — is the authoritative document id space. BM25, the + /// This, NOT `vectors`, is the authoritative document id space. BM25, the /// knowledge graph and content lookup all key off it; `vectors` is a subset, /// since `add`'s zip truncates whenever fewer embeddings come back than /// document ids were sent. @@ -1783,9 +1755,6 @@ fn generate_rag_sbx_mixin( header_name: &str, value_format: &str, ) -> Result<()> { - // The client reaches the store through `normalize_base_url`, so deriving the - // allow entry from that same URL keeps the whitelist and the actual dialled - // port from drifting apart. let base_url = QdrantProvider::normalize_base_url(host); let Some(allow_entry) = mcp_credentials::allow_entry_for_url(&base_url) else { eprintln!( @@ -1793,6 +1762,7 @@ fn generate_rag_sbx_mixin( grammar, so no sandbox mixin was written for RAG '{service_name}'. \ Queries to this RAG will be blocked inside the sandbox." ); + return Ok(()); }; @@ -1826,12 +1796,11 @@ fn generate_rag_sbx_mixin( mixin_path.display() ) })?; + println!("✓ Sandbox mixin: '{}'.", mixin_path.display()); Ok(()) } -/// Bearer credentials are spelled as a `scheme`, everything else as an explicit -/// `header`; the two are mutually exclusive in the inject grammar. fn rag_inject_rule( domain: &str, header_name: &str, @@ -1856,8 +1825,6 @@ fn rag_inject_rule( } } -/// Derives a deterministic env var name from a RAG name: -/// `company-docs` → `COMPANY_DOCS_API_KEY`. fn rag_env_var_name(rag_name: &str) -> String { format!( "{}_API_KEY", @@ -1865,8 +1832,6 @@ fn rag_env_var_name(rag_name: &str) -> String { ) } -/// How a driver authenticates its HTTP requests. Qdrant uses a bare `api-key` -/// header rather than `Authorization: Bearer`. fn driver_auth_header(driver: &str) -> (&'static str, &'static str) { match driver { "qdrant" => ("api-key", "%s"), @@ -2104,8 +2069,6 @@ fn find_hash_skip( /// so the pool is bounded by `top_k * query_chunks`, and `reciprocal_rank_fusion` /// truncates to `top_k` itself. Capping here would let whichever query chunk has /// the strongest absolute scores crowd out every other chunk's hits. -/// -/// Free function (not a method) so it is unit-testable without an embeddings client. fn merge_vector_results(mut results: Vec<(DocumentId, f32)>) -> Vec<(DocumentId, f32)> { debug_assert!( results.iter().all(|(_, score)| score.is_finite()), @@ -2161,16 +2124,17 @@ fn embedding_dim_for_model(model_id: &str) -> usize { #[cfg(test)] mod tests { use super::*; + use std::path::PathBuf; + use std::time::{SystemTime, UNIX_EPOCH}; - /// Scratch directory for tests that must write a real file. struct TempDir { - path: std::path::PathBuf, + path: PathBuf, } impl TempDir { fn new(tag: &str) -> Self { - let unique = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) .unwrap() .as_nanos(); let path = env::temp_dir().join(format!("coyote-rag-{tag}-{unique}")); @@ -2272,10 +2236,6 @@ mod tests { .collect() } - /// The generated mixin must carry the kit v2 schema envelope: - /// `wrap_mixin_as_kit` copies it verbatim to `spec.yaml` for - /// `sbx create --kit`, with no `kind` rewrite and no validation on Coyote's - /// side, so a stale envelope breaks the launch itself. #[test] fn generated_sbx_mixin_carries_the_schema_envelope() { let (text, parsed) = render_rag_mixin( @@ -2290,14 +2250,11 @@ mod tests { text.starts_with("schemaVersion:"), "envelope must come first:\n{text}" ); - // Quoted string, never the numeric 2 — the kit rejects an int here. assert_eq!(parsed["schemaVersion"].as_str(), Some("2")); assert_eq!(parsed["kind"].as_str(), Some("mixin")); assert_eq!(parsed["name"].as_str(), Some("rag-company-docs")); assert!(parsed["description"].as_str().is_some()); - // https/443 is the only bare-host case; 6333 must carry its port, and - // the bare host must NOT also be listed. assert_eq!(allow_list(&parsed), vec!["rag.example.com:6333"]); let credential = &parsed["credentials"][0]; @@ -2313,14 +2270,11 @@ mod tests { assert_eq!(inject["header"].as_str(), Some("api-key")); assert_eq!(inject["format"].as_str(), Some("%s")); - // sbx does not derive allow entries from inject rules; an inject - // domain that is not allowed is a dead rule. assert!( allow_list(&parsed).contains(&inject["domain"].as_str().unwrap().to_string()), "every inject domain must also appear in allow:\n{text}" ); - // The v1 vocabulary is gone, not merely unused. for dead in ["allowedDomains", "serviceDomains", "serviceAuth"] { assert!(!text.contains(dead), "v1 key '{dead}' survived:\n{text}"); } @@ -2334,8 +2288,6 @@ mod tests { ); } - /// The allow entry has to name the port the client actually dials, which is - /// whatever `normalize_base_url` resolves to — not a Qdrant-specific guess. #[test] fn generated_sbx_mixin_allows_the_port_the_client_dials() { let cases = [ @@ -2361,8 +2313,6 @@ mod tests { } } - /// `Authorization: Bearer` is spelled as a scheme; `header` and `scheme` are - /// mutually exclusive in the inject grammar. #[test] fn generated_sbx_mixin_spells_bearer_as_a_scheme() { let (_, parsed) = render_rag_mixin( @@ -2372,16 +2322,14 @@ mod tests { "Authorization", "Bearer %s", ); + let inject = &parsed["credentials"][0]["apiKey"]["inject"][0]; + assert_eq!(inject["scheme"].as_str(), Some("bearer")); assert!(inject["header"].is_null()); assert!(inject["format"].is_null()); } - /// The bind in `inject_rag_secrets` and the `service` declared here both run - /// the RAG name through `secret_service_id`. If they disagreed, the proxy - /// would hold a value under one id and an inject rule under another, and the - /// header would never be rewritten. #[test] fn generated_sbx_mixin_service_id_matches_the_host_side_bind() { let (_, parsed) = render_rag_mixin( @@ -2391,9 +2339,10 @@ mod tests { "api-key", "%s", ); + assert_eq!( parsed["credentials"][0]["service"].as_str(), - Some(crate::sandbox::mcp_credentials::secret_service_id("My_Docs").as_str()) + Some(mcp_credentials::secret_service_id("My_Docs").as_str()) ); assert_eq!( parsed["credentials"][0]["service"].as_str(), @@ -2401,12 +2350,11 @@ mod tests { ); } - /// A store with no API key still needs egress, but declaring a credential - /// nothing ever binds would leave sbx waiting on a binding that never comes. #[test] fn generated_sbx_mixin_omits_credentials_when_there_is_no_api_key() { let (text, parsed) = render_rag_mixin("https://store.example.com", "docs", None, "api-key", "%s"); + assert_eq!(allow_list(&parsed), vec!["store.example.com"]); assert!( parsed["credentials"].is_null(), @@ -2423,7 +2371,6 @@ mod tests { #[test] fn driver_auth_header_uses_a_bare_api_key_for_qdrant() { - // Qdrant's REST API reads `api-key`, NOT `Authorization: Bearer`. assert_eq!(driver_auth_header("qdrant"), ("api-key", "%s")); assert_eq!( driver_auth_header("something-else"), @@ -2431,8 +2378,6 @@ mod tests { ); } - /// An attached RAG has no local `files`, so the citation helpers would - /// otherwise emit "unknown" and an empty source list for every result. #[test] fn attached_rag_citation_helpers_do_not_fall_back_to_the_empty_file_index() { let mut data = RagData { @@ -2446,7 +2391,6 @@ mod tests { .insert("collection".into(), "company-kb".into()); assert!(data.files.is_empty()); - // The real helper — `resolve_source`/`format_sources` both delegate here. assert_eq!( data.attached_source_label(), "[external collection: company-kb]" @@ -2580,6 +2524,7 @@ mod tests { .iter_documents() .map(|(id, doc)| (id, doc.page_content.as_str())) .collect(); + assert_eq!( documents, vec![ @@ -2600,12 +2545,10 @@ mod tests { None, GraphRagConfig::default(), ); + assert_eq!(data.iter_documents().count(), 0); } - /// The document id space is `files`, never `vectors`: `add`'s zip truncates - /// silently, so a vector may exist for an id no file provides. Content lookup - /// and BM25 both key off this iterator and must agree. #[test] fn rag_data_iter_documents_ignores_vector_only_ids() { let mut data = RagData::new( @@ -2876,14 +2819,17 @@ mod tests { #[test] fn merge_vector_results_empty_input() { - let result = super::merge_vector_results(vec![]); + let result = merge_vector_results(vec![]); + assert!(result.is_empty(), "empty input should produce empty output"); } #[test] fn merge_vector_results_keeps_best_score_per_document() { let doc = DocumentId::new(0, 0); - let result = super::merge_vector_results(vec![(doc, 0.2), (doc, 0.9)]); + + let result = merge_vector_results(vec![(doc, 0.2), (doc, 0.9)]); + assert_eq!(result.len(), 1, "a document must not be double-counted"); assert_eq!(result[0].0, doc); assert_eq!( @@ -2897,10 +2843,10 @@ mod tests { let doc_a = DocumentId::new(0, 0); let doc_b = DocumentId::new(1, 0); let doc_c = DocumentId::new(2, 0); - // Interleaved as two per-chunk hit lists would arrive: concatenating them - // would yield a, c, b — only a global sort produces c, a, b. - let result = super::merge_vector_results(vec![(doc_a, 0.5), (doc_c, 0.9), (doc_b, 0.1)]); + + let result = merge_vector_results(vec![(doc_a, 0.5), (doc_c, 0.9), (doc_b, 0.1)]); let ids: Vec = result.iter().map(|(id, _)| *id).collect(); + assert_eq!(ids, vec![doc_c, doc_a, doc_b]); } @@ -2909,7 +2855,9 @@ mod tests { let input: Vec<(DocumentId, f32)> = (0..10) .map(|i| (DocumentId::new(i, 0), i as f32 / 10.0)) .collect(); - let result = super::merge_vector_results(input); + + let result = merge_vector_results(input); + assert_eq!( result.len(), 10, @@ -2935,6 +2883,7 @@ mod tests { #[test] fn force_reingest_re_embeds_hash_identical_files() { let (files, to_deleted) = hash_skip_fixture(); + assert_eq!( find_hash_skip(true, &to_deleted, &files, "abc", "test.txt"), None, @@ -2945,6 +2894,7 @@ mod tests { #[test] fn refresh_without_force_still_hash_skips() { let (files, to_deleted) = hash_skip_fixture(); + assert_eq!( find_hash_skip(false, &to_deleted, &files, "abc", "test.txt"), Some((0, 7)), @@ -2955,6 +2905,7 @@ mod tests { #[test] fn find_hash_skip_returns_none_on_path_change() { let (files, to_deleted) = hash_skip_fixture(); + assert_eq!( find_hash_skip(false, &to_deleted, &files, "abc", "moved.txt"), None @@ -2976,6 +2927,7 @@ mod tests { None, GraphRagConfig::default(), ); + assert_eq!(data.driver, "yaml"); assert!(!data.attached); } @@ -2992,7 +2944,9 @@ document_paths: [] files: {} vectors: {} "; + let data: RagData = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(data.driver, "yaml"); assert!(!data.attached); } @@ -3013,6 +2967,7 @@ vectors: {} let yaml = serde_yaml::to_string(&data).unwrap(); let restored: RagData = serde_yaml::from_str(&yaml).unwrap(); + assert_eq!(restored.driver, "qdrant"); assert!(restored.attached); } @@ -3029,7 +2984,9 @@ vectors: {} GraphRagConfig::default(), ); data.attached = true; + let err = data.validate().unwrap_err().to_string(); + assert!(err.contains("cannot be attached"), "got: {err}"); } @@ -3046,6 +3003,7 @@ vectors: {} ); data.driver = "qdrant".to_string(); data.attached = true; + assert!(data.validate().is_ok()); } diff --git a/src/rag/provider.rs b/src/rag/provider.rs index 6889e81..3c75fd2 100644 --- a/src/rag/provider.rs +++ b/src/rag/provider.rs @@ -4,11 +4,6 @@ use async_trait::async_trait; /// Abstracts where RAG vector data is stored and queried. /// -/// Implementors: -/// - YamlProvider: HNSW in-memory, state derived from RagData.vectors/files -/// - DuckDbProvider: DuckDB on-disk vector index + document store -/// - QdrantProvider: remote Qdrant collection -/// /// The Rag orchestrator owns: embeddings, chunking, BM25 keyword search, graph RAG, /// entity extraction, RRF merging. Providers own: vector storage and content retrieval. #[async_trait] @@ -26,7 +21,7 @@ pub trait RagProvider: Send + Sync { /// /// **Ordering contract:** implementations MUST return results in the same /// relative order as the input `ids` slice. `hybrid_search` passes an - /// RRF-ranked list and feeds the result straight to the LLM — a provider + /// RRF-ranked list and feeds the result straight to the LLM. A provider /// that returns rows in storage order (e.g. Qdrant `get_points`, DuckDB /// `WHERE id IN (...)`) would silently discard the ranking. Implementations /// that query an unordered backend must re-sort by input position before @@ -43,34 +38,28 @@ pub trait RagProvider: Send + Sync { /// Called once at the end of every sync_documents pass. /// /// `full_rebuild` mirrors `sync_documents`' `refresh` parameter: - /// - `true` — a full re-index (`.rebuild rag`, `--rebuild-rag`, initial build). + /// - `true`: a full re-index (`.rebuild rag`, `--rebuild-rag`, initial build). /// Destructive strategies (wipe-then-reindex) are permitted. - /// - `false` — an incremental change (`.edit rag-docs` adding/removing a file). + /// - `false`: an incremental change (`.edit rag-docs` adding/removing a file). /// Implementations MUST NOT wipe existing state; upsert only. /// /// The parameter is part of the signature from the outset so it is fixed - /// while there is exactly one implementor. Yaml/DuckDb ignore it — + /// while there is exactly one implementor. Yaml/DuckDb ignore it, /// rebuilding their local state wholesale is fast and always correct. /// Only a remote provider is destructive enough to care. - /// - /// YamlProvider: rebuilds HNSW + content map from data.vectors/files. - /// DuckDbProvider: writes new rows to DuckDB, deletes removed rows. - /// QdrantProvider: no-op while attach-only — remote data is unchanged. async fn rebuild_indexes(&mut self, data: &RagData, full_rebuild: bool) -> Result<()>; /// Keyword / full-text search. Returns (DocumentId, BM25-style score) sorted desc. /// - /// Default impl returns `Ok(vec![])` — callers fall back to `Rag.bm25` (local in-memory - /// BM25 built from `data.files`). DuckDbProvider overrides this with a native FTS query - /// (DuckDB's `fts` extension, installed once at schema-creation time). + /// Default impl returns `Ok(vec![])`. Callers fall back to `Rag.bm25` (local in-memory + /// BM25 built from `data.files`). /// /// Callers check `has_native_keyword_search()` before deciding which path to take: /// - true → call this method; skip `Rag.bm25` /// - false → call `Rag.keyword_search()` which uses `Rag.bm25` (sync, infallible) - /// - /// YamlProvider and QdrantProvider do NOT override this (return empty). async fn keyword_search(&self, query: &str, top_k: usize) -> Result> { let _ = (query, top_k); + Ok(vec![]) } diff --git a/src/rag/providers/duckdb.rs b/src/rag/providers/duckdb.rs index f5e9f07..105eed9 100644 --- a/src/rag/providers/duckdb.rs +++ b/src/rag/providers/duckdb.rs @@ -1,9 +1,11 @@ use crate::rag::provider::RagProvider; use crate::rag::{DocumentId, RagData}; +use std::collections::HashMap; use anyhow::{Context, Result, anyhow, bail}; use async_trait::async_trait; use duckdb::Connection; +use duckdb::types::Value; use indexmap::IndexMap; use log::warn; use std::path::{Path, PathBuf}; @@ -136,9 +138,6 @@ impl DuckDbProvider { pub(crate) fn read_all_vectors(&self) -> Result>> { let conn = self.lock_conn()?; let mut stmt = conn.prepare("SELECT doc_id, embedding FROM vectors")?; - // Collect into a Result, NOT a filter_map. `.filter_map(|r| r.ok())` here would - // turn a systematic decode failure (e.g. a schema written by a different duckdb - // version) into a silently short map, indistinguishable from an empty store. let raw: Vec<(u64, Vec)> = stmt .query_map([], |row| { let id: u64 = row.get(0)?; @@ -146,12 +145,12 @@ impl DuckDbProvider { // column is read as a `Value` and destructured. A FLOAT[N] column yields // `Value::Array`, a FLOAT[] column yields `Value::List`; match both so // the reader survives a file written under either schema. - let embedding: Vec = match row.get::<_, duckdb::types::Value>(1)? { - duckdb::types::Value::Array(vals) | duckdb::types::Value::List(vals) => vals + let embedding: Vec = match row.get::<_, Value>(1)? { + Value::Array(vals) | Value::List(vals) => vals .into_iter() .map(|v| match v { - duckdb::types::Value::Float(f) => f, - duckdb::types::Value::Double(d) => d as f32, + Value::Float(f) => f, + Value::Double(d) => d as f32, _ => f32::NAN, }) .collect(), @@ -184,6 +183,7 @@ impl DuckDbProvider { } out.insert(DocumentId(id as usize), embedding); } + Ok(out) } @@ -192,11 +192,10 @@ impl DuckDbProvider { /// /// A schema-existence check alone is NOT sufficient. After /// `CREATE OR REPLACE TABLE documents`, the `fts_main_documents` schema still - /// exists and `match_bm25` still SUCCEEDS — but returns zero rows for every term. + /// exists and `match_bm25` still SUCCEEDS, but returns zero rows for every term. /// The index is silently dead. The probe therefore asserts that a KNOWN row comes /// back rather than merely that the call did not error. fn probe_fts_index(conn: &Connection) -> bool { - // 1. Structural check — cheap, and short-circuits a never-built index. let schema_exists = conn .query_row( "SELECT COUNT(*) FROM duckdb_schemas() WHERE schema_name = 'fts_main_documents'", @@ -208,11 +207,6 @@ impl DuckDbProvider { if !schema_exists { return false; } - // 2. Liveness check — pull a real token out of a real row and confirm the index - // scores that same row. A live index returns >= 1; a stale one returns 0 - // without erroring. An empty `documents` table cannot be probed, and - // reporting false is correct: there is nothing to keyword-search, and the - // next rebuild_indexes sets the flag directly. conn.query_row( "SELECT COUNT(*) FROM documents d WHERE fts_main_documents.match_bm25( @@ -250,7 +244,6 @@ impl RagProvider for DuckDbProvider { top_k: usize, min_score: f32, ) -> Result> { - // Validate before building the SQL literal — a NaN would produce malformed SQL. if embedding.iter().any(|f| !f.is_finite()) { bail!("Query embedding contains a non-finite value (NaN or infinity)"); } @@ -276,11 +269,11 @@ impl RagProvider for DuckDbProvider { let id: u64 = row.get(0)?; // `array_cosine_distance` on a FLOAT[N] column returns FLOAT (f32), NOT // DOUBLE. Reading it as f64 raises InvalidColumnType INSIDE the closure, - // which a bare `.filter_map(|r| r.ok())` would silently discard — + // which a bare `.filter_map(|r| r.ok())` would silently discard, // yielding ZERO results with no error and no log line. - let distance: f32 = match row.get::<_, duckdb::types::Value>(1)? { - duckdb::types::Value::Float(f) => f, - duckdb::types::Value::Double(d) => d as f32, + let distance: f32 = match row.get::<_, Value>(1)? { + Value::Float(f) => f, + Value::Double(d) => d as f32, other => { warn!("unexpected distance type from DuckDB: {other:?}"); return Err(duckdb::Error::InvalidQuery); @@ -299,6 +292,7 @@ impl RagProvider for DuckDbProvider { }) .filter(|(_, score)| *score > min_score) .collect(); + Ok(results) } @@ -310,10 +304,7 @@ impl RagProvider for DuckDbProvider { let placeholders = ids.iter().map(|_| "?").collect::>().join(", "); let sql = format!("SELECT doc_id, page_content FROM documents WHERE doc_id IN ({placeholders})"); - let params: Vec = ids - .iter() - .map(|id| duckdb::types::Value::UBigInt(id.0 as u64)) - .collect(); + let params: Vec = ids.iter().map(|id| Value::UBigInt(id.0 as u64)).collect(); let mut stmt = conn.prepare(&sql)?; let mut rows: Vec<(DocumentId, String)> = stmt .query_map(duckdb::params_from_iter(params.iter()), |row| { @@ -329,22 +320,21 @@ impl RagProvider for DuckDbProvider { } }) .collect(); - // Ordering contract: `WHERE doc_id IN (...)` returns rows in storage order, NOT - // in the order of `ids`. Since `ids` is the RRF-ranked list, returning storage - // order would silently discard the ranking. Re-sort by input position. - let position: std::collections::HashMap = + let position: HashMap = ids.iter().enumerate().map(|(i, id)| (*id, i)).collect(); + rows.sort_by_key(|(id, _)| position.get(id).copied().unwrap_or(usize::MAX)); + Ok(rows) } async fn rebuild_indexes(&mut self, data: &RagData, _full_rebuild: bool) -> Result<()> { - // Local on-disk state — a wholesale table reset plus re-INSERT is always + // Local on-disk state. A wholesale table reset plus re-INSERT is always // correct, so the incremental/full distinction is ignored. // // 🔴 ANTI-WIPE GUARD. The CREATE OR REPLACE TABLE below writes exactly what // `data.vectors` holds. An empty map on a RAG that HAS indexed files, against a - // store that ALREADY holds vectors, is always a bug — a failed/skipped + // store that ALREADY holds vectors, is always a bug; a failed/skipped // hydration, or a caller that emptied the live map. Refuse rather than commit // the loss. Every legitimate empty-vector rebuild also has an empty `files` // (a fresh RAG; the zero-document tests), so this cannot fire on a correct call. @@ -353,7 +343,7 @@ impl RagProvider for DuckDbProvider { if data.vectors.is_empty() && !data.files.is_empty() { let existing: i64 = { // Scoped: the guard MUST be dropped before `lock_conn()` is taken again - // below. `Mutex` is not reentrant — holding both self-deadlocks at + // below. `Mutex` is not reentrant; holding both self-deadlocks at // runtime, with no compile error. let conn = self.lock_conn()?; conn.query_row("SELECT count(*) FROM vectors", [], |r| r.get(0)) @@ -382,7 +372,6 @@ impl RagProvider for DuckDbProvider { } } let dim = self.dim; - // `Connection::transaction()` takes `&mut self`, so this binding must be `mut`. let mut conn = self.lock_conn()?; let tx = conn .transaction() @@ -408,7 +397,7 @@ impl RagProvider for DuckDbProvider { { // Plain INSERT: tables are empty after CREATE OR REPLACE, so no PK conflict. - // The embedding is bound as TEXT and cast in SQL — the duckdb crate's + // The embedding is bound as TEXT and cast in SQL. The duckdb crate's // `bind_parameter` has no arm for List/Array, so binding a vector directly // fails at runtime with "binding List parameters is not yet supported". let mut vstmt = tx.prepare(&format!( @@ -418,12 +407,12 @@ impl RagProvider for DuckDbProvider { tx.prepare("INSERT INTO documents (doc_id, page_content) VALUES (?, ?)")?; // 🔴 TWO INDEPENDENT LOOPS. `documents` is keyed on `files`, `vectors` on - // `vectors` — they are DIFFERENT key sets and neither is a subset of the + // `vectors`. They are DIFFERENT key sets and neither is a subset of the // other. Do not merge these into one loop over `&data.vectors` gated on a // lookup into `files`: that produces keys(documents) ⊆ keys(vectors), and // every id in `files \ vectors` (which `RagData::add`'s zip truncation // really does produce) becomes a live, rankable id from BM25 and - // graph_search that resolves to nothing — no error, no log line. + // graph_search that resolves to nothing; no error, no log line. for (id, doc) in data.iter_documents() { dstmt.execute(duckdb::params![id.0 as u64, doc.page_content.as_str()])?; } @@ -461,7 +450,7 @@ impl RagProvider for DuckDbProvider { // Rebuild the FTS index (must be outside the transaction). This rebuild is // MANDATORY on every pass, not an optimization: CREATE OR REPLACE TABLE above - // leaves the old fts_main_documents schema in place but DEAD — match_bm25 keeps + // leaves the old fts_main_documents schema in place but DEAD; match_bm25 keeps // succeeding while returning zero rows for every term. // GUARDED on a non-empty table: FTS cannot index an empty table, and // rebuild_indexes is legitimately called with zero documents (a fresh RAG, and @@ -480,7 +469,7 @@ impl RagProvider for DuckDbProvider { // Live only if an index was actually built. With zero documents there is no FTS // index, so `has_native_keyword_search()` must stay false and hybrid_search must - // fall back to local BM25 — which is also empty, and therefore correct. + // fall back to local BM25, which is also empty, and therefore correct. self.fts_ready.store(doc_count > 0, Ordering::Relaxed); Ok(()) @@ -501,11 +490,11 @@ impl RagProvider for DuckDbProvider { let id: u64 = row.get(0)?; // Same hazard as vector_search's distance column: guessing the width // wrong raises InvalidColumnType INSIDE the closure, which a bare - // `.filter_map(|r| r.ok())` silently discards — yielding ZERO keyword + // `.filter_map(|r| r.ok())` silently discards, yielding ZERO keyword // hits, indistinguishable from "the query matched nothing". - let score: f32 = match row.get::<_, duckdb::types::Value>(1)? { - duckdb::types::Value::Double(d) => d as f32, - duckdb::types::Value::Float(f) => f, + let score: f32 = match row.get::<_, Value>(1)? { + Value::Double(d) => d as f32, + Value::Float(f) => f, other => { warn!("unexpected match_bm25 score type from DuckDB: {other:?}"); return Err(duckdb::Error::InvalidQuery); @@ -543,7 +532,7 @@ impl RagProvider for DuckDbProvider { // This means DuckDbProvider clones are NOT independent snapshots: state lives on // disk and a rebuild through one handle is immediately visible to the other. // That is unavoidable for any on-disk store and is handled by the discipline - // documented on `Rag`'s Clone impl — the pre-clone instance must be discarded. + // documented on `Rag`'s Clone impl, the pre-clone instance must be discarded. Box::new(DuckDbProvider { path: self.path.clone(), conn: Arc::clone(&self.conn), @@ -556,44 +545,37 @@ impl RagProvider for DuckDbProvider { #[cfg(test)] mod tests { use super::*; - // Trait methods are only callable with the trait in scope. use crate::rag::provider::RagProvider; - // `RagFile` / `RagDocument` have private fields; struct-literal construction - // compiles only because this module is a descendant of `crate::rag`. They are - // deliberately not in the non-test `use` block — unused there, and `--deny warnings` - // rejects that. use crate::rag::{RagDocument, RagFile}; + use std::time::{SystemTime, UNIX_EPOCH}; + use std::{env, fs}; - /// Unique temp path per test. `tempfile` is not a dev-dependency; this mirrors the - /// house pattern used elsewhere in the tree. struct TempDb { path: PathBuf, } impl TempDb { fn new(tag: &str) -> Self { - let unique = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) .unwrap() .as_nanos(); - let path = std::env::temp_dir().join(format!("coyote-duckdb-{tag}-{unique}.duckdb")); + let path = env::temp_dir().join(format!("coyote-duckdb-{tag}-{unique}.duckdb")); Self { path } } } impl Drop for TempDb { fn drop(&mut self) { - let _ = std::fs::remove_file(&self.path); - // DuckDB writes a `.wal` sidecar; remove it too or /tmp accumulates - // one per test run. - let _ = std::fs::remove_file(self.path.with_extension("duckdb.wal")); + let _ = fs::remove_file(&self.path); + let _ = fs::remove_file(self.path.with_extension("duckdb.wal")); } } /// ⚠️ NOTE THE EMPTY `files`. This fixture describes a RAG with ZERO documents. /// Since `rebuild_indexes` populates the `documents` table from /// `data.iter_documents()` (keyed on `files`), any test built on this helper alone - /// exercises the `vectors` table and NOTHING ELSE — no `documents` rows, no FTS + /// exercises the `vectors` table and NOTHING ELSE. No `documents` rows, no FTS /// index. That is correct for the rebuild tests below, and is exactly why the /// FTS-flag test and the `fetch_content` tests use `populated_rag_data()` instead. /// Do not "simplify" them back onto this helper. @@ -609,7 +591,7 @@ mod tests { } } - /// Two files, one document each — the minimum fixture that produces `documents` + /// Two files, one document each; the minimum fixture that produces `documents` /// rows. fn populated_rag_data() -> RagData { let mut data = minimal_rag_data(); @@ -644,12 +626,14 @@ mod tests { let db = TempDb::new("schema"); let provider = DuckDbProvider::open(&db.path, 3).unwrap(); let conn = provider.conn.lock().unwrap(); + let v: i64 = conn .query_row("SELECT COUNT(*) FROM vectors", [], |r| r.get(0)) .unwrap(); let d: i64 = conn .query_row("SELECT COUNT(*) FROM documents", [], |r| r.get(0)) .unwrap(); + assert_eq!(v, 0); assert_eq!(d, 0); } @@ -668,10 +652,12 @@ mod tests { ) .unwrap(); } + let results = provider .vector_search(&[0.1, 0.2, 0.3], 5, 0.0) .await .unwrap(); + assert_eq!(results.len(), 1); assert_eq!(results[0].0.0, 0); assert!(results[0].1 > 0.99); @@ -689,7 +675,9 @@ mod tests { ) .unwrap(); } + let results = provider.fetch_content(&[DocumentId(42)]).await.unwrap(); + assert_eq!(results.len(), 1); assert_eq!(results[0].1, "hello world"); } @@ -700,19 +688,15 @@ mod tests { let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); let data = minimal_rag_data(); provider.rebuild_indexes(&data, true).await.unwrap(); + let results = provider .vector_search(&[0.1, 0.2, 0.3], 5, 0.0) .await .unwrap(); - assert!(results.is_empty()); // no vectors in empty data + + assert!(results.is_empty()); } - /// Rebuilding TWICE must succeed. - /// - /// The first rebuild always passes because the tables start empty. The bug this - /// guards against — `DELETE FROM` plus re-INSERT of the same PKs inside one - /// transaction — only fires on the SECOND rebuild, with "Duplicate key ... violates - /// primary key constraint". A single-rebuild test cannot catch it. #[tokio::test] async fn rebuild_indexes_is_idempotent_across_two_passes() { let db = TempDb::new("idempotent"); @@ -722,7 +706,7 @@ mod tests { data.vectors.insert(DocumentId(1), vec![0.4, 0.5, 0.6]); provider.rebuild_indexes(&data, true).await.unwrap(); - // Second pass over the SAME doc_ids — this is the assertion that matters. + provider .rebuild_indexes(&data, true) .await @@ -735,27 +719,16 @@ mod tests { assert_eq!(count, 2, "rebuild must replace rows, not duplicate them"); } - /// Guards the incremental data-loss bug: `sync_documents` hash-skips unchanged - /// files, so on an incremental pass `data.vectors` holds ONLY the newly embedded - /// chunks. If the live map is ever emptied, the `CREATE OR REPLACE TABLE` in - /// rebuild_indexes writes only those, destroying every previously indexed vector. - /// Hydration via `read_all_vectors()` is what prevents it. - /// - /// This simulates the real restart-then-edit sequence: build, drop the in-memory - /// map, rehydrate from disk, add one vector, rebuild incrementally. #[tokio::test] async fn rebuild_indexes_preserves_prior_vectors_on_incremental_pass() { let db = TempDb::new("incremental"); let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); - // Pass 1 — initial full build with two vectors. let mut data = minimal_rag_data(); data.vectors.insert(DocumentId(0), vec![0.1, 0.2, 0.3]); data.vectors.insert(DocumentId(1), vec![0.4, 0.5, 0.6]); provider.rebuild_indexes(&data, true).await.unwrap(); - // Simulate a restart: the YAML on disk carries NO vectors, so a fresh RagData - // starts empty. create() hydrates it back from the sidecar. let mut reloaded = minimal_rag_data(); assert!( reloaded.vectors.is_empty(), @@ -768,7 +741,6 @@ mod tests { "hydration must restore both vectors" ); - // Pass 2 — incremental add of ONE new chunk. reloaded.vectors.insert(DocumentId(2), vec![0.7, 0.8, 0.9]); provider.rebuild_indexes(&reloaded, false).await.unwrap(); @@ -783,14 +755,6 @@ mod tests { ); } - /// `has_native_keyword_search()` must be false before any FTS index exists, - /// otherwise hybrid_search routes into keyword_search and the `?` turns a DuckDB - /// catalog error into a total query failure. - /// - /// 🔴 THE FIXTURE MUST CARRY DOCUMENTS. `rebuild_indexes` builds the FTS index only - /// when the `documents` table is non-empty, and that table is filled from - /// `data.iter_documents()` (keyed on `files`), never from `vectors`. With `files` - /// empty the pragma is skipped and `fts_ready` stores `false`. #[tokio::test] async fn keyword_search_is_not_advertised_before_first_rebuild() { let db = TempDb::new("ftsflag"); @@ -800,13 +764,10 @@ mod tests { "a freshly opened DB has no FTS index yet" ); - // 2 files × 1 document → 2 `documents` rows → doc_count > 0 → pragma runs. let mut data = populated_rag_data(); data.vectors.insert(DocumentId(0), vec![0.1, 0.2, 0.3]); provider.rebuild_indexes(&data, true).await.unwrap(); { - // Pin the precondition explicitly. If this ever reads 0 the assertion below - // is vacuous and the FTS hazard is unguarded again. let conn = provider.conn.lock().unwrap(); let docs: i64 = conn .query_row("SELECT COUNT(*) FROM documents", [], |r| r.get(0)) @@ -819,23 +780,11 @@ mod tests { ); } - /// The `documents` table is keyed on `files`, NOT on `vectors`. - /// - /// `fetch_content` is the sink for BOTH the BM25 and graph_search paths, and both - /// enumerate `files` via `iter_documents()`. If `rebuild_indexes` only inserts a - /// documents row for ids that also appear in `data.vectors`, every id in - /// `files \ vectors` resolves to nothing: healthy BM25 scores, healthy graph hits, - /// zero results, no error. - /// - /// The incremental test CANNOT catch this — its fixture has an empty `files`, so the - /// documents table is empty in every assertion either way. #[tokio::test] async fn fetch_content_resolves_documents_without_vectors() { let db = TempDb::new("docsnovec"); let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); - // Two files, one document each — but a vector for ONLY THE FIRST. This is the - // `RagData::add` zip-truncation shape. let mut data = populated_rag_data(); let id_a = DocumentId::new(0, 0); let id_b = DocumentId::new(1, 0); @@ -858,13 +807,6 @@ mod tests { assert_eq!(got[1].1, "beta keyword"); } - /// `fetch_content` must return rows in INPUT order, not storage order. - /// - /// `YamlProvider` satisfies this contract structurally — it maps over `ids` — so the - /// existing yaml test proves nothing about DuckDB. DuckDB's `WHERE doc_id IN (...)` - /// returns storage order and relies on an explicit positional re-sort, which is what - /// can actually regress. `ids` is the RRF-ranked list, so losing the order silently - /// discards the ranking while still returning the right documents. #[tokio::test] async fn duckdb_fetch_content_preserves_input_order() { let db = TempDb::new("order"); @@ -872,11 +814,9 @@ mod tests { let data = populated_rag_data(); provider.rebuild_indexes(&data, true).await.unwrap(); - let id_a = DocumentId::new(0, 0); // inserted first → storage order 0 - let id_b = DocumentId::new(1, 0); // inserted second → storage order 1 + let id_a = DocumentId::new(0, 0); + let id_b = DocumentId::new(1, 0); - // REVERSED relative to storage order. Without the positional re-sort this - // returns ["alpha keyword", "beta keyword"] and the assertion fails. let got = provider.fetch_content(&[id_b, id_a]).await.unwrap(); assert_eq!(got.len(), 2); assert_eq!(got[0].0, id_b, "input order must win over storage order"); @@ -885,19 +825,11 @@ mod tests { assert_eq!(got[1].1, "alpha keyword"); } - /// The anti-wipe guard. - /// - /// `rebuild_indexes` does `CREATE OR REPLACE TABLE` and writes exactly what - /// `data.vectors` holds. An empty map on a RAG that HAS indexed files, against a - /// store that already holds vectors, is always a bug (failed hydration, or a caller - /// that emptied the live map) and must be refused rather than committed. The loss - /// would otherwise be permanent: nothing re-embeds it back. #[tokio::test] async fn rebuild_indexes_refuses_to_wipe_when_vectors_are_empty_but_files_are_not() { let db = TempDb::new("nowipe"); let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); - // A healthy store: two files with documents, two vectors. let mut data = populated_rag_data(); data.vectors .insert(DocumentId::new(0, 0), vec![0.1, 0.2, 0.3]); @@ -905,8 +837,7 @@ mod tests { .insert(DocumentId::new(1, 0), vec![0.4, 0.5, 0.6]); provider.rebuild_indexes(&data, true).await.unwrap(); - // Now the failure state: vectors lost in memory, files intact. - let broken = populated_rag_data(); // same files, NO vectors + let broken = populated_rag_data(); assert!( broken.vectors.is_empty() && !broken.files.is_empty(), "precondition" @@ -918,7 +849,6 @@ mod tests { "got: {err}" ); - // The store must be untouched — this is the whole point. let conn = provider.conn.lock().unwrap(); let count: i64 = conn .query_row("SELECT COUNT(*) FROM vectors", [], |r| r.get(0)) @@ -929,31 +859,23 @@ mod tests { ); } - /// The guard must NOT fire on a legitimately empty RAG — otherwise a fresh - /// `Rag::init()` cannot complete. Empty `vectors` AND empty `files` is fine. #[tokio::test] async fn rebuild_indexes_allows_empty_vectors_when_files_are_also_empty() { let db = TempDb::new("emptyok"); let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); - let data = minimal_rag_data(); // no files, no vectors + let data = minimal_rag_data(); provider .rebuild_indexes(&data, true) .await .expect("a fresh RAG with nothing indexed must rebuild cleanly"); } - /// `duplicate()` must clone the Arc, NOT call `open()` again. - /// - /// This asserts SHARED state, which is the actual contract. It deliberately does NOT - /// use `fetch_content(&[])` — that early-returns before ever touching the - /// connection, so it would pass even against a broken double-opening implementation. #[tokio::test] async fn duplicate_shares_the_same_connection() { let db = TempDb::new("dup"); let provider = DuckDbProvider::open(&db.path, 3).unwrap(); let dup = provider.duplicate(&minimal_rag_data()); - // Write through the ORIGINAL... { let conn = provider.conn.lock().unwrap(); conn.execute( @@ -962,8 +884,9 @@ mod tests { ) .unwrap(); } - // ...and read it back through the DUPLICATE. Only possible if they share state. + let via_dup = dup.fetch_content(&[DocumentId(7)]).await.unwrap(); + assert_eq!( via_dup.len(), 1, @@ -972,8 +895,6 @@ mod tests { assert_eq!(via_dup[0].1, "shared row"); } - /// A decode failure must abort hydration rather than yield a thinned map: a short - /// map is what the next CREATE OR REPLACE commits as the new truth. #[tokio::test] async fn read_all_vectors_rejects_non_finite_embeddings() { let db = TempDb::new("nonfinite"); @@ -1005,6 +926,7 @@ mod tests { #[test] fn duckdb_path_from_yaml_swaps_extension() { let p = duckdb_path_from_yaml(Path::new("/tmp/rags/docs.yaml")); + assert_eq!(p, PathBuf::from("/tmp/rags/docs.duckdb")); } } diff --git a/src/rag/providers/mod.rs b/src/rag/providers/mod.rs index 6fd4472..1a54ca8 100644 --- a/src/rag/providers/mod.rs +++ b/src/rag/providers/mod.rs @@ -1,15 +1,8 @@ mod yaml; -// Use `self::` on every re-export in this file. Once a `mod duckdb;` sits here -// alongside a dependency on the `duckdb` CRATE, a bare `pub use duckdb::...` -// is ambiguous (E0659) — `use` paths resolve against both this module's items -// and the extern prelude, and `use` declarations may not shadow. pub use self::yaml::YamlProvider; mod duckdb; pub use self::duckdb::DuckDbProvider; -// `create()` in rag/mod.rs derives the sidecar path through this. It is `pub(crate)` -// in providers/duckdb.rs, and the re-export must be `pub(crate)` too — a `pub use` of -// a `pub(crate)` item is E0364/E0365. pub(crate) use self::duckdb::duckdb_path_from_yaml; mod qdrant; diff --git a/src/rag/providers/qdrant.rs b/src/rag/providers/qdrant.rs index a2d2c34..563a6f8 100644 --- a/src/rag/providers/qdrant.rs +++ b/src/rag/providers/qdrant.rs @@ -3,6 +3,9 @@ use crate::rag::{DocumentId, RagData}; use anyhow::{Context, Result, bail}; use async_trait::async_trait; +use reqwest::header::{HeaderMap, HeaderValue}; +use reqwest::{Client, Response, StatusCode}; +use serde_json::Value; use std::collections::HashMap; /// Render Qdrant's error envelope into a human-readable message. @@ -15,22 +18,18 @@ use std::collections::HashMap; /// * routing-level 404s (a wrong HTTP verb) return an EMPTY body with no JSON at /// all, which without the length check surfaces as "EOF while parsing a value" /// instead of the actual 404. -fn format_error_body(status: reqwest::StatusCode, body: &str) -> String { +fn format_error_body(status: StatusCode, body: &str) -> String { if body.is_empty() { return format!("HTTP {status} (empty body — check the HTTP verb and path)"); } - serde_json::from_str::(body) + serde_json::from_str::(body) .ok() .and_then(|v| v["status"]["error"].as_str().map(str::to_string)) .unwrap_or_else(|| format!("HTTP {status}: {body}")) } /// Read the vector dimension out of a parsed `GET /collections/{name}` response. -/// -/// Unnamed collections put `size` directly under `vectors`; named ones nest it -/// under the vector's name. Both shapes occur in the wild, so try the flat one -/// first and fall back to the first named entry. -fn vector_dimension_from_collection(body: &serde_json::Value) -> Result { +fn vector_dimension_from_collection(body: &Value) -> Result { let params = &body["result"]["config"]["params"]; params["vectors"]["size"] .as_u64() @@ -47,12 +46,12 @@ fn vector_dimension_from_collection(body: &serde_json::Value) -> Result { /// (multi-vector) collection. /// /// `vector_search` posts an unnamed vector, which a named-vector collection -/// rejects with HTTP 400 on every query — so attaching one yields a RAG that is +/// rejects with HTTP 400 on every query, so attaching one yields a RAG that is /// silently 100% broken. A named collection holding a SINGLE vector is /// structurally a map, identical in kind to the multi-named case, and rejects /// the same way; testing for a numeric `size` directly under `vectors` catches /// it, whereas counting keys (`len() > 1`) would wrongly accept it. -fn is_multi_vector_config(body: &serde_json::Value) -> bool { +fn is_multi_vector_config(body: &Value) -> bool { body["result"]["config"]["params"]["vectors"]["size"] .as_u64() .is_none() @@ -63,27 +62,21 @@ fn is_multi_vector_config(body: &serde_json::Value) -> bool { /// Attach-only: this provider never writes to the remote collection. Coyote does /// not own the data, and `rebuild_indexes` refuses rather than pretending to. pub struct QdrantProvider { - /// `reqwest::Client` is Arc-backed, so `clone()` is O(1) and shares both the - /// connection pool and the `api-key` default header injected at build time. - client: reqwest::Client, - /// Includes the scheme, e.g. `http://qdrant.example.com:6333`. + client: Client, base_url: String, collection: String, } impl QdrantProvider { - /// The resolved API key is injected as a default header here and is - /// deliberately NOT stored on the struct: the plaintext value stays a local - /// of the caller and never outlives it. - fn make_client(api_key: Option<&str>) -> Result { - let mut headers = reqwest::header::HeaderMap::new(); + fn make_client(api_key: Option<&str>) -> Result { + let mut headers = HeaderMap::new(); if let Some(key) = api_key { - let mut value = reqwest::header::HeaderValue::from_str(key) - .context("api-key header value is not valid ASCII")?; + let mut value = + HeaderValue::from_str(key).context("api-key header value is not valid ASCII")?; value.set_sensitive(true); headers.insert("api-key", value); } - reqwest::Client::builder() + Client::builder() .default_headers(headers) .build() .context("Failed to build reqwest client") @@ -97,7 +90,7 @@ impl QdrantProvider { } } - async fn error_message(resp: reqwest::Response) -> String { + async fn error_message(resp: Response) -> String { let status = resp.status(); let body = resp.text().await.unwrap_or_default(); format_error_body(status, &body) @@ -109,7 +102,7 @@ impl QdrantProvider { host: &str, collection: &str, api_key: Option<&str>, - ) -> Result { + ) -> Result { let base_url = Self::normalize_base_url(host); let client = Self::make_client(api_key)?; let resp = client @@ -123,13 +116,13 @@ impl QdrantProvider { Self::error_message(resp).await ); } + Ok(resp.json().await?) } pub async fn new(host: &str, collection: &str, api_key: Option<&str>) -> Result { let base_url = Self::normalize_base_url(host); let client = Self::make_client(api_key)?; - // Preflight: confirm the collection exists and we may read it. let resp = client .get(format!("{base_url}/collections/{collection}")) .send() @@ -141,6 +134,7 @@ impl QdrantProvider { Self::error_message(resp).await ); } + Ok(Self { client, base_url, @@ -162,13 +156,15 @@ impl QdrantProvider { Self::error_message(resp).await ); } - let body: serde_json::Value = resp.json().await?; + + let body: Value = resp.json().await?; let names = body["result"]["collections"] .as_array() .context("Unexpected /collections response shape")? .iter() .filter_map(|v| v["name"].as_str().map(str::to_string)) .collect(); + Ok(names) } @@ -178,6 +174,7 @@ impl QdrantProvider { api_key: Option<&str>, ) -> Result { let body = Self::fetch_collection(host, collection, api_key).await?; + vector_dimension_from_collection(&body) } @@ -187,11 +184,10 @@ impl QdrantProvider { api_key: Option<&str>, ) -> Result { let body = Self::fetch_collection(host, collection, api_key).await?; + Ok(is_multi_vector_config(&body)) } - /// Peek at one point to learn how its ID is typed. Returns the raw JSON - /// rendering, so a string ID comes back quoted and an integer one bare. pub async fn sample_point_id( host: &str, collection: &str, @@ -201,23 +197,27 @@ impl QdrantProvider { let client = Self::make_client(api_key)?; let url = format!("{base_url}/collections/{collection}/points/scroll"); let body = serde_json::json!({ "limit": 1, "with_payload": false }); + let resp = client .post(&url) .json(&body) .send() .await .with_context(|| format!("Failed to connect to {host}"))?; + if !resp.status().is_success() { bail!( "Failed to sample a point from '{collection}': {}", Self::error_message(resp).await ); } - let data: serde_json::Value = resp.json().await?; + + let data: Value = resp.json().await?; let id_val = data["result"]["points"] .as_array() .and_then(|pts| pts.first()) .map(|pt| pt["id"].to_string()); + Ok(id_val) } } @@ -251,7 +251,7 @@ impl RagProvider for QdrantProvider { Self::error_message(resp).await ); } - let data: serde_json::Value = resp.json().await?; + let data: Value = resp.json().await?; let results = data["result"] .as_array() .context("Unexpected /points/search response shape")? @@ -266,6 +266,7 @@ impl RagProvider for QdrantProvider { }) .filter(|(_, score)| *score > min_score) .collect(); + Ok(results) } @@ -279,7 +280,9 @@ impl RagProvider for QdrantProvider { "ids": id_list, "with_payload": true, }); + let resp = self.client.post(&url).json(&body).send().await?; + if !resp.status().is_success() { bail!( "Qdrant point fetch on '{}' failed: {}", @@ -287,7 +290,7 @@ impl RagProvider for QdrantProvider { Self::error_message(resp).await ); } - let data: serde_json::Value = resp.json().await?; + let data: Value = resp.json().await?; let mut rows: Vec<(DocumentId, String)> = data["result"] .as_array() .context("Unexpected /points response shape")? @@ -303,13 +306,14 @@ impl RagProvider for QdrantProvider { let position: HashMap = ids.iter().enumerate().map(|(i, id)| (*id, i)).collect(); rows.sort_by_key(|(id, _)| position.get(id).copied().unwrap_or(usize::MAX)); + Ok(rows) } async fn rebuild_indexes(&mut self, data: &RagData, _full_rebuild: bool) -> Result<()> { // Both arms refuse. A silent `Ok(())` would make `.rebuild rag` and // `.edit rag-docs` look like they worked while writing nothing to the - // remote — leaving the user believing the collection was updated. + // remote, leaving the user believing the collection was updated. if data.attached { bail!( "This RAG is attached to an external Qdrant collection. Coyote does not own \ @@ -340,7 +344,7 @@ mod tests { fn error_message_reads_the_object_status_envelope() { let body = r#"{"status": {"error": "Wrong input: Not existing vector name error:"}, "time": 0.0}"#; - let msg = format_error_body(reqwest::StatusCode::BAD_REQUEST, body); + let msg = format_error_body(StatusCode::BAD_REQUEST, body); assert!(msg.contains("Not existing vector name"), "got: {msg}"); assert!( !msg.contains("EOF"), @@ -350,15 +354,14 @@ mod tests { #[test] fn error_message_survives_the_string_status_and_the_empty_body() { - // Success envelope: `status` is a bare string, so the object lookup misses - // and we must fall back rather than panic or invent an error text. - let ok = format_error_body(reqwest::StatusCode::OK, r#"{"status": "ok", "time": 0.0}"#); + let ok = format_error_body(StatusCode::OK, r#"{"status": "ok", "time": 0.0}"#); assert!( ok.contains("200"), "no `status.error` present → fall back to status+body: {ok}" ); - // Routing-level 404 from a wrong HTTP verb: empty body, no JSON at all. - let empty = format_error_body(reqwest::StatusCode::NOT_FOUND, ""); + + let empty = format_error_body(StatusCode::NOT_FOUND, ""); + assert!(empty.contains("empty body"), "got: {empty}"); assert!( empty.contains("verb"), @@ -384,15 +387,11 @@ mod tests { #[test] fn is_multi_vector_rejects_the_named_single_collection() { - // The only supported shape: a single unnamed vector. let unnamed = serde_json::json!({ "result": {"config": {"params": {"vectors": {"size": 1536, "distance": "Cosine"}}}} }); assert!(!is_multi_vector_config(&unnamed)); - // Named but SINGLE — structurally a map, and writes to it fail with - // `400 "Wrong input: Not existing vector name error:"`. A `len() > 1` check - // would wrongly accept this one; that is the bug this case exists to catch. let named_single = serde_json::json!({ "result": {"config": {"params": {"vectors": {"text": {"size": 1536}}}}} }); @@ -426,7 +425,7 @@ mod tests { #[tokio::test] async fn rebuild_indexes_refuses_for_attached_and_unattached_alike() { let mut provider = QdrantProvider { - client: reqwest::Client::new(), + client: Client::new(), base_url: "http://localhost:6333".to_string(), collection: "c".to_string(), }; @@ -459,13 +458,12 @@ mod tests { #[tokio::test] async fn fetch_content_short_circuits_on_an_empty_id_list() { - // No network is touched: the early return happens before any request, which - // is why this can assert against an unreachable host. let provider = QdrantProvider { - client: reqwest::Client::new(), + client: Client::new(), base_url: "http://127.0.0.1:1".to_string(), collection: "c".to_string(), }; + assert!(provider.fetch_content(&[]).await.unwrap().is_empty()); } @@ -475,6 +473,7 @@ mod tests { let collections = QdrantProvider::list_collections("http://localhost:6333", None) .await .unwrap(); + assert!(!collections.is_empty()); } @@ -485,7 +484,9 @@ mod tests { .await .unwrap(); let embedding = vec![0.0f32; 1536]; + let results = provider.vector_search(&embedding, 5, 0.0).await.unwrap(); + assert!(results.len() <= 5); } } diff --git a/src/rag/providers/yaml.rs b/src/rag/providers/yaml.rs index df1f6f5..e56b396 100644 --- a/src/rag/providers/yaml.rs +++ b/src/rag/providers/yaml.rs @@ -20,9 +20,6 @@ impl YamlProvider { } fn build_content_map(data: &RagData) -> IndexMap { - // Keyed on `files`, NOT `vectors`: this is the exact replacement for the - // per-id document lookup it supersedes, and it must resolve every id that - // BM25 or graph_search can produce — both of which enumerate `files`. data.iter_documents() .map(|(id, doc)| (id, doc.page_content.clone())) .collect() @@ -52,12 +49,11 @@ impl RagProvider for YamlProvider { }) }) .collect(); + Ok(results) } async fn fetch_content(&self, ids: &[DocumentId]) -> Result> { - // Iterating `ids` (not `content_map`) satisfies the trait's ordering - // contract for free — output order mirrors input order. Ok(ids .iter() .filter_map(|id| self.content_map.get(id).map(|text| (*id, text.clone()))) @@ -65,10 +61,10 @@ impl RagProvider for YamlProvider { } async fn rebuild_indexes(&mut self, data: &RagData, _full_rebuild: bool) -> Result<()> { - // Local in-memory state — a wholesale rebuild is fast and always correct, - // so the incremental/full distinction is irrelevant here. self.hnsw = data.build_hnsw(); + self.content_map = Self::build_content_map(data); + Ok(()) } @@ -80,14 +76,9 @@ impl RagProvider for YamlProvider { #[cfg(test)] mod provider_tests { use super::*; - // `RagFile` and `RagDocument` are not used by the impl above, so they are - // imported here rather than at module scope. Both have private fields, which - // is why these tests must live in-crate rather than under `tests/`. use crate::rag::{RagDocument, RagFile}; fn minimal_rag_data() -> RagData { - // `..Default::default()` rather than an exhaustive struct literal so that - // later additions to `RagData` do not break this helper. RagData { embedding_model: "text-embedding-3-small".to_string(), chunk_size: 1024, @@ -99,7 +90,7 @@ mod provider_tests { } } - /// Two files, one chunk each, with vectors — the minimum needed to exercise + /// Two files, one chunk each, with vectors, the minimum needed to exercise /// `build_content_map` and the `fetch_content` ordering contract. /// `DocumentId::new(f, d)` packs (file_index, document_index); `RagData::add` /// is the real insertion path but a direct literal is sufficient and avoids @@ -143,13 +134,12 @@ mod provider_tests { async fn yaml_provider_empty_data_returns_nothing() { let data = minimal_rag_data(); let provider = YamlProvider::from_data(&data); + let results = provider.fetch_content(&[]).await.unwrap(); + assert!(results.is_empty()); } - /// `fetch_content` MUST return results in the same relative order as the input - /// ids. The reversed-input case is the one that fails if an implementation ever - /// iterates its own map instead of `ids`. #[tokio::test] async fn yaml_provider_fetch_content_preserves_input_order() { let data = populated_rag_data(); @@ -163,8 +153,8 @@ mod provider_tests { assert_eq!(forward[0].1, "alpha"); assert_eq!(forward[1].1, "beta"); - // Reversed input must produce reversed output — NOT storage order. let reversed = provider.fetch_content(&[b, a]).await.unwrap(); + assert_eq!( reversed[0].1, "beta", "fetch_content must honor input order" @@ -172,8 +162,6 @@ mod provider_tests { assert_eq!(reversed[1].1, "alpha"); } - /// A missing id is skipped, not an error, and does not disturb the order of - /// the ids that DO resolve. #[tokio::test] async fn yaml_provider_fetch_content_skips_missing_ids() { let data = populated_rag_data(); @@ -189,23 +177,16 @@ mod provider_tests { assert_eq!(out[1].1, "beta"); } - /// `YamlProvider::duplicate()` rebuilds from `data`, so the clone is a genuine - /// independent snapshot. Providers backed by a shared store deliberately are not. #[tokio::test] async fn yaml_provider_duplicate_returns_equivalent_content() { - // MUST be populated_rag_data(): on minimal_rag_data() both providers hold an - // EMPTY content map, so `assert_eq!(r1, r2)` compares two empty vectors and - // passes against a duplicate() that returns nothing at all. let data = populated_rag_data(); let provider = YamlProvider::from_data(&data); let dup = provider.duplicate(&data); let ids = [DocumentId::new(0, 0), DocumentId::new(1, 0)]; - // Query with REAL ids, not `&[]` — an empty slice is answered without ever - // touching the content map, so it would pass against a broken duplicate(). + let r1 = provider.fetch_content(&ids).await.unwrap(); let r2 = dup.fetch_content(&ids).await.unwrap(); - // Guard against the vacuous case: if both sides resolved nothing, the equality - // below proves nothing. Assert the fixture actually produced content first. + assert_eq!(r1.len(), 2, "fixture must resolve both documents"); assert_eq!( r1, r2, @@ -213,11 +194,6 @@ mod provider_tests { ); } - /// The content store is keyed on `files`, never on `vectors`. A vector may exist - /// for an id with no backing file (a stale entry, or a file dropped mid-sync); - /// keying on `vectors` would surface such an id with empty text instead of - /// dropping it. The shared fixture only ever inserts vectors for ids that also - /// have files, so this case has to be constructed here. #[tokio::test] async fn yaml_provider_content_is_keyed_on_files_not_vectors() { let mut data = populated_rag_data(); @@ -232,7 +208,6 @@ mod provider_tests { "an id present only in `vectors` must not resolve to content" ); - // The file-backed ids still resolve, so the assertion above is not vacuous. let real = provider .fetch_content(&[DocumentId::new(0, 0), DocumentId::new(1, 0)]) .await diff --git a/src/render/markdown.rs b/src/render/markdown.rs index cc93c2c..be54fa0 100644 --- a/src/render/markdown.rs +++ b/src/render/markdown.rs @@ -1749,11 +1749,6 @@ std::error::Error>> { ); } - /// Removes CSI escape sequences so only printable content is measured. - /// - /// Deliberately tolerant of malformed input: a sequence that was sliced - /// mid-escape swallows the following characters, which is precisely the - /// corruption `render_table_pads_columns_by_display_width` exists to catch. fn strip_ansi(text: &str) -> String { let mut out = String::with_capacity(text.len()); let mut chars = text.chars(); @@ -1780,12 +1775,6 @@ std::error::Error>> { assert_eq!(strip_ansi("plain"), "plain"); } - /// `render_table` hands comfy-table pre-styled cells that already contain - /// ANSI escapes, and `colorize_box_chars` adds more afterwards. Column - /// widths are therefore only correct if the escapes are excluded from the - /// width calculation. When they are not, the table still renders and every - /// other assertion in this file still passes -- only the alignment silently - /// degrades -- so this is the sole guard over that behaviour. #[test] fn render_table_pads_columns_by_display_width() { use unicode_width::UnicodeWidthStr; diff --git a/src/repl/mod.rs b/src/repl/mod.rs index 68a871b..7ade0a6 100644 --- a/src/repl/mod.rs +++ b/src/repl/mod.rs @@ -219,7 +219,7 @@ static REPL_COMMANDS: LazyLock<[ReplCommand; 60]> = LazyLock::new(|| { ), ReplCommand::new( ".rag attach", - "Attach to a pre-existing external RAG (Qdrant)", + "Attach to a pre-existing external RAG", AssertState::False(StateFlags::AGENT), ), ReplCommand::new( @@ -889,22 +889,17 @@ pub async fn run_repl_command( let version = args.map(|s| s.trim().to_string()); task::spawn_blocking(move || config::run_self_update(version, false)).await??; } - ".rag" => { - // `split_first_arg` rather than `starts_with("attach ")`: the latter - // misses a bare `.rag attach`, which would silently create a RAG - // literally named "attach". - match split_first_arg(args) { - Some(("attach", rest)) => match rest { - Some(name) if !name.trim().is_empty() => { - ctx.attach_rag(name.trim()).await?; - } - _ => println!("Usage: .rag attach "), - }, - _ => { - ctx.use_rag(args, abort_signal.clone()).await?; + ".rag" => match split_first_arg(args) { + Some(("attach", rest)) => match rest { + Some(name) if !name.trim().is_empty() => { + ctx.attach_rag(name.trim()).await?; } + _ => println!("Usage: .rag attach "), + }, + _ => { + ctx.use_rag(args, abort_signal.clone()).await?; } - } + }, ".agent" => match split_first_arg(args) { Some((agent_name, args)) => { let (new_args, _) = split_args_text(args.unwrap_or_default(), cfg!(windows)); diff --git a/src/sandbox/mcp_credentials.rs b/src/sandbox/mcp_credentials.rs index a16d8e6..8e73be3 100644 --- a/src/sandbox/mcp_credentials.rs +++ b/src/sandbox/mcp_credentials.rs @@ -387,10 +387,6 @@ pub(crate) fn collect_server_allow_entries( out.into_iter().collect() } -/// The single definition of the sbx kit v2 allow-list entry grammar: https on -/// the default port yields a bare host, anything else is spelled `host:port`. -/// Bracketed IPv6 hosts and non-http(s) schemes have no representation in the -/// grammar and yield `None`. pub(crate) fn allow_entry_for_url(raw: &str) -> Option { let url = Url::parse(raw).ok()?; let scheme = url.scheme(); @@ -465,12 +461,6 @@ struct Network { allow: Vec, } -/// Serializes one sbx kit v2 mixin document. -/// -/// Every `inject[].domain` is unioned into `permissions.network.allow`: sbx -/// does not derive allow entries from inject rules, so a rule whose domain is -/// not allowed would be dead. Enforcing it here keeps the invariant in one -/// place for every mixin Coyote generates. pub(crate) fn render_mixin_document( name: &str, description: &str, diff --git a/src/sandbox/mixins.rs b/src/sandbox/mixins.rs index 57365ae..697716a 100644 --- a/src/sandbox/mixins.rs +++ b/src/sandbox/mixins.rs @@ -73,10 +73,6 @@ pub fn discover() -> Result> { for path in collect_subdir_mixins(&paths::agents_data_dir()) { out.push(read_mixin(path)?); } - // RAG sidecars are FLAT files named `.sbx-mixin.yaml` inside rags/, not - // the `/sbx-mixin.yaml` shape the two scans above walk. Loaded - // unconditionally, mirroring agents/*: a RAG mixin only adds an outbound - // allowlist entry for that RAG's host and opens no inbound rules. for path in collect_flat_mixins(&paths::rags_dir()) { out.push(read_mixin(path)?); } @@ -184,8 +180,6 @@ fn collect_subdir_mixins(dir: &Path) -> Vec { result } -/// Mixins stored as flat `.sbx-mixin.yaml` files directly inside `dir`, -/// matched by suffix rather than by exact filename. fn collect_flat_mixins(dir: &Path) -> Vec { let mut result = Vec::new(); let Ok(rd) = read_dir(dir) else { return result }; @@ -516,16 +510,13 @@ network: } } - /// RAG sidecars are flat `.sbx-mixin.yaml` files, matched by SUFFIX. #[test] fn collect_flat_mixins_matches_rag_sidecars_by_suffix() { let root = unique_root("flat-mixins"); fs::write(root.join("company-docs.sbx-mixin.yaml"), "kind: mixin\n").unwrap(); fs::write(root.join("alpha.sbx-mixin.yaml"), "kind: mixin\n").unwrap(); - // The RAGs themselves must not be picked up, only their sidecars. fs::write(root.join("company-docs.yaml"), "driver: qdrant\n").unwrap(); fs::write(root.join("notes.yaml"), "driver: yaml\n").unwrap(); - // A directory whose name ends in the suffix is not a mixin file. fs::create_dir_all(root.join("decoy.sbx-mixin.yaml")).unwrap(); let found = collect_flat_mixins(&root); @@ -533,7 +524,6 @@ network: .iter() .map(|p| p.file_name().unwrap().to_str().unwrap()) .collect(); - // Sorted by file name, so the order is deterministic. assert_eq!( names, vec!["alpha.sbx-mixin.yaml", "company-docs.sbx-mixin.yaml"] diff --git a/src/sandbox/mod.rs b/src/sandbox/mod.rs index 4fb9e08..94c9889 100644 --- a/src/sandbox/mod.rs +++ b/src/sandbox/mod.rs @@ -314,11 +314,6 @@ fn inject_mcp_secrets(vault: &Vault, registered: &HashSet) -> Result) -> Result<()> { let rags_dir = paths::rags_dir(); if !rags_dir.exists() { @@ -330,7 +325,6 @@ fn inject_rag_secrets(vault: &Vault, registered: &HashSet) -> Result<()> continue; } let stem = match path.file_stem().and_then(|s| s.to_str()) { - // Skip sidecars ("myrag.sbx-mixin.yaml" has stem "myrag.sbx-mixin"). Some(s) if !paths::is_rag_sidecar_name(s) => s.to_string(), _ => continue, }; @@ -346,10 +340,6 @@ fn inject_rag_secrets(vault: &Vault, registered: &HashSet) -> Result<()> let Some(placeholder) = data.driver_config.get("api_key") else { continue; }; - // The sidecar mixin declares `credentials[].service` under the same - // derivation, so the bound value and the inject rule that consumes it - // always name the same service. Passing the raw stem here would produce - // an id sbx rejects for any RAG whose name is not already a valid id. let service_id = mcp_credentials::secret_service_id(&stem); if service_id.is_empty() || registered.contains(&service_id) { continue; @@ -358,9 +348,7 @@ fn inject_rag_secrets(vault: &Vault, registered: &HashSet) -> Result<()> .trim_start_matches("{{") .trim_end_matches("}}") .trim(); - // Degrade rather than abort: one stale RAG key must not block the whole - // sandbox launch. Queries to that RAG fail with a 401 at runtime, which - // is recoverable without a restart. + match vault.get_secret(secret_name, false) { Ok(secret_value) => { sbx_secret_set(&service_id, &secret_value) @@ -375,6 +363,7 @@ fn inject_rag_secrets(vault: &Vault, registered: &HashSet) -> Result<()> } } } + Ok(()) } From 93a934439b2f174a9526f5984cba1e34f0b6aeb3 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 11 Aug 2026 13:44:00 -0600 Subject: [PATCH 10/36] fix(rag): fail loudly when a RAG's vault secret is missing `interpolate_secrets` does not error on a secret the vault cannot resolve: it substitutes the empty string and returns the name in its second tuple element. `load_async` discarded that vec, so a typo'd or deleted vault secret produced `api_key = ""` and an unexplained 401 from Qdrant. Bail instead, naming the RAG and the missing secrets, matching what global config loading already does. --- src/rag/mod.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/rag/mod.rs b/src/rag/mod.rs index dc20877..032830d 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -377,10 +377,22 @@ impl Rag { let api_key: Option = match data.driver_config.get("api_key") { Some(placeholder) => { - let (resolved, _) = - interpolate_secrets(placeholder, vault).with_context(|| { + let (resolved, missing) = interpolate_secrets(placeholder, vault) + .with_context(|| { format!("Failed to resolve api_key secret for RAG '{name}'") })?; + // A secret the vault does not hold is NOT an error inside + // `interpolate_secrets`: it substitutes an empty string and + // only reports the name. Accepting that silently attaches with + // `api_key = ""`, and the user sees an unexplained 401 from the + // server instead of the typo they made. + if !missing.is_empty() { + bail!( + "RAG '{name}' references secrets that are missing from the vault: {}. \ + Add them with `coyote --add-secret `, then try again.", + missing.join(", ") + ); + } Some(resolved) } None => None, From 7f90710427be7a46656ac9f29190393de128a33f Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 11 Aug 2026 13:46:32 -0600 Subject: [PATCH 11/36] refactor(rag): interpolate every driver_config value, not just api_key Only `driver_config["api_key"]` was interpolated, so any credential-bearing driver field added later would have shipped its raw `{{PLACEHOLDER}}` to the server. Resolve every value instead, via `resolve_driver_config`. Resolution still happens into a function-local copy and never touches `RagData`: `save()` serializes `self.data` and is called by `.set rag_top_k` and friends, so a resolved credential parked there would be written to the RAG's YAML in plaintext. The literal `{{NAME}}` also has to survive on disk because sandbox credential provisioning parses it back out to learn which vault secret to bind. Scope stays `driver_config` deliberately: the rest of a RAG file is ingested document text, where `{{...}}` is ordinary content. Covered by a test that saves after a load and asserts the placeholder, not the secret, is what reaches the file. --- src/rag/mod.rs | 221 ++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 190 insertions(+), 31 deletions(-) diff --git a/src/rag/mod.rs b/src/rag/mod.rs index 032830d..47d1bce 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -355,50 +355,27 @@ impl Rag { let raw_content = fs::read_to_string(path).with_context(err)?; // Parsed WITHOUT secret interpolation, so `driver_config` keeps its - // `{{...}}` placeholders. Interpolating here would bake the resolved API - // key into `self.data`, which `save()` then writes back to disk in - // plaintext. + // `{{...}}` placeholders in `self.data`. Resolution happens below, into a + // function-local copy only — see `resolve_driver_config` for why the + // resolved values must never travel back into `data`. let data: RagData = serde_yaml::from_str(&raw_content).with_context(err)?; data.validate().with_context(err)?; match data.driver.as_str() { "qdrant" => { - let host = data - .driver_config + let driver_config = resolve_driver_config(&data.driver_config, vault, name)?; + let host = driver_config .get("host") .context("qdrant driver requires 'host' in driver_config")? .clone(); - let collection = data - .driver_config + let collection = driver_config .get("collection") .context("qdrant driver requires 'collection' in driver_config")? .clone(); + let api_key = driver_config.get("api_key").map(String::as_str); - let api_key: Option = match data.driver_config.get("api_key") { - Some(placeholder) => { - let (resolved, missing) = interpolate_secrets(placeholder, vault) - .with_context(|| { - format!("Failed to resolve api_key secret for RAG '{name}'") - })?; - // A secret the vault does not hold is NOT an error inside - // `interpolate_secrets`: it substitutes an empty string and - // only reports the name. Accepting that silently attaches with - // `api_key = ""`, and the user sees an unexplained 401 from the - // server instead of the typo they made. - if !missing.is_empty() { - bail!( - "RAG '{name}' references secrets that are missing from the vault: {}. \ - Add them with `coyote --add-secret `, then try again.", - missing.join(", ") - ); - } - Some(resolved) - } - None => None, - }; - - let provider = QdrantProvider::new(&host, &collection, api_key.as_deref()).await?; + let provider = QdrantProvider::new(&host, &collection, api_key).await?; let embedding_model = Model::retrieve_model(app, &data.embedding_model, ModelType::Embedding)?; Ok(Rag { @@ -2133,6 +2110,72 @@ fn embedding_dim_for_model(model_id: &str) -> usize { } } +/// Resolves `{{SECRET}}` placeholders in every `driver_config` value against the +/// vault, returning a DETACHED copy. +/// +/// Three properties this must preserve, each of which has already bitten: +/// +/// 1. The resolved values never go back into `RagData`. `Rag::save()` +/// serializes `self.data`, and `.set rag_top_k`, `.set rag_reranker_model` +/// and every post-sync save call it — so a resolved credential parked in +/// `data.driver_config` gets written to the RAG's YAML file in plaintext the +/// next time the user changes any setting. +/// 2. The literal `{{NAME}}` text survives in `data` and on disk. Sandbox +/// credential provisioning parses that placeholder back out of the file to +/// learn which vault secret to bind into the sandbox; resolve it away and +/// provisioning silently finds nothing to register. +/// 3. Only `driver_config` is interpolated, never the whole file. The rest of a +/// RAG file is ingested document text and vectors — where `{{...}}` is +/// ordinary content (Jinja, Mustache, Vue, Go templates) that would be read +/// as a secret reference, blanked to `""`, and persisted on the next save. +/// `driver_config` is small and is the only place credentials live. +fn resolve_driver_config( + driver_config: &IndexMap, + vault: &Vault, + rag_name: &str, +) -> Result> { + resolve_driver_config_with(driver_config, rag_name, |value| { + interpolate_secrets(value, vault) + }) +} + +/// Interpolation core, taking the resolver as an argument so it can be exercised +/// without a vault. Mirrors `interpolate_secrets` / `interpolate_secrets_with`. +fn resolve_driver_config_with( + driver_config: &IndexMap, + rag_name: &str, + mut interpolate: F, +) -> Result> +where + F: FnMut(&str) -> Result<(String, Vec)>, +{ + let mut resolved = IndexMap::with_capacity(driver_config.len()); + let mut missing: Vec = Vec::new(); + for (key, value) in driver_config { + let (value, value_missing) = interpolate(value).with_context(|| { + format!("Failed to resolve '{key}' in driver_config for RAG '{rag_name}'") + })?; + missing.extend(value_missing); + resolved.insert(key.clone(), value); + } + + // A secret the vault does not hold is NOT an error inside + // `interpolate_secrets`: it substitutes the empty string and only reports the + // name. Accepting that ships an empty credential, and the user sees an + // unexplained 401 from the server instead of the typo they made. + if !missing.is_empty() { + missing.sort(); + missing.dedup(); + bail!( + "RAG '{rag_name}' references secrets that are missing from the vault: {}. \ + Add them with `coyote --add-secret `, then try again.", + missing.join(", ") + ); + } + + Ok(resolved) +} + #[cfg(test)] mod tests { use super::*; @@ -2183,6 +2226,122 @@ mod tests { assert!(yaml.contains("{{QDRANT_API_KEY}}")); } + const FAKE_SECRET: &str = "sk-live-fake-value-for-tests"; + + fn attached_qdrant_data() -> RagData { + let mut data = RagData { + driver: "qdrant".to_string(), + attached: true, + embedding_model: "text-embedding-3-small".to_string(), + top_k: 5, + ..Default::default() + }; + data.driver_config + .insert("host".into(), "localhost:6333".into()); + data.driver_config.insert("collection".into(), "c".into()); + data.driver_config + .insert("api_key".into(), "{{QDRANT_API_KEY}}".into()); + data + } + + /// THE invariant behind `resolve_driver_config` returning a detached copy. + /// + /// `save()` serializes `self.data`, and `.set rag_top_k`, `.set + /// rag_reranker_model` and every post-sync save call it. If load ever bakes + /// the resolved credential into `data.driver_config`, the next trivial + /// setting change writes the user's plaintext API key into the RAG's YAML + /// file. The literal placeholder must also survive, because sandbox + /// credential provisioning parses it back off disk. + #[test] + fn a_save_after_load_writes_the_placeholder_not_the_resolved_secret() { + let dir = TempDir::new("driver-config-secret"); + let path = dir.path.join("kb.yaml"); + let data = attached_qdrant_data(); + + // Exactly what `load_async` does with the parsed data. + let resolved = resolve_driver_config_with(&data.driver_config, "kb", |value| { + Ok((value.replace("{{QDRANT_API_KEY}}", FAKE_SECRET), vec![])) + }) + .unwrap(); + assert_eq!( + resolved["api_key"], FAKE_SECRET, + "the live client still has to receive the real key" + ); + assert_eq!( + data.driver_config["api_key"], "{{QDRANT_API_KEY}}", + "resolution must not mutate the RagData that save() serializes" + ); + + let rag = Rag { + app_config: Arc::new(AppConfig::default()), + name: "kb".to_string(), + path: path.display().to_string(), + embedding_model: Model::new("openai", "text-embedding-3-small"), + bm25: data.build_bm25(), + provider: Box::new(YamlProvider::from_data(&data)), + node_to_docs: IndexMap::new(), + data, + last_sources: RwLock::new(None), + }; + assert!(rag.save().unwrap()); + + let on_disk = fs::read_to_string(&path).unwrap(); + assert!( + on_disk.contains("{{QDRANT_API_KEY}}"), + "sandbox provisioning parses this placeholder back off disk: {on_disk}" + ); + assert!( + !on_disk.contains(FAKE_SECRET), + "a save after load leaked the plaintext secret to {}", + path.display() + ); + } + + /// Every value is interpolated, not just `api_key` — a credential-bearing + /// field added later must not ship its raw placeholder to the server. + #[test] + fn resolution_covers_every_driver_config_value() { + let mut driver_config = IndexMap::new(); + driver_config.insert("host".to_string(), "{{QDRANT_HOST}}".to_string()); + driver_config.insert("collection".to_string(), "c".to_string()); + driver_config.insert("api_key".to_string(), "{{QDRANT_API_KEY}}".to_string()); + + let resolved = resolve_driver_config_with(&driver_config, "kb", |value| { + let out = value + .replace("{{QDRANT_HOST}}", "qdrant.internal:6333") + .replace("{{QDRANT_API_KEY}}", FAKE_SECRET); + Ok((out, vec![])) + }) + .unwrap(); + + assert_eq!(resolved["host"], "qdrant.internal:6333"); + assert_eq!(resolved["collection"], "c"); + assert_eq!(resolved["api_key"], FAKE_SECRET); + } + + /// Missing secrets are reported together, deduplicated, and name the RAG. + #[test] + fn missing_secrets_fail_the_load_instead_of_resolving_to_empty() { + let mut driver_config = IndexMap::new(); + driver_config.insert("host".to_string(), "{{QDRANT_HOST}}".to_string()); + driver_config.insert("api_key".to_string(), "{{QDRANT_API_KEY}}".to_string()); + + let err = resolve_driver_config_with(&driver_config, "kb", |value| { + // What `interpolate_secrets` really does for an absent secret: blank it + // out and report the name rather than returning Err. + Ok(( + String::new(), + vec![value.trim_matches(['{', '}']).to_string()], + )) + }) + .expect_err("an empty API key must not be accepted as a successful load"); + + let msg = err.to_string(); + assert!(msg.contains("kb"), "the RAG must be named: {msg}"); + assert!(msg.contains("QDRANT_HOST"), "got: {msg}"); + assert!(msg.contains("QDRANT_API_KEY"), "got: {msg}"); + } + /// A qdrant RAG's vectors MUST survive serialization. /// /// `save()` omits vectors only for `driver == "duckdb"`. Qdrant must not join From 860566bf50b013942a7a703a246b17c41a679949 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 11 Aug 2026 13:46:59 -0600 Subject: [PATCH 12/36] feat: let workflow rag nodes select a RAG driver `RagNode` gains an optional `driver`, forwarded into `RagInitConfig` so a graph node can build its knowledge base on duckdb instead of yaml. Nodes that name no driver forward `None`, which still resolves to yaml, so existing workflows are unaffected. An unknown driver is rejected up front rather than at construction time. `Rag::create` dispatches unknown drivers to its yaml catch-all, so a typo would otherwise embed every document and persist the bogus string, after which every subsequent load fails validation and the agent cannot start. The check asks `RagData::validate()` through a probe value instead of restating the list of valid drivers, so the two cannot drift. --- src/config/agent.rs | 87 ++++++++++++++++++++++++++++------ src/graph/types.rs | 103 +++++++++++++++++++++++++++++++++++++++++ src/graph/validator.rs | 101 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 276 insertions(+), 15 deletions(-) diff --git a/src/config/agent.rs b/src/config/agent.rs index a6c051a..569ec48 100644 --- a/src/config/agent.rs +++ b/src/config/agent.rs @@ -12,6 +12,7 @@ use crate::config::prompts::{ DEFAULT_SPAWN_INSTRUCTIONS, DEFAULT_TEAMMATE_INSTRUCTIONS, DEFAULT_TODO_INSTRUCTIONS, DEFAULT_USER_INTERACTION_INSTRUCTIONS, }; +use crate::graph::types::RagNode; use crate::graph::{Graph, GraphParser, NodeType}; use crate::rag::RagInitConfig; use crate::vault::SECRET_RE; @@ -952,6 +953,30 @@ fn resolve_document_paths( Ok(document_paths) } +/// How a graph rag node describes the knowledge base it wants built. +/// +/// `driver` is forwarded as-is: `None` means the node did not ask for one, which +/// `RagInitConfig` resolves to yaml, so workflows written before drivers existed +/// keep their current storage. +/// +/// Every field is now named explicitly, so adding one to `RagInitConfig` breaks +/// this literal. That is deliberate: the new field then gets a decision about +/// whether a rag node can drive it, instead of silently taking its default. +fn rag_init_config(rag_node: &RagNode) -> RagInitConfig { + RagInitConfig { + embedding_model: rag_node.embedding_model.clone(), + chunk_size: rag_node.chunk_size, + chunk_overlap: rag_node.chunk_overlap, + reranker_model: rag_node.reranker_model.clone(), + top_k: rag_node.top_k, + batch_size: rag_node.batch_size, + extractor_model: rag_node.extractor_model.clone(), + extractor_prompt: rag_node.extractor_prompt.clone(), + graph_hops: rag_node.graph_hops, + driver: rag_node.driver.clone(), + } +} + #[allow(clippy::too_many_arguments)] async fn init_graph_rags( app: &AppConfig, @@ -989,21 +1014,18 @@ async fn init_graph_rags( }) .await? } else { - let config = RagInitConfig { - embedding_model: rag_node.embedding_model.clone(), - chunk_size: rag_node.chunk_size, - chunk_overlap: rag_node.chunk_overlap, - reranker_model: rag_node.reranker_model.clone(), - top_k: rag_node.top_k, - batch_size: rag_node.batch_size, - extractor_model: rag_node.extractor_model.clone(), - extractor_prompt: rag_node.extractor_prompt.clone(), - graph_hops: rag_node.graph_hops, - // Graph-node RAGs are yaml-only: `RagNode` has no `driver` field, so - // there is nothing to forward. The rest-pattern also keeps this literal - // from breaking on future `RagInitConfig` additions. - ..Default::default() - }; + // Checked before anything is built: an unknown driver would otherwise + // fall through `Rag::create`'s catch-all to a yaml store, embed every + // document, and persist the bogus driver string. The RAG would then be + // rejected on every subsequent load, leaving the agent unstartable. + // Graph validation catches this too, but it is skipped when + // `validate_before_run` is off, so this guard is the load-bearing one. + if let Some(driver) = &rag_node.driver + && let Some(message) = crate::graph::validator::rag_driver_error(driver) + { + bail!("rag node '{node_id}': {message}"); + } + let config = rag_init_config(rag_node); let fully_specified = config.embedding_model.is_some() && config.chunk_size.is_some() && config.chunk_overlap.is_some(); @@ -1337,4 +1359,39 @@ version: "1.0" assert_eq!(meta.description, ""); } + + #[test] + fn rag_init_config_forwards_an_explicit_driver() { + let node: RagNode = + serde_yaml::from_str("documents: [\"./docs\"]\ndriver: duckdb\n").unwrap(); + + assert_eq!(rag_init_config(&node).driver.as_deref(), Some("duckdb")); + } + + /// A node that names no driver must forward `None`, which `RagInitConfig` + /// documents as "yaml". Existing workflows therefore keep their yaml store. + #[test] + fn rag_init_config_leaves_the_driver_unset_by_default() { + let node: RagNode = serde_yaml::from_str("documents: [\"./docs\"]\n").unwrap(); + + assert_eq!(rag_init_config(&node).driver, None); + } + + /// The driver must ride alongside the rest of the node's settings, not + /// replace them. + #[test] + fn rag_init_config_forwards_the_other_settings_too() { + let node: RagNode = serde_yaml::from_str( + "documents: [\"./docs\"]\ndriver: duckdb\nchunk_size: 512\nchunk_overlap: 64\ntop_k: 7\nembedding_model: some:model\n", + ) + .unwrap(); + + let config = rag_init_config(&node); + + assert_eq!(config.driver.as_deref(), Some("duckdb")); + assert_eq!(config.chunk_size, Some(512)); + assert_eq!(config.chunk_overlap, Some(64)); + assert_eq!(config.top_k, Some(7)); + assert_eq!(config.embedding_model.as_deref(), Some("some:model")); + } } diff --git a/src/graph/types.rs b/src/graph/types.rs index 020badf..fcd3292 100644 --- a/src/graph/types.rs +++ b/src/graph/types.rs @@ -367,6 +367,13 @@ pub struct RagNode { #[serde(default, skip_serializing_if = "Option::is_none")] pub graph_hops: Option, + /// Storage driver for this node's knowledge base ("yaml", "duckdb"). `None` + /// means "yaml". Only honored when the knowledge base is first built; + /// changing it afterwards has no effect until the RAG is deleted and + /// re-initialized. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub driver: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub state_updates: Option>, @@ -1152,4 +1159,100 @@ nodes: assert!(triage.next.as_ref().unwrap().is_fan_out()); assert_eq!(triage.next.as_ref().unwrap().as_slice().len(), 2); } + + fn rag_node_of(graph: &Graph, id: &str) -> RagNode { + match &graph.get_node(id).unwrap().node_type { + NodeType::Rag(r) => r.clone(), + other => panic!("expected a rag node, got {other:?}"), + } + } + + #[test] + fn rag_node_deserializes_an_explicit_driver() { + let yaml = r#" +name: kb +start: research +nodes: + research: + type: rag + documents: ["./docs"] + driver: duckdb + next: done + done: + type: end + output: ok +"#; + let graph: Graph = serde_yaml::from_str(yaml).unwrap(); + + assert_eq!( + rag_node_of(&graph, "research").driver.as_deref(), + Some("duckdb") + ); + } + + /// Workflows written before drivers existed must keep parsing, and must keep + /// asking for nothing, so `RagInitConfig` resolves them to the yaml default. + #[test] + fn rag_node_without_a_driver_stays_unset() { + let yaml = r#" +name: kb +start: research +nodes: + research: + type: rag + documents: ["./docs"] + next: done + done: + type: end + output: ok +"#; + let graph: Graph = serde_yaml::from_str(yaml).unwrap(); + + assert_eq!(rag_node_of(&graph, "research").driver, None); + } + + #[test] + fn rag_node_driver_survives_a_serialize_round_trip() { + let yaml = r#" +name: kb +start: research +nodes: + research: + type: rag + documents: ["./docs"] + driver: duckdb + next: done + done: + type: end + output: ok +"#; + let graph: Graph = serde_yaml::from_str(yaml).unwrap(); + let reparsed: Graph = + serde_yaml::from_str(&serde_yaml::to_string(&graph).unwrap()).unwrap(); + + assert_eq!( + rag_node_of(&reparsed, "research").driver.as_deref(), + Some("duckdb") + ); + } + + /// `skip_serializing_if` must keep `driver:` out of graphs that never set it. + #[test] + fn rag_node_without_a_driver_omits_the_key_when_serialized() { + let yaml = r#" +name: kb +start: research +nodes: + research: + type: rag + documents: ["./docs"] + next: done + done: + type: end + output: ok +"#; + let graph: Graph = serde_yaml::from_str(yaml).unwrap(); + + assert!(!serde_yaml::to_string(&graph).unwrap().contains("driver")); + } } diff --git a/src/graph/validator.rs b/src/graph/validator.rs index f655c8e..fe438f7 100644 --- a/src/graph/validator.rs +++ b/src/graph/validator.rs @@ -2,6 +2,7 @@ use super::state::template_root_keys; use super::types::{Graph, Node, NodeType}; use crate::client::{Model, ModelType}; use crate::config::{Agent, AppConfig, paths}; +use crate::rag::{GraphRagConfig, RagData}; use anyhow::{Result, bail}; use std::collections::{BTreeMap, HashSet, VecDeque}; use std::path::PathBuf; @@ -96,6 +97,51 @@ pub struct GraphValidator { skill_exists: fn(&str) -> bool, } +/// A minimal `RagData` whose only interesting field is `driver`. The numeric +/// arguments are the smallest values that satisfy `validate()`'s unrelated +/// floors (top_k >= 1, and chunk_size >= 1 with chunk_overlap < chunk_size for +/// a non-attached RAG). `RagData::new` sets `attached: false`, which is the +/// correct shape here: a graph rag node always builds its own local knowledge +/// base from `documents` and can never be attached. +fn rag_driver_probe(driver: &str) -> RagData { + let mut data = RagData::new( + String::new(), + 1, + 0, + None, + 1, + None, + GraphRagConfig::default(), + ); + data.driver = driver.to_string(); + data +} + +/// `Some(message)` when `driver` is one that `RagData::validate()` would reject. +/// +/// The set of valid drivers is defined in exactly one place, `RagData::validate()`, +/// so this asks that function rather than restating the list here. +/// +/// Fails open on purpose: the first probe below uses the default driver, which is +/// valid by definition. If even that one is rejected, `validate()` has grown a +/// precondition the probe fixture no longer satisfies, and every verdict from here +/// would be a false positive that rejects working graphs. In that case we decline +/// to judge and leave enforcement to RAG construction. The +/// `rag_driver_probe_fixture_is_accepted` test turns that silent degradation into a +/// loud failure. Both `validate()` calls are load-bearing; neither is redundant. +pub(crate) fn rag_driver_error(driver: &str) -> Option { + if rag_driver_probe(&RagData::default().driver) + .validate() + .is_err() + { + return None; + } + rag_driver_probe(driver) + .validate() + .err() + .map(|err| err.to_string()) +} + impl GraphValidator { pub fn new(base_dir: impl Into) -> Self { Self { @@ -154,6 +200,11 @@ impl GraphValidator { not be written to state", )); } + if let Some(driver) = &r.driver + && let Some(message) = rag_driver_error(driver) + { + result.error(ValidationError::with_node(node_id, message)); + } } } } @@ -1031,6 +1082,7 @@ mod tests { extractor_model: None, extractor_prompt: None, graph_hops: None, + driver: None, state_updates, timeout: None, }), @@ -1385,6 +1437,55 @@ mod tests { ); } + /// Guards the fail-open branch in `rag_driver_error`. If this fails, + /// `RagData::validate()` grew a precondition the probe fixture no longer + /// satisfies and rag-node driver validation has silently switched itself off. + /// Repair the fixture in `rag_driver_probe`; do not delete this test. + #[test] + fn rag_driver_probe_fixture_is_accepted() { + let default_driver = RagData::default().driver; + assert!( + rag_driver_probe(&default_driver).validate().is_ok(), + "probe fixture rejected for the default driver '{default_driver}'" + ); + } + + #[test] + fn rag_driver_error_defers_to_ragdata_validate() { + assert_eq!(rag_driver_error("yaml"), None); + assert_eq!(rag_driver_error("duckdb"), None); + + let message = rag_driver_error("duckdbb").expect("unknown driver must be rejected"); + assert!(message.contains("duckdbb"), "got: {message}"); + } + + #[test] + fn rag_node_with_unknown_driver_errors_naming_the_node() { + let mut node = rag_node("kb", &["./docs"], true); + if let NodeType::Rag(ref mut r) = node.node_type { + r.driver = Some("postgres".into()); + } + let graph = graph_with(vec![("kb", node), ("end", end_node("end"))], "kb"); + + let result = validator().validate(&graph); + + assert!(!result.is_valid()); + let err = result.into_result().unwrap_err().to_string(); + assert!(err.contains("[kb]"), "must name the node: {err}"); + assert!(err.contains("postgres"), "must name the driver: {err}"); + } + + #[test] + fn rag_node_with_duckdb_driver_produces_no_findings() { + let mut node = rag_node("kb", &["./docs"], true); + if let NodeType::Rag(ref mut r) = node.node_type { + r.driver = Some("duckdb".into()); + } + let graph = graph_with(vec![("kb", node), ("end", end_node("end"))], "kb"); + + assert!(validator().validate(&graph).is_valid()); + } + fn agent_node(id: &str, agent: &str, next: Option<&str>) -> Node { Node { id: id.into(), From c458ca93a9aed02a6c07b54da4fe2c9e693a806d Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 11 Aug 2026 13:50:33 -0600 Subject: [PATCH 13/36] feat(rag): create the API key secret inline in the attach wizard The wizard hard-errored with "Secret 'X' not found in vault. Run `coyote --add-secret X` first.", throwing away every answer the user had already given it. Offer to create the secret in place instead, deferring to `Vault::add_secret` for the masked prompt, the provider write and the confirmation line, then read it back. Only a genuine `SecretError::NotFound` triggers the offer. An auth failure, a provider outage, or the vault being disabled inside a sandbox all propagate with their own message, because prompting for a value that cannot be stored would fail one step later and bury the real cause. Declining the offer fails with both ways out spelled: add the secret up front, or answer "no" to the API-key question. --- src/rag/mod.rs | 82 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 76 insertions(+), 6 deletions(-) diff --git a/src/rag/mod.rs b/src/rag/mod.rs index 47d1bce..daf4173 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -18,6 +18,7 @@ use crate::vault::{Vault, interpolate_secrets}; use anyhow::{Context, Result, anyhow, bail}; use bm25::{Language, SearchEngine, SearchEngineBuilder}; +use gman::SecretError; use hnsw_rs::prelude::*; use indexmap::{IndexMap, IndexSet}; use inquire::{Confirm, Select, Text, required, validator::Validation}; @@ -429,12 +430,7 @@ impl Rag { .with_default("QDRANT_API_KEY") .with_validator(required!("This field is required")) .prompt()?; - let resolved = vault.get_secret(&secret_name, false).with_context(|| { - format!( - "Secret '{secret_name}' not found in vault. \ - Run `coyote --add-secret {secret_name}` first." - ) - })?; + let resolved = resolve_or_create_api_key_secret(vault, &secret_name)?; Some((secret_name, resolved)) } else { None @@ -2110,6 +2106,57 @@ fn embedding_dim_for_model(model_id: &str) -> usize { } } +/// True only for "the vault does not hold this key". +/// +/// Everything else — an auth failure, a provider outage, or the vault being +/// disabled because Coyote is running inside a sandbox — must NOT be treated as +/// a missing secret. Offering to create one in those cases would prompt for a +/// value that cannot be stored and bury the real reason. +fn is_missing_secret(err: &anyhow::Error) -> bool { + matches!( + err.downcast_ref::(), + Some(SecretError::NotFound { .. }) + ) +} + +/// Reads `secret_name` out of the vault, offering to create it in place when the +/// vault simply does not hold it yet. +/// +/// Sending the user off to run `coyote --add-secret` mid-wizard discarded every +/// answer they had already given. `Vault::add_secret` does the masked prompt, +/// the provider write and the confirmation line, so this defers to it rather +/// than collecting or storing the value itself. +fn resolve_or_create_api_key_secret(vault: &Vault, secret_name: &str) -> Result { + let read_err = match vault.get_secret(secret_name, false) { + Ok(secret) => return Ok(secret), + Err(err) => err, + }; + if !is_missing_secret(&read_err) { + return Err(read_err) + .with_context(|| format!("Cannot read secret '{secret_name}' from the vault")); + } + + let create = Confirm::new(&format!( + "Secret '{secret_name}' is not in the vault. Create it now?" + )) + .with_default(true) + .prompt()?; + if !create { + bail!( + "This instance needs an API key, so '{secret_name}' has to exist before \ + attaching. Add it with `coyote --add-secret {secret_name}` and re-run, or \ + re-run and answer 'no' when asked whether the instance requires an API key." + ); + } + + vault + .add_secret(secret_name) + .with_context(|| format!("Failed to store secret '{secret_name}' in the vault"))?; + vault + .get_secret(secret_name, false) + .with_context(|| format!("Secret '{secret_name}' is unreadable after being stored")) +} + /// Resolves `{{SECRET}}` placeholders in every `driver_config` value against the /// vault, returning a DETACHED copy. /// @@ -2342,6 +2389,29 @@ mod tests { assert!(msg.contains("QDRANT_API_KEY"), "got: {msg}"); } + /// Only a genuine NotFound may trigger the attach wizard's "create it now?" + /// offer. The vault is disabled wholesale inside a sandbox, where creating a + /// secret is impossible — misreading that as "missing" would prompt for a + /// value that cannot be stored and hide why. + #[test] + fn only_a_not_found_error_counts_as_a_missing_secret() { + let not_found = anyhow::Error::new(SecretError::NotFound { + key: "QDRANT_API_KEY".to_string(), + provider: "local", + }); + assert!(is_missing_secret(¬_found)); + + let auth_failed = anyhow::Error::new(SecretError::AuthFailed { + provider: "local", + source: anyhow!("bad vault password"), + }); + assert!(!is_missing_secret(&auth_failed)); + + // What `Vault::get_secret` returns in sandbox mode: a plain anyhow error. + let sandboxed = anyhow!("Vault management is disabled in sandbox mode."); + assert!(!is_missing_secret(&sandboxed)); + } + /// A qdrant RAG's vectors MUST survive serialization. /// /// `save()` omits vectors only for `driver == "duckdb"`. Qdrant must not join From 5e2b9c98ad2d47d2665dc6af826597da265c0cc9 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 11 Aug 2026 13:55:59 -0600 Subject: [PATCH 14/36] fix(rag): stop the attach wizard from silently accepting an empty collection `sample_point_id` returns `None` for a collection with no points, so the UUID guard's `if let Some(..)` fell straight through and the wizard attached happily. The result is a RAG that answers every query with zero hits and never says why. Sample once, then check for emptiness explicitly. This warns and asks rather than hard-failing: an empty collection is not necessarily a mistake, since another tool may be about to populate it, and none of the wizard's remaining probes can distinguish that from a misconfiguration. The confirmation defaults to "no" so it cannot be walked past by accident, and `attach` already refuses to run non-interactively, so no unattended path reaches it. --- src/rag/mod.rs | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/rag/mod.rs b/src/rag/mod.rs index daf4173..cbfd133 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -453,10 +453,32 @@ impl Rag { let collection = Select::new("Select collection:", collections).prompt()?; + let sample_id = QdrantProvider::sample_point_id(&host, &collection, api_key).await?; + + // `None` means the scroll came back with no points at all: the collection + // is empty. Attaching is not necessarily wrong — another tool may be about + // to fill it — but accepting it silently yields a RAG that answers every + // query with nothing and never explains why, and none of the checks below + // can tell that apart from a misconfiguration. Ask, defaulting to no, so it + // cannot happen by accident. (`attach` already refuses to run + // non-interactively, so there is no unattended path through this prompt.) + if sample_id.is_none() { + println!( + "⚠️ Collection '{collection}' contains no points. Queries will return \ + nothing until something writes to it." + ); + let attach_anyway = Confirm::new("Attach to this empty collection anyway?") + .with_default(false) + .prompt()?; + if !attach_anyway { + bail!("Collection '{collection}' is empty; nothing to attach to."); + } + } + // Point IDs are read with `as_u64()`, which yields None for a JSON string. // A UUID-keyed collection would therefore return zero hits with no error, // so refuse it here instead of attaching something silently broken. - if let Some(raw_id) = QdrantProvider::sample_point_id(&host, &collection, api_key).await? + if let Some(raw_id) = &sample_id && raw_id.starts_with('"') { bail!( From dc677a252944f75c58faab4f8fd5aa7d193bc5fa Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 11 Aug 2026 14:05:31 -0600 Subject: [PATCH 15/36] fix(sandbox): discover agent-scoped RAG mixin sidecars An agent-scoped RAG writes its config to /agents//.yaml, so its sbx mixin sidecar lands beside it as .sbx-mixin.yaml. Discovery scanned the agents directory only for a file named exactly sbx-mixin.yaml, and scanned for suffixed sidecars only in the top-level rags directory, so a RAG attached while an agent was active contributed no network allow rule and no credential to the sandbox. The failure was silent: the sandbox launched and the RAG was simply unreachable from inside it. The two collectors differed only in the filename shape they matched, so they are now one scan that takes the set of layouts to look for. The agents directory asks for both its own sbx-mixin.yaml and the suffixed sidecars one level in, which is the shape that was missing. Discovery order is unchanged, and it is load-bearing: each mixin becomes a --kit in list order and later ones layer over earlier ones, so the workspace mixin must stay last. --- src/sandbox/mixins.rs | 204 +++++++++++++++++++++++++++++++++--------- 1 file changed, 164 insertions(+), 40 deletions(-) diff --git a/src/sandbox/mixins.rs b/src/sandbox/mixins.rs index 697716a..3123062 100644 --- a/src/sandbox/mixins.rs +++ b/src/sandbox/mixins.rs @@ -67,13 +67,16 @@ pub fn discover() -> Result> { push_if_exists(&mut out, paths::sbx_mixin_file())?; push_if_exists(&mut out, paths::global_tools_sbx_mixin_file())?; - for path in collect_subdir_mixins(&paths::functions_dir()) { + for path in collect_mixins(&paths::functions_dir(), &[ScanMode::SubdirNamed]) { out.push(read_mixin(path)?); } - for path in collect_subdir_mixins(&paths::agents_data_dir()) { + for path in collect_mixins( + &paths::agents_data_dir(), + &[ScanMode::SubdirNamed, ScanMode::SubdirFlat], + ) { out.push(read_mixin(path)?); } - for path in collect_flat_mixins(&paths::rags_dir()) { + for path in collect_mixins(&paths::rags_dir(), &[ScanMode::Flat]) { out.push(read_mixin(path)?); } @@ -160,27 +163,54 @@ fn read_mixin(path: PathBuf) -> Result { }) } -fn collect_subdir_mixins(dir: &Path) -> Vec { +/// One on-disk layout a mixin scan can look for. A scan takes a set of these, +/// and each mode contributes only the shape it names. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ScanMode { + /// `/*.sbx-mixin.yaml` + Flat, + /// `/*/sbx-mixin.yaml` + SubdirNamed, + /// `/*/*.sbx-mixin.yaml` + SubdirFlat, +} + +/// Collects mixin paths under `dir` for every requested layout. Missing or +/// unreadable directories yield nothing rather than an error — these paths are +/// all optional on disk. +/// +/// Order is deterministic: flat matches first (sorted by file name), then each +/// subdirectory in sorted order, contributing its named mixin before its +/// suffixed ones. +fn collect_mixins(dir: &Path, modes: &[ScanMode]) -> Vec { let mut result = Vec::new(); - let Ok(rd) = read_dir(dir) else { return result }; - let mut entries: Vec<_> = rd - .flatten() - .filter(|e| e.file_type().map(|t| t.is_dir()).unwrap_or(false)) - .collect(); - entries.sort_by_key(|e| e.file_name()); + if modes.contains(&ScanMode::Flat) { + result.extend(suffixed_mixins_in(dir)); + } - for entry in entries { - let candidate = entry.path().join(SBX_MIXIN_FILE_NAME); - if candidate.exists() { - result.push(candidate); + let named = modes.contains(&ScanMode::SubdirNamed); + let subdir_flat = modes.contains(&ScanMode::SubdirFlat); + if !named && !subdir_flat { + return result; + } + + for subdir in subdirs_of(dir) { + if named { + let candidate = subdir.join(SBX_MIXIN_FILE_NAME); + if candidate.exists() { + result.push(candidate); + } + } + if subdir_flat { + result.extend(suffixed_mixins_in(&subdir)); } } result } -fn collect_flat_mixins(dir: &Path) -> Vec { +fn suffixed_mixins_in(dir: &Path) -> Vec { let mut result = Vec::new(); let Ok(rd) = read_dir(dir) else { return result }; @@ -195,10 +225,21 @@ fn collect_flat_mixins(dir: &Path) -> Vec { .collect(); entries.sort_by_key(|e| e.file_name()); - for entry in entries { - result.push(entry.path()); - } + result.extend(entries.into_iter().map(|e| e.path())); + result +} +fn subdirs_of(dir: &Path) -> Vec { + let mut result = Vec::new(); + let Ok(rd) = read_dir(dir) else { return result }; + + let mut entries: Vec<_> = rd + .flatten() + .filter(|e| e.file_type().map(|t| t.is_dir()).unwrap_or(false)) + .collect(); + entries.sort_by_key(|e| e.file_name()); + + result.extend(entries.into_iter().map(|e| e.path())); result } @@ -218,6 +259,13 @@ mod tests { root } + fn file_names(paths: &[PathBuf]) -> Vec<&str> { + paths + .iter() + .map(|p| p.file_name().unwrap().to_str().unwrap()) + .collect() + } + #[test] fn summarize_counts_installs_and_domains() { let root = unique_root("sbx-mixin-counts"); @@ -301,7 +349,7 @@ network: } #[test] - fn collect_subdir_mixins_sorts_and_skips_missing() { + fn subdir_named_scan_sorts_and_skips_missing() { let root = unique_root("sbx-mixin-subdirs"); for name in ["zebra", "apple", "no-mixin", "mango"] { let dir = root.join(name); @@ -311,7 +359,7 @@ network: } } - let found = collect_subdir_mixins(&root); + let found = collect_mixins(&root, &[ScanMode::SubdirNamed]); let names: Vec = found .iter() .map(|p| { @@ -329,9 +377,9 @@ network: } #[test] - fn collect_subdir_mixins_returns_empty_for_missing_dir() { + fn subdir_named_scan_returns_empty_for_missing_dir() { let absent = env::temp_dir().join("coyote-definitely-not-here-xyz"); - let found = collect_subdir_mixins(&absent); + let found = collect_mixins(&absent, &[ScanMode::SubdirNamed]); assert!(found.is_empty()); } @@ -511,7 +559,7 @@ network: } #[test] - fn collect_flat_mixins_matches_rag_sidecars_by_suffix() { + fn flat_scan_matches_rag_sidecars_by_suffix() { let root = unique_root("flat-mixins"); fs::write(root.join("company-docs.sbx-mixin.yaml"), "kind: mixin\n").unwrap(); fs::write(root.join("alpha.sbx-mixin.yaml"), "kind: mixin\n").unwrap(); @@ -519,39 +567,115 @@ network: fs::write(root.join("notes.yaml"), "driver: yaml\n").unwrap(); fs::create_dir_all(root.join("decoy.sbx-mixin.yaml")).unwrap(); - let found = collect_flat_mixins(&root); - let names: Vec<_> = found - .iter() - .map(|p| p.file_name().unwrap().to_str().unwrap()) - .collect(); + let found = collect_mixins(&root, &[ScanMode::Flat]); assert_eq!( - names, + file_names(&found), vec!["alpha.sbx-mixin.yaml", "company-docs.sbx-mixin.yaml"] ); let _ = fs::remove_dir_all(&root); } - /// Why `collect_flat_mixins` had to be written: the existing collector walks - /// SUBDIRECTORIES for a file named exactly `sbx-mixin.yaml`, so it cannot see - /// a flat sidecar. If this ever starts finding them, the new collector is - /// redundant — but until then, removing it silently drops every RAG mixin. + /// Every scan site in `discover()` picks its modes assuming each mode owns + /// exactly one layout and nothing else. `agents_data_dir()` requests two + /// modes at once, so an overlap would collect the same file twice and + /// `create_sandbox` would pass it as two `--kit` flags. #[test] - fn collect_subdir_mixins_cannot_see_flat_rag_sidecars() { - let root = unique_root("flat-vs-subdir"); - fs::write(root.join("company-docs.sbx-mixin.yaml"), "kind: mixin\n").unwrap(); + fn each_scan_mode_owns_exactly_one_layout() { + let root = unique_root("scan-mode-ownership"); + let agent = root.join("researcher"); + fs::create_dir_all(&agent).unwrap(); + let flat = root.join("company-docs.sbx-mixin.yaml"); + let subdir_named = agent.join("sbx-mixin.yaml"); + let subdir_flat = agent.join("handbook.sbx-mixin.yaml"); + for path in [&flat, &subdir_named, &subdir_flat] { + fs::write(path, "kind: mixin\n").unwrap(); + } - assert!(collect_subdir_mixins(&root).is_empty()); - assert_eq!(collect_flat_mixins(&root).len(), 1); + assert_eq!(collect_mixins(&root, &[ScanMode::Flat]), vec![flat.clone()]); + assert_eq!( + collect_mixins(&root, &[ScanMode::SubdirNamed]), + vec![subdir_named.clone()] + ); + assert_eq!( + collect_mixins(&root, &[ScanMode::SubdirFlat]), + vec![subdir_flat.clone()] + ); + + let all = collect_mixins( + &root, + &[ScanMode::Flat, ScanMode::SubdirNamed, ScanMode::SubdirFlat], + ); + assert_eq!(all, vec![flat, subdir_named, subdir_flat]); + + let mut deduped = all.clone(); + deduped.sort(); + deduped.dedup(); + assert_eq!( + deduped.len(), + all.len(), + "no mixin may be collected twice: {all:?}" + ); let _ = fs::remove_dir_all(&root); } #[test] - fn collect_flat_mixins_tolerates_a_missing_directory() { + fn flat_scan_tolerates_a_missing_directory() { let root = unique_root("flat-missing"); let absent = root.join("nope"); - assert!(collect_flat_mixins(&absent).is_empty()); + assert!(collect_mixins(&absent, &[ScanMode::Flat]).is_empty()); + + let _ = fs::remove_dir_all(&root); + } + + /// `generate_rag_sbx_mixin` writes an agent-scoped RAG sidecar next to the + /// rag yaml, at `//.sbx-mixin.yaml`. Before `SubdirFlat` + /// existed, nothing scanned that shape and attaching a Qdrant RAG from + /// inside an agent produced no network allow rule and no credential. + #[test] + fn agent_scoped_rag_sidecar_is_discovered() { + let root = unique_root("agent-scoped-rag"); + let agent = root.join("researcher"); + fs::create_dir_all(&agent).unwrap(); + fs::write(agent.join("company-docs.sbx-mixin.yaml"), "kind: mixin\n").unwrap(); + fs::write(agent.join("company-docs.yaml"), "driver: qdrant\n").unwrap(); + + let found = collect_mixins(&root, &[ScanMode::SubdirNamed, ScanMode::SubdirFlat]); + assert_eq!(found, vec![agent.join("company-docs.sbx-mixin.yaml")]); + + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn agent_level_mixin_and_rag_sidecars_are_both_discovered() { + let root = unique_root("agent-both-shapes"); + let agent = root.join("researcher"); + fs::create_dir_all(&agent).unwrap(); + fs::write(agent.join("sbx-mixin.yaml"), "kind: mixin\n").unwrap(); + fs::write(agent.join("zebra.sbx-mixin.yaml"), "kind: mixin\n").unwrap(); + fs::write(agent.join("alpha.sbx-mixin.yaml"), "kind: mixin\n").unwrap(); + + let found = collect_mixins(&root, &[ScanMode::SubdirNamed, ScanMode::SubdirFlat]); + assert_eq!( + file_names(&found), + vec![ + "sbx-mixin.yaml", + "alpha.sbx-mixin.yaml", + "zebra.sbx-mixin.yaml" + ] + ); + + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn subdir_flat_scan_ignores_a_directory_named_like_a_mixin() { + let root = unique_root("subdir-flat-decoy"); + let agent = root.join("researcher"); + fs::create_dir_all(agent.join("decoy.sbx-mixin.yaml")).unwrap(); + + assert!(collect_mixins(&root, &[ScanMode::SubdirFlat]).is_empty()); let _ = fs::remove_dir_all(&root); } From 118c346345c7c7396b655ab0460c98ac6785b1d9 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 11 Aug 2026 14:05:38 -0600 Subject: [PATCH 16/36] feat(rag): support string and UUID Qdrant point IDs Point ids were read with as_u64() inside a filter_map, so a string id was silently dropped and a UUID-keyed collection returned zero hits with no error. The attach wizard therefore refused such collections and told the user to rebuild with integer ids, which defeats the purpose of attaching to a collection someone else already built. LangChain, a common way to populate Qdrant, uses UUIDs by default. The integer id was never load-bearing for this driver. DocumentId is a packed (file, chunk) pair used positionally by the local drivers, but an attached RAG holds no local files or vectors and every positional consumer already returns early on it, so the id only has to survive the round trip from search back to the content fetch. Ids that cannot make that trip as a u64 are interned behind a synthetic handle and restored when the fetch is issued, leaving collections that already use integer ids on exactly the path they used before. --- src/rag/mod.rs | 12 -- src/rag/providers/qdrant.rs | 303 ++++++++++++++++++++++++++++++++---- 2 files changed, 277 insertions(+), 38 deletions(-) diff --git a/src/rag/mod.rs b/src/rag/mod.rs index cbfd133..0b8a72c 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -475,18 +475,6 @@ impl Rag { } } - // Point IDs are read with `as_u64()`, which yields None for a JSON string. - // A UUID-keyed collection would therefore return zero hits with no error, - // so refuse it here instead of attaching something silently broken. - if let Some(raw_id) = &sample_id - && raw_id.starts_with('"') - { - bail!( - "Collection '{collection}' uses string (UUID) point IDs. \ - Coyote requires integer point IDs. Rebuild the collection with integer IDs \ - (e.g. LangChain: pass ids=list(range(len(docs))) to add_documents())." - ); - } println!("ℹ This collection must store document text in a 'page_content' payload field."); let dim = QdrantProvider::get_vector_dimension(&host, &collection, api_key) diff --git a/src/rag/providers/qdrant.rs b/src/rag/providers/qdrant.rs index 563a6f8..1f42b0a 100644 --- a/src/rag/providers/qdrant.rs +++ b/src/rag/providers/qdrant.rs @@ -3,10 +3,124 @@ use crate::rag::{DocumentId, RagData}; use anyhow::{Context, Result, bail}; use async_trait::async_trait; +use parking_lot::RwLock; use reqwest::header::{HeaderMap, HeaderValue}; use reqwest::{Client, Response, StatusCode}; use serde_json::Value; use std::collections::HashMap; +use std::sync::Arc; + +/// Marks a `DocumentId` that stands in for a point id Coyote cannot carry +/// directly. Qdrant accepts UUID strings as point ids, and that is what +/// LangChain writes by default. +/// +/// `DocumentId` packs `(file_index, document_index)` into one `usize` with the +/// file index in the high half, so this bit is only reachable at a file index of +/// 2^31. Nothing local gets near that, and an attached RAG builds no local index +/// at all — `data.files` and `data.vectors` stay empty and every +/// `DocumentId::split` caller early-returns on `data.attached`. Along the +/// attached path the id is an opaque key carried through RRF, which is what +/// makes a synthetic one safe here and nowhere else. +const SYNTHETIC_ID_TAG: usize = 1 << (usize::BITS - 1); + +/// Two-way map between a raw Qdrant point id and the `DocumentId` the retrieval +/// pipeline sees. +/// +/// Only ids that cannot survive the round trip are interned. A plain `u64` that +/// fits below the tag keeps mapping to itself, so integer-keyed collections +/// behave exactly as they did before this map existed. +#[derive(Default)] +struct PointIdInterner { + handles: HashMap, + raw: HashMap, + next: usize, +} + +impl PointIdInterner { + /// The `DocumentId` for a raw point id, minting a handle if one is needed. + /// + /// `None` only for a missing id, which is a malformed response. + fn document_id(&mut self, raw: &Value) -> Option { + if raw.is_null() { + return None; + } + // The pre-existing integer path, unchanged. `try_from` rather than `as` + // so a value too wide for the target's `usize` is interned instead of + // silently truncated into a different point. + if let Some(n) = raw.as_u64() + && let Ok(n) = usize::try_from(n) + && n & SYNTHETIC_ID_TAG == 0 + { + return Some(DocumentId(n)); + } + Some(self.intern(raw)) + } + + fn intern(&mut self, raw: &Value) -> DocumentId { + // Keyed on the JSON rendering, so the string "1" and the integer 1 are + // not conflated into one point. + let key = raw.to_string(); + if let Some(handle) = self.handles.get(&key) { + return *handle; + } + let handle = DocumentId(SYNTHETIC_ID_TAG | self.next); + self.next += 1; + self.handles.insert(key, handle); + self.raw.insert(handle, raw.clone()); + handle + } + + /// The original id for a handle, or `None` when the id was never interned — + /// i.e. it is a plain integer that is already its own id. + fn raw_id(&self, handle: DocumentId) -> Option<&Value> { + self.raw.get(&handle) + } + + /// Builds the `ids` array for an outbound `/points` fetch. Every entry is the + /// id Qdrant issued, integer or string; a synthetic handle must never leave + /// this process. + fn outbound_ids(&self, ids: &[DocumentId]) -> Vec { + ids.iter() + .map(|id| match self.raw_id(*id) { + Some(raw) => raw.clone(), + None => Value::from(id.0 as u64), + }) + .collect() + } +} + +fn parse_search_hits( + interner: &mut PointIdInterner, + body: &Value, + min_score: f32, +) -> Result> { + let hits = body["result"] + .as_array() + .context("Unexpected /points/search response shape")?; + + Ok(hits + .iter() + .filter_map(|pt| { + let score = pt["score"].as_f64()? as f32; + Some((interner.document_id(&pt["id"])?, score)) + }) + .filter(|(_, score)| *score > min_score) + .collect()) +} + +fn parse_points(interner: &mut PointIdInterner, body: &Value) -> Result> { + let points = body["result"] + .as_array() + .context("Unexpected /points response shape")?; + + Ok(points + .iter() + .filter_map(|pt| { + let text = pt["payload"]["page_content"].as_str()?.to_string(); + Some((interner.document_id(&pt["id"])?, text)) + }) + .collect()) +} /// Render Qdrant's error envelope into a human-readable message. /// @@ -65,6 +179,7 @@ pub struct QdrantProvider { client: Client, base_url: String, collection: String, + point_ids: Arc>, } impl QdrantProvider { @@ -139,6 +254,7 @@ impl QdrantProvider { client, base_url, collection: collection.to_string(), + point_ids: Arc::default(), }) } @@ -252,22 +368,11 @@ impl RagProvider for QdrantProvider { ); } let data: Value = resp.json().await?; - let results = data["result"] - .as_array() - .context("Unexpected /points/search response shape")? - .iter() - .filter_map(|pt| { - // String (UUID) IDs yield None here and are dropped. The attach - // wizard rejects such collections up front so this cannot silently - // become "zero results, no error". - let id = pt["id"].as_u64()? as usize; - let score = pt["score"].as_f64()? as f32; - Some((DocumentId(id), score)) - }) - .filter(|(_, score)| *score > min_score) - .collect(); + // The interner is what lets a UUID-keyed collection work: a string id gets + // a synthetic handle here and the original is replayed by `fetch_content`. + let mut interner = self.point_ids.write(); - Ok(results) + parse_search_hits(&mut interner, &data, min_score) } async fn fetch_content(&self, ids: &[DocumentId]) -> Result> { @@ -275,7 +380,8 @@ impl RagProvider for QdrantProvider { return Ok(vec![]); } let url = format!("{}/collections/{}/points", self.base_url, self.collection); - let id_list: Vec = ids.iter().map(|d| d.0 as u64).collect(); + // Qdrant is asked for the ids it issued, never for a synthetic handle. + let id_list = self.point_ids.read().outbound_ids(ids); let body = serde_json::json!({ "ids": id_list, "with_payload": true, @@ -291,16 +397,10 @@ impl RagProvider for QdrantProvider { ); } let data: Value = resp.json().await?; - let mut rows: Vec<(DocumentId, String)> = data["result"] - .as_array() - .context("Unexpected /points response shape")? - .iter() - .filter_map(|pt| { - let id = pt["id"].as_u64()? as usize; - let text = pt["payload"]["page_content"].as_str()?.to_string(); - Some((DocumentId(id), text)) - }) - .collect(); + let mut rows = { + let mut interner = self.point_ids.write(); + parse_points(&mut interner, &data)? + }; // `/points` does not guarantee response order matches request order, and the // caller's RRF ranking is carried by that order. Restore it. let position: HashMap = @@ -328,10 +428,17 @@ impl RagProvider for QdrantProvider { // Cloning the client shares the connection pool and the injected api-key // header. Sharing is correct: both handles address the same remote // collection, and neither of them writes to it. + // + // The point-id map is shared for the same reason, and because it MUST be: + // `Rag::clone()` hands the clone `DocumentId`s that the original minted, + // so a fresh map would resolve them to nothing and `fetch_content` would + // ask Qdrant for a synthetic handle — zero results, no error. Resetting it + // would also re-mint handles for ids the original still holds. Box::new(Self { client: self.client.clone(), base_url: self.base_url.clone(), collection: self.collection.clone(), + point_ids: Arc::clone(&self.point_ids), }) } } @@ -428,6 +535,7 @@ mod tests { client: Client::new(), base_url: "http://localhost:6333".to_string(), collection: "c".to_string(), + point_ids: Arc::default(), }; let attached = RagData { @@ -462,11 +570,154 @@ mod tests { client: Client::new(), base_url: "http://127.0.0.1:1".to_string(), collection: "c".to_string(), + point_ids: Arc::default(), }; assert!(provider.fetch_content(&[]).await.unwrap().is_empty()); } + /// A UUID-keyed collection has to survive the whole `vector_search` → + /// `fetch_content` round trip, and the fetch must ask Qdrant for the ORIGINAL + /// string id. Parsing ids with `as_u64()` used to drop these hits inside a + /// `filter_map`, i.e. zero results and no error. + #[test] + fn uuid_point_ids_round_trip_and_are_requested_verbatim() { + let mut interner = PointIdInterner::default(); + let first_uuid = "3f1b0c2e-1111-4000-8000-000000000001"; + let second_uuid = "3f1b0c2e-2222-4000-8000-000000000002"; + + let search = serde_json::json!({ + "result": [ + {"id": first_uuid, "score": 0.91}, + {"id": second_uuid, "score": 0.42}, + ] + }); + let hits = parse_search_hits(&mut interner, &search, 0.0).unwrap(); + assert_eq!(hits.len(), 2, "string ids must not be silently dropped"); + + let ids: Vec = hits.iter().map(|(id, _)| *id).collect(); + assert_eq!( + interner.outbound_ids(&ids), + vec![Value::from(first_uuid), Value::from(second_uuid)], + "the fetch must send the ids Qdrant issued, not the handles" + ); + + // Qdrant may answer /points in any order; the handles still map back and + // the caller's RRF ranking is recoverable. + let points = serde_json::json!({ + "result": [ + {"id": second_uuid, "payload": {"page_content": "second"}}, + {"id": first_uuid, "payload": {"page_content": "first"}}, + ] + }); + let mut rows = parse_points(&mut interner, &points).unwrap(); + let position: HashMap = + ids.iter().enumerate().map(|(i, id)| (*id, i)).collect(); + rows.sort_by_key(|(id, _)| position.get(id).copied().unwrap_or(usize::MAX)); + assert_eq!( + rows, + vec![ + (ids[0], "first".to_string()), + (ids[1], "second".to_string()) + ] + ); + } + + /// Integer-keyed collections must be untouched by the interner: the id maps to + /// itself on the way in and goes back out as the same integer. + #[test] + fn integer_point_ids_are_passed_through_untouched() { + let mut interner = PointIdInterner::default(); + let search = serde_json::json!({ + "result": [{"id": 7, "score": 0.9}, {"id": 0, "score": 0.5}] + }); + + let hits = parse_search_hits(&mut interner, &search, 0.0).unwrap(); + assert_eq!( + hits, + vec![(DocumentId(7), 0.9_f32), (DocumentId(0), 0.5_f32)] + ); + + let ids: Vec = hits.iter().map(|(id, _)| *id).collect(); + assert_eq!( + interner.outbound_ids(&ids), + vec![Value::from(7_u64), Value::from(0_u64)], + "integer ids must not be regressed into synthetic handles" + ); + assert!( + interner.raw_id(DocumentId(7)).is_none(), + "a plain integer id is its own id and needs no map entry" + ); + } + + /// Synthetic handles are stable per point id and live in a range no packed + /// `DocumentId` can reach. + #[test] + fn synthetic_handles_are_stable_and_never_collide_with_packed_ids() { + let mut interner = PointIdInterner::default(); + let uuid = Value::from("9d2f0a11-3333-4000-8000-00000000000a"); + + let handle = interner.document_id(&uuid).unwrap(); + assert_eq!( + interner.document_id(&uuid).unwrap(), + handle, + "the same point id must keep the same handle across queries" + ); + assert_ne!( + interner.document_id(&Value::from("other")).unwrap(), + handle, + "distinct point ids must not share a handle" + ); + assert_ne!(handle.0 & SYNTHETIC_ID_TAG, 0, "a handle carries the tag"); + + // A packed (file_index, document_index) never sets the tag bit: it is the + // top bit of the file index, which would take 2^31 indexed files. + for (file_index, document_index) in [(0, 0), (1, 0), (0, 4242), (1_000_000, 999)] { + assert_eq!( + DocumentId::new(file_index, document_index).0 & SYNTHETIC_ID_TAG, + 0, + "packed ({file_index}, {document_index}) must stay out of the handle range" + ); + } + + // The one integer id that WOULD land on the tag is interned instead of + // being handed back as itself, so it cannot alias a handle. + let collides = Value::from(SYNTHETIC_ID_TAG as u64); + let interned = interner.document_id(&collides).unwrap(); + assert_eq!(interner.raw_id(interned), Some(&collides)); + assert_eq!( + interner.outbound_ids(&[interned]), + vec![collides], + "the original integer must still be what Qdrant is asked for" + ); + } + + /// `duplicate()` shares the map rather than resetting it: `Rag::clone()` hands + /// the clone `DocumentId`s the original minted, and a fresh map would turn + /// those into requests for a synthetic handle — zero results, no error. + #[test] + fn duplicate_shares_the_point_id_map() { + let provider = QdrantProvider { + client: Client::new(), + base_url: "http://127.0.0.1:1".to_string(), + collection: "c".to_string(), + point_ids: Arc::default(), + }; + let uuid = Value::from("c0ffee00-4444-4000-8000-000000000007"); + let handle = provider.point_ids.write().document_id(&uuid).unwrap(); + + let dup = provider.duplicate(&RagData { + driver: "qdrant".to_string(), + attached: true, + ..Default::default() + }); + // Downcasting is not available through `dyn RagProvider`, so go via the + // shared Arc: the clone must observe the original's interning. + assert_eq!(Arc::strong_count(&provider.point_ids), 2); + assert_eq!(provider.point_ids.read().raw_id(handle), Some(&uuid)); + drop(dup); + } + #[tokio::test] #[ignore] async fn qdrant_list_collections_requires_running_instance() { From c0067d387c49a550411b3c87839f6103e32a7091 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 11 Aug 2026 14:06:57 -0600 Subject: [PATCH 17/36] refactor(rag): drop the hardcoded embedding model hint from attach The attach wizard mapped a collection's vector dimension to a hardcoded list of model names and printed them as likely candidates. The list was never checked against the models the user actually has configured, so it could recommend a model they cannot select, and one entry was a parenthetical note rather than a model id and so could never match anything. Any list like this rots as models are released. The dimension itself comes from the server and is worth stating, so it is still printed, as is the warning that a mismatched embedding model returns bad results. Deriving real candidates would need a dimension recorded against each configured model, which the model config does not carry today. --- src/rag/mod.rs | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/src/rag/mod.rs b/src/rag/mod.rs index 0b8a72c..0d76481 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -491,13 +491,7 @@ impl Rag { ); } if dim > 0 { - let candidates = embedding_model_candidates_for_dimension(dim); - if !candidates.is_empty() { - println!( - "Collection uses {dim}-dim vectors. Likely models: {}", - candidates.join(", ") - ); - } + println!("Collection uses {dim}-dim vectors."); } println!( "⚠️ If the embedding model doesn't match what built this collection, \ @@ -1834,21 +1828,6 @@ fn driver_auth_header(driver: &str) -> (&'static str, &'static str) { } } -/// Embedding models known to produce a given vector dimension, used to hint the -/// user toward a model compatible with the collection they just picked. -fn embedding_model_candidates_for_dimension(dim: u64) -> Vec<&'static str> { - match dim { - 1536 => vec!["text-embedding-3-small", "text-embedding-ada-002"], - 3072 => vec!["text-embedding-3-large"], - 768 => vec!["nomic-embed-text", "all-minilm-l6-v2"], - 1024 => vec![ - "text-embedding-3-small (matryoshka-1024)", - "jina-embeddings-v2-base", - ], - _ => vec![], - } -} - fn select_embedding_model(models: &[&Model]) -> Result { let max_width = models.iter().map(|v| v.id().len()).max().unwrap_or(0); let models: Vec<_> = models From e006e29ff1c49864f0b193501a52fbb1fe467783 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 11 Aug 2026 14:58:18 -0600 Subject: [PATCH 18/36] feat(rag): let several Coyote processes query one duckdb RAG at once The DuckDB store was always opened read-write, which takes an exclusive file lock, so a second Coyote process could not even read the RAG. Querying does not write, and DuckDB permits many concurrent readers as long as no writer is attached, so the store is now opened read-only whenever it already carries a complete schema. Creating or initializing the store still writes, as does rebuilding, so those paths take the exclusive handle. The rebuild path upgrades a read-only connection in place, which every clone of the handle observes because the mode lives behind the shared mutex rather than beside it. Extension loads and the HNSW persistence setting are per-connection and are re-established on the upgraded connection. An upgrade that loses the race for the write lock reports that another process holds the RAG and that nothing was written, then reopens read-only so the session can keep querying. A read-only handle also refuses writes outright, so a missed upgrade cannot silently discard an ingest. --- src/rag/providers/duckdb.rs | 612 +++++++++++++++++++++++++++++++++--- 1 file changed, 561 insertions(+), 51 deletions(-) diff --git a/src/rag/providers/duckdb.rs b/src/rag/providers/duckdb.rs index 105eed9..aa9bfff 100644 --- a/src/rag/providers/duckdb.rs +++ b/src/rag/providers/duckdb.rs @@ -4,8 +4,8 @@ use std::collections::HashMap; use anyhow::{Context, Result, anyhow, bail}; use async_trait::async_trait; -use duckdb::Connection; use duckdb::types::Value; +use duckdb::{AccessMode, Config, Connection}; use indexmap::IndexMap; use log::warn; use std::path::{Path, PathBuf}; @@ -17,8 +17,12 @@ use std::sync::{Arc, Mutex, MutexGuard}; /// `~/.duckdb/extensions/...`. Two threads installing the same extension at once /// both perform that move; on Windows the loser's move targets a file the winner /// already holds open and fails with "Access is denied", where POSIX would let the -/// replacement through. Guards nothing but the install step, so it is never held -/// across a `DuckDbProvider::conn` guard and cannot invert lock order. +/// replacement through. Guards nothing but the install step. +/// +/// Lock order is `DuckDbProvider::conn` -> INSTALL_LOCK, never the reverse: +/// `ensure_writable` reopens the connection, and so may install, while holding the +/// `ConnHandle` guard, whereas nothing ever acquires a `ConnHandle` guard while +/// holding this lock. The cycle that would deadlock cannot form. static INSTALL_LOCK: Mutex<()> = Mutex::new(()); /// Derive the DuckDB sidecar path from a RAG's YAML path: `docs.yaml` -> `docs.duckdb`. @@ -26,9 +30,44 @@ pub(crate) fn duckdb_path_from_yaml(yaml_path: &Path) -> PathBuf { yaml_path.with_extension("duckdb") } +/// The shared connection together with the access mode it was opened with. +/// +/// `conn` is an `Option` only so that an upgrade can DROP the read-only connection +/// before asking DuckDB for a read-write one. It is `Some` at every point an outside +/// caller can observe, and is never left `None` on a path that returns `Ok`. +struct ConnHandle { + conn: Option, + /// True when `conn` was opened READ_WRITE. This lives behind the same mutex as the + /// connection itself rather than next to it in `DuckDbProvider`, so that a + /// `duplicate()` clone sharing the `Arc` observes an upgrade performed through any + /// other handle instead of keeping its own stale copy of the mode. + writable: bool, +} + +impl ConnHandle { + fn conn(&self) -> Result<&Connection> { + self.conn.as_ref().ok_or_else(Self::lost) + } + + fn conn_mut(&mut self) -> Result<&mut Connection> { + self.conn.as_mut().ok_or_else(Self::lost) + } + + /// Only reachable when a read-write upgrade failed AND reopening read-only failed + /// too. Returning an error beats panicking inside a locked scope, which would + /// poison the mutex for the remaining life of the process. + fn lost() -> anyhow::Error { + anyhow!( + "The DuckDB connection was lost: upgrading it to read-write failed and the \ + store could not be reopened read-only afterwards. Another process is \ + holding the file; retry once it has released it." + ) + } +} + pub struct DuckDbProvider { path: PathBuf, - conn: Arc>, + conn: Arc>, /// Embedding dimension; fixed at open time because the `FLOAT[N]` column depends on it. dim: usize, /// True once an FTS index has been built on `documents`. Until then @@ -40,28 +79,140 @@ pub struct DuckDbProvider { impl DuckDbProvider { /// Open (or create) the DuckDB file. `dim` is the embedding vector dimension, /// supplied by the caller who knows the model. + /// + /// Opens READ-ONLY whenever the file already carries a complete schema, so that any + /// number of Coyote processes can query the same RAG at the same time. DuckDB allows + /// many concurrent readers XOR exactly one writer, so the exclusive read-write handle + /// is taken only when there is actually something to write: when the store has to be + /// created or initialized here, or lazily through `ensure_writable` on the rebuild + /// path. pub fn open(db_path: &Path, dim: usize) -> Result { - let conn = Connection::open(db_path).with_context(|| { + let (conn, writable) = Self::open_for_workload(db_path, dim)?; + // A reopened file may already carry a live FTS index from a previous session, + // in which case keyword search works immediately. + let fts_exists = Self::probe_fts_index(&conn); + Ok(Self { + path: db_path.to_path_buf(), + conn: Arc::new(Mutex::new(ConnHandle { + conn: Some(conn), + writable, + })), + dim, + fts_ready: AtomicBool::new(fts_exists), + }) + } + + /// Pick the weakest access mode that can serve this store, returning the connection + /// and whether it came back writable. + fn open_for_workload(db_path: &Path, dim: usize) -> Result<(Connection, bool)> { + if db_path.exists() + && let Ok(conn) = Self::open_read_only(db_path) + && Self::store_is_initialized(&conn) + { + return Ok((conn, false)); + } + // Three cases land here: the file does not exist yet, it could not be opened + // read-only (another process holds it read-write), or it carries no usable + // schema. All of them need a read-write handle, and the read-write attempt is + // also what produces the actionable lock error for the middle case. + let conn = Self::open_read_write(db_path, dim)?; + Ok((conn, true)) + } + + /// Is this file already a fully initialized Coyote store? + /// + /// This gate decides whether a read-only open is viable, so it must be exact: every + /// statement in `init_schema` is rejected outright on a read-only handle, INCLUDING + /// `CREATE TABLE IF NOT EXISTS` against a table that already exists, which DuckDB + /// refuses rather than treating as a no-op. Anything missing therefore forces a + /// read-write open. The HNSW index is part of the check because a store whose tables + /// survived but whose index did not would otherwise be opened read-only and silently + /// serve every `vector_search` from a full scan. + fn store_is_initialized(conn: &Connection) -> bool { + let tables: i64 = conn + .query_row( + "SELECT count(*) FROM duckdb_tables() \ + WHERE table_name IN ('vectors', 'documents')", + [], + |r| r.get(0), + ) + .unwrap_or(0); + if tables < 2 { + return false; + } + conn.query_row( + "SELECT count(*) FROM duckdb_indexes() WHERE index_name = 'hnsw_idx'", + [], + |r| r.get::<_, i64>(0), + ) + .map(|n| n > 0) + .unwrap_or(false) + } + + /// Open the store read-only. Many processes may hold such a handle at once. + fn open_read_only(db_path: &Path) -> Result { + let config = Config::default() + .access_mode(AccessMode::ReadOnly) + .context("Failed to build a read-only DuckDB configuration")?; + let conn = Connection::open_with_flags(db_path, config).with_context(|| { format!( - "Failed to open the DuckDB store at '{}'. If another Coyote process (or \ - another window) has this RAG open, close it and retry — a duckdb RAG can \ - only be open in ONE process at a time. Unlike the yaml driver, its data \ - lives in a single file with an exclusive lock.", + "Failed to open the DuckDB store at '{}' read-only", db_path.display() ) })?; - // Statement order is load-bearing. `hnsw_enable_experimental_persistence` is - // registered BY the vss extension, so setting it before `LOAD vss` fails with - // "Setting with name ... is not in the catalog, but it exists in the vss - // extension". Omitting it entirely makes CREATE INDEX ... USING HNSW on a - // file-backed database fail with "HNSW index persistence is not yet supported - // by default". ensure vss (installing it if missing) -> ensure fts -> SET -> - // CREATE INDEX. - Self::ensure_extension(&conn, "vss")?; - Self::ensure_extension(&conn, "fts")?; + Self::establish_session(&conn)?; + Ok(conn) + } + + /// Open the store read-write and make sure its schema exists. Exactly one process + /// may hold such a handle, and no reader from another process may hold it meanwhile. + fn open_read_write(db_path: &Path, dim: usize) -> Result { + let config = Config::default() + .access_mode(AccessMode::ReadWrite) + .context("Failed to build a read-write DuckDB configuration")?; + let conn = Connection::open_with_flags(db_path, config).with_context(|| { + format!( + "Failed to open the DuckDB store at '{}' for writing. Another Coyote \ + process (or another window) has this RAG open: a duckdb RAG supports MANY \ + concurrent READERS, but only ONE writer at a time, and a writer excludes \ + readers in other processes. Close that process, or wait for its sync to \ + finish, and retry.", + db_path.display() + ) + })?; + Self::establish_session(&conn)?; + Self::init_schema(&conn, dim)?; + Ok(conn) + } + + /// Install the per-connection session state that every connection needs, whatever + /// its access mode. + /// + /// Extension `LOAD`s and `SET` are per-CONNECTION, not per-database: a connection + /// opened later — an upgrade, in particular — starts with none of this and must run + /// it again. None of these statements write to the database, so they all succeed on + /// a read-only handle. + /// + /// Statement order is load-bearing. `hnsw_enable_experimental_persistence` is + /// registered BY the vss extension, so setting it before `LOAD vss` fails with + /// "Setting with name ... is not in the catalog, but it exists in the vss + /// extension". ensure vss (installing it if missing) -> ensure fts -> SET. + fn establish_session(conn: &Connection) -> Result<()> { + Self::ensure_extension(conn, "vss")?; + Self::ensure_extension(conn, "fts")?; + conn.execute_batch("SET hnsw_enable_experimental_persistence = true;") + .context("Failed to enable DuckDB HNSW index persistence") + } + + /// Create the tables and the vector index. Every statement here WRITES, so this only + /// ever runs on a read-write connection. + /// + /// Must be preceded by `establish_session`: without the `SET` it performs, a + /// CREATE INDEX ... USING HNSW on a file-backed database fails with "HNSW index + /// persistence is not yet supported by default". + fn init_schema(conn: &Connection, dim: usize) -> Result<()> { conn.execute_batch(&format!( - "SET hnsw_enable_experimental_persistence = true; - CREATE TABLE IF NOT EXISTS vectors ( + "CREATE TABLE IF NOT EXISTS vectors ( doc_id UBIGINT PRIMARY KEY, embedding FLOAT[{dim}] ); @@ -73,16 +224,44 @@ impl DuckDbProvider { page_content TEXT NOT NULL );" )) - .context("Failed to initialize DuckDB schema")?; - // A reopened file may already carry a live FTS index from a previous session, - // in which case keyword search works immediately. - let fts_exists = Self::probe_fts_index(&conn); - Ok(Self { - path: db_path.to_path_buf(), - conn: Arc::new(Mutex::new(conn)), - dim, - fts_ready: AtomicBool::new(fts_exists), - }) + .context("Failed to initialize DuckDB schema") + } + + /// Guarantee the shared connection is read-write, upgrading it in place if it is not. + /// EVERY write path must call this before touching the store. + /// + /// The upgrade replaces the `Connection` INSIDE the shared `Arc>`, so + /// `duplicate()` clones, which share that `Arc`, see it too. The read-only connection + /// is dropped before the read-write open because DuckDB tracks the file lock per + /// database instance and the old handle still holds one. + /// + /// On failure the store is reopened read-only so that queries keep working, and the + /// error is propagated so the caller aborts instead of writing. A failed upgrade must + /// leave the provider degraded, never bricked, and never silently read-only-with-a- + /// caller-that-thinks-it-wrote. + fn ensure_writable(&self) -> Result<()> { + let mut handle = self.lock_conn()?; + if handle.writable { + return Ok(()); + } + drop(handle.conn.take()); + match Self::open_read_write(&self.path, self.dim) { + Ok(conn) => { + handle.conn = Some(conn); + handle.writable = true; + Ok(()) + } + Err(e) => { + handle.conn = Self::open_read_only(&self.path).ok(); + Err(e.context(format!( + "Cannot write to the DuckDB RAG at '{}': it is open read-only and could \ + not be upgraded to read-write, because another Coyote process has this \ + RAG open. NOTHING WAS WRITTEN. Close the other process, or wait for it \ + to finish, and retry.", + self.path.display() + ))) + } + } } /// Make a DuckDB extension available on `conn`, installing it if this machine does @@ -136,8 +315,10 @@ impl DuckDbProvider { /// `rebuild_indexes` does `CREATE OR REPLACE TABLE` and writes exactly what it is /// given, so a thinned map is committed as the new truth on the next sync. pub(crate) fn read_all_vectors(&self) -> Result>> { - let conn = self.lock_conn()?; - let mut stmt = conn.prepare("SELECT doc_id, embedding FROM vectors")?; + let handle = self.lock_conn()?; + let mut stmt = handle + .conn()? + .prepare("SELECT doc_id, embedding FROM vectors")?; let raw: Vec<(u64, Vec)> = stmt .query_map([], |row| { let id: u64 = row.get(0)?; @@ -226,7 +407,7 @@ impl DuckDbProvider { /// Never `.lock().unwrap()` here: a panic anywhere inside a locked scope poisons the /// mutex permanently, and an unwrap would then turn every subsequent RAG query into /// a panic for the remaining life of the process. - fn lock_conn(&self) -> Result> { + fn lock_conn(&self) -> Result> { self.conn .lock() .map_err(|e| anyhow!("DuckDB connection mutex was poisoned: {e}")) @@ -253,7 +434,7 @@ impl RagProvider for DuckDbProvider { .collect::>() .join(", "); let dim = self.dim; - let conn = self.lock_conn()?; + let handle = self.lock_conn()?; // array_cosine_distance requires a FLOAT[N] ARRAY, not the LIST type FLOAT[]. // ORDER BY distance ASC is required for the planner to use hnsw_idx; the // similarity form (DESC) does NOT trigger the ANN index. Distance is converted @@ -263,7 +444,7 @@ impl RagProvider for DuckDbProvider { array_cosine_distance(embedding, [{vals}]::FLOAT[{dim}]) AS distance \ FROM vectors ORDER BY distance ASC LIMIT {top_k}" ); - let mut stmt = conn.prepare(&sql)?; + let mut stmt = handle.conn()?.prepare(&sql)?; let results = stmt .query_map([], |row| { let id: u64 = row.get(0)?; @@ -300,12 +481,12 @@ impl RagProvider for DuckDbProvider { if ids.is_empty() { return Ok(vec![]); } - let conn = self.lock_conn()?; + let handle = self.lock_conn()?; let placeholders = ids.iter().map(|_| "?").collect::>().join(", "); let sql = format!("SELECT doc_id, page_content FROM documents WHERE doc_id IN ({placeholders})"); let params: Vec = ids.iter().map(|id| Value::UBigInt(id.0 as u64)).collect(); - let mut stmt = conn.prepare(&sql)?; + let mut stmt = handle.conn()?.prepare(&sql)?; let mut rows: Vec<(DocumentId, String)> = stmt .query_map(duckdb::params_from_iter(params.iter()), |row| { let id: u64 = row.get(0)?; @@ -345,8 +526,10 @@ impl RagProvider for DuckDbProvider { // Scoped: the guard MUST be dropped before `lock_conn()` is taken again // below. `Mutex` is not reentrant; holding both self-deadlocks at // runtime, with no compile error. - let conn = self.lock_conn()?; - conn.query_row("SELECT count(*) FROM vectors", [], |r| r.get(0)) + let handle = self.lock_conn()?; + handle + .conn()? + .query_row("SELECT count(*) FROM vectors", [], |r| r.get(0)) .context("Failed to count existing vectors before rebuild")? }; if existing > 0 { @@ -372,7 +555,16 @@ impl RagProvider for DuckDbProvider { } } let dim = self.dim; - let mut conn = self.lock_conn()?; + // THE write path. Everything above this line only reads, so the upgrade happens + // here, after both guards have had their say: a rebuild that is going to be + // refused must not first take the exclusive lock away from other processes. + // + // This is also the point that makes a silently-dropped write impossible. If the + // upgrade fails, `?` aborts the rebuild before a single statement is issued and + // the caller gets the error. Nothing below can run on a read-only connection. + self.ensure_writable()?; + let mut handle = self.lock_conn()?; + let conn = handle.conn_mut()?; let tx = conn .transaction() .context("Failed to begin DuckDB transaction")?; @@ -476,9 +668,9 @@ impl RagProvider for DuckDbProvider { } async fn keyword_search(&self, query: &str, top_k: usize) -> Result> { - let conn = self.lock_conn()?; + let handle = self.lock_conn()?; // match_bm25 returns NULL for non-matching rows; WHERE filters them out. - let mut stmt = conn.prepare( + let mut stmt = handle.conn()?.prepare( "SELECT doc_id, fts_main_documents.match_bm25(doc_id, ?) AS score FROM documents WHERE score IS NOT NULL @@ -533,6 +725,10 @@ impl RagProvider for DuckDbProvider { // disk and a rebuild through one handle is immediately visible to the other. // That is unavoidable for any on-disk store and is handled by the discipline // documented on `Rag`'s Clone impl, the pre-clone instance must be discarded. + // + // Sharing the Arc also shares the ACCESS MODE, which lives inside the ConnHandle + // rather than beside it: when one handle upgrades itself to read-write, every + // clone is upgraded with it and none is left holding a stale "read-only" belief. Box::new(DuckDbProvider { path: self.path.clone(), conn: Arc::clone(&self.conn), @@ -547,6 +743,7 @@ mod tests { use super::*; use crate::rag::provider::RagProvider; use crate::rag::{RagDocument, RagFile}; + use std::process::Command; use std::time::{SystemTime, UNIX_EPOCH}; use std::{env, fs}; @@ -625,7 +822,8 @@ mod tests { async fn open_creates_schema() { let db = TempDb::new("schema"); let provider = DuckDbProvider::open(&db.path, 3).unwrap(); - let conn = provider.conn.lock().unwrap(); + let handle = provider.conn.lock().unwrap(); + let conn = handle.conn().unwrap(); let v: i64 = conn .query_row("SELECT COUNT(*) FROM vectors", [], |r| r.get(0)) @@ -643,7 +841,8 @@ mod tests { let db = TempDb::new("vsearch"); let provider = DuckDbProvider::open(&db.path, 3).unwrap(); { - let conn = provider.conn.lock().unwrap(); + let handle = provider.conn.lock().unwrap(); + let conn = handle.conn().unwrap(); // The ::FLOAT[3] cast is REQUIRED: a bare [0.1, 0.2, 0.3] literal infers // DOUBLE[], which does not match the FLOAT[N] ARRAY column type. conn.execute( @@ -668,7 +867,8 @@ mod tests { let db = TempDb::new("fetch"); let provider = DuckDbProvider::open(&db.path, 3).unwrap(); { - let conn = provider.conn.lock().unwrap(); + let handle = provider.conn.lock().unwrap(); + let conn = handle.conn().unwrap(); conn.execute( "INSERT INTO documents (doc_id, page_content) VALUES (42, 'hello world')", [], @@ -712,7 +912,8 @@ mod tests { .await .expect("second rebuild must not violate the primary key constraint"); - let conn = provider.conn.lock().unwrap(); + let handle = provider.conn.lock().unwrap(); + let conn = handle.conn().unwrap(); let count: i64 = conn .query_row("SELECT COUNT(*) FROM vectors", [], |r| r.get(0)) .unwrap(); @@ -744,7 +945,8 @@ mod tests { reloaded.vectors.insert(DocumentId(2), vec![0.7, 0.8, 0.9]); provider.rebuild_indexes(&reloaded, false).await.unwrap(); - let conn = provider.conn.lock().unwrap(); + let handle = provider.conn.lock().unwrap(); + let conn = handle.conn().unwrap(); let count: i64 = conn .query_row("SELECT COUNT(*) FROM vectors", [], |r| r.get(0)) .unwrap(); @@ -768,7 +970,8 @@ mod tests { data.vectors.insert(DocumentId(0), vec![0.1, 0.2, 0.3]); provider.rebuild_indexes(&data, true).await.unwrap(); { - let conn = provider.conn.lock().unwrap(); + let handle = provider.conn.lock().unwrap(); + let conn = handle.conn().unwrap(); let docs: i64 = conn .query_row("SELECT COUNT(*) FROM documents", [], |r| r.get(0)) .unwrap(); @@ -849,7 +1052,8 @@ mod tests { "got: {err}" ); - let conn = provider.conn.lock().unwrap(); + let handle = provider.conn.lock().unwrap(); + let conn = handle.conn().unwrap(); let count: i64 = conn .query_row("SELECT COUNT(*) FROM vectors", [], |r| r.get(0)) .unwrap(); @@ -877,7 +1081,8 @@ mod tests { let dup = provider.duplicate(&minimal_rag_data()); { - let conn = provider.conn.lock().unwrap(); + let handle = provider.conn.lock().unwrap(); + let conn = handle.conn().unwrap(); conn.execute( "INSERT INTO documents (doc_id, page_content) VALUES (7, 'shared row')", [], @@ -900,7 +1105,8 @@ mod tests { let db = TempDb::new("nonfinite"); let provider = DuckDbProvider::open(&db.path, 3).unwrap(); { - let conn = provider.conn.lock().unwrap(); + let handle = provider.conn.lock().unwrap(); + let conn = handle.conn().unwrap(); conn.execute( "INSERT INTO vectors (doc_id, embedding) VALUES (0, [0.1, 0.2, 0.3]::FLOAT[3])", [], @@ -923,6 +1129,310 @@ mod tests { ); } + /// Was the shared connection opened read-write? Reads the flag that lives inside the + /// shared handle, which is the same one `ensure_writable` flips. + fn is_writable(provider: &DuckDbProvider) -> bool { + provider.conn.lock().unwrap().writable + } + + /// Build a fully initialized store, then let the read-write handle go so the file is + /// unlocked for the next opener. + async fn seed_store(path: &Path) { + let mut provider = DuckDbProvider::open(path, 3).unwrap(); + assert!(is_writable(&provider), "a fresh file must open read-write"); + let mut data = populated_rag_data(); + data.vectors + .insert(DocumentId::new(0, 0), vec![0.1, 0.2, 0.3]); + provider.rebuild_indexes(&data, true).await.unwrap(); + } + + #[tokio::test] + async fn a_fresh_file_is_opened_read_write() { + let db = TempDb::new("freshrw"); + let provider = DuckDbProvider::open(&db.path, 3).unwrap(); + + assert!( + is_writable(&provider), + "the schema has to be created, which writes, so a missing file must open \ + read-write" + ); + } + + #[tokio::test] + async fn an_initialized_store_is_reopened_read_only() { + let db = TempDb::new("reopenro"); + seed_store(&db.path).await; + + let provider = DuckDbProvider::open(&db.path, 3).unwrap(); + + assert!( + !is_writable(&provider), + "a store that needs no schema work must open read-only, so that other Coyote \ + processes can query it at the same time" + ); + } + + #[tokio::test] + async fn a_file_without_the_coyote_schema_is_opened_read_write() { + let db = TempDb::new("noschema"); + { + // A valid DuckDB file that is not one of ours. Opening it read-only would + // strand it forever: the init batch is refused on a read-only handle. + let conn = Connection::open(&db.path).unwrap(); + conn.execute_batch("CREATE TABLE unrelated (x INTEGER);") + .unwrap(); + } + + let provider = DuckDbProvider::open(&db.path, 3).unwrap(); + + assert!( + is_writable(&provider), + "a schema-less file must open read-write" + ); + let handle = provider.conn.lock().unwrap(); + let tables: i64 = handle + .conn() + .unwrap() + .query_row( + "SELECT count(*) FROM duckdb_tables() \ + WHERE table_name IN ('vectors', 'documents')", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(tables, 2, "the schema must have been created"); + } + + #[tokio::test] + async fn a_read_only_store_still_serves_vector_and_keyword_search() { + let db = TempDb::new("rosearch"); + seed_store(&db.path).await; + + let provider = DuckDbProvider::open(&db.path, 3).unwrap(); + assert!(!is_writable(&provider), "precondition: opened read-only"); + + let hits = provider + .vector_search(&[0.1, 0.2, 0.3], 5, 0.0) + .await + .unwrap(); + assert_eq!( + hits.len(), + 1, + "the persisted HNSW index must be queryable read-only" + ); + assert!(hits[0].1 > 0.99); + + assert!( + provider.has_native_keyword_search(), + "the FTS index built by the previous session must still be detected on a \ + read-only handle" + ); + let kw = provider.keyword_search("alpha", 5).await.unwrap(); + assert_eq!(kw.len(), 1, "keyword search must work read-only"); + + let docs = provider + .fetch_content(&[DocumentId::new(0, 0)]) + .await + .unwrap(); + assert_eq!(docs[0].1, "alpha keyword"); + } + + #[tokio::test] + async fn a_read_only_handle_refuses_a_direct_write() { + let db = TempDb::new("rorefuse"); + seed_store(&db.path).await; + let provider = DuckDbProvider::open(&db.path, 3).unwrap(); + assert!(!is_writable(&provider), "precondition: opened read-only"); + + let handle = provider.conn.lock().unwrap(); + let err = handle + .conn() + .unwrap() + .execute( + "INSERT INTO documents (doc_id, page_content) VALUES (99, 'nope')", + [], + ) + .unwrap_err(); + + // The backstop, not the primary defence: `rebuild_indexes` upgrades first and + // never reaches a write on a read-only handle. It matters anyway because DuckDB + // lets `transaction()` open and `commit()` return Ok on a read-only connection, + // so a write that slipped through would look like it had succeeded. + assert!( + err.to_string().contains("read-only mode"), + "a read-only handle must refuse writes loudly; got: {err}" + ); + } + + #[tokio::test] + async fn rebuild_indexes_upgrades_a_read_only_connection() { + let db = TempDb::new("upgrade"); + seed_store(&db.path).await; + + let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); + assert!(!is_writable(&provider), "precondition: opened read-only"); + + let mut data = populated_rag_data(); + data.vectors = provider.read_all_vectors().unwrap(); + data.vectors + .insert(DocumentId::new(1, 0), vec![0.4, 0.5, 0.6]); + provider.rebuild_indexes(&data, false).await.unwrap(); + + assert!( + is_writable(&provider), + "the write path must have upgraded the connection in place" + ); + + // The upgraded connection is a NEW connection, so the per-connection session + // state has to have been re-established on it. Without the re-run `SET`, the + // CREATE INDEX ... USING HNSW inside rebuild_indexes would already have failed. + let persisted: String = { + let handle = provider.conn.lock().unwrap(); + handle + .conn() + .unwrap() + .query_row( + "SELECT CAST(current_setting('hnsw_enable_experimental_persistence') \ + AS VARCHAR)", + [], + |r| r.get(0), + ) + .unwrap() + }; + assert_eq!( + persisted, "true", + "the upgraded connection must re-run the SET; session state does not carry \ + over from the dropped read-only connection" + ); + + drop(provider); + let reopened = DuckDbProvider::open(&db.path, 3).unwrap(); + let all = reopened.read_all_vectors().unwrap(); + assert_eq!(all.len(), 2, "the upgraded write must have reached disk"); + } + + #[tokio::test] + async fn an_upgrade_is_visible_through_duplicate_clones() { + let db = TempDb::new("upgradedup"); + seed_store(&db.path).await; + + let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); + assert!(!is_writable(&provider), "precondition: opened read-only"); + let dup = provider.duplicate(&minimal_rag_data()); + + let mut data = populated_rag_data(); + data.vectors = provider.read_all_vectors().unwrap(); + provider.rebuild_indexes(&data, false).await.unwrap(); + + // `duplicate()` shares the Arc, and the access mode lives inside it, so the clone + // must observe the upgrade rather than keep believing it is read-only. + let via_dup = dup.fetch_content(&[DocumentId::new(0, 0)]).await.unwrap(); + assert_eq!( + via_dup.len(), + 1, + "the clone must still read after an upgrade" + ); + assert!( + is_writable(&provider), + "the shared handle must report writable to every clone" + ); + } + + /// Two REAL OS processes reading one store at the same time. + /// + /// Ignored by default because it re-executes the test binary as a child process, + /// which is heavier and more environment-dependent than the rest of the suite. Run it + /// with: + /// cargo test --all -- --ignored duckdb_store_is_shared_across_processes + /// + /// It cannot be written as an ordinary in-process test: DuckDB keeps ONE database + /// instance per process, so a second open in the same process bypasses the file lock + /// entirely (a read-write open succeeds even while this process holds a read-only + /// one). Only separate processes exercise the lock this feature exists to avoid. + #[tokio::test] + #[ignore = "spawns a second OS process; run explicitly with --ignored"] + async fn duckdb_store_is_shared_across_processes() { + const CHILD_DB: &str = "COYOTE_DUCKDB_MULTIPROC_DB"; + const CHILD_EXPECT: &str = "COYOTE_DUCKDB_MULTIPROC_EXPECT"; + const TEST_NAME: &str = + "rag::providers::duckdb::tests::duckdb_store_is_shared_across_processes"; + + if let Ok(path) = env::var(CHILD_DB) { + let expect = env::var(CHILD_EXPECT).unwrap_or_default(); + let opened = DuckDbProvider::open(Path::new(&path), 3); + match expect.as_str() { + "readable" => { + let provider = opened.expect( + "a second process must be able to open a store that another \ + process holds READ-ONLY", + ); + assert!(!is_writable(&provider), "the child must land read-only"); + let hits = provider + .vector_search(&[0.1, 0.2, 0.3], 5, 0.0) + .await + .unwrap(); + assert_eq!(hits.len(), 1, "the child must read the seeded vector"); + } + "blocked" => { + let err = opened.err().expect( + "a second process must NOT be able to open a store that another \ + process holds READ-WRITE", + ); + let msg = format!("{err:#}"); + assert!( + msg.contains("concurrent READERS") && msg.contains("only ONE writer"), + "the lock error must explain the reader/writer rule; got: {msg}" + ); + } + other => panic!("unknown child expectation {other:?}"), + } + return; + } + + let db = TempDb::new("multiproc"); + seed_store(&db.path).await; + + let run_child = |expect: &str| { + Command::new(env::current_exe().unwrap()) + .args(["--exact", "--ignored", "--nocapture", TEST_NAME]) + .env(CHILD_DB, &db.path) + .env(CHILD_EXPECT, expect) + .output() + .expect("failed to spawn the child test process") + }; + + // Phase 1: this process holds a READ-ONLY handle. The child must get one too. + let provider = DuckDbProvider::open(&db.path, 3).unwrap(); + assert!(!is_writable(&provider), "precondition: parent is read-only"); + let out = run_child("readable"); + assert!( + out.status.success(), + "child could not share the read-only store:\n{}\n{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + let hits = provider + .vector_search(&[0.1, 0.2, 0.3], 5, 0.0) + .await + .unwrap(); + assert_eq!( + hits.len(), + 1, + "the parent must still read after the child ran" + ); + + // Phase 2: upgrade this process to READ-WRITE. The child must now be refused, + // with the message that explains why. + provider.ensure_writable().unwrap(); + let out = run_child("blocked"); + assert!( + out.status.success(), + "a writer must exclude other processes, with an actionable error:\n{}\n{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + } + #[test] fn duckdb_path_from_yaml_swaps_extension() { let p = duckdb_path_from_yaml(Path::new("/tmp/rags/docs.yaml")); From 912e00a627329a850eb712bea45f9f311d8b87e8 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 11 Aug 2026 16:38:55 -0600 Subject: [PATCH 19/36] docs(rag): correct the duckdb concurrency note in the driver prompt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver picker still told users a duckdb RAG can only be open in one Coyote process at a time. That stopped being true once the store began opening read-only for queries: any number of processes can now query it concurrently. The restriction that remains is narrower and only bites while writing, so the prompt now states that instead — several processes can query at once, but an ingest or rebuild locks the others out until it finishes. --- src/rag/mod.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/rag/mod.rs b/src/rag/mod.rs index 0d76481..0c49e2f 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -274,15 +274,17 @@ impl Rag { let driver = if prompt_for_driver { let options = vec![ "yaml — portable, in-memory HNSW; usable from several Coyote processes at once (default)", - "duckdb — persistent on-disk store; vectors and content survive restarts; HNSW approximate search. Can only be open in ONE Coyote process at a time, and its driver cannot be changed later without recreating the RAG", + "duckdb — persistent on-disk store; vectors and content survive restarts; HNSW approximate search. Several Coyote processes can query it at once, but ingesting or rebuilding it locks the others out until that finishes, and its driver cannot be changed later without recreating the RAG", ]; let sel = Select::new("RAG storage driver:", options) .with_starting_cursor(0) .prompt()?; if sel.starts_with("duckdb") { println!( - "Note: a duckdb RAG can only be open in one Coyote process at a time, \ - and changing its driver later means deleting and recreating the RAG." + "Note: several Coyote processes can query a duckdb RAG at the same time, \ + but while one process is ingesting or rebuilding it the others cannot \ + read it until that finishes. Changing its driver later means deleting \ + and recreating the RAG." ); "duckdb" } else { From d6c114fe58eb46a4a3e4c8f516e76ec1a1cd024b Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 11 Aug 2026 16:50:50 -0600 Subject: [PATCH 20/36] feat: simplified the duckdb selection prompt --- src/rag/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rag/mod.rs b/src/rag/mod.rs index 0c49e2f..ac1ba52 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -274,7 +274,7 @@ impl Rag { let driver = if prompt_for_driver { let options = vec![ "yaml — portable, in-memory HNSW; usable from several Coyote processes at once (default)", - "duckdb — persistent on-disk store; vectors and content survive restarts; HNSW approximate search. Several Coyote processes can query it at once, but ingesting or rebuilding it locks the others out until that finishes, and its driver cannot be changed later without recreating the RAG", + "duckdb — persistent on-disk store; vectors and content survive restarts; HNSW approximate search.", ]; let sel = Select::new("RAG storage driver:", options) .with_starting_cursor(0) From 7b1c0342b496beef335db354a727ad750f9bc6d0 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 11 Aug 2026 16:54:50 -0600 Subject: [PATCH 21/36] fix(rag): delete the DuckDB write-ahead log alongside the store Deleting a RAG removed its .duckdb file but left the sibling .duckdb.wal behind. DuckDB only removes that log on a clean close, so any RAG whose process was killed left one on disk, and creating a new RAG under the same name let it inherit a write-ahead log describing someone else's data. The test helper already cleaned the log up after itself, which is why no test noticed the production path did not. --- src/config/paths.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/config/paths.rs b/src/config/paths.rs index f49f688..bbbf6b1 100644 --- a/src/config/paths.rs +++ b/src/config/paths.rs @@ -437,6 +437,14 @@ pub(crate) fn remove_rag_sidecars(dir: &Path, name: &str) -> Result<()> { if duckdb_path.exists() { let _ = remove_file(&duckdb_path); } + // DuckDB keeps its write-ahead log in a sibling file and only removes it on a + // clean close, so a crash or a kill leaves one behind. Deleting the database + // without it strands a `.wal` that the next RAG created under the same name + // would inherit as if it were its own. + let wal_path = dir.join(format!("{name}.duckdb.wal")); + if wal_path.exists() { + let _ = remove_file(&wal_path); + } let mixin_path = dir.join(format!("{name}.sbx-mixin.yaml")); if mixin_path.exists() { remove_file(&mixin_path).with_context(|| { @@ -894,16 +902,20 @@ mod tests { } #[test] - fn remove_rag_sidecars_removes_both() { + fn remove_rag_sidecars_removes_duckdb_wal_and_mixin() { let root = sidecar_temp_dir("rag-sidecars-both"); let duckdb = root.join("docs.duckdb"); + // DuckDB leaves this behind whenever it was not closed cleanly. + let wal = root.join("docs.duckdb.wal"); let mixin = root.join("docs.sbx-mixin.yaml"); fs::write(&duckdb, "db").unwrap(); + fs::write(&wal, "wal").unwrap(); fs::write(&mixin, "mixin").unwrap(); remove_rag_sidecars(&root, "docs").unwrap(); assert!(!duckdb.exists(), "the .duckdb sidecar must be removed"); + assert!(!wal.exists(), "the .duckdb.wal sidecar must be removed"); assert!( !mixin.exists(), "the .sbx-mixin.yaml sidecar must be removed" From 6d0a5550fec905eed97f74b1328547076a7d7fe5 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 11 Aug 2026 20:56:51 -0600 Subject: [PATCH 22/36] style: Removed some redundant comments --- src/config/paths.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/config/paths.rs b/src/config/paths.rs index bbbf6b1..44589d9 100644 --- a/src/config/paths.rs +++ b/src/config/paths.rs @@ -437,10 +437,6 @@ pub(crate) fn remove_rag_sidecars(dir: &Path, name: &str) -> Result<()> { if duckdb_path.exists() { let _ = remove_file(&duckdb_path); } - // DuckDB keeps its write-ahead log in a sibling file and only removes it on a - // clean close, so a crash or a kill leaves one behind. Deleting the database - // without it strands a `.wal` that the next RAG created under the same name - // would inherit as if it were its own. let wal_path = dir.join(format!("{name}.duckdb.wal")); if wal_path.exists() { let _ = remove_file(&wal_path); @@ -905,7 +901,6 @@ mod tests { fn remove_rag_sidecars_removes_duckdb_wal_and_mixin() { let root = sidecar_temp_dir("rag-sidecars-both"); let duckdb = root.join("docs.duckdb"); - // DuckDB leaves this behind whenever it was not closed cleanly. let wal = root.join("docs.duckdb.wal"); let mixin = root.join("docs.sbx-mixin.yaml"); fs::write(&duckdb, "db").unwrap(); From 74bc613d943632cba0a64d1f46d7691d221ba80a Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 11 Aug 2026 21:04:21 -0600 Subject: [PATCH 23/36] fix(rag): address Copilot review findings on the driver abstraction Five review comments, all real: - hybrid_search ran its vector and keyword legs sequentially after the provider refactor; main ran them under tokio::join!. Restores the concurrency while keeping the degrade-on-error keyword behaviour, so a remote provider no longer pays two serial round trips per query. - inject_rag_secrets derived a vault secret name by trimming braces, which leaves a literal key untouched. A RAG holding a plaintext api_key therefore looked the secret up by its own value and printed it to stderr on failure. Parsing is now strict and a non-placeholder is skipped with a warning that names no credential. - validate() now refuses a driver_config.api_key that is not a {{NAME}} placeholder, so a plaintext key cannot reach the RAG YAML at all. - Rag::create's catch-all arm treated any unrecognised driver as yaml. A typo built a yaml store, paid to embed the corpus, persisted the bad driver and only failed on the next run. Unknown drivers now fail immediately. - The qdrant arm's error was written for a developer; it now tells the user that only attached collections are readable and points at .rag attach. --- src/rag/mod.rs | 137 +++++++++++++++++++++++++++++++++++++++++---- src/sandbox/mod.rs | 20 +++++-- 2 files changed, 142 insertions(+), 15 deletions(-) diff --git a/src/rag/mod.rs b/src/rag/mod.rs index ac1ba52..8522e05 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -592,13 +592,23 @@ impl Rag { (Box::new(duck), bm25) } "qdrant" => bail!( - "Qdrant RAGs cannot be constructed via Rag::create(); \ - use Rag::attach() or Rag::load_async() instead" + "RAG '{name}' uses driver 'qdrant' without `attached: true`. \ + Coyote can currently only READ a pre-existing Qdrant \ + collection — attach one with `.rag attach`. Writing to a \ + Coyote-owned Qdrant collection is not supported yet." ), - _ => { + "yaml" => { let bm25 = data.build_bm25(); (Box::new(YamlProvider::from_data(&data)), bm25) } + // Explicitly NOT a catch-all falling through to yaml. A typo'd driver + // used to build a yaml store, pay to embed the whole corpus, persist + // the bad driver, and only fail on the NEXT run — leaving the RAG + // unusable without hand-editing the YAML. + other => bail!( + "Unknown RAG driver '{other}' for RAG '{name}'. \ + Valid drivers: yaml, duckdb, qdrant." + ), }; let node_to_docs = data.knowledge_graph.build_node_to_docs(); let embedding_model = @@ -1169,12 +1179,12 @@ impl Rag { top_k: usize, rerank_model: Option<&str>, ) -> Result> { - let vector_search_results = self.vector_search(query, top_k, 0.0).await?; - debug!("vector_search_results: {vector_search_results:?}",); - let vector_search_ids: Vec = - vector_search_results.into_iter().map(|(v, _)| v).collect(); - - let keyword_search_results: Vec<(DocumentId, f32)> = + // The two legs run CONCURRENTLY. Both can be network round trips on a + // remote provider (embedding the query, then the vector search; a native + // keyword search), so awaiting them in sequence roughly doubles the + // latency of every hybrid query. The local BM25 branch is synchronous and + // simply runs inline inside the future. + let keyword_leg = async { if self.provider.has_native_keyword_search() { self.provider .keyword_search(query, top_k) @@ -1185,7 +1195,16 @@ impl Rag { }) } else { self.keyword_search(query, top_k, 0.0) - }; + } + }; + let (vector_search_results, keyword_search_results) = + tokio::join!(self.vector_search(query, top_k, 0.0), keyword_leg); + + let vector_search_results = vector_search_results?; + debug!("vector_search_results: {vector_search_results:?}",); + let vector_search_ids: Vec = + vector_search_results.into_iter().map(|(v, _)| v).collect(); + debug!("keyword_search_results: {keyword_search_results:?}",); let keyword_search_ids: Vec = keyword_search_results.into_iter().map(|(v, _)| v).collect(); @@ -1568,6 +1587,23 @@ impl RagData { } } + // An api_key must be a `{{NAME}}` reference, never the credential itself. + // Two reasons, both load-bearing: a literal key here gets committed to the + // RAG YAML in plaintext, and sandbox provisioning parses this value back + // out to learn which vault secret to bind, so a literal one silently + // provisions nothing (and used to be echoed to stderr on failure). + if let Some(api_key) = self.driver_config.get("api_key") + && placeholder_secret_name(api_key).is_none() + { + bail!( + "driver_config.api_key must be a secret placeholder of the form \ + '{{{{NAME}}}}', not a literal key. Store the credential with \ + `coyote --add-secret ` and reference it by name; a literal \ + key would be written to this RAG's YAML in plaintext and cannot \ + be provisioned into the sandbox." + ); + } + match (self.driver.as_str(), self.attached) { ("yaml", false) => Ok(()), ("duckdb", false) => Ok(()), @@ -2177,6 +2213,21 @@ fn resolve_driver_config( }) } +/// The secret NAME inside a `{{NAME}}` placeholder, or `None` for anything else. +/// +/// Deliberately strict, and shared with sandbox provisioning so both agree on +/// what a placeholder is. A RAG's `driver_config.api_key` is supposed to hold a +/// placeholder, never a credential, but nothing stops a hand-edited or older +/// config from holding the literal key. Consumers report failures *by name*, so +/// treating a literal value as a name leaks the credential into stderr and logs. +pub(crate) fn placeholder_secret_name(value: &str) -> Option<&str> { + let inner = value.trim().strip_prefix("{{")?.strip_suffix("}}")?.trim(); + if inner.is_empty() || inner.contains(['{', '}']) { + return None; + } + Some(inner) +} + /// Interpolation core, taking the resolver as an argument so it can be exercised /// without a vault. Mirrors `interpolate_secrets` / `interpolate_secrets_with`. fn resolve_driver_config_with( @@ -3239,6 +3290,72 @@ vectors: {} assert!(data.validate().is_ok()); } + /// A literal credential in `driver_config.api_key` is refused outright: it + /// would be persisted to the RAG YAML in plaintext, and sandbox + /// provisioning parses this field expecting a placeholder. + #[test] + fn ragdata_validate_rejects_a_literal_api_key() { + let mut data = RagData::new( + "m".into(), + 1024, + 50, + None, + 5, + None, + GraphRagConfig::default(), + ); + data.driver = "qdrant".to_string(); + data.attached = true; + data.driver_config + .insert("api_key".to_string(), "sk-a-real-looking-key".to_string()); + + let err = data.validate().unwrap_err().to_string(); + + assert!(err.contains("must be a secret placeholder"), "got: {err}"); + assert!( + !err.contains("sk-a-real-looking-key"), + "the error must never echo the credential back: {err}" + ); + } + + #[test] + fn ragdata_validate_accepts_a_placeholder_api_key() { + let mut data = RagData::new( + "m".into(), + 1024, + 50, + None, + 5, + None, + GraphRagConfig::default(), + ); + data.driver = "qdrant".to_string(); + data.attached = true; + data.driver_config + .insert("api_key".to_string(), "{{QDRANT_KEY}}".to_string()); + + assert!(data.validate().is_ok()); + } + + /// The parser is the single thing standing between a hand-edited literal key + /// and a "could not load secret ''" line in the user's terminal. + #[test] + fn placeholder_secret_name_accepts_only_well_formed_placeholders() { + assert_eq!(placeholder_secret_name("{{NAME}}"), Some("NAME")); + assert_eq!(placeholder_secret_name(" {{ NAME }} "), Some("NAME")); + + // Every one of these used to survive the old trim_matches unchanged and + // then be used as a secret name. + assert_eq!(placeholder_secret_name("sk-literal-key"), None); + assert_eq!(placeholder_secret_name(""), None); + assert_eq!(placeholder_secret_name("{{}}"), None); + assert_eq!(placeholder_secret_name("{{ }}"), None); + assert_eq!(placeholder_secret_name("{{A}}{{B}}"), None); + assert_eq!(placeholder_secret_name("prefix{{NAME}}"), None); + assert_eq!(placeholder_secret_name("{{NAME"), None); + assert_eq!(placeholder_secret_name("NAME}}"), None); + } + #[test] fn ragdata_validate_rejects_zero_top_k_from_a_truncated_yaml() { let yaml = " diff --git a/src/sandbox/mod.rs b/src/sandbox/mod.rs index 94c9889..e97fcb1 100644 --- a/src/sandbox/mod.rs +++ b/src/sandbox/mod.rs @@ -19,7 +19,7 @@ use crate::config::AppConfig; use crate::config::Config; use crate::config::VAULT_DATA_FILE_NAME; use crate::config::paths; -use crate::rag::RagData; +use crate::rag::{RagData, placeholder_secret_name}; use crate::sandbox::mcp_credentials::MCP_MIXIN_NAME; use crate::sandbox::mixins::DiscoveredMixin; use crate::utils::run_command_with_output; @@ -344,10 +344,20 @@ fn inject_rag_secrets(vault: &Vault, registered: &HashSet) -> Result<()> if service_id.is_empty() || registered.contains(&service_id) { continue; } - let secret_name = placeholder - .trim_start_matches("{{") - .trim_end_matches("}}") - .trim(); + // A literal key must NOT be mistaken for a secret NAME. The trims that + // used to stand here leave a non-placeholder value completely untouched, + // so the vault lookup below would run with the credential as the "name" + // and the warning would then print that credential to stderr. + let Some(secret_name) = placeholder_secret_name(placeholder) else { + eprintln!( + "Warning: RAG '{stem}' has a driver_config.api_key that is not a \ + secret placeholder, so no credential can be provisioned to the \ + sandbox and queries to this RAG will fail inside it. Store the \ + key with `coyote --add-secret `, then set api_key to the \ + matching placeholder in the RAG YAML." + ); + continue; + }; match vault.get_secret(secret_name, false) { Ok(secret_value) => { From de91ffa5175465a664d63d06efd7a168f5f6371c Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 11 Aug 2026 21:07:20 -0600 Subject: [PATCH 24/36] fix(rag): treat a zero min_score as no floor on Qdrant searches parse_search_hits filtered on score > min_score, and the only caller passes 0.0. Qdrant Euclid collections score by negative distance, so every hit was dropped and an attached Euclid collection returned nothing at all, silently. This is the same trap the surrounding code already documents: score_threshold is deliberately not sent because it is metric-aware and a 0.0 floor filters everything out on Euclid. The local filter then reproduced it exactly. Only a positive floor is now treated as a floor. --- src/rag/providers/qdrant.rs | 42 +++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/src/rag/providers/qdrant.rs b/src/rag/providers/qdrant.rs index 1f42b0a..bcc980c 100644 --- a/src/rag/providers/qdrant.rs +++ b/src/rag/providers/qdrant.rs @@ -104,7 +104,10 @@ fn parse_search_hits( let score = pt["score"].as_f64()? as f32; Some((interner.document_id(&pt["id"])?, score)) }) - .filter(|(_, score)| *score > min_score) + // Only a positive floor is a floor. Euclid collections score by negative + // distance, so a 0.0 floor would drop every hit — the exact failure the + // caller avoids `score_threshold` to prevent. + .filter(|(_, score)| min_score <= 0.0 || *score > min_score) .collect()) } @@ -353,7 +356,8 @@ impl RagProvider for QdrantProvider { // `score_threshold` is deliberately NOT sent. It is metric-aware: on Cosine // collections 0.0 means "no floor" as expected, but Euclid collections score // by negative distance, where 0.0 filters everything out. The attach wizard - // does not pin the distance metric, so filter locally instead. + // does not pin the distance metric, so filter locally instead — where a + // 0.0 floor is correctly treated as "no floor" (see `parse_search_hits`). let body = serde_json::json!({ "vector": embedding, "limit": top_k, @@ -576,6 +580,40 @@ mod tests { assert!(provider.fetch_content(&[]).await.unwrap().is_empty()); } + /// Euclid collections score by NEGATIVE distance, so the 0.0 the caller + /// passes must mean "no floor". Filtering on it drops every hit — the exact + /// bug that keeps Qdrant's own `score_threshold` off the wire. + #[test] + fn a_zero_floor_keeps_negative_euclid_scores() { + let mut interner = PointIdInterner::default(); + let search = serde_json::json!({ + "result": [ + {"id": 1, "score": -0.12}, + {"id": 2, "score": -8.5}, + ] + }); + + let hits = parse_search_hits(&mut interner, &search, 0.0).unwrap(); + + assert_eq!(hits.len(), 2, "a 0.0 floor must not drop negative scores"); + } + + #[test] + fn a_positive_floor_still_filters() { + let mut interner = PointIdInterner::default(); + let search = serde_json::json!({ + "result": [ + {"id": 1, "score": 0.9}, + {"id": 2, "score": 0.2}, + ] + }); + + let hits = parse_search_hits(&mut interner, &search, 0.5).unwrap(); + + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].0, DocumentId(1)); + } + /// A UUID-keyed collection has to survive the whole `vector_search` → /// `fetch_content` round trip, and the fetch must ask Qdrant for the ORIGINAL /// string id. Parsing ids with `as_u64()` used to drop these hits inside a From 1322d73c7b70c46af85f1c23c73e77853c50c6a5 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 11 Aug 2026 21:50:46 -0600 Subject: [PATCH 25/36] style: further cleanup --- src/rag/mod.rs | 20 +------------------- src/rag/providers/qdrant.rs | 5 +---- src/sandbox/mod.rs | 4 ---- 3 files changed, 2 insertions(+), 27 deletions(-) diff --git a/src/rag/mod.rs b/src/rag/mod.rs index 8522e05..f1e1255 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -603,7 +603,7 @@ impl Rag { } // Explicitly NOT a catch-all falling through to yaml. A typo'd driver // used to build a yaml store, pay to embed the whole corpus, persist - // the bad driver, and only fail on the NEXT run — leaving the RAG + // the bad driver, and only fail on the NEXT run, leaving the RAG // unusable without hand-editing the YAML. other => bail!( "Unknown RAG driver '{other}' for RAG '{name}'. \ @@ -1179,11 +1179,6 @@ impl Rag { top_k: usize, rerank_model: Option<&str>, ) -> Result> { - // The two legs run CONCURRENTLY. Both can be network round trips on a - // remote provider (embedding the query, then the vector search; a native - // keyword search), so awaiting them in sequence roughly doubles the - // latency of every hybrid query. The local BM25 branch is synchronous and - // simply runs inline inside the future. let keyword_leg = async { if self.provider.has_native_keyword_search() { self.provider @@ -1587,11 +1582,6 @@ impl RagData { } } - // An api_key must be a `{{NAME}}` reference, never the credential itself. - // Two reasons, both load-bearing: a literal key here gets committed to the - // RAG YAML in plaintext, and sandbox provisioning parses this value back - // out to learn which vault secret to bind, so a literal one silently - // provisions nothing (and used to be echoed to stderr on failure). if let Some(api_key) = self.driver_config.get("api_key") && placeholder_secret_name(api_key).is_none() { @@ -3290,9 +3280,6 @@ vectors: {} assert!(data.validate().is_ok()); } - /// A literal credential in `driver_config.api_key` is refused outright: it - /// would be persisted to the RAG YAML in plaintext, and sandbox - /// provisioning parses this field expecting a placeholder. #[test] fn ragdata_validate_rejects_a_literal_api_key() { let mut data = RagData::new( @@ -3337,15 +3324,10 @@ vectors: {} assert!(data.validate().is_ok()); } - /// The parser is the single thing standing between a hand-edited literal key - /// and a "could not load secret ''" line in the user's terminal. #[test] fn placeholder_secret_name_accepts_only_well_formed_placeholders() { assert_eq!(placeholder_secret_name("{{NAME}}"), Some("NAME")); assert_eq!(placeholder_secret_name(" {{ NAME }} "), Some("NAME")); - - // Every one of these used to survive the old trim_matches unchanged and - // then be used as a secret name. assert_eq!(placeholder_secret_name("sk-literal-key"), None); assert_eq!(placeholder_secret_name(""), None); assert_eq!(placeholder_secret_name("{{}}"), None); diff --git a/src/rag/providers/qdrant.rs b/src/rag/providers/qdrant.rs index bcc980c..b77d2cb 100644 --- a/src/rag/providers/qdrant.rs +++ b/src/rag/providers/qdrant.rs @@ -104,9 +104,6 @@ fn parse_search_hits( let score = pt["score"].as_f64()? as f32; Some((interner.document_id(&pt["id"])?, score)) }) - // Only a positive floor is a floor. Euclid collections score by negative - // distance, so a 0.0 floor would drop every hit — the exact failure the - // caller avoids `score_threshold` to prevent. .filter(|(_, score)| min_score <= 0.0 || *score > min_score) .collect()) } @@ -356,7 +353,7 @@ impl RagProvider for QdrantProvider { // `score_threshold` is deliberately NOT sent. It is metric-aware: on Cosine // collections 0.0 means "no floor" as expected, but Euclid collections score // by negative distance, where 0.0 filters everything out. The attach wizard - // does not pin the distance metric, so filter locally instead — where a + // does not pin the distance metric, so filter locally instead; i.e. where a // 0.0 floor is correctly treated as "no floor" (see `parse_search_hits`). let body = serde_json::json!({ "vector": embedding, diff --git a/src/sandbox/mod.rs b/src/sandbox/mod.rs index e97fcb1..39f362b 100644 --- a/src/sandbox/mod.rs +++ b/src/sandbox/mod.rs @@ -344,10 +344,6 @@ fn inject_rag_secrets(vault: &Vault, registered: &HashSet) -> Result<()> if service_id.is_empty() || registered.contains(&service_id) { continue; } - // A literal key must NOT be mistaken for a secret NAME. The trims that - // used to stand here leave a non-placeholder value completely untouched, - // so the vault lookup below would run with the credential as the "name" - // and the warning would then print that credential to stderr. let Some(secret_name) = placeholder_secret_name(placeholder) else { eprintln!( "Warning: RAG '{stem}' has a driver_config.api_key that is not a \ From 64d594f4eeecb65d86f0adf33d08e08e250a5a22 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 11 Aug 2026 22:03:45 -0600 Subject: [PATCH 26/36] refactor(rag): discover driver_config secrets by grammar, not field name Sandbox provisioning only ever looked at driver_config["api_key"], so a driver whose credential is called anything else would have been silently unprovisioned inside a sandbox. It now scans every driver_config value and treats any that is a secret placeholder as a credential, which is the same rule resolve_driver_config already used at point of use. The first one binds to the RAG's own service id, which is what the generated mixin declares; any others register under their own names, as MCP secrets do. The mixin still carries a single credential entry, so a driver needing two bound secrets remains a follow-up. Also drops the placeholder parser added in 74bc613. crate::vault::SECRET_RE is already the canonical definition and was already imported here, so that was a third implementation of the same grammar. Requiring the whole value to match is what keeps a literal key from being read as a secret name and printed. The api_key check is gone from RagData::validate: a generic config validator should not know a provider's field names. --- src/rag/mod.rs | 85 ------------------------- src/sandbox/mod.rs | 155 +++++++++++++++++++++++++++++++++++++-------- 2 files changed, 128 insertions(+), 112 deletions(-) diff --git a/src/rag/mod.rs b/src/rag/mod.rs index f1e1255..fc6f306 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -1582,18 +1582,6 @@ impl RagData { } } - if let Some(api_key) = self.driver_config.get("api_key") - && placeholder_secret_name(api_key).is_none() - { - bail!( - "driver_config.api_key must be a secret placeholder of the form \ - '{{{{NAME}}}}', not a literal key. Store the credential with \ - `coyote --add-secret ` and reference it by name; a literal \ - key would be written to this RAG's YAML in plaintext and cannot \ - be provisioned into the sandbox." - ); - } - match (self.driver.as_str(), self.attached) { ("yaml", false) => Ok(()), ("duckdb", false) => Ok(()), @@ -2203,21 +2191,6 @@ fn resolve_driver_config( }) } -/// The secret NAME inside a `{{NAME}}` placeholder, or `None` for anything else. -/// -/// Deliberately strict, and shared with sandbox provisioning so both agree on -/// what a placeholder is. A RAG's `driver_config.api_key` is supposed to hold a -/// placeholder, never a credential, but nothing stops a hand-edited or older -/// config from holding the literal key. Consumers report failures *by name*, so -/// treating a literal value as a name leaks the credential into stderr and logs. -pub(crate) fn placeholder_secret_name(value: &str) -> Option<&str> { - let inner = value.trim().strip_prefix("{{")?.strip_suffix("}}")?.trim(); - if inner.is_empty() || inner.contains(['{', '}']) { - return None; - } - Some(inner) -} - /// Interpolation core, taking the resolver as an argument so it can be exercised /// without a vault. Mirrors `interpolate_secrets` / `interpolate_secrets_with`. fn resolve_driver_config_with( @@ -3280,64 +3253,6 @@ vectors: {} assert!(data.validate().is_ok()); } - #[test] - fn ragdata_validate_rejects_a_literal_api_key() { - let mut data = RagData::new( - "m".into(), - 1024, - 50, - None, - 5, - None, - GraphRagConfig::default(), - ); - data.driver = "qdrant".to_string(); - data.attached = true; - data.driver_config - .insert("api_key".to_string(), "sk-a-real-looking-key".to_string()); - - let err = data.validate().unwrap_err().to_string(); - - assert!(err.contains("must be a secret placeholder"), "got: {err}"); - assert!( - !err.contains("sk-a-real-looking-key"), - "the error must never echo the credential back: {err}" - ); - } - - #[test] - fn ragdata_validate_accepts_a_placeholder_api_key() { - let mut data = RagData::new( - "m".into(), - 1024, - 50, - None, - 5, - None, - GraphRagConfig::default(), - ); - data.driver = "qdrant".to_string(); - data.attached = true; - data.driver_config - .insert("api_key".to_string(), "{{QDRANT_KEY}}".to_string()); - - assert!(data.validate().is_ok()); - } - - #[test] - fn placeholder_secret_name_accepts_only_well_formed_placeholders() { - assert_eq!(placeholder_secret_name("{{NAME}}"), Some("NAME")); - assert_eq!(placeholder_secret_name(" {{ NAME }} "), Some("NAME")); - assert_eq!(placeholder_secret_name("sk-literal-key"), None); - assert_eq!(placeholder_secret_name(""), None); - assert_eq!(placeholder_secret_name("{{}}"), None); - assert_eq!(placeholder_secret_name("{{ }}"), None); - assert_eq!(placeholder_secret_name("{{A}}{{B}}"), None); - assert_eq!(placeholder_secret_name("prefix{{NAME}}"), None); - assert_eq!(placeholder_secret_name("{{NAME"), None); - assert_eq!(placeholder_secret_name("NAME}}"), None); - } - #[test] fn ragdata_validate_rejects_zero_top_k_from_a_truncated_yaml() { let yaml = " diff --git a/src/sandbox/mod.rs b/src/sandbox/mod.rs index 39f362b..b75ce09 100644 --- a/src/sandbox/mod.rs +++ b/src/sandbox/mod.rs @@ -19,7 +19,7 @@ use crate::config::AppConfig; use crate::config::Config; use crate::config::VAULT_DATA_FILE_NAME; use crate::config::paths; -use crate::rag::{RagData, placeholder_secret_name}; +use crate::rag::RagData; use crate::sandbox::mcp_credentials::MCP_MIXIN_NAME; use crate::sandbox::mixins::DiscoveredMixin; use crate::utils::run_command_with_output; @@ -337,35 +337,25 @@ fn inject_rag_secrets(vault: &Vault, registered: &HashSet) -> Result<()> if !data.attached { continue; } - let Some(placeholder) = data.driver_config.get("api_key") else { - continue; - }; - let service_id = mcp_credentials::secret_service_id(&stem); - if service_id.is_empty() || registered.contains(&service_id) { - continue; - } - let Some(secret_name) = placeholder_secret_name(placeholder) else { - eprintln!( - "Warning: RAG '{stem}' has a driver_config.api_key that is not a \ - secret placeholder, so no credential can be provisioned to the \ - sandbox and queries to this RAG will fail inside it. Store the \ - key with `coyote --add-secret `, then set api_key to the \ - matching placeholder in the RAG YAML." - ); + let secret_names = driver_config_secret_names(&data); + let Some((primary, extra)) = secret_names.split_first() else { continue; }; - match vault.get_secret(secret_name, false) { - Ok(secret_value) => { - sbx_secret_set(&service_id, &secret_value) - .context("Failed to register RAG secret with sbx")?; - } - Err(e) => { - eprintln!( - "Warning: could not load secret '{secret_name}' for RAG '{stem}': {e}. \ - Queries to this RAG will fail inside the sandbox. \ - Run `coyote --add-secret {secret_name}` to fix." - ); + // The generated mixin declares one credential per RAG, keyed on the RAG's + // own service id, so that is where the first one binds. + let service_id = mcp_credentials::secret_service_id(&stem); + if !service_id.is_empty() && !registered.contains(&service_id) { + bind_rag_secret(vault, &service_id, primary, &stem)?; + } + + // Anything beyond the first is registered under its own name, the way MCP + // secrets are, so a hand-written mixin can reference it. The generated + // mixin cannot yet: it carries a single credential entry. + for name in extra { + let id = mcp_credentials::secret_service_id(name); + if !id.is_empty() && !registered.contains(&id) { + bind_rag_secret(vault, &id, name, &stem)?; } } } @@ -373,6 +363,53 @@ fn inject_rag_secrets(vault: &Vault, registered: &HashSet) -> Result<()> Ok(()) } +/// Every distinct vault secret referenced by a RAG's `driver_config`. +/// +/// Deliberately keyed on the placeholder grammar rather than on field names: a +/// driver may call its credential `api_key`, `token` or anything else, and this +/// path should not have to learn each one. Plain values such as `host` and +/// `collection` never match, so they are skipped. +/// +/// A value only counts when it is a placeholder and *nothing else*. That is what +/// keeps a literal credential from being read as a secret NAME — the caller +/// reports failures by name, so a literal would otherwise be printed to stderr. +fn driver_config_secret_names(data: &RagData) -> Vec { + let mut names: Vec = Vec::new(); + for value in data.driver_config.values() { + let trimmed = value.trim(); + let Ok(Some(caps)) = SECRET_RE.captures(trimmed) else { + continue; + }; + if caps.get(0).map(|m| m.as_str()) != Some(trimmed) { + continue; + } + let Some(name) = caps.get(1).map(|m| m.as_str().trim()) else { + continue; + }; + if !name.is_empty() && !names.iter().any(|n| n == name) { + names.push(name.to_string()); + } + } + names +} + +fn bind_rag_secret(vault: &Vault, service_id: &str, secret_name: &str, stem: &str) -> Result<()> { + match vault.get_secret(secret_name, false) { + Ok(secret_value) => { + sbx_secret_set(service_id, &secret_value) + .context("Failed to register RAG secret with sbx")?; + } + Err(e) => { + eprintln!( + "Warning: could not load secret '{secret_name}' for RAG '{stem}': {e}. \ + Queries to this RAG will fail inside the sandbox. \ + Run `coyote --add-secret {secret_name}` to fix." + ); + } + } + Ok(()) +} + fn provider_to_sbx_service(provider_type: &str, client_name: Option<&str>) -> String { match provider_type { "claude" => "anthropic".to_string(), @@ -652,6 +689,70 @@ fn chown_agent_recursive(sandbox: &str, path: &str) -> Result<()> { #[cfg(test)] mod tests { use super::*; + + fn rag_with(driver_config: &[(&str, &str)]) -> RagData { + let mut data = RagData::new("m".into(), 1024, 50, None, 5, None, Default::default()); + data.driver = "qdrant".to_string(); + data.attached = true; + for (k, v) in driver_config { + data.driver_config.insert(k.to_string(), v.to_string()); + } + data + } + + /// The whole point of keying on the placeholder grammar: a driver may call + /// its credential anything, and this path must not have to know the name. + #[test] + fn secret_names_are_found_whatever_the_field_is_called() { + let data = rag_with(&[ + ("host", "qdrant.example.com:6333"), + ("collection", "docs"), + ("token", "{{SOME_TOKEN}}"), + ]); + + assert_eq!(driver_config_secret_names(&data), vec!["SOME_TOKEN"]); + } + + /// A literal credential must never be read as a secret NAME. Callers report + /// failures by name, so doing so prints the credential to stderr. + #[test] + fn a_literal_credential_is_not_treated_as_a_secret_name() { + let data = rag_with(&[("api_key", "sk-a-real-looking-key")]); + + assert!(driver_config_secret_names(&data).is_empty()); + } + + #[test] + fn plain_values_are_never_mistaken_for_secrets() { + let data = rag_with(&[("host", "localhost:6333"), ("collection", "docs")]); + + assert!(driver_config_secret_names(&data).is_empty()); + } + + /// A value that merely *contains* a placeholder is not the credential: the + /// sbx proxy injects the whole secret as the header value. + #[test] + fn a_partial_placeholder_is_not_a_credential() { + let data = rag_with(&[("api_key", "Bearer {{KEY}}")]); + + assert!(driver_config_secret_names(&data).is_empty()); + } + + #[test] + fn several_secrets_are_all_found_and_deduped() { + let data = rag_with(&[ + ("api_key", "{{QDRANT_KEY}}"), + ("host", "localhost:6333"), + ("token", "{{ OTHER_TOKEN }}"), + ("fallback_key", "{{QDRANT_KEY}}"), + ]); + + assert_eq!( + driver_config_secret_names(&data), + vec!["QDRANT_KEY", "OTHER_TOKEN"], + "order follows driver_config, and a repeat is not registered twice" + ); + } use std::time::{SystemTime, UNIX_EPOCH}; #[test] From 78740db17089f09391c7779730d309bf70bcefb1 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 11 Aug 2026 22:03:45 -0600 Subject: [PATCH 27/36] chore: ignore the .coyote workspace directory It holds generated workspace state and should never be committed. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 54413f1..87207b0 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ .idea/ /coyote.iml /.idea/ +.coyote From 6f586bd5353b524bda71aaaf424e16860381a43c Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 11 Aug 2026 22:07:35 -0600 Subject: [PATCH 28/36] style: cleanup --- src/sandbox/mod.rs | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/src/sandbox/mod.rs b/src/sandbox/mod.rs index b75ce09..e7b0b67 100644 --- a/src/sandbox/mod.rs +++ b/src/sandbox/mod.rs @@ -342,16 +342,11 @@ fn inject_rag_secrets(vault: &Vault, registered: &HashSet) -> Result<()> continue; }; - // The generated mixin declares one credential per RAG, keyed on the RAG's - // own service id, so that is where the first one binds. let service_id = mcp_credentials::secret_service_id(&stem); if !service_id.is_empty() && !registered.contains(&service_id) { bind_rag_secret(vault, &service_id, primary, &stem)?; } - // Anything beyond the first is registered under its own name, the way MCP - // secrets are, so a hand-written mixin can reference it. The generated - // mixin cannot yet: it carries a single credential entry. for name in extra { let id = mcp_credentials::secret_service_id(name); if !id.is_empty() && !registered.contains(&id) { @@ -363,16 +358,6 @@ fn inject_rag_secrets(vault: &Vault, registered: &HashSet) -> Result<()> Ok(()) } -/// Every distinct vault secret referenced by a RAG's `driver_config`. -/// -/// Deliberately keyed on the placeholder grammar rather than on field names: a -/// driver may call its credential `api_key`, `token` or anything else, and this -/// path should not have to learn each one. Plain values such as `host` and -/// `collection` never match, so they are skipped. -/// -/// A value only counts when it is a placeholder and *nothing else*. That is what -/// keeps a literal credential from being read as a secret NAME — the caller -/// reports failures by name, so a literal would otherwise be printed to stderr. fn driver_config_secret_names(data: &RagData) -> Vec { let mut names: Vec = Vec::new(); for value in data.driver_config.values() { @@ -700,8 +685,6 @@ mod tests { data } - /// The whole point of keying on the placeholder grammar: a driver may call - /// its credential anything, and this path must not have to know the name. #[test] fn secret_names_are_found_whatever_the_field_is_called() { let data = rag_with(&[ @@ -713,8 +696,6 @@ mod tests { assert_eq!(driver_config_secret_names(&data), vec!["SOME_TOKEN"]); } - /// A literal credential must never be read as a secret NAME. Callers report - /// failures by name, so doing so prints the credential to stderr. #[test] fn a_literal_credential_is_not_treated_as_a_secret_name() { let data = rag_with(&[("api_key", "sk-a-real-looking-key")]); @@ -729,8 +710,6 @@ mod tests { assert!(driver_config_secret_names(&data).is_empty()); } - /// A value that merely *contains* a placeholder is not the credential: the - /// sbx proxy injects the whole secret as the header value. #[test] fn a_partial_placeholder_is_not_a_credential() { let data = rag_with(&[("api_key", "Bearer {{KEY}}")]); From af9622d31c2f054197c2c845c73a193ff2d3034d Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 11 Aug 2026 22:15:04 -0600 Subject: [PATCH 29/36] fix(rag): stop routing Qdrant requests through an ambient proxy make_client used a bare reqwest builder, which honours whatever proxy the environment advertises. That made it the only HTTP client in Coyote to do so: utils::set_proxy discards ambient settings and applies only Coyote's configured proxy, and every other client goes through it. The symptom is that a perfectly healthy Qdrant is unreachable and the error belongs to the interposing proxy, not the store, so it reads as a Coyote or Qdrant fault. Locally an installed Socket Firewall answered `.rag attach` against 127.0.0.1:6333 with an HTML 'Connection Required' page and HTTP 405. Both #[ignore]d live tests now pass against a real Qdrant; they failed with that same 405 before this change, which is the first time either has run green. A remote store that genuinely needs Coyote's configured proxy is a follow-up: that means threading the proxy config into the provider. --- src/rag/providers/qdrant.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/rag/providers/qdrant.rs b/src/rag/providers/qdrant.rs index b77d2cb..515fb6f 100644 --- a/src/rag/providers/qdrant.rs +++ b/src/rag/providers/qdrant.rs @@ -193,6 +193,11 @@ impl QdrantProvider { } Client::builder() .default_headers(headers) + // reqwest picks up ambient proxy settings by default, which routes even + // a loopback Qdrant through whatever proxy the environment dictates and + // fails with that proxy's error rather than Qdrant's. Every other client + // in Coyote discards them the same way, in `utils::set_proxy`. + .no_proxy() .build() .context("Failed to build reqwest client") } From 4dd6e794b22a84bcee8684b9e833d548ac5726d8 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Tue, 11 Aug 2026 22:19:14 -0600 Subject: [PATCH 30/36] docs: removed redundant comment --- src/rag/providers/qdrant.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/rag/providers/qdrant.rs b/src/rag/providers/qdrant.rs index 515fb6f..7dac07e 100644 --- a/src/rag/providers/qdrant.rs +++ b/src/rag/providers/qdrant.rs @@ -193,10 +193,6 @@ impl QdrantProvider { } Client::builder() .default_headers(headers) - // reqwest picks up ambient proxy settings by default, which routes even - // a loopback Qdrant through whatever proxy the environment dictates and - // fails with that proxy's error rather than Qdrant's. Every other client - // in Coyote discards them the same way, in `utils::set_proxy`. .no_proxy() .build() .context("Failed to build reqwest client") From 54685be9a22df582e4dac14a3ffe1468d87f0400 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Wed, 12 Aug 2026 10:33:03 -0600 Subject: [PATCH 31/36] fix: keep loopback and LAN traffic off an ambient proxy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-existing on main, not introduced by the driver work, but it makes a local RAG backend unusable so it belongs with this change. build_client only called set_proxy when a client had configured one of its own. With nothing configured, reqwest's own detection applied, which sends every request through a *_PROXY variable including ones bound for 127.0.0.1 or a LAN address. A proxy cannot usefully forward those, and anything that intercepts proxied traffic answers on behalf of a service that is running perfectly well, so the error names the proxy rather than the store and reads as a Coyote fault. Concretely, an installed Socket Firewall exports HTTP_PROXY to the processes it wraps and rejects hosts outside its allow list. That turned a healthy Ollama on the LAN into 'error decoding response body: expected value at line 2 column 1' — its HTML refusal page parsed as JSON — and a loopback Qdrant into an HTTP 405. Proxy handling is now always applied and always exempts loopback and private ranges, with NO_PROXY merged in since replacing reqwest's detection also replaces its handling of that variable. HTTP_PROXY and HTTPS_PROXY are kept separate because they are allowed to differ. An explicitly configured proxy still wins, and '-' still means none. This also supersedes the unconditional no_proxy() added to the Qdrant client in af9622d: that made it the only client to ignore a proxy outright, on a justification I got wrong. It now shares this path, so a remote store behind a real proxy keeps working. --- src/client/common.rs | 4 +- src/rag/providers/qdrant.rs | 5 +- src/utils/mod.rs | 118 ++++++++++++++++++++++++++++++++++-- 3 files changed, 115 insertions(+), 12 deletions(-) diff --git a/src/client/common.rs b/src/client/common.rs index d927440..8b4fe84 100644 --- a/src/client/common.rs +++ b/src/client/common.rs @@ -56,9 +56,7 @@ pub trait Client: Sync + Send { let mut builder = ReqwestClient::builder(); let extra = self.extra_config(); let timeout = extra.and_then(|v| v.connect_timeout).unwrap_or(10); - if let Some(proxy) = extra.and_then(|v| v.proxy.as_deref()) { - builder = set_proxy(builder, proxy)?; - } + builder = apply_proxy(builder, extra.and_then(|v| v.proxy.as_deref()))?; if let Some(user_agent) = self.app_config().user_agent.as_ref() { builder = builder.user_agent(user_agent); } diff --git a/src/rag/providers/qdrant.rs b/src/rag/providers/qdrant.rs index 7dac07e..29f5690 100644 --- a/src/rag/providers/qdrant.rs +++ b/src/rag/providers/qdrant.rs @@ -1,5 +1,6 @@ use crate::rag::provider::RagProvider; use crate::rag::{DocumentId, RagData}; +use crate::utils::apply_proxy; use anyhow::{Context, Result, bail}; use async_trait::async_trait; @@ -191,9 +192,7 @@ impl QdrantProvider { value.set_sensitive(true); headers.insert("api-key", value); } - Client::builder() - .default_headers(headers) - .no_proxy() + apply_proxy(Client::builder().default_headers(headers), None)? .build() .context("Failed to build reqwest client") } diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 428e882..2e8dc5b 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -295,18 +295,83 @@ pub fn is_url(path: &str) -> bool { path.starts_with("http://") || path.starts_with("https://") } -pub fn set_proxy( +/// 127.0.0.1 means something different to a proxy than it does to us, so a tool +/// that intercepts proxied traffic answers for a local service that is running +/// fine, and the failure reads as a fault in Coyote or in that service. +const LOCAL_NO_PROXY: &str = + "localhost,127.0.0.0/8,::1,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.local"; + +/// Applies proxy settings, keeping local traffic direct. `configured` is a +/// client's own `extra.proxy`, where `"-"` means no proxy at all; with nothing +/// configured an ambient `*_PROXY` is still honoured for public hosts. +pub fn apply_proxy( mut builder: reqwest::ClientBuilder, - proxy: &str, + configured: Option<&str>, ) -> Result { + // reqwest offers no way to add rules to the proxies it auto-detects, so + // detection is disabled and redone below. builder = builder.no_proxy(); - if !proxy.is_empty() && proxy != "-" { - builder = builder - .proxy(reqwest::Proxy::all(proxy).with_context(|| format!("Invalid proxy `{proxy}`"))?); - }; + + let configured = configured.map(str::trim).filter(|p| !p.is_empty()); + if configured == Some("-") { + return Ok(builder); + } + let exempt = no_proxy_rules(); + if let Some(url) = configured { + let proxy = reqwest::Proxy::all(url) + .with_context(|| format!("Invalid proxy `{url}`"))? + .no_proxy(reqwest::NoProxy::from_string(&exempt)); + return Ok(builder.proxy(proxy)); + } + + // Split per scheme, because HTTP_PROXY and HTTPS_PROXY are allowed to differ. + for (is_https, keys) in [ + ( + true, + ["HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy"], + ), + ( + false, + ["HTTP_PROXY", "http_proxy", "ALL_PROXY", "all_proxy"], + ), + ] { + let Some(url) = first_env(&keys) else { + continue; + }; + let proxy = if is_https { + reqwest::Proxy::https(&url) + } else { + reqwest::Proxy::http(&url) + }; + let proxy = proxy + .with_context(|| format!("Invalid proxy `{url}`"))? + .no_proxy(reqwest::NoProxy::from_string(&exempt)); + builder = builder.proxy(proxy); + } Ok(builder) } +fn first_env(keys: &[&str]) -> Option { + keys.iter() + .find_map(|key| env::var(key).ok()) + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} + +/// Replacing reqwest's auto-detection loses its `NO_PROXY` handling, so that is +/// merged back in here. +fn no_proxy_rules() -> String { + let mut rules = LOCAL_NO_PROXY.to_string(); + let extra = env::var("NO_PROXY") + .or_else(|_| env::var("no_proxy")) + .unwrap_or_default(); + if !extra.trim().is_empty() { + rules.push(','); + rules.push_str(extra.trim()); + } + rules +} + pub fn decode_bin(data: &[u8]) -> Result { let (v, _) = bincode::serde::decode_from_slice(data, bincode::config::legacy())?; Ok(v) @@ -316,6 +381,47 @@ pub fn decode_bin(data: &[u8]) -> Result { mod tests { use super::*; + /// Invalid rule syntax makes `from_string` return `None`, which silently drops + /// every exemption and sends local traffic back through the proxy. + #[test] + fn the_local_no_proxy_rules_are_valid() { + assert!( + reqwest::NoProxy::from_string(LOCAL_NO_PROXY).is_some(), + "reqwest rejected LOCAL_NO_PROXY, so nothing would be exempt" + ); + } + + #[test] + fn local_rules_cover_loopback_and_private_ranges() { + for host in [ + "localhost", + "127.0.0.0/8", + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", + ] { + assert!( + LOCAL_NO_PROXY.contains(host), + "{host} must stay exempt from proxying" + ); + } + } + + #[test] + fn a_dash_means_no_proxy_at_all() { + assert!(apply_proxy(reqwest::ClientBuilder::new(), Some("-")).is_ok()); + assert!(apply_proxy(reqwest::ClientBuilder::new(), Some(" - ")).is_ok()); + } + + #[test] + fn an_unparseable_proxy_is_reported() { + let err = apply_proxy(reqwest::ClientBuilder::new(), Some("not a url")) + .unwrap_err() + .to_string(); + + assert!(err.contains("Invalid proxy"), "got: {err}"); + } + #[test] #[cfg(not(target_os = "windows"))] fn test_safe_join_path() { From b837f82d7e09fdb62f43234720c41275157eb533 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Wed, 12 Aug 2026 10:52:31 -0600 Subject: [PATCH 32/36] fix(rag): keep a local Qdrant off an ambient proxy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts the global proxy rework in 54685be and narrows it to the provider. That commit took over proxy detection for every client in order to exempt loopback and private ranges. Too broad: reqwest's detection also reads macOS System Settings and the Windows registry behind its system-proxy feature, which sits in its default set. Coyote disables default features today, so hand-rolling the environment lookup happened to match — but re-enabling defaults later would silently restore that support for main and not for the hand-rolled version. It also made an explicitly configured proxy skip local hosts, which nobody asked for: a proxy named for a LAN endpoint should be used. build_client and utils are byte-identical to main again. The bypass now lives in QdrantProvider::make_client, which is the only place that knows the target host, and applies solely when that host is loopback, link-local, private or .local. A public or cloud-hosted store keeps whatever the environment configures. Also drops apply_proxy: with build_client reverted there was one caller left, and set_proxy already covers it. Both #[ignore]d live tests still pass against a Qdrant on loopback while an ambient proxy that rejects it is in force. --- src/client/common.rs | 4 +- src/rag/providers/qdrant.rs | 71 +++++++++++++++++++--- src/utils/mod.rs | 118 ++---------------------------------- 3 files changed, 71 insertions(+), 122 deletions(-) diff --git a/src/client/common.rs b/src/client/common.rs index 8b4fe84..d927440 100644 --- a/src/client/common.rs +++ b/src/client/common.rs @@ -56,7 +56,9 @@ pub trait Client: Sync + Send { let mut builder = ReqwestClient::builder(); let extra = self.extra_config(); let timeout = extra.and_then(|v| v.connect_timeout).unwrap_or(10); - builder = apply_proxy(builder, extra.and_then(|v| v.proxy.as_deref()))?; + if let Some(proxy) = extra.and_then(|v| v.proxy.as_deref()) { + builder = set_proxy(builder, proxy)?; + } if let Some(user_agent) = self.app_config().user_agent.as_ref() { builder = builder.user_agent(user_agent); } diff --git a/src/rag/providers/qdrant.rs b/src/rag/providers/qdrant.rs index 29f5690..fd1e0ad 100644 --- a/src/rag/providers/qdrant.rs +++ b/src/rag/providers/qdrant.rs @@ -1,6 +1,5 @@ use crate::rag::provider::RagProvider; use crate::rag::{DocumentId, RagData}; -use crate::utils::apply_proxy; use anyhow::{Context, Result, bail}; use async_trait::async_trait; @@ -10,6 +9,7 @@ use reqwest::{Client, Response, StatusCode}; use serde_json::Value; use std::collections::HashMap; use std::sync::Arc; +use url::{Host, Url}; /// Marks a `DocumentId` that stands in for a point id Coyote cannot carry /// directly. Qdrant accepts UUID strings as point ids, and that is what @@ -184,7 +184,25 @@ pub struct QdrantProvider { } impl QdrantProvider { - fn make_client(api_key: Option<&str>) -> Result { + /// A proxy cannot usefully forward to an address that means something different + /// on its side, and anything that intercepts proxied traffic answers for a store + /// that is running fine, so the failure names the proxy rather than Qdrant. + fn skips_proxy(base_url: &str) -> bool { + let Ok(url) = Url::parse(base_url) else { + return false; + }; + match url.host() { + Some(Host::Domain(name)) => { + name == "localhost" || name.ends_with(".localhost") || name.ends_with(".local") + } + Some(Host::Ipv4(ip)) => ip.is_loopback() || ip.is_private() || ip.is_link_local(), + // No stable is_unique_local, so fc00::/7 is matched directly. + Some(Host::Ipv6(ip)) => ip.is_loopback() || ip.segments()[0] & 0xfe00 == 0xfc00, + None => false, + } + } + + fn make_client(base_url: &str, api_key: Option<&str>) -> Result { let mut headers = HeaderMap::new(); if let Some(key) = api_key { let mut value = @@ -192,9 +210,11 @@ impl QdrantProvider { value.set_sensitive(true); headers.insert("api-key", value); } - apply_proxy(Client::builder().default_headers(headers), None)? - .build() - .context("Failed to build reqwest client") + let mut builder = Client::builder().default_headers(headers); + if Self::skips_proxy(base_url) { + builder = builder.no_proxy(); + } + builder.build().context("Failed to build reqwest client") } pub(crate) fn normalize_base_url(host: &str) -> String { @@ -219,7 +239,7 @@ impl QdrantProvider { api_key: Option<&str>, ) -> Result { let base_url = Self::normalize_base_url(host); - let client = Self::make_client(api_key)?; + let client = Self::make_client(&base_url, api_key)?; let resp = client .get(format!("{base_url}/collections/{collection}")) .send() @@ -237,7 +257,7 @@ impl QdrantProvider { pub async fn new(host: &str, collection: &str, api_key: Option<&str>) -> Result { let base_url = Self::normalize_base_url(host); - let client = Self::make_client(api_key)?; + let client = Self::make_client(&base_url, api_key)?; let resp = client .get(format!("{base_url}/collections/{collection}")) .send() @@ -260,7 +280,7 @@ impl QdrantProvider { pub async fn list_collections(host: &str, api_key: Option<&str>) -> Result> { let base_url = Self::normalize_base_url(host); - let client = Self::make_client(api_key)?; + let client = Self::make_client(&base_url, api_key)?; let resp = client .get(format!("{base_url}/collections")) .send() @@ -310,7 +330,7 @@ impl QdrantProvider { api_key: Option<&str>, ) -> Result> { let base_url = Self::normalize_base_url(host); - let client = Self::make_client(api_key)?; + let client = Self::make_client(&base_url, api_key)?; let url = format!("{base_url}/collections/{collection}/points/scroll"); let body = serde_json::json!({ "limit": 1, "with_payload": false }); @@ -577,6 +597,39 @@ mod tests { assert!(provider.fetch_content(&[]).await.unwrap().is_empty()); } + #[test] + fn local_and_private_hosts_skip_the_proxy() { + for host in [ + "http://localhost:6333", + "http://127.0.0.1:6333", + "http://192.168.0.56:6333", + "http://10.1.2.3:6333", + "http://172.16.4.5:6333", + "http://qdrant.local:6333", + "http://[::1]:6333", + ] { + assert!( + QdrantProvider::skips_proxy(host), + "{host} should not be proxied" + ); + } + } + + #[test] + fn public_hosts_still_honour_the_environment() { + for host in [ + "https://qdrant.example.com", + "http://8.8.8.8:6333", + "https://xyz.eu-central.aws.cloud.qdrant.io:6333", + "http://172.32.0.1:6333", + ] { + assert!( + !QdrantProvider::skips_proxy(host), + "{host} must keep the environment's proxy" + ); + } + } + /// Euclid collections score by NEGATIVE distance, so the 0.0 the caller /// passes must mean "no floor". Filtering on it drops every hit — the exact /// bug that keeps Qdrant's own `score_threshold` off the wire. diff --git a/src/utils/mod.rs b/src/utils/mod.rs index 2e8dc5b..428e882 100644 --- a/src/utils/mod.rs +++ b/src/utils/mod.rs @@ -295,83 +295,18 @@ pub fn is_url(path: &str) -> bool { path.starts_with("http://") || path.starts_with("https://") } -/// 127.0.0.1 means something different to a proxy than it does to us, so a tool -/// that intercepts proxied traffic answers for a local service that is running -/// fine, and the failure reads as a fault in Coyote or in that service. -const LOCAL_NO_PROXY: &str = - "localhost,127.0.0.0/8,::1,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.local"; - -/// Applies proxy settings, keeping local traffic direct. `configured` is a -/// client's own `extra.proxy`, where `"-"` means no proxy at all; with nothing -/// configured an ambient `*_PROXY` is still honoured for public hosts. -pub fn apply_proxy( +pub fn set_proxy( mut builder: reqwest::ClientBuilder, - configured: Option<&str>, + proxy: &str, ) -> Result { - // reqwest offers no way to add rules to the proxies it auto-detects, so - // detection is disabled and redone below. builder = builder.no_proxy(); - - let configured = configured.map(str::trim).filter(|p| !p.is_empty()); - if configured == Some("-") { - return Ok(builder); - } - let exempt = no_proxy_rules(); - if let Some(url) = configured { - let proxy = reqwest::Proxy::all(url) - .with_context(|| format!("Invalid proxy `{url}`"))? - .no_proxy(reqwest::NoProxy::from_string(&exempt)); - return Ok(builder.proxy(proxy)); - } - - // Split per scheme, because HTTP_PROXY and HTTPS_PROXY are allowed to differ. - for (is_https, keys) in [ - ( - true, - ["HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy"], - ), - ( - false, - ["HTTP_PROXY", "http_proxy", "ALL_PROXY", "all_proxy"], - ), - ] { - let Some(url) = first_env(&keys) else { - continue; - }; - let proxy = if is_https { - reqwest::Proxy::https(&url) - } else { - reqwest::Proxy::http(&url) - }; - let proxy = proxy - .with_context(|| format!("Invalid proxy `{url}`"))? - .no_proxy(reqwest::NoProxy::from_string(&exempt)); - builder = builder.proxy(proxy); - } + if !proxy.is_empty() && proxy != "-" { + builder = builder + .proxy(reqwest::Proxy::all(proxy).with_context(|| format!("Invalid proxy `{proxy}`"))?); + }; Ok(builder) } -fn first_env(keys: &[&str]) -> Option { - keys.iter() - .find_map(|key| env::var(key).ok()) - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()) -} - -/// Replacing reqwest's auto-detection loses its `NO_PROXY` handling, so that is -/// merged back in here. -fn no_proxy_rules() -> String { - let mut rules = LOCAL_NO_PROXY.to_string(); - let extra = env::var("NO_PROXY") - .or_else(|_| env::var("no_proxy")) - .unwrap_or_default(); - if !extra.trim().is_empty() { - rules.push(','); - rules.push_str(extra.trim()); - } - rules -} - pub fn decode_bin(data: &[u8]) -> Result { let (v, _) = bincode::serde::decode_from_slice(data, bincode::config::legacy())?; Ok(v) @@ -381,47 +316,6 @@ pub fn decode_bin(data: &[u8]) -> Result { mod tests { use super::*; - /// Invalid rule syntax makes `from_string` return `None`, which silently drops - /// every exemption and sends local traffic back through the proxy. - #[test] - fn the_local_no_proxy_rules_are_valid() { - assert!( - reqwest::NoProxy::from_string(LOCAL_NO_PROXY).is_some(), - "reqwest rejected LOCAL_NO_PROXY, so nothing would be exempt" - ); - } - - #[test] - fn local_rules_cover_loopback_and_private_ranges() { - for host in [ - "localhost", - "127.0.0.0/8", - "10.0.0.0/8", - "172.16.0.0/12", - "192.168.0.0/16", - ] { - assert!( - LOCAL_NO_PROXY.contains(host), - "{host} must stay exempt from proxying" - ); - } - } - - #[test] - fn a_dash_means_no_proxy_at_all() { - assert!(apply_proxy(reqwest::ClientBuilder::new(), Some("-")).is_ok()); - assert!(apply_proxy(reqwest::ClientBuilder::new(), Some(" - ")).is_ok()); - } - - #[test] - fn an_unparseable_proxy_is_reported() { - let err = apply_proxy(reqwest::ClientBuilder::new(), Some("not a url")) - .unwrap_err() - .to_string(); - - assert!(err.contains("Invalid proxy"), "got: {err}"); - } - #[test] #[cfg(not(target_os = "windows"))] fn test_safe_join_path() { From 6f7defe25f6184ecad23a948d54be0336380f39b Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Wed, 12 Aug 2026 10:56:34 -0600 Subject: [PATCH 33/36] style: removed redundant comment --- src/rag/providers/qdrant.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/rag/providers/qdrant.rs b/src/rag/providers/qdrant.rs index fd1e0ad..da3b84b 100644 --- a/src/rag/providers/qdrant.rs +++ b/src/rag/providers/qdrant.rs @@ -184,9 +184,6 @@ pub struct QdrantProvider { } impl QdrantProvider { - /// A proxy cannot usefully forward to an address that means something different - /// on its side, and anything that intercepts proxied traffic answers for a store - /// that is running fine, so the failure names the proxy rather than Qdrant. fn skips_proxy(base_url: &str) -> bool { let Ok(url) = Url::parse(base_url) else { return false; From 81ed769f8a562401672384bbe7525031dcfb8926 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Wed, 12 Aug 2026 11:26:16 -0600 Subject: [PATCH 34/36] fix(rag): warn when a duckdb store is empty but files are indexed A duckdb RAG is two files. The .yaml deliberately carries no vectors, and open() runs CREATE TABLE IF NOT EXISTS, so a .yaml copied or synced without its .duckdb sidecar produces a fresh empty store, hydrates to nothing, and answers every query with nothing while .info rag still lists every indexed file. Neither existing guard catches it: the anti-wipe check in rebuild_indexes needs existing > 0, and the mandatory ? on hydration needs a genuine error, while an absent store is the same Ok(empty) as a RAG with nothing indexed yet. Warn rather than bail, so a store deleted on purpose still loads and can be rebuilt. --- src/rag/mod.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/rag/mod.rs b/src/rag/mod.rs index fc6f306..69e6ea9 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -586,6 +586,17 @@ impl Rag { if data.vectors.is_empty() { data.vectors = duck.read_all_vectors()?; } + if data.vectors.is_empty() && !data.files.is_empty() { + println!( + "{} RAG '{name}' lists {} indexed file(s), but its vector store \ + '{}' holds no vectors, so every search will return nothing. A \ + duckdb RAG is two files: bring the .duckdb sidecar along with \ + the .yaml, or re-embed with `.rebuild rag`.", + warning_text("WARNING:"), + data.files.len(), + db_path.display() + ); + } // data.files is always populated for duckdb, so build_bm25() is the only // path; there is no from-DuckDB fallback. let bm25 = data.build_bm25(); From c84f9522e9087f2f81c56e5dcad4594ed111db71 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Wed, 12 Aug 2026 12:02:15 -0600 Subject: [PATCH 35/36] feat(rag): offer the storage driver when an agent initializes its RAG Agent startup and graph rag nodes both run an interactive wizard when their knowledge base has not been built, but neither offered the driver choice that interactive named-RAG creation has, so both silently produced a yaml store. A plain agent was the worse of the two: AgentConfig carries only documents, so there was no way to get a duckdb RAG for one, interactively or declaratively. A graph node could at least declare driver: in the workflow. Agent startup now passes prompt_for_driver, and a rag node whose wizard runs is asked too. The prompt is skipped when the node already declares a driver, and sits inside the not-fully-specified branch after the non-interactive bail, so declarative workflows and headless runs are unchanged. Temp RAGs still pass false: they are deleted on the next run, so a persistent store would only leave a sidecar behind. The prompt moves to select_rag_driver rather than being duplicated. --- src/config/agent.rs | 8 ++++++-- src/rag/mod.rs | 48 ++++++++++++++++++++++----------------------- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/src/config/agent.rs b/src/config/agent.rs index 569ec48..2630129 100644 --- a/src/config/agent.rs +++ b/src/config/agent.rs @@ -185,7 +185,7 @@ impl Agent { &rag_path_clone, &document_paths, abort, - false, + true, ) .await }) @@ -1025,7 +1025,7 @@ async fn init_graph_rags( { bail!("rag node '{node_id}': {message}"); } - let config = rag_init_config(rag_node); + let mut config = rag_init_config(rag_node); let fully_specified = config.embedding_model.is_some() && config.chunk_size.is_some() && config.chunk_overlap.is_some(); @@ -1051,6 +1051,10 @@ async fn init_graph_rags( initialized. RAG initialization is required for this agent." ); } + + if config.driver.is_none() { + config.driver = Some(crate::rag::select_rag_driver()?); + } } let document_paths = diff --git a/src/rag/mod.rs b/src/rag/mod.rs index 69e6ea9..a71d7b7 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -267,31 +267,10 @@ impl Rag { } println!("⚙ Initializing RAG..."); let (embedding_model, chunk_size, chunk_overlap) = Self::create_config(app)?; - // Only interactive named-RAG creation offers a driver choice. Temp RAGs and - // agent startup pass `false`; an explicit flag is used rather than inferring - // from the name because the agent path passes the literal name "rag", which is - // indistinguishable from a user creating a RAG genuinely named `rag`. let driver = if prompt_for_driver { - let options = vec![ - "yaml — portable, in-memory HNSW; usable from several Coyote processes at once (default)", - "duckdb — persistent on-disk store; vectors and content survive restarts; HNSW approximate search.", - ]; - let sel = Select::new("RAG storage driver:", options) - .with_starting_cursor(0) - .prompt()?; - if sel.starts_with("duckdb") { - println!( - "Note: several Coyote processes can query a duckdb RAG at the same time, \ - but while one process is ingesting or rebuilding it the others cannot \ - read it until that finishes. Changing its driver later means deleting \ - and recreating the RAG." - ); - "duckdb" - } else { - "yaml" - } + select_rag_driver()? } else { - "yaml" + "yaml".to_string() }; let reranker_model = app.rag_reranker_model.clone(); let top_k = app.rag_top_k; @@ -318,7 +297,7 @@ impl Rag { graph_hops: Some(graph_hops), }, ); - data.driver = driver.to_string(); + data.driver = driver; let mut rag = Self::create(app, name, save_path, data)?; let mut paths = doc_paths.to_vec(); if paths.is_empty() { @@ -1867,6 +1846,27 @@ fn select_embedding_model(models: &[&Model]) -> Result { Ok(result.value) } +pub(crate) fn select_rag_driver() -> Result { + let options = vec![ + "yaml — portable, in-memory HNSW; usable from several Coyote processes at once (default)", + "duckdb — persistent on-disk store; vectors and content survive restarts; HNSW approximate search.", + ]; + let sel = Select::new("RAG storage driver:", options) + .with_starting_cursor(0) + .prompt()?; + if sel.starts_with("duckdb") { + println!( + "Note: several Coyote processes can query a duckdb RAG at the same time, \ + but while one process is ingesting or rebuilding it the others cannot \ + read it until that finishes. Changing its driver later means deleting \ + and recreating the RAG." + ); + Ok("duckdb".to_string()) + } else { + Ok("yaml".to_string()) + } +} + const EXTRACTOR_SKIP: &str = "Skip"; fn select_extractor_model(app: &AppConfig) -> Result> { From ebba976a27353d18e44dcd74d1b1b3f8802baf5f Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Wed, 12 Aug 2026 12:07:38 -0600 Subject: [PATCH 36/36] fmt: applied formatting --- src/config/agent.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/config/agent.rs b/src/config/agent.rs index 2630129..6b993dc 100644 --- a/src/config/agent.rs +++ b/src/config/agent.rs @@ -4,6 +4,7 @@ use crate::{ client::Model, config::memory, function::{Functions, run_llm_function}, + graph, rag, }; use super::rag_cache::RagKey; @@ -1021,7 +1022,7 @@ async fn init_graph_rags( // Graph validation catches this too, but it is skipped when // `validate_before_run` is off, so this guard is the load-bearing one. if let Some(driver) = &rag_node.driver - && let Some(message) = crate::graph::validator::rag_driver_error(driver) + && let Some(message) = graph::validator::rag_driver_error(driver) { bail!("rag node '{node_id}': {message}"); } @@ -1053,7 +1054,7 @@ async fn init_graph_rags( } if config.driver.is_none() { - config.driver = Some(crate::rag::select_rag_driver()?); + config.driver = Some(rag::select_rag_driver()?); } }