diff --git a/src/config/agent.rs b/src/config/agent.rs index 16081ff..a6c051a 100644 --- a/src/config/agent.rs +++ b/src/config/agent.rs @@ -146,11 +146,18 @@ impl Agent { let rag = if rag_path.exists() { let key = RagKey::Agent(name.to_string()); let app_clone = app.clone(); + let vault_clone = app_state.vault.clone(); let rag_path_clone = rag_path.clone(); let rag = app_state .rag_cache .load_with(key, || async move { - Rag::load(&app_clone, DEFAULT_AGENT_NAME, &rag_path_clone) + Rag::load_async( + &app_clone, + &vault_clone, + DEFAULT_AGENT_NAME, + &rag_path_clone, + ) + .await }) .await?; Some(rag) @@ -972,12 +979,13 @@ async fn init_graph_rags( }; let rag = if rag_path.exists() { let app_clone = app.clone(); + let vault_clone = app_state.vault.clone(); let path_clone = rag_path.clone(); let name_clone = node_id.clone(); app_state .rag_cache .load_with(key, || async move { - Rag::load(&app_clone, &name_clone, &path_clone) + Rag::load_async(&app_clone, &vault_clone, &name_clone, &path_clone).await }) .await? } else { diff --git a/src/config/request_context.rs b/src/config/request_context.rs index e2f3883..5260518 100644 --- a/src/config/request_context.rs +++ b/src/config/request_context.rs @@ -4122,6 +4122,9 @@ impl RequestContext { } let app = self.app.config.clone(); + // Hoisted: `rag_cache` below borrows `self`, so the loader closure cannot + // reach through `self` for the vault. `GlobalVault` is an Arc, so this is cheap. + let vault = self.app.vault.clone(); let rag_cache = self.rag_cache(); let working_mode = self.working_mode; @@ -4158,6 +4161,7 @@ impl RequestContext { let loaded = rag_cache .load_with(key.clone(), || { let app = app.clone(); + let vault = vault.clone(); let rag_path = rag_path.clone(); let abort_signal = abort_signal.clone(); async move { @@ -4168,7 +4172,7 @@ impl RequestContext { Rag::init(&app, name, &rag_path, &[], abort_signal.clone(), true) .await } else { - Rag::load(&app, name, &rag_path) + Rag::load_async(&app, &vault, name, &rag_path).await } } }) @@ -4181,6 +4185,30 @@ impl RequestContext { Ok(()) } + pub async fn attach_rag(&mut self, name: &str) -> Result<()> { + let rag_path = self.rag_file(name); + if rag_path.exists() { + bail!( + "RAG '{name}' already exists at '{}'. \ + Use a different name, or delete the existing file first.", + rag_path.display() + ); + } + let app = self.app.config.as_ref(); + let vault = self.app.vault.clone(); + let rag = Rag::attach(app, &vault, name, &rag_path).await?; + let rag = Arc::new(rag); + // Populate the cache so a later `.rag ` reuses this instance rather + // than re-running the network preflight. Attach is always a global RAG. + let key = RagKey::Named(name.to_string()); + self.rag_cache().insert(key.clone(), &rag); + self.rag = Some(rag); + // Carried so invalidation in rebuild_rag()/edit_rag_docs() can find this + // entry; without it a stale Arc would linger in the cache all session. + self.rag_key = Some(key); + Ok(()) + } + pub async fn edit_rag_docs(&mut self, abort_signal: AbortSignal) -> Result<()> { let mut rag = match self.rag.clone() { Some(v) => v.as_ref().clone(), diff --git a/src/rag/mod.rs b/src/rag/mod.rs index 6c8d41a..abe5319 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -15,7 +15,8 @@ use self::provider::RagProvider; // `providers::duckdb_path_from_yaml(path)` is called through the module path in // `create()`, so `providers` itself must stay in scope — do not collapse it into // the `use` below. -use self::providers::{DuckDbProvider, YamlProvider}; +use self::providers::{DuckDbProvider, QdrantProvider, YamlProvider}; +use crate::vault::{Vault, interpolate_secrets}; use anyhow::{Context, Result, anyhow, bail}; use bm25::{Language, SearchEngine, SearchEngineBuilder}; @@ -348,6 +349,232 @@ impl Rag { Self::create(app, name, path, data) } + /// Loads a RAG from a YAML file. External drivers need an async constructor + /// because building their provider performs a network preflight. + pub async fn load_async( + app: &AppConfig, + vault: &Vault, + name: &str, + path: &Path, + ) -> Result { + let err = || format!("Failed to load rag '{name}' at '{}'", path.display()); + let raw_content = fs::read_to_string(path).with_context(err)?; + + // Parsed WITHOUT secret interpolation, so `driver_config` keeps its + // `{{...}}` placeholders. Interpolating here would bake the resolved API + // key into `self.data`, which `save()` then writes back to disk in + // plaintext. + let data: RagData = serde_yaml::from_str(&raw_content).with_context(err)?; + + // Validated before the match so the rule applies to every driver, including + // the sync fallthrough below. + data.validate().with_context(err)?; + + match data.driver.as_str() { + "qdrant" => { + let host = data + .driver_config + .get("host") + .context("qdrant driver requires 'host' in driver_config")? + .clone(); + let collection = data + .driver_config + .get("collection") + .context("qdrant driver requires 'collection' in driver_config")? + .clone(); + + // Resolved out of band and kept in a local; it never enters `data`. + let api_key: Option = match data.driver_config.get("api_key") { + Some(placeholder) => { + let (resolved, _) = + interpolate_secrets(placeholder, vault).with_context(|| { + format!("Failed to resolve api_key secret for RAG '{name}'") + })?; + Some(resolved) + } + None => None, + }; + + let provider = QdrantProvider::new(&host, &collection, api_key.as_deref()).await?; + let embedding_model = + Model::retrieve_model(app, &data.embedding_model, ModelType::Embedding)?; + Ok(Rag { + app_config: Arc::new(app.clone()), + name: name.to_string(), + path: path.display().to_string(), + embedding_model, + bm25: data.build_bm25(), + provider: Box::new(provider), + node_to_docs: data.knowledge_graph.build_node_to_docs(), + data, + last_sources: RwLock::new(None), + }) + } + // yaml/duckdb take the sync path. It re-reads and re-parses the file; + // that cost is accepted to keep every existing caller untouched. + _ => Self::load(app, name, path), + } + } + + /// Connects to a pre-existing external collection. Coyote is a query-only + /// client here: it never indexes documents into it. + pub async fn attach( + app: &AppConfig, + vault: &Vault, + name: &str, + save_path: &Path, + ) -> Result { + if !*IS_STDOUT_TERMINAL { + bail!("Cannot run attach wizard in non-interactive mode"); + } + println!("⚙ Attaching to external RAG..."); + + let driver = Select::new("Select driver:", vec!["qdrant"]).prompt()?; + + let host = Text::new("Host (e.g. qdrant.company.com:6333):") + .with_validator(required!("This field is required")) + .with_validator(|input: &str| { + // Bracketed IPv6 literals would produce a malformed sandbox + // allowedDomains entry, so refuse them at the prompt. + Ok(if input.contains('[') || input.contains(']') { + Validation::Invalid( + "Bracketed IPv6 literals are not supported; use a hostname.".into(), + ) + } else { + Validation::Valid + }) + }) + .prompt()?; + + let api_key_entry: Option<(String, String)> = { + let needs_key = Confirm::new("Does this instance require an API key?") + .with_default(true) + .prompt()?; + if needs_key { + let secret_name = Text::new("Vault secret name for API key:") + .with_default("QDRANT_API_KEY") + .with_validator(required!("This field is required")) + .prompt()?; + let resolved = vault.get_secret(&secret_name, false).with_context(|| { + format!( + "Secret '{secret_name}' not found in vault. \ + Run `coyote --add-secret {secret_name}` first." + ) + })?; + Some((secret_name, resolved)) + } else { + None + } + }; + + println!("⚙ Connecting to {host}..."); + let api_key = api_key_entry.as_ref().map(|(_, v)| v.as_str()); + let collections = QdrantProvider::list_collections(&host, api_key) + .await + .with_context(|| format!("Failed to connect to {host}. Check host and API key."))?; + + if collections.is_empty() { + bail!("No collections found in this Qdrant instance"); + } + println!( + "✓ Connected. {} collection(s) available.", + collections.len() + ); + + let collection = Select::new("Select collection:", collections).prompt()?; + + // Point IDs are read with `as_u64()`, which yields None for a JSON string. + // A UUID-keyed collection would therefore return zero hits with no error, + // so refuse it here instead of attaching something silently broken. + if let Some(raw_id) = QdrantProvider::sample_point_id(&host, &collection, api_key).await? + && raw_id.starts_with('"') + { + bail!( + "Collection '{collection}' uses string (UUID) point IDs. \ + Coyote requires integer point IDs. Rebuild the collection with integer IDs \ + (e.g. LangChain: pass ids=list(range(len(docs))) to add_documents())." + ); + } + println!("ℹ This collection must store document text in a 'page_content' payload field."); + + let dim = QdrantProvider::get_vector_dimension(&host, &collection, api_key) + .await + .unwrap_or(0); + // Queries send a single unnamed vector, which a named/multi-vector + // collection rejects with HTTP 400 every time. Checked separately from + // `dim` because `dim == 0` also means "the request failed". + if QdrantProvider::is_multi_vector(&host, &collection, api_key).await? { + bail!( + "Collection '{collection}' uses named (multi-vector) configuration. \ + Coyote queries with a single unnamed vector and would fail with HTTP 400 \ + on every request. Attach a single-vector collection instead." + ); + } + if dim > 0 { + let candidates = embedding_model_candidates_for_dimension(dim); + if !candidates.is_empty() { + println!( + "Collection uses {dim}-dim vectors. Likely models: {}", + candidates.join(", ") + ); + } + } + println!( + "⚠️ If the embedding model doesn't match what built this collection, \ + queries will return bad results." + ); + let models = list_models(app, ModelType::Embedding); + if models.is_empty() { + bail!("No available embedding model"); + } + let embedding_model_id = select_embedding_model(&models)?; + + let mut driver_config = IndexMap::new(); + driver_config.insert("host".to_string(), host.clone()); + driver_config.insert("collection".to_string(), collection.clone()); + if let Some((secret_name, _)) = &api_key_entry { + driver_config.insert("api_key".to_string(), format!("{{{{{secret_name}}}}}")); + } + + let data = RagData { + driver: driver.to_string(), + attached: true, + driver_config, + embedding_model: embedding_model_id, + chunk_size: app.rag_chunk_size.unwrap_or(1024), + chunk_overlap: app.rag_chunk_overlap.unwrap_or(50), + // A top_k of 0 makes every query return nothing. + top_k: app.rag_top_k.max(1), + ..RagData::default() + }; + data.validate()?; + + let embedding_model = + Model::retrieve_model(app, &data.embedding_model, ModelType::Embedding)?; + let provider = QdrantProvider::new(&host, &collection, api_key).await?; + let rag = Rag { + app_config: Arc::new(app.clone()), + name: name.to_string(), + path: save_path.display().to_string(), + embedding_model, + // Both empty: an attached RAG holds no local text and no local graph. + bm25: data.build_bm25(), + node_to_docs: IndexMap::new(), + provider: Box::new(provider), + data, + last_sources: RwLock::new(None), + }; + + rag.save()?; + println!("✓ Attached '{name}' → collection '{collection}' on {host}."); + + let env_var = rag_env_var_name(name); + let (header_name, value_format) = driver_auth_header(driver); + generate_rag_sbx_mixin(save_path, &host, name, &env_var, header_name, value_format)?; + + Ok(rag) + } + /// `mut data` — the duckdb arm rehydrates `data.vectors` from the sidecar. pub fn create(app: &AppConfig, name: &str, path: &Path, mut data: RagData) -> Result { // Deliberately does NOT call rebuild_indexes: both callers construct the Rag @@ -491,6 +718,13 @@ impl Rag { } pub fn set_last_sources(&self, ids: &[DocumentId]) { + if self.data.attached { + // `data.files` is empty for an attached RAG; the local index is not the + // source of truth. A static label is honest, an empty list is not. + *self.last_sources.write() = + Some("[attached RAG — source list unavailable]".to_string()); + return; + } let mut sources: IndexMap> = IndexMap::new(); for id in ids { let (file_index, _) = id.split(); @@ -665,6 +899,9 @@ impl Rag { } fn resolve_source(&self, id: &DocumentId) -> String { + if self.data.attached { + return self.data.attached_source_label(); + } let (file_index, _) = id.split(); self.data .files @@ -674,6 +911,9 @@ impl Rag { } fn format_sources(&self, ids: &[DocumentId]) -> String { + if self.data.attached { + return format!("- {}", self.data.attached_source_label()); + } let mut seen = IndexSet::new(); for id in ids { let (file_index, _) = id.split(); @@ -1231,6 +1471,11 @@ pub struct RagData { pub driver: String, #[serde(default)] pub attached: bool, + /// Driver-specific connection parameters (qdrant: `host`, `collection`, `api_key`). + /// Secret-bearing values are stored as `{{SECRET_NAME}}` placeholders and resolved + /// out of band at load time, so a resolved credential never reaches disk. + #[serde(default, skip_serializing_if = "IndexMap::is_empty")] + pub driver_config: IndexMap, pub embedding_model: String, #[serde(default)] @@ -1268,6 +1513,7 @@ impl Debug for RagData { f.debug_struct("RagData") .field("driver", &self.driver) .field("attached", &self.attached) + .field("driver_config", &self.driver_config) .field("embedding_model", &self.embedding_model) .field("chunk_size", &self.chunk_size) .field("chunk_overlap", &self.chunk_overlap) @@ -1297,6 +1543,7 @@ impl RagData { Self { driver: "yaml".to_string(), attached: false, + driver_config: Default::default(), embedding_model, chunk_size, chunk_overlap, @@ -1318,6 +1565,15 @@ impl RagData { "yaml".to_string() } + /// Citation label for an attached RAG. Its documents live in a remote + /// collection, so there is no local file path to cite. + fn attached_source_label(&self) -> String { + match self.driver_config.get("collection") { + Some(collection) => format!("[external collection: {collection}]"), + None => "[external collection]".to_string(), + } + } + pub fn validate(&self) -> Result<()> { if self.top_k == 0 { bail!( @@ -1497,6 +1753,131 @@ impl DocumentId { } } +/// Writes the per-RAG sandbox sidecar that whitelists the external host and tells +/// the sbx proxy which header to rewrite with the stored credential. +/// +/// Two details are load-bearing and fail silently if guessed: +/// 1. The schema envelope is mandatory. `wrap_mixin_as_kit` copies this file +/// byte-for-byte to `spec.yaml` inside a kit dir handed to `sbx create --kit`, +/// with no `kind` rewrite — so an envelope-less file can break the launch +/// itself, not merely this RAG's traffic. +/// 2. `allowedDomains` entries carry a port; `serviceDomains` keys are bare +/// hostnames. The asymmetry is deliberate. +fn generate_rag_sbx_mixin( + rag_yaml_path: &Path, + host: &str, + service_name: &str, + env_var: &str, + header_name: &str, + value_format: &str, +) -> Result<()> { + let (bare_host, allowed_domain) = sbx_domain_forms(host); + let mixin_path = rag_yaml_path.with_extension("sbx-mixin.yaml"); + let content = format!( + r#"schemaVersion: "1" +kind: mixin +name: rag-{service_name} +description: > + Auto-generated by the Coyote attach wizard for RAG '{service_name}'. Allows + outbound traffic to its external vector store and tells the sbx proxy which + header to rewrite with the stored credential. Do not edit manually. + +network: + allowedDomains: + - "{allowed_domain}" + serviceDomains: + {bare_host}: {service_name} + serviceAuth: + {service_name}: + headerName: {header_name} + valueFormat: "{value_format}" + +credentials: + sources: + {service_name}: + env: + - {env_var} + +environment: + proxyManaged: + - {env_var} +"# + ); + fs::write(&mixin_path, &content).with_context(|| { + format!( + "Failed to write sandbox mixin to '{}'", + mixin_path.display() + ) + })?; + println!("✓ Sandbox mixin: '{}'.", mixin_path.display()); + Ok(()) +} + +/// Splits a user-supplied host into the two forms sbx needs: +/// `(bare_host, allowed_domain)`. +/// +/// `bare_host` is the `serviceDomains` KEY — hostname only, no scheme, no port. +/// `allowed_domain` is an `allowedDomains` ENTRY — always `host:port`. +/// +/// The rule: emit the host verbatim when it already carries a numeric port, +/// otherwise append the default for the scheme. Stripping the port fails +/// silently — the sandbox just denies the connection, with no compile or test +/// signal. Defaults match `normalize_base_url`, which assumes `http://` when no +/// scheme is given: plain host → 6333, explicit `https://` → 443. +fn sbx_domain_forms(host: &str) -> (String, String) { + let is_https = host.starts_with("https://"); + let hostport = host + .strip_prefix("https://") + .or_else(|| host.strip_prefix("http://")) + .unwrap_or(host) + .trim_end_matches('/') + // A trailing ':' with no digits is a typo, not a port. Trim it so the + // default-port arm cannot emit "host::6333". + .trim_end_matches(':'); + match hostport.rsplit_once(':') { + Some((h, p)) if !p.is_empty() && p.bytes().all(|b| b.is_ascii_digit()) => { + (h.to_string(), hostport.to_string()) + } + _ => { + let port = if is_https { 443 } else { 6333 }; + (hostport.to_string(), format!("{hostport}:{port}")) + } + } +} + +/// Derives a deterministic env var name from a RAG name: +/// `company-docs` → `COMPANY_DOCS_API_KEY`. +fn rag_env_var_name(rag_name: &str) -> String { + format!( + "{}_API_KEY", + rag_name.to_uppercase().replace(['-', ' '], "_") + ) +} + +/// How a driver authenticates its HTTP requests. Qdrant uses a bare `api-key` +/// header rather than `Authorization: Bearer`. +fn driver_auth_header(driver: &str) -> (&'static str, &'static str) { + match driver { + "qdrant" => ("api-key", "%s"), + _ => ("Authorization", "Bearer %s"), + } +} + +/// Embedding models known to produce a given vector dimension, used to hint the +/// user toward a model compatible with the collection they just picked. +fn embedding_model_candidates_for_dimension(dim: u64) -> Vec<&'static str> { + match dim { + 1536 => vec!["text-embedding-3-small", "text-embedding-ada-002"], + 3072 => vec!["text-embedding-3-large"], + 768 => vec!["nomic-embed-text", "all-minilm-l6-v2"], + 1024 => vec![ + "text-embedding-3-small (matryoshka-1024)", + "jina-embeddings-v2-base", + ], + _ => vec![], + } +} + fn select_embedding_model(models: &[&Model]) -> Result { let max_width = models.iter().map(|v| v.id().len()).max().unwrap_or(0); let models: Vec<_> = models @@ -1770,6 +2151,221 @@ fn embedding_dim_for_model(model_id: &str) -> usize { mod tests { use super::*; + /// Scratch directory for tests that must write a real file. + struct TempDir { + path: std::path::PathBuf, + } + + impl TempDir { + fn new(tag: &str) -> Self { + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = env::temp_dir().join(format!("coyote-rag-{tag}-{unique}")); + fs::create_dir_all(&path).unwrap(); + Self { path } + } + } + + impl Drop for TempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } + } + + #[test] + fn attached_ragdata_serializes_driver_and_attached() { + let mut data = RagData { + driver: "qdrant".to_string(), + attached: true, + embedding_model: "text-embedding-3-small".to_string(), + top_k: 5, + ..Default::default() + }; + data.driver_config + .insert("host".into(), "localhost:6333".into()); + data.driver_config.insert("collection".into(), "c".into()); + data.driver_config + .insert("api_key".into(), "{{QDRANT_API_KEY}}".into()); + data.validate().unwrap(); + let yaml = serde_yaml::to_string(&data).unwrap(); + assert!(yaml.contains("attached: true")); + assert!(yaml.contains("driver: qdrant")); + // The placeholder is what reaches disk — never a resolved secret. + assert!(yaml.contains("{{QDRANT_API_KEY}}")); + } + + /// A qdrant RAG's vectors MUST survive serialization. + /// + /// `save()` omits vectors only for `driver == "duckdb"`. Qdrant must not join + /// that guard: Cosine collections L2-normalize on write, so the YAML copy is + /// the only place the unnormalized originals survive. This fails loudly the + /// day someone "tidies" the guard into `matches!(driver, "duckdb" | "qdrant")`. + #[test] + fn save_round_trips_qdrant_vectors_intact() { + let mut data = RagData { + driver: "qdrant".to_string(), + attached: true, + embedding_model: "text-embedding-3-small".to_string(), + top_k: 5, + ..Default::default() + }; + // Deliberately NOT unit-length: magnitude 5, so any normalization is visible. + data.vectors.insert(DocumentId(0), vec![3.0, 0.0, 0.0, 4.0]); + let yaml = serde_yaml::to_string(&data).unwrap(); + assert!( + yaml.contains("vectors:"), + "qdrant vectors must be serialized, not omitted" + ); + let back: RagData = serde_yaml::from_str(&yaml).unwrap(); + assert_eq!( + back.vectors.get(&DocumentId(0)), + Some(&vec![3.0, 0.0, 0.0, 4.0]), + "magnitudes must survive — Qdrant normalizes, the YAML copy must not" + ); + } + + /// `allowedDomains` entries are host:PORT; `serviceDomains` keys are bare + /// hostnames. Stripping the port breaks sandbox whitelisting silently — no + /// compile error, no runtime error on the host, just a denied connection + /// inside the sandbox. This test is the only signal. + #[test] + fn sbx_domain_forms_keeps_explicit_ports_and_defaults_the_rest() { + assert_eq!( + sbx_domain_forms("rag.example.com:6333"), + ("rag.example.com".into(), "rag.example.com:6333".into()) + ); + // A non-default explicit port must survive. + assert_eq!( + sbx_domain_forms("rag.example.com:7777").1, + "rag.example.com:7777" + ); + // Bare host → Qdrant's REST default, matching normalize_base_url. + assert_eq!( + sbx_domain_forms("rag.example.com"), + ("rag.example.com".into(), "rag.example.com:6333".into()) + ); + // The scheme is stripped; http keeps 6333. + assert_eq!( + sbx_domain_forms("http://localhost:6333").1, + "localhost:6333" + ); + // https with no port → 443 (the Qdrant Cloud shape). + assert_eq!( + sbx_domain_forms("https://xyz.cloud.qdrant.io"), + ( + "xyz.cloud.qdrant.io".into(), + "xyz.cloud.qdrant.io:443".into() + ) + ); + // A dangling colon is a typo, not a port: trimmed, then defaulted. + assert_eq!( + sbx_domain_forms("rag.example.com:").1, + "rag.example.com:6333" + ); + } + + /// The generated mixin must carry the schema envelope: `wrap_mixin_as_kit` + /// copies it verbatim to `spec.yaml` for `sbx create --kit`, with no `kind` + /// rewrite, so an envelope-less file can break the launch itself. + #[test] + fn generated_sbx_mixin_carries_the_schema_envelope() { + let dir = TempDir::new("mixin"); + let yaml_path = dir.path.join("company-docs.yaml"); + generate_rag_sbx_mixin( + &yaml_path, + "rag.example.com", + "company-docs", + "COMPANY_DOCS_API_KEY", + "api-key", + "%s", + ) + .unwrap(); + let text = fs::read_to_string(dir.path.join("company-docs.sbx-mixin.yaml")).unwrap(); + + assert!( + text.starts_with("schemaVersion:"), + "envelope must come first:\n{text}" + ); + assert!( + text.contains("kind: mixin"), + "kind must be `mixin`, not `sandbox`" + ); + assert!(text.contains("name: rag-company-docs")); + assert!(text.contains("description:")); + + // It must parse as YAML at all — a broken format! escape is invisible otherwise. + let parsed: serde_yaml::Value = serde_yaml::from_str(&text).unwrap(); + assert_eq!(parsed["kind"].as_str(), Some("mixin")); + // The list entry carries the port; the map key does not. + assert_eq!( + parsed["network"]["allowedDomains"][0].as_str(), + Some("rag.example.com:6333") + ); + assert_eq!( + parsed["network"]["serviceDomains"]["rag.example.com"].as_str(), + Some("company-docs") + ); + // The proxy rewrites this header with the credential it holds. + assert_eq!( + parsed["network"]["serviceAuth"]["company-docs"]["headerName"].as_str(), + Some("api-key") + ); + assert_eq!( + parsed["credentials"]["sources"]["company-docs"]["env"][0].as_str(), + Some("COMPANY_DOCS_API_KEY") + ); + assert_eq!( + parsed["environment"]["proxyManaged"][0].as_str(), + Some("COMPANY_DOCS_API_KEY") + ); + } + + #[test] + fn rag_env_var_name_uppercases_and_underscores() { + assert_eq!(rag_env_var_name("company-docs"), "COMPANY_DOCS_API_KEY"); + assert_eq!(rag_env_var_name("my rag"), "MY_RAG_API_KEY"); + assert_eq!(rag_env_var_name("docs"), "DOCS_API_KEY"); + } + + #[test] + fn driver_auth_header_uses_a_bare_api_key_for_qdrant() { + // Qdrant's REST API reads `api-key`, NOT `Authorization: Bearer`. + assert_eq!(driver_auth_header("qdrant"), ("api-key", "%s")); + assert_eq!( + driver_auth_header("something-else"), + ("Authorization", "Bearer %s") + ); + } + + /// An attached RAG has no local `files`, so the citation helpers would + /// otherwise emit "unknown" and an empty source list for every result. + #[test] + fn attached_rag_citation_helpers_do_not_fall_back_to_the_empty_file_index() { + let mut data = RagData { + driver: "qdrant".to_string(), + attached: true, + embedding_model: "text-embedding-3-small".to_string(), + top_k: 5, + ..Default::default() + }; + data.driver_config + .insert("collection".into(), "company-kb".into()); + assert!(data.files.is_empty()); + + // The real helper — `resolve_source`/`format_sources` both delegate here. + assert_eq!( + data.attached_source_label(), + "[external collection: company-kb]" + ); + + // Degrades to a generic label rather than "unknown" when the collection + // name is absent. + data.driver_config.shift_remove("collection"); + assert_eq!(data.attached_source_label(), "[external collection]"); + } + #[test] fn embedding_dim_for_model_maps_known_models() { assert_eq!(embedding_dim_for_model("text-embedding-3-large"), 3072); diff --git a/src/rag/providers/mod.rs b/src/rag/providers/mod.rs index 19d9c9a..6fd4472 100644 --- a/src/rag/providers/mod.rs +++ b/src/rag/providers/mod.rs @@ -11,3 +11,6 @@ pub use self::duckdb::DuckDbProvider; // in providers/duckdb.rs, and the re-export must be `pub(crate)` too — a `pub use` of // a `pub(crate)` item is E0364/E0365. pub(crate) use self::duckdb::duckdb_path_from_yaml; + +mod qdrant; +pub use self::qdrant::QdrantProvider; diff --git a/src/rag/providers/qdrant.rs b/src/rag/providers/qdrant.rs new file mode 100644 index 0000000..ffff33a --- /dev/null +++ b/src/rag/providers/qdrant.rs @@ -0,0 +1,491 @@ +use crate::rag::provider::RagProvider; +use crate::rag::{DocumentId, RagData}; + +use anyhow::{Context, Result, bail}; +use async_trait::async_trait; +use std::collections::HashMap; + +/// Render Qdrant's error envelope into a human-readable message. +/// +/// `body` is the raw response text. Two shapes have to be tolerated: +/// * application-level errors carry `{"status": {"error": "..."}, "time": 0.0}`, +/// while successful responses carry a bare string `{"status": "ok", ...}` — so +/// `status` is string-or-object and a struct with `status: String` fails to +/// parse every error body; +/// * routing-level 404s (a wrong HTTP verb) return an EMPTY body with no JSON at +/// all, which without the length check surfaces as "EOF while parsing a value" +/// instead of the actual 404. +fn format_error_body(status: reqwest::StatusCode, body: &str) -> String { + if body.is_empty() { + return format!("HTTP {status} (empty body — check the HTTP verb and path)"); + } + serde_json::from_str::(body) + .ok() + .and_then(|v| v["status"]["error"].as_str().map(str::to_string)) + .unwrap_or_else(|| format!("HTTP {status}: {body}")) +} + +/// Read the vector dimension out of a parsed `GET /collections/{name}` response. +/// +/// Unnamed collections put `size` directly under `vectors`; named ones nest it +/// under the vector's name. Both shapes occur in the wild, so try the flat one +/// first and fall back to the first named entry. +fn vector_dimension_from_collection(body: &serde_json::Value) -> Result { + let params = &body["result"]["config"]["params"]; + params["vectors"]["size"] + .as_u64() + .or_else(|| { + params["vectors"] + .as_object() + .and_then(|m| m.values().next()) + .and_then(|v| v["size"].as_u64()) + }) + .context("Could not determine vector dimension from collection config") +} + +/// True if a parsed `GET /collections/{name}` response describes a NAMED +/// (multi-vector) collection. +/// +/// `vector_search` posts an unnamed vector, which a named-vector collection +/// rejects with HTTP 400 on every query — so attaching one yields a RAG that is +/// silently 100% broken. A named collection holding a SINGLE vector is +/// structurally a map, identical in kind to the multi-named case, and rejects +/// the same way; testing for a numeric `size` directly under `vectors` catches +/// it, whereas counting keys (`len() > 1`) would wrongly accept it. +fn is_multi_vector_config(body: &serde_json::Value) -> bool { + body["result"]["config"]["params"]["vectors"]["size"] + .as_u64() + .is_none() +} + +/// Query-only client for an external Qdrant collection. +/// +/// Attach-only: this provider never writes to the remote collection. Coyote does +/// not own the data, and `rebuild_indexes` refuses rather than pretending to. +pub struct QdrantProvider { + /// `reqwest::Client` is Arc-backed, so `clone()` is O(1) and shares both the + /// connection pool and the `api-key` default header injected at build time. + client: reqwest::Client, + /// Includes the scheme, e.g. `http://qdrant.example.com:6333`. + base_url: String, + collection: String, +} + +impl QdrantProvider { + /// The resolved API key is injected as a default header here and is + /// deliberately NOT stored on the struct: the plaintext value stays a local + /// of the caller and never outlives it. + fn make_client(api_key: Option<&str>) -> Result { + let mut headers = reqwest::header::HeaderMap::new(); + if let Some(key) = api_key { + let mut value = reqwest::header::HeaderValue::from_str(key) + .context("api-key header value is not valid ASCII")?; + value.set_sensitive(true); + headers.insert("api-key", value); + } + reqwest::Client::builder() + .default_headers(headers) + .build() + .context("Failed to build reqwest client") + } + + fn normalize_base_url(host: &str) -> String { + if host.starts_with("http://") || host.starts_with("https://") { + host.to_string() + } else { + format!("http://{host}") + } + } + + async fn error_message(resp: reqwest::Response) -> String { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + format_error_body(status, &body) + } + + /// Shared `GET /collections/{name}` fetch. Both the dimension and the + /// multi-vector probe discriminate on this same response. + async fn fetch_collection( + host: &str, + collection: &str, + api_key: Option<&str>, + ) -> Result { + let base_url = Self::normalize_base_url(host); + let client = Self::make_client(api_key)?; + let resp = client + .get(format!("{base_url}/collections/{collection}")) + .send() + .await + .with_context(|| format!("Failed to connect to {host}"))?; + if !resp.status().is_success() { + bail!( + "Failed to read collection '{collection}': {}", + Self::error_message(resp).await + ); + } + Ok(resp.json().await?) + } + + pub async fn new(host: &str, collection: &str, api_key: Option<&str>) -> Result { + let base_url = Self::normalize_base_url(host); + let client = Self::make_client(api_key)?; + // Preflight: confirm the collection exists and we may read it. + let resp = client + .get(format!("{base_url}/collections/{collection}")) + .send() + .await + .with_context(|| format!("Failed to connect to {host}"))?; + if !resp.status().is_success() { + bail!( + "Collection '{collection}' not accessible at {host}: {}", + Self::error_message(resp).await + ); + } + Ok(Self { + client, + base_url, + collection: collection.to_string(), + }) + } + + pub async fn list_collections(host: &str, api_key: Option<&str>) -> Result> { + let base_url = Self::normalize_base_url(host); + let client = Self::make_client(api_key)?; + let resp = client + .get(format!("{base_url}/collections")) + .send() + .await + .with_context(|| format!("Failed to connect to {host}"))?; + if !resp.status().is_success() { + bail!( + "Failed to list collections: {}", + Self::error_message(resp).await + ); + } + let body: serde_json::Value = resp.json().await?; + let names = body["result"]["collections"] + .as_array() + .context("Unexpected /collections response shape")? + .iter() + .filter_map(|v| v["name"].as_str().map(str::to_string)) + .collect(); + Ok(names) + } + + pub async fn get_vector_dimension( + host: &str, + collection: &str, + api_key: Option<&str>, + ) -> Result { + let body = Self::fetch_collection(host, collection, api_key).await?; + vector_dimension_from_collection(&body) + } + + pub async fn is_multi_vector( + host: &str, + collection: &str, + api_key: Option<&str>, + ) -> Result { + let body = Self::fetch_collection(host, collection, api_key).await?; + Ok(is_multi_vector_config(&body)) + } + + /// Peek at one point to learn how its ID is typed. Returns the raw JSON + /// rendering, so a string ID comes back quoted and an integer one bare. + pub async fn sample_point_id( + host: &str, + collection: &str, + api_key: Option<&str>, + ) -> Result> { + let base_url = Self::normalize_base_url(host); + let client = Self::make_client(api_key)?; + let url = format!("{base_url}/collections/{collection}/points/scroll"); + let body = serde_json::json!({ "limit": 1, "with_payload": false }); + let resp = client + .post(&url) + .json(&body) + .send() + .await + .with_context(|| format!("Failed to connect to {host}"))?; + if !resp.status().is_success() { + bail!( + "Failed to sample a point from '{collection}': {}", + Self::error_message(resp).await + ); + } + let data: serde_json::Value = resp.json().await?; + let id_val = data["result"]["points"] + .as_array() + .and_then(|pts| pts.first()) + .map(|pt| pt["id"].to_string()); + Ok(id_val) + } +} + +#[async_trait] +impl RagProvider for QdrantProvider { + async fn vector_search( + &self, + embedding: &[f32], + top_k: usize, + min_score: f32, + ) -> Result> { + let url = format!( + "{}/collections/{}/points/search", + self.base_url, self.collection + ); + // `score_threshold` is deliberately NOT sent. It is metric-aware: on Cosine + // collections 0.0 means "no floor" as expected, but Euclid collections score + // by negative distance, where 0.0 filters everything out. The attach wizard + // does not pin the distance metric, so filter locally instead. + let body = serde_json::json!({ + "vector": embedding, + "limit": top_k, + "with_payload": false, + }); + let resp = self.client.post(&url).json(&body).send().await?; + if !resp.status().is_success() { + bail!( + "Qdrant search on '{}' failed: {}", + self.collection, + Self::error_message(resp).await + ); + } + let data: serde_json::Value = resp.json().await?; + let results = data["result"] + .as_array() + .context("Unexpected /points/search response shape")? + .iter() + .filter_map(|pt| { + // String (UUID) IDs yield None here and are dropped. The attach + // wizard rejects such collections up front so this cannot silently + // become "zero results, no error". + let id = pt["id"].as_u64()? as usize; + let score = pt["score"].as_f64()? as f32; + Some((DocumentId(id), score)) + }) + .filter(|(_, score)| *score > min_score) + .collect(); + Ok(results) + } + + async fn fetch_content(&self, ids: &[DocumentId]) -> Result> { + if ids.is_empty() { + return Ok(vec![]); + } + let url = format!("{}/collections/{}/points", self.base_url, self.collection); + let id_list: Vec = ids.iter().map(|d| d.0 as u64).collect(); + let body = serde_json::json!({ + "ids": id_list, + "with_payload": true, + }); + let resp = self.client.post(&url).json(&body).send().await?; + if !resp.status().is_success() { + bail!( + "Qdrant point fetch on '{}' failed: {}", + self.collection, + Self::error_message(resp).await + ); + } + let data: serde_json::Value = resp.json().await?; + let mut rows: Vec<(DocumentId, String)> = data["result"] + .as_array() + .context("Unexpected /points response shape")? + .iter() + .filter_map(|pt| { + let id = pt["id"].as_u64()? as usize; + let text = pt["payload"]["page_content"].as_str()?.to_string(); + Some((DocumentId(id), text)) + }) + .collect(); + // `/points` does not guarantee response order matches request order, and the + // caller's RRF ranking is carried by that order. Restore it. + let position: HashMap = + ids.iter().enumerate().map(|(i, id)| (*id, i)).collect(); + rows.sort_by_key(|(id, _)| position.get(id).copied().unwrap_or(usize::MAX)); + Ok(rows) + } + + async fn rebuild_indexes(&mut self, data: &RagData, _full_rebuild: bool) -> Result<()> { + // Both arms refuse. A silent `Ok(())` would make `.rebuild rag` and + // `.edit rag-docs` look like they worked while writing nothing to the + // remote — leaving the user believing the collection was updated. + if data.attached { + bail!( + "This RAG is attached to an external Qdrant collection. Coyote does not own \ + its documents and cannot rebuild it. Manage the collection directly, or \ + create a Coyote-owned RAG with `.rag `." + ); + } + bail!("Writing to Qdrant is not supported yet (attach-only)."); + } + + fn duplicate(&self, _data: &RagData) -> Box { + // Cloning the client shares the connection pool and the injected api-key + // header. Sharing is correct: both handles address the same remote + // collection, and neither of them writes to it. + Box::new(Self { + client: self.client.clone(), + base_url: self.base_url.clone(), + collection: self.collection.clone(), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn error_message_reads_the_object_status_envelope() { + let body = + r#"{"status": {"error": "Wrong input: Not existing vector name error:"}, "time": 0.0}"#; + let msg = format_error_body(reqwest::StatusCode::BAD_REQUEST, body); + assert!(msg.contains("Not existing vector name"), "got: {msg}"); + assert!( + !msg.contains("EOF"), + "must not fall through to a parse error" + ); + } + + #[test] + fn error_message_survives_the_string_status_and_the_empty_body() { + // Success envelope: `status` is a bare string, so the object lookup misses + // and we must fall back rather than panic or invent an error text. + let ok = format_error_body(reqwest::StatusCode::OK, r#"{"status": "ok", "time": 0.0}"#); + assert!( + ok.contains("200"), + "no `status.error` present → fall back to status+body: {ok}" + ); + // Routing-level 404 from a wrong HTTP verb: empty body, no JSON at all. + let empty = format_error_body(reqwest::StatusCode::NOT_FOUND, ""); + assert!(empty.contains("empty body"), "got: {empty}"); + assert!( + empty.contains("verb"), + "the message must point at the likely cause: {empty}" + ); + } + + #[test] + fn vector_dimension_handles_both_collection_shapes() { + let unnamed = serde_json::json!({ + "result": {"config": {"params": {"vectors": {"size": 1536, "distance": "Cosine"}}}} + }); + assert_eq!(vector_dimension_from_collection(&unnamed).unwrap(), 1536); + + let named = serde_json::json!({ + "result": {"config": {"params": {"vectors": {"text": {"size": 768, "distance": "Cosine"}}}}} + }); + assert_eq!(vector_dimension_from_collection(&named).unwrap(), 768); + + let junk = serde_json::json!({"result": {"config": {"params": {}}}}); + assert!(vector_dimension_from_collection(&junk).is_err()); + } + + #[test] + fn is_multi_vector_rejects_the_named_single_collection() { + // The only supported shape: a single unnamed vector. + let unnamed = serde_json::json!({ + "result": {"config": {"params": {"vectors": {"size": 1536, "distance": "Cosine"}}}} + }); + assert!(!is_multi_vector_config(&unnamed)); + + // Named but SINGLE — structurally a map, and writes to it fail with + // `400 "Wrong input: Not existing vector name error:"`. A `len() > 1` check + // would wrongly accept this one; that is the bug this case exists to catch. + let named_single = serde_json::json!({ + "result": {"config": {"params": {"vectors": {"text": {"size": 1536}}}}} + }); + assert!( + is_multi_vector_config(&named_single), + "named-single must be rejected too" + ); + + let named_multi = serde_json::json!({ + "result": {"config": {"params": {"vectors": {"text": {"size": 1536}, "image": {"size": 512}}}}} + }); + assert!(is_multi_vector_config(&named_multi)); + } + + #[test] + fn normalize_base_url_only_adds_a_scheme_when_missing() { + assert_eq!( + QdrantProvider::normalize_base_url("qdrant.example.com:6333"), + "http://qdrant.example.com:6333" + ); + assert_eq!( + QdrantProvider::normalize_base_url("https://xyz.cloud.qdrant.io"), + "https://xyz.cloud.qdrant.io" + ); + assert_eq!( + QdrantProvider::normalize_base_url("http://localhost:6333"), + "http://localhost:6333" + ); + } + + #[tokio::test] + async fn rebuild_indexes_refuses_for_attached_and_unattached_alike() { + let mut provider = QdrantProvider { + client: reqwest::Client::new(), + base_url: "http://localhost:6333".to_string(), + collection: "c".to_string(), + }; + + let attached = RagData { + driver: "qdrant".to_string(), + attached: true, + ..Default::default() + }; + let err = provider + .rebuild_indexes(&attached, true) + .await + .expect_err("an attached qdrant RAG must never report a successful rebuild"); + assert!(err.to_string().contains("cannot rebuild"), "got: {err}"); + + // `attached: false` is reserved for the (unimplemented) write path. It must + // also refuse: silently succeeding would run a full paid embedding pass and + // then discard every vector. + let owned = RagData { + driver: "qdrant".to_string(), + attached: false, + ..Default::default() + }; + let err = provider + .rebuild_indexes(&owned, true) + .await + .expect_err("writing to qdrant is unimplemented and must fail loudly"); + assert!(err.to_string().contains("not supported yet"), "got: {err}"); + } + + #[tokio::test] + async fn fetch_content_short_circuits_on_an_empty_id_list() { + // No network is touched: the early return happens before any request, which + // is why this can assert against an unreachable host. + let provider = QdrantProvider { + client: reqwest::Client::new(), + base_url: "http://127.0.0.1:1".to_string(), + collection: "c".to_string(), + }; + assert!(provider.fetch_content(&[]).await.unwrap().is_empty()); + } + + #[tokio::test] + #[ignore] + async fn qdrant_list_collections_requires_running_instance() { + let collections = QdrantProvider::list_collections("http://localhost:6333", None) + .await + .unwrap(); + assert!(!collections.is_empty()); + } + + #[tokio::test] + #[ignore] + async fn qdrant_vector_search_returns_results() { + let provider = QdrantProvider::new("http://localhost:6333", "test-collection", None) + .await + .unwrap(); + let embedding = vec![0.0f32; 1536]; + let results = provider.vector_search(&embedding, 5, 0.0).await.unwrap(); + assert!(results.len() <= 5); + } +} diff --git a/src/repl/mod.rs b/src/repl/mod.rs index c33d067..68a871b 100644 --- a/src/repl/mod.rs +++ b/src/repl/mod.rs @@ -53,7 +53,7 @@ pub const DEFAULT_CONTINUATION_PROMPT: &str = indoc! {" 4. Continue with the next pending item now. Call tools immediately." }; -static REPL_COMMANDS: LazyLock<[ReplCommand; 59]> = LazyLock::new(|| { +static REPL_COMMANDS: LazyLock<[ReplCommand; 60]> = LazyLock::new(|| { [ ReplCommand::new(".help", "Show this help guide", AssertState::pass()), ReplCommand::new(".info", "Show system info", AssertState::pass()), @@ -217,6 +217,11 @@ static REPL_COMMANDS: LazyLock<[ReplCommand; 59]> = LazyLock::new(|| { "Initialize or access RAG", AssertState::False(StateFlags::AGENT), ), + ReplCommand::new( + ".rag attach", + "Attach to a pre-existing external RAG (Qdrant)", + AssertState::False(StateFlags::AGENT), + ), ReplCommand::new( ".edit rag-docs", "Add or remove documents from an existing RAG", @@ -885,7 +890,20 @@ pub async fn run_repl_command( task::spawn_blocking(move || config::run_self_update(version, false)).await??; } ".rag" => { - ctx.use_rag(args, abort_signal.clone()).await?; + // `split_first_arg` rather than `starts_with("attach ")`: the latter + // misses a bare `.rag attach`, which would silently create a RAG + // literally named "attach". + match split_first_arg(args) { + Some(("attach", rest)) => match rest { + Some(name) if !name.trim().is_empty() => { + ctx.attach_rag(name.trim()).await?; + } + _ => println!("Usage: .rag attach "), + }, + _ => { + ctx.use_rag(args, abort_signal.clone()).await?; + } + } } ".agent" => match split_first_arg(args) { Some((agent_name, args)) => { @@ -1711,8 +1729,8 @@ mod tests { } #[test] - fn repl_commands_has_59_entries() { - assert_eq!(REPL_COMMANDS.len(), 59); + fn repl_commands_has_60_entries() { + assert_eq!(REPL_COMMANDS.len(), 60); } #[test] diff --git a/src/sandbox/mixins.rs b/src/sandbox/mixins.rs index 9a5d8f2..e9574a9 100644 --- a/src/sandbox/mixins.rs +++ b/src/sandbox/mixins.rs @@ -10,6 +10,7 @@ use sha2::{Digest, Sha256}; use crate::config::paths; const SBX_MIXIN_FILE_NAME: &str = "sbx-mixin.yaml"; +const SBX_MIXIN_FILE_SUFFIX: &str = ".sbx-mixin.yaml"; const KIT_SPEC_FILE_NAME: &str = "spec.yaml"; #[derive(Debug, Clone)] @@ -72,6 +73,13 @@ pub fn discover() -> Result> { for path in collect_subdir_mixins(&paths::agents_data_dir()) { out.push(read_mixin(path)?); } + // RAG sidecars are FLAT files named `.sbx-mixin.yaml` inside rags/, not + // the `/sbx-mixin.yaml` shape the two scans above walk. Loaded + // unconditionally, mirroring agents/*: a RAG mixin only adds an outbound + // allowlist entry for that RAG's host and opens no inbound rules. + for path in collect_flat_mixins(&paths::rags_dir()) { + out.push(read_mixin(path)?); + } if let Ok(cwd) = env::current_dir() && let Some(path) = paths::find_workspace_sbx_mixin(&cwd) @@ -173,6 +181,30 @@ fn collect_subdir_mixins(dir: &Path) -> Vec { result } +/// Mixins stored as flat `.sbx-mixin.yaml` files directly inside `dir`, +/// matched by suffix rather than by exact filename. +fn collect_flat_mixins(dir: &Path) -> Vec { + let mut result = Vec::new(); + let Ok(rd) = read_dir(dir) else { return result }; + + let mut entries: Vec<_> = rd + .flatten() + .filter(|e| e.file_type().map(|t| t.is_file()).unwrap_or(false)) + .filter(|e| { + e.file_name() + .to_str() + .is_some_and(|n| n.ends_with(SBX_MIXIN_FILE_SUFFIX)) + }) + .collect(); + entries.sort_by_key(|e| e.file_name()); + + for entry in entries { + result.push(entry.path()); + } + + result +} + #[cfg(test)] mod tests { use super::*; @@ -439,4 +471,54 @@ network: ); } } + + /// RAG sidecars are flat `.sbx-mixin.yaml` files, matched by SUFFIX. + #[test] + fn collect_flat_mixins_matches_rag_sidecars_by_suffix() { + let root = unique_root("flat-mixins"); + fs::write(root.join("company-docs.sbx-mixin.yaml"), "kind: mixin\n").unwrap(); + fs::write(root.join("alpha.sbx-mixin.yaml"), "kind: mixin\n").unwrap(); + // The RAGs themselves must not be picked up, only their sidecars. + fs::write(root.join("company-docs.yaml"), "driver: qdrant\n").unwrap(); + fs::write(root.join("notes.yaml"), "driver: yaml\n").unwrap(); + // A directory whose name ends in the suffix is not a mixin file. + fs::create_dir_all(root.join("decoy.sbx-mixin.yaml")).unwrap(); + + let found = collect_flat_mixins(&root); + let names: Vec<_> = found + .iter() + .map(|p| p.file_name().unwrap().to_str().unwrap()) + .collect(); + // Sorted by file name, so the order is deterministic. + assert_eq!( + names, + vec!["alpha.sbx-mixin.yaml", "company-docs.sbx-mixin.yaml"] + ); + + let _ = fs::remove_dir_all(&root); + } + + /// Why `collect_flat_mixins` had to be written: the existing collector walks + /// SUBDIRECTORIES for a file named exactly `sbx-mixin.yaml`, so it cannot see + /// a flat sidecar. If this ever starts finding them, the new collector is + /// redundant — but until then, removing it silently drops every RAG mixin. + #[test] + fn collect_subdir_mixins_cannot_see_flat_rag_sidecars() { + let root = unique_root("flat-vs-subdir"); + fs::write(root.join("company-docs.sbx-mixin.yaml"), "kind: mixin\n").unwrap(); + + assert!(collect_subdir_mixins(&root).is_empty()); + assert_eq!(collect_flat_mixins(&root).len(), 1); + + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn collect_flat_mixins_tolerates_a_missing_directory() { + let root = unique_root("flat-missing"); + let absent = root.join("nope"); + assert!(collect_flat_mixins(&absent).is_empty()); + + let _ = fs::remove_dir_all(&root); + } } diff --git a/src/sandbox/mod.rs b/src/sandbox/mod.rs index 19a9fc4..91fc33b 100644 --- a/src/sandbox/mod.rs +++ b/src/sandbox/mod.rs @@ -16,6 +16,7 @@ use crate::config::AppConfig; use crate::config::Config; use crate::config::VAULT_DATA_FILE_NAME; use crate::config::paths; +use crate::rag::RagData; use crate::sandbox::mixins::DiscoveredMixin; use crate::utils::run_command_with_output; use crate::vault::SECRET_RE; @@ -51,6 +52,7 @@ pub fn launch(name: Option, fresh: bool) -> Result<()> { inject_llm_secret(&config_content, &vault, ®istered)?; if !fresh { inject_mcp_secrets(&vault, ®istered)?; + inject_rag_secrets(&vault, ®istered)?; } let discovered = mixins::discover()?; @@ -301,6 +303,65 @@ fn inject_mcp_secrets(vault: &Vault, registered: &HashSet) -> Result<()> Ok(()) } +/// Registers the API key of every attached RAG with the sbx proxy. +/// +/// `launch()` has no notion of an active RAG — that is runtime state set by +/// `--rag` / `.rag` and never persisted — so every attached RAG is scanned +/// unconditionally, exactly as `inject_mcp_secrets` does for MCP servers. +fn inject_rag_secrets(vault: &Vault, registered: &HashSet) -> Result<()> { + let rags_dir = paths::rags_dir(); + if !rags_dir.exists() { + return Ok(()); + } + for entry in fs::read_dir(&rags_dir)?.flatten() { + let path = entry.path(); + if path.extension().and_then(|e| e.to_str()) != Some("yaml") { + continue; + } + let stem = match path.file_stem().and_then(|s| s.to_str()) { + // Skip sidecars ("myrag.sbx-mixin.yaml" has stem "myrag.sbx-mixin"). + Some(s) if !paths::is_rag_sidecar_name(s) => s.to_string(), + _ => continue, + }; + let Ok(raw) = fs::read_to_string(&path) else { + continue; + }; + let Ok(data) = serde_yaml::from_str::(&raw) else { + continue; + }; + if !data.attached { + continue; + } + let Some(placeholder) = data.driver_config.get("api_key") else { + continue; + }; + if registered.contains(&stem) { + continue; + } + let secret_name = placeholder + .trim_start_matches("{{") + .trim_end_matches("}}") + .trim(); + // Degrade rather than abort: one stale RAG key must not block the whole + // sandbox launch. Queries to that RAG fail with a 401 at runtime, which + // is recoverable without a restart. + match vault.get_secret(secret_name, false) { + Ok(secret_value) => { + sbx_secret_set(&stem, &secret_value) + .context("Failed to register RAG secret with sbx")?; + } + Err(e) => { + eprintln!( + "Warning: could not load secret '{secret_name}' for RAG '{stem}': {e}. \ + Queries to this RAG will fail inside the sandbox. \ + Run `coyote --add-secret {secret_name}` to fix." + ); + } + } + } + Ok(()) +} + fn provider_to_sbx_service(provider_type: &str, client_name: Option<&str>) -> String { match provider_type { "claude" => "anthropic".to_string(),