Compare commits

Author SHA1 Message Date
Dark-Alex-17 2fe6704fbc fix: fs tools now output better error handling to guide the model more effectively
CI / All (ubuntu-latest) (push) Failing after 25s
CI / All (macos-latest) (push) Has been cancelled
CI / All (windows-latest) (push) Has been cancelled
2026-07-14 12:47:43 -06:00
Dark-Alex-17 dd40892ad5 fix: Make fs_read more tolerant of various arg invocation formats. 2026-07-14 12:31:24 -06:00
Dark-Alex-17 ed86b7bfc3 Merge branch 'main' of github.com:Dark-Alex-17/coyote 2026-07-14 11:31:21 -06:00
Dark-Alex-17 f32d72a3f2 feat: Made fs_patch more flexible for different model preferences of patch formats 2026-07-14 11:31:11 -06:00
Dark-Alex-17 7b00638476 style: removed redundant '&' from functions module 2026-07-13 18:11:03 -06:00
Dark-Alex-17 6733b3600f test: Fixed flaky python AST parser test for macOS
CI / All (ubuntu-latest) (push) Failing after 24s
CI / All (macos-latest) (push) Has been cancelled
CI / All (windows-latest) (push) Has been cancelled
2026-07-13 17:46:34 -06:00
Dark-Alex-17 de6010d525 docs: Organized coyote --help output to be more readable
CI / All (ubuntu-latest) (push) Failing after 25s
CI / All (macos-latest) (push) Has been cancelled
CI / All (windows-latest) (push) Has been cancelled
2026-07-13 17:31:20 -06:00
Dark-Alex-17 9b0e26bade feat: Installed nano into the sandbox so that users can edit config files in the sandbox directly 2026-07-13 17:29:10 -06:00
Dark-Alex-17 ac40043c00 style: Removed outdated implementation plan 2026-07-13 17:25:20 -06:00
Dark-Alex-17 d8eec1d427 docs: Documented the new no_workspace_mcp configuration property that disables workspace-local MCP configurations
CI / All (ubuntu-latest) (push) Failing after 24s
CI / All (macos-latest) (push) Has been cancelled
CI / All (windows-latest) (push) Has been cancelled
2026-07-13 17:14:06 -06:00
Dark-Alex-17 382916c3ee style: Removed redundant '&' from paths module function calls 2026-07-13 17:12:58 -06:00
Dark-Alex-17 bc3cc10a7b feat: Support workspace-local skill definitions and MCP configurations 2026-07-13 17:12:34 -06:00
Dark-Alex-17 b91f738209 docs: updated the configuratino examples for graph-based RAG
CI / All (ubuntu-latest) (push) Failing after 24s
CI / All (macos-latest) (push) Has been cancelled
CI / All (windows-latest) (push) Has been cancelled
2026-07-13 16:55:18 -06:00
Dark-Alex-17 4f0dae9b49 feat: fully functional graph-based RAG
CI / All (ubuntu-latest) (push) Failing after 26s
CI / All (macos-latest) (push) Has been cancelled
CI / All (windows-latest) (push) Has been cancelled
2026-07-13 16:50:07 -06:00
Dark-Alex-17 deb673ebc9 fmt: applied some formatting changes 2026-07-13 16:07:19 -06:00
24 changed files with 941 additions and 654 deletions
-371
View File
@@ -1,371 +0,0 @@
# Graph RAG Design Spec
## Status: COMPLETE
### Verified From Code (all claims backed by actual file reads)
---
## Goal
Extend the existing two-signal hybrid search (vector HNSW + BM25 → RRF) to a three-signal hybrid
(vector + BM25 + knowledge graph → RRF). The graph captures entity/relationship knowledge extracted
from documents at ingestion time via an LLM call per chunk. At query time, graph traversal expands
context beyond semantic similarity.
---
## Verified Current Architecture
### `Rag` struct (`src/rag/mod.rs:48`)
```rust
pub struct Rag {
app_config: Arc<AppConfig>,
name: String,
path: String,
embedding_model: Model,
hnsw: Hnsw<'static, f32, DistCosine>, // ephemeral, rebuilt on load
bm25: SearchEngine<DocumentId>, // ephemeral, rebuilt on load
data: RagData, // serialized to YAML
last_sources: RwLock<Option<String>>,
}
```
### `RagData` struct (`src/rag/mod.rs:892`)
```rust
pub struct RagData {
pub embedding_model: String,
pub chunk_size: usize,
pub chunk_overlap: usize,
pub reranker_model: Option<String>,
pub top_k: usize,
pub batch_size: Option<usize>,
pub next_file_id: FileId,
pub document_paths: Vec<String>,
pub files: IndexMap<FileId, RagFile>,
#[serde(with = "serde_vectors")]
pub vectors: IndexMap<DocumentId, Vec<f32>>,
}
```
### `RagData::new` callers (both need updating):
1. `Rag::init` (`src/rag/mod.rs:219`) — interactive init path
2. `Rag::resolve_init_data` (`src/rag/mod.rs:195`) — config-driven init path
### `Rag::create` (`src/rag/mod.rs:253`) — all init paths converge here:
```rust
pub fn create(app: &AppConfig, name: &str, path: &Path, data: RagData) -> Result<Self> {
let hnsw = data.build_hnsw();
let bm25 = data.build_bm25();
let embedding_model = Model::retrieve_model(app, &data.embedding_model, ModelType::Embedding)?;
let rag = Rag { app_config: Arc::new(app.clone()), name: name.to_string(),
path: path.display().to_string(), data, embedding_model, hnsw, bm25,
last_sources: RwLock::new(None) };
Ok(rag)
}
```
### `hybrid_search` (`src/rag/mod.rs:710`)
```rust
async fn hybrid_search(&self, query: &str, top_k: usize, rerank_model: Option<&str>)
-> Result<Vec<(DocumentId, String)>>
```
Runs `vector_search` + `keyword_search` in parallel via `tokio::join!`, then either reranks or
applies `reciprocal_rank_fusion(vec![vector_ids, keyword_ids], vec![1.125, 1.0], top_k)`.
### `reciprocal_rank_fusion` (`src/rag/mod.rs:1186`) — standalone fn, already weight-parameterized:
```rust
fn reciprocal_rank_fusion(
list_of_document_ids: Vec<Vec<DocumentId>>,
list_of_weights: Vec<f32>,
top_k: usize,
) -> Vec<DocumentId>
```
### `RagData::del` (`src/rag/mod.rs:953`):
```rust
pub fn del(&mut self, file_ids: Vec<FileId>) {
for file_id in file_ids {
if let Some(file) = self.files.swap_remove(&file_id) {
for (document_index, _) in file.documents.iter().enumerate() {
let document_id = DocumentId::new(file_id, document_index);
self.vectors.swap_remove(&document_id);
}
}
}
}
```
### `RagNode` (`src/graph/types.rs:331`):
```rust
pub struct RagNode {
pub documents: Vec<String>,
pub query: Option<String>,
pub top_k: Option<usize>,
pub embedding_model: Option<String>,
pub chunk_size: Option<usize>,
pub chunk_overlap: Option<usize>,
pub reranker_model: Option<String>,
pub batch_size: Option<usize>,
pub state_updates: Option<HashMap<String, String>>,
pub timeout: Option<u64>,
}
```
### `Client` trait (`src/client/common.rs:40`):
- `async fn chat_completions(&self, input: Input) -> Result<ChatCompletionsOutput>` — needs `Input`
- `async fn chat_completions_inner(&self, client: &ReqwestClient, data: ChatCompletionsData) -> Result<ChatCompletionsOutput>` — accessible on `Box<dyn Client>` via vtable
- `async fn embeddings(&self, data: &EmbeddingsData) -> Result<Vec<Vec<f32>>>`
- `async fn rerank(&self, data: &RerankData) -> Result<RerankOutput>`
- `fn build_client(&self) -> Result<ReqwestClient>`
- `fn model(&self) -> &Model`
**Key finding**: `Input` cannot be constructed without `RequestContext` (which `Rag` doesn't have).
Instead, `extract_entities` uses `chat_completions_inner` directly with manually built
`ChatCompletionsData`. This is accessible via `Box<dyn Client>`.
### `Message` (`src/client/message.rs:22`):
```rust
pub fn new(role: MessageRole, content: MessageContent) -> Self
```
`MessageRole::User`, `MessageContent::Text(String)` — both confirmed.
### `AppConfig` RAG fields (`src/config/app_config.rs:71`):
```rust
pub rag_embedding_model: Option<String>,
pub rag_reranker_model: Option<String>,
pub rag_top_k: usize, // default: 5
pub rag_chunk_size: Option<usize>,
pub rag_chunk_overlap: Option<usize>,
pub rag_template: Option<String>,
```
### `patch_messages` — confirmed exported from `crate::client::*` (used in `input.rs:5`)
### `init_client(app_config, model)` — works for any `ModelType`, including `Chat`
### `ModelType` variants: `Chat`, `Embedding`, `Reranker` (confirmed in `model.rs`)
### petgraph serde: `NodeIndex` serializes as inner `u32`; `StableGraph` preserves index positions
through roundtrip. `IndexMap<DocumentId, Vec<NodeIndex>>` safe for YAML (DocumentId is newtype over
usize, serializes as integer key).
---
## New Dependency
```toml
petgraph = { version = "0.7", features = ["serde-1"] }
```
---
## New File: `src/rag/graph.rs`
All graph types and extraction logic. Module declared in `mod.rs` as `mod graph; use self::graph::*;`.
### Types:
- `Entity { name: String, entity_type: String, description: Option<String> }`
- `Relationship { relation_type: String, weight: f32 }`
- `ExtractionResult { entities: Vec<ExtractedEntity>, relationships: Vec<ExtractedRelationship> }`
- `ExtractedEntity { name: String, r#type: String, description: Option<String> }`
- `ExtractedRelationship { from: String, to: String, r#type: String, weight: Option<f32> }`
- `KnowledgeGraph { graph: StableGraph<Entity, Relationship>, entity_index: IndexMap<String, NodeIndex>, document_entities: IndexMap<DocumentId, Vec<NodeIndex>> }`
### Key methods on `KnowledgeGraph`:
- `merge(doc_id: DocumentId, result: ExtractionResult)` — merges extraction into graph
- `remove_documents(ids: &[DocumentId])` — removes entities exclusive to deleted documents
- `build_node_to_docs(&self) -> IndexMap<NodeIndex, Vec<DocumentId>>` — ephemeral reverse map
### `extract_entities(client: &dyn Client, chunk: &str) -> Result<ExtractionResult>`:
- Builds `ChatCompletionsData` manually (no `Input` needed)
- Calls `patch_messages` then `client.chat_completions_inner(&reqwest_client, data).await`
- Strips markdown code fences from response before JSON parse
- Temperature: `Some(0.0)` for deterministic extraction
### Extraction prompt: structured JSON output requesting entities + relationships
---
## Changes to `src/rag/mod.rs`
### `Rag` struct — add one ephemeral field:
```rust
node_to_docs: IndexMap<NodeIndex, Vec<DocumentId>>, // ephemeral, rebuilt on load
```
### `Rag::create` — build node_to_docs before moving data:
```rust
let node_to_docs = data.knowledge_graph.build_node_to_docs();
// then add to struct literal
```
### `Rag` Clone impl — add:
```rust
node_to_docs: self.data.knowledge_graph.build_node_to_docs(),
```
### `RagData` struct — three new fields (all `#[serde(default)]` for backward compat):
```rust
#[serde(default)]
pub graph_enabled: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub extractor_model: Option<String>,
#[serde(default)]
pub knowledge_graph: KnowledgeGraph,
```
### `RagData::new` — two new params: `graph_enabled: bool, extractor_model: Option<String>`
### `RagData::del` — collect doc_ids during existing loop, call `remove_documents` at end:
```rust
let mut doc_ids_to_remove = vec![];
for file_id in file_ids {
if let Some(file) = self.files.swap_remove(&file_id) {
for (document_index, _) in file.documents.iter().enumerate() {
let document_id = DocumentId::new(file_id, document_index);
self.vectors.swap_remove(&document_id);
doc_ids_to_remove.push(document_id);
}
}
}
self.knowledge_graph.remove_documents(&doc_ids_to_remove);
```
### `Rag::init` (line 219) — add two params to `RagData::new`:
```rust
app.rag_graph_enabled,
app.rag_extractor_model.clone(),
```
### `resolve_init_data` — resolve from config+app, pass to `RagData::new`:
```rust
let graph_enabled = config.graph_enabled.unwrap_or(app.rag_graph_enabled);
let extractor_model = config.extractor_model.clone().or_else(|| app.rag_extractor_model.clone());
```
### `sync_documents` — entity extraction block after `rag_files` built, before embedding:
```rust
if self.data.graph_enabled {
if let Some(extractor_model_id) = self.data.extractor_model.clone() {
let model = Model::retrieve_model(&self.app_config, &extractor_model_id, ModelType::Chat)?;
let client = self.create_embeddings_client(model)?;
let total_chunks: usize = rag_files.iter().map(|f| f.documents.len()).sum();
let mut chunk_num = 0;
let file_offset = next_file_id;
for (batch_file_idx, rag_file) in rag_files.iter().enumerate() {
let file_id = file_offset + batch_file_idx;
for (doc_idx, doc) in rag_file.documents.iter().enumerate() {
chunk_num += 1;
progress(&spinner, format!("Extracting entities [{chunk_num}/{total_chunks}]"));
let doc_id = DocumentId::new(file_id, doc_idx);
match extract_entities(client.as_ref(), &doc.page_content).await {
Ok(result) => self.data.knowledge_graph.merge(doc_id, result),
Err(e) => debug!("Entity extraction failed for {doc_id:?}: {e}"),
}
}
}
}
}
```
### After line 705 (after hnsw/bm25 rebuild in sync_documents):
```rust
self.node_to_docs = self.data.knowledge_graph.build_node_to_docs();
```
### `hybrid_search` — add third signal:
```rust
let graph_search_ids: Vec<DocumentId> = if self.data.graph_enabled
&& !self.data.knowledge_graph.entity_index.is_empty()
{
self.graph_search(query, &keyword_search_ids, top_k)
} else {
vec![]
};
// RRF: extend to 3-way when graph has results, fall back to 2-way otherwise
```
### New `graph_search` method (sync):
```rust
fn graph_search(&self, query: &str, bm25_anchor_ids: &[DocumentId], top_k: usize) -> Vec<DocumentId>
```
Phase 1: entity names from query via substring match in `entity_index`.
Phase 2: fallback — entities from top BM25 document chunks.
Phase 3: expand 1-hop neighbors in `StableGraph`.
Phase 4: score docs by entity overlap ratio, return top_k.
### `RagInitConfig` — two new fields:
```rust
pub graph_enabled: Option<bool>,
pub extractor_model: Option<String>,
```
---
## Changes to `src/config/app_config.rs`
New fields alongside existing `rag_*` block:
```rust
pub rag_graph_enabled: bool, // default: false
pub rag_extractor_model: Option<String>, // default: None
```
Defaults, env var overrides, and propagation all follow the same pattern as existing `rag_*` fields.
---
## Changes to `src/graph/types.rs` — `RagNode`
```rust
#[serde(default, skip_serializing_if = "Option::is_none")]
pub graph_enabled: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub extractor_model: Option<String>,
```
---
## Changes to `src/config/agent.rs`
Pass new fields through to `RagInitConfig`:
```rust
graph_enabled: rag_node.graph_enabled,
extractor_model: rag_node.extractor_model.clone(),
```
---
## Backward Compatibility
- All new `RagData` fields have `#[serde(default)]` — old YAML files load without migration
- `graph_enabled` defaults `false` — existing RAG instances unchanged
- `graph_search_ids` empty → 2-way RRF runs (identical to current behavior)
- `node_to_docs` rebuild on `create()` is O(n) over empty map for old instances
---
## V1 Scope Exclusions
- LLM entity extraction from query at search time (V1 uses substring match + BM25 anchoring)
- Multi-hop traversal (field reserved, 1-hop only in V1)
- Entity embeddings / fuzzy entity lookup
- Bincode for large-corpus graph storage
- Gleaning / multi-pass extraction
---
## Implementation Progress
- [x] Cargo.toml — petgraph dependency
- [x] src/rag/graph.rs — new file
- [x] src/rag/mod.rs — mod/use, Rag struct, create, clone
- [x] src/rag/mod.rs — RagData fields, new, del
- [x] src/rag/mod.rs — Rag::init, resolve_init_data
- [x] src/rag/mod.rs — sync_documents extraction block
- [x] src/rag/mod.rs — hybrid_search + graph_search
- [x] src/rag/mod.rs — RagInitConfig fields
- [x] src/config/app_config.rs — new fields
- [x] src/config/mod.rs — propagation
- [x] src/graph/types.rs — RagNode fields
- [x] src/config/agent.rs — propagation
- [x] cargo check — clean (0 warnings, 1065 tests passing)
+10 -2
View File
@@ -10,5 +10,13 @@ set -e
main() {
# shellcheck disable=SC2154
cat "$argc_path" >> "$LLM_OUTPUT" 2>&1 || echo "No such file or path: $argc_path" >> "$LLM_OUTPUT"
}
local path="$argc_path"
# An empty result is shown to the model as the opaque literal "DONE"; emit a note instead.
if [[ -f "$path" && ! -s "$path" ]]; then
echo "(empty file: $path)" >> "$LLM_OUTPUT"
return 0
fi
cat "$path" >> "$LLM_OUTPUT" 2>&1 || echo "No such file or path: $path" >> "$LLM_OUTPUT"
}
+2 -2
View File
@@ -17,8 +17,8 @@ main() {
local search_path="${argc_path:-.}"
if [[ ! -d "$search_path" ]]; then
echo "Error: directory not found: $search_path" >> "$LLM_OUTPUT"
return 1
echo "Error: directory not found: $search_path" >&2
exit 1
fi
local results
+2 -2
View File
@@ -21,8 +21,8 @@ main() {
local include_filter="${argc_include:-}"
if [[ ! -e "$search_path" ]]; then
echo "Error: path not found: $search_path" >> "$LLM_OUTPUT"
return 1
echo "Error: path not found: $search_path" >&2
exit 1
fi
local grep_args=(-nH --color=never)
+15 -2
View File
@@ -9,5 +9,18 @@ set -e
main() {
# shellcheck disable=SC2154
ls -1 "$argc_path" >> "$LLM_OUTPUT" 2>&1 || echo "No such path: $argc_path" >> "$LLM_OUTPUT"
}
local path="$argc_path"
local output
if ! output=$(ls -1 "$path" 2>&1); then
echo "$output" >> "$LLM_OUTPUT"
return 0
fi
# An empty result is shown to the model as the opaque literal "DONE"; emit a note instead.
if [[ -z "$output" ]]; then
echo "(empty directory: $path)" >> "$LLM_OUTPUT"
else
echo "$output" >> "$LLM_OUTPUT"
fi
}
+18 -6
View File
@@ -8,8 +8,8 @@ set -e
# Use the grep tool to find specific content before reading, then read with offset to target the relevant section.
# @option --path! The absolute path to the file or directory to read
# @option --offset The line number to start reading from (1-indexed, default: 1)
# @option --limit The maximum number of lines to read (default: 2000)
# @option --offset <INT> The line number to start reading from (1-indexed, default: 1)
# @option --limit <INT> The maximum number of lines to read (default: 2000)
# @env LLM_OUTPUT=/dev/stdout The output path
@@ -23,8 +23,8 @@ main() {
local limit="${argc_limit:-2000}"
if [[ ! -e "$target" ]]; then
echo "Error: path not found: $target" >> "$LLM_OUTPUT"
return 1
echo "Error: path not found: $target" >&2
exit 1
fi
if [[ -d "$target" ]]; then
@@ -33,9 +33,20 @@ main() {
fi
local total_lines file_bytes
total_lines=$(wc -l < "$target" 2>/dev/null || echo 0)
# awk counts a final line that lacks a trailing newline; wc -l would undercount it by one.
total_lines=$(awk 'END { print NR }' "$target" 2>/dev/null || echo 0)
file_bytes=$(wc -c < "$target" 2>/dev/null || echo 0)
if [[ "$total_lines" -eq 0 ]]; then
echo "(file is empty: $target)" >> "$LLM_OUTPUT"
return 0
fi
if [[ "$offset" -gt "$total_lines" ]]; then
echo "(offset $offset is past the end of the file, which has $total_lines lines)" >> "$LLM_OUTPUT"
return 0
fi
if [[ "$file_bytes" -gt "$MAX_BYTES" ]] && [[ "$offset" -eq 1 ]] && [[ "$limit" -ge 2000 ]]; then
{
echo "Warning: Large file (${file_bytes} bytes, ${total_lines} lines). Showing first ${limit} lines."
@@ -48,7 +59,8 @@ main() {
sed -n "${offset},${end_line}p" "$target" 2>/dev/null | {
local line_num=$offset
while IFS= read -r line; do
# `|| [[ -n "$line" ]]` keeps the final line when the file has no trailing newline.
while IFS= read -r line || [[ -n "$line" ]]; do
if [[ ${#line} -gt $MAX_LINE_LENGTH ]]; then
line="${line:0:$MAX_LINE_LENGTH}... (truncated)"
fi
+5 -1
View File
@@ -552,7 +552,7 @@ patch_file() {
continue
}
if (line ~ /^@@ /) {
if (line ~ /^@@/) {
mode = "hunk"
hunkIndex++
patchLineIndex++
@@ -585,6 +585,10 @@ patch_file() {
if (hunkIndex == 0) {
print "error: no patch" > "/dev/stderr"
print "" > "/dev/stderr"
print "No hunk header was found. Each hunk must start with a line beginning \"@@\"" > "/dev/stderr"
print "(for example \"@@ ... @@\" or \"@@ -1,4 +1,4 @@\"). Inside a hunk, context lines" > "/dev/stderr"
print "start with a single space, removed lines with \"-\", and added lines with \"+\"." > "/dev/stderr"
exit 1
}
+55 -53
View File
@@ -5,7 +5,7 @@
# sbx cp $HOME/.config/coyote/ testing:/home/agent/.config/
# sbx cp $HOME/.coyote_password testing:/home/agent/
# sbx run testing --kit ./sbx-kit/
schemaVersion: "1"
schemaVersion: '1'
kind: sandbox
name: coyote
displayName: Coyote
@@ -14,10 +14,10 @@ description: >
CLI & REPL mode, RAG, AI tools & agents, MCP servers, skills, and macros.
sandbox:
image: "docker/sandbox-templates:shell-docker"
image: 'docker/sandbox-templates:shell-docker'
aiFilename: COYOTE.md
entrypoint:
run: ["bash", "-lc", "exec /home/agent/.cargo/bin/coyote"]
run: ['bash', '-lc', 'exec /home/agent/.cargo/bin/coyote']
network:
# Proxy-managed LLM providers: the proxy substitutes `proxy-managed` for
@@ -50,96 +50,96 @@ network:
serviceAuth:
openai:
headerName: Authorization
valueFormat: "Bearer %s"
valueFormat: 'Bearer %s'
anthropic:
headerName: x-api-key
valueFormat: "%s"
valueFormat: '%s'
gemini:
headerName: x-goog-api-key
valueFormat: "%s"
valueFormat: '%s'
cohere:
headerName: Authorization
valueFormat: "Bearer %s"
valueFormat: 'Bearer %s'
groq:
headerName: Authorization
valueFormat: "Bearer %s"
valueFormat: 'Bearer %s'
openrouter:
headerName: Authorization
valueFormat: "Bearer %s"
valueFormat: 'Bearer %s'
ai21:
headerName: Authorization
valueFormat: "Bearer %s"
valueFormat: 'Bearer %s'
cloudflare:
headerName: Authorization
valueFormat: "Bearer %s"
valueFormat: 'Bearer %s'
deepinfra:
headerName: Authorization
valueFormat: "Bearer %s"
valueFormat: 'Bearer %s'
deepseek:
headerName: Authorization
valueFormat: "Bearer %s"
valueFormat: 'Bearer %s'
mistral:
headerName: Authorization
valueFormat: "Bearer %s"
valueFormat: 'Bearer %s'
perplexity:
headerName: Authorization
valueFormat: "Bearer %s"
valueFormat: 'Bearer %s'
voyageai:
headerName: Authorization
valueFormat: "Bearer %s"
valueFormat: 'Bearer %s'
xai:
headerName: Authorization
valueFormat: "Bearer %s"
valueFormat: 'Bearer %s'
jina:
headerName: Authorization
valueFormat: "Bearer %s"
valueFormat: 'Bearer %s'
ernie:
headerName: Authorization
valueFormat: "Bearer %s"
valueFormat: 'Bearer %s'
hunyuan:
headerName: Authorization
valueFormat: "Bearer %s"
valueFormat: 'Bearer %s'
minimax:
headerName: Authorization
valueFormat: "Bearer %s"
valueFormat: 'Bearer %s'
moonshot:
headerName: Authorization
valueFormat: "Bearer %s"
valueFormat: 'Bearer %s'
qianwen:
headerName: Authorization
valueFormat: "Bearer %s"
valueFormat: 'Bearer %s'
zhipuai:
headerName: Authorization
valueFormat: "Bearer %s"
valueFormat: 'Bearer %s'
allowedDomains:
# Coyote release + self-update + model-registry sync
- "github.com:443"
- "api.github.com:443"
- "raw.githubusercontent.com:443"
- "objects.githubusercontent.com:443"
- "*.githubusercontent.com:443"
- 'github.com:443'
- 'api.github.com:443'
- 'raw.githubusercontent.com:443'
- 'objects.githubusercontent.com:443'
- '*.githubusercontent.com:443'
# Coyote install paths (cargo install + uv + rustup + Python tool deps at runtime)
- "crates.io:443"
- "static.crates.io:443"
- "pypi.org:443"
- "files.pythonhosted.org:443"
- "astral.sh:443"
- "sh.rustup.rs:443"
- "static.rust-lang.org:443"
- 'crates.io:443'
- 'static.crates.io:443'
- 'pypi.org:443'
- 'files.pythonhosted.org:443'
- 'astral.sh:443'
- 'sh.rustup.rs:443'
- 'static.rust-lang.org:443'
# LLM model OAuth + API endpoints
- "claude.ai:443"
- "console.anthropic.com:443"
- "accounts.google.com:443"
- 'claude.ai:443'
- 'console.anthropic.com:443'
- 'accounts.google.com:443'
# *.googleapis.com covers oauth2 + userinfo + VertexAI regional endpoints
# (*-aiplatform.googleapis.com). Do not narrow without re-checking VertexAI.
- "*.googleapis.com:443"
- '*.googleapis.com:443'
# Bedrock and GitHub Models use signed / GitHub-PAT auth that the proxy
# cannot rewrite. Domains are allow-listed; credentials must be injected
# separately (see README "Extending").
- "*.amazonaws.com:443"
- "models.inference.ai.azure.com:443"
- '*.amazonaws.com:443'
- 'models.inference.ai.azure.com:443'
credentials:
sources:
@@ -210,9 +210,10 @@ credentials:
environment:
variables:
IS_SANDBOX: "1"
IS_SANDBOX: '1'
COYOTE_LOG_LEVEL: INFO
COYOTE_CONFIG_DIR: /home/agent/.config/coyote
EDITOR: nano
proxyManaged:
- OPENAI_API_KEY
- ANTHROPIC_API_KEY
@@ -249,8 +250,9 @@ commands:
musl-tools \
libssl-dev \
pandoc \
bzip2
user: "1000"
bzip2 \
nano
user: '1000'
description: Install system prerequisites (including pandoc for fetch_url_via_curl)
- command: |
curl -LsSf https://astral.sh/uv/install.sh | sh
@@ -258,7 +260,7 @@ commands:
printf '#!/bin/sh\nexec uv tool run "$@"\n' > "$HOME/.local/bin/uvx"
chmod +x "$HOME/.local/bin/uvx"
fi
user: "1000"
user: '1000'
description: Install uv and write a uvx shell wrapper (the installer may place a macOS binary at this path on Docker-for-Mac hosts, which the Linux container cannot execute)
- command: |
set -euo pipefail
@@ -274,7 +276,7 @@ commands:
curl -fsSL --retry 3 "https://github.com/xo/usql/releases/download/v${USQL_VERSION}/usql_static-${USQL_VERSION}-linux-${USQL_ARCH}.tar.bz2" -o "$TMPDIR/usql.tar.bz2"
tar -xjf "$TMPDIR/usql.tar.bz2" -C "$TMPDIR"
sudo install -m 0755 "$TMPDIR/usql_static" /usr/local/bin/usql
user: "1000"
user: '1000'
description: Install the usql universal SQL CLI (used by the built-in sql agent and execute_sql_code tool)
- command: |
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
@@ -284,27 +286,27 @@ commands:
--target x86_64-unknown-linux-musl
. "$HOME/.cargo/env"
cargo install --locked coyote-ai
user: "1000"
user: '1000'
description: Install Coyote AI CLI via Rust's Cargo
- command: |
. "$HOME/.cargo/env"
cargo install --locked iwec
user: "1000"
user: '1000'
description: Install the IWE MCP server binary (iwec) used by the built-in iwe MCP server and iwe-knowledge-base skill
- command: |
. "$HOME/.cargo/env"
cargo install --locked ast-grep
user: "1000"
user: '1000'
description: Install ast-grep, used by the built-in ast_grep structural code search tool (and the explore agent)
startup:
- command:
[
"sh",
"-c",
'sh',
'-c',
'test -f "$HOME/.config/coyote/config.yaml" || coyote --info >/dev/null 2>&1 || true',
]
user: "1000"
user: '1000'
background: false
description: Bootstrap Coyote config directory on first sandbox start
+5 -1
View File
@@ -134,6 +134,10 @@ enabled_mcp_servers: null # Which MCP servers to enable by default.
# - slack
# Example (comma-separated form):
# enabled_mcp_servers: github,slack,ddg-search
no_workspace_mcp: false # Disable loading workspace-local MCP servers from .coyote/mcp.json (default: false).
# When false (the default), Coyote merges .coyote/mcp.json from the current directory
# into the global MCP registry at startup. Workspace entries shadow global ones on
# name collision. Set to true (or pass --no-workspace-mcp) to skip this entirely.
# ---- Skills ----
# Skills are modular knowledge or capability packs the LLM can load and unload mid-conversation.
@@ -199,7 +203,7 @@ rag_chunk_size: null # Defines the size of chunks for document proce
rag_chunk_overlap: null # Defines the overlap between chunks
rag_extractor_model: null # LLM model for graph-based entity/relationship extraction; when set, enables a graph RAG signal alongside vector and BM25
rag_extractor_prompt: null # Custom extraction prompt template; must contain __CHUNK__ placeholder; defaults to built-in prompt when null
rag_graph_hops: 1 # Number of hops to expand from matched entities at query time (1 = direct neighbors; increase for denser graphs)
rag_graph_hops: 1 # Number of hops to expand from matched entities at query time (0 = seed nodes only; 1 = direct neighbors; increase for denser graphs)
# Defines the query structure using variables like __CONTEXT__, __SOURCES__, and __INPUT__ to tailor searches to specific needs
rag_template: |
Answer the query based on the context while respecting the rules. (user query, some textual context and rules, all inside xml tags)
+1 -1
View File
@@ -227,7 +227,7 @@ nodes:
reranker_model: null # Optional reranker for hybrid-search results
extractor_model: null # Optional chat model for graph-based entity/relationship extraction; enables graph RAG signal when set
extractor_prompt: null # Optional custom extraction prompt; must contain __CHUNK__ placeholder; uses built-in prompt when null
graph_hops: 1 # Graph expansion depth at query time (1 = direct neighbors; increase for denser knowledge graphs)
graph_hops: 1 # Graph expansion depth at query time (0 = seed nodes only; 1 = direct neighbors; increase for denser knowledge graphs)
batch_size: 100 # Optional embedding-request batch size
state_updates: # {{output}} = { context: <str>, sources: [<path>, ...] }
context: "{{output.context}}" # writes `context` -> `reducers.context = concat`
+152 -109
View File
@@ -43,6 +43,10 @@ use std::io::{Read, stdin};
),
)]
pub struct Cli {
/// Input text
#[arg(trailing_var_arg = true)]
text: Vec<String>,
/// Select a LLM model
#[arg(short, long, add = ArgValueCompleter::new(model_completer))]
pub model: Option<String>,
@@ -52,30 +56,6 @@ pub struct Cli {
/// Select a role
#[arg(short, long, add = ArgValueCompleter::new(role_completer))]
pub role: Option<String>,
/// Start or join a session
#[arg(short = 's', long, add = ArgValueCompleter::new(session_completer))]
pub session: Option<Option<String>>,
/// Ensure the session is empty
#[arg(long)]
pub empty_session: bool,
/// Ensure the new conversation is saved to the session
#[arg(long)]
pub save_session: bool,
/// Start an agent
#[arg(short = 'a', long, add = ArgValueCompleter::new(agent_completer))]
pub agent: Option<String>,
/// Set agent variables
#[arg(long, value_names = ["NAME", "VALUE"], num_args = 2)]
pub agent_variable: Vec<String>,
/// Start a RAG
#[arg(long, add = ArgValueCompleter::new(rag_completer))]
pub rag: Option<String>,
/// Rebuild the RAG to sync document changes
#[arg(long)]
pub rebuild_rag: bool,
/// Execute a macro
#[arg(long = "macro", value_name = "MACRO", add = ArgValueCompleter::new(macro_completer))]
pub macro_name: Option<String>,
/// Execute commands in natural language
#[arg(short = 'e', long)]
pub execute: bool,
@@ -88,113 +68,176 @@ pub struct Cli {
/// Turn off stream mode
#[arg(short = 'S', long)]
pub no_stream: bool,
/// Display the message without sending it
#[arg(long)]
pub dry_run: bool,
/// Disable loading workspace MCP servers from .coyote/mcp.json
#[arg(long)]
pub no_workspace_mcp: bool,
/// Disable memory for this invocation
#[arg(long)]
pub no_memory: bool,
/// Skip permission prompts by setting AUTO_CONFIRM for all tools (dangerous!)
#[arg(long)]
pub dangerously_skip_permissions: bool,
/// Start or join a session
#[arg(short = 's', long, help_heading = "Session & Memory", add = ArgValueCompleter::new(session_completer))]
pub session: Option<Option<String>>,
/// Ensure the session is empty
#[arg(long, help_heading = "Session & Memory")]
pub empty_session: bool,
/// Ensure the new conversation is saved to the session
#[arg(long, help_heading = "Session & Memory")]
pub save_session: bool,
/// Bootstrap a memory marker so coyote begins loading memory next run
#[arg(long, value_name = "SCOPE", value_enum)]
#[arg(
long,
value_name = "SCOPE",
value_enum,
help_heading = "Session & Memory"
)]
pub init_memory: Option<MemoryScope>,
/// Display the message without sending it
#[arg(long)]
pub dry_run: bool,
/// Display information
#[arg(long)]
pub info: bool,
/// Build all configured Bash tool scripts
#[arg(long)]
pub build_tools: bool,
/// Reinstall bundled assets, overwriting any local changes
#[arg(long, value_name = "CATEGORY", value_enum)]
pub install: Option<AssetCategory>,
/// Install assets from a remote git repository (URL may be suffixed with #<ref>)
#[arg(long, value_name = "GIT_URL")]
pub install_from: Option<String>,
/// Restrict --install-from to a single asset category
#[arg(long, value_name = "CATEGORY", value_enum, requires = "install_from")]
pub filter: Option<InstallFilter>,
/// Overwrite all conflicts without prompting (used with --install-from)
#[arg(long, requires = "install_from")]
pub install_force: bool,
/// Sync models updates
#[arg(long)]
pub sync_models: bool,
/// List all available chat models
#[arg(long)]
pub list_models: bool,
/// List all roles
#[arg(long)]
pub list_roles: bool,
/// List all sessions
#[arg(long)]
pub list_sessions: bool,
/// List all agents
#[arg(long)]
pub list_agents: bool,
/// List all RAGs
#[arg(long)]
pub list_rags: bool,
/// List all macros
#[arg(long)]
pub list_macros: bool,
/// List all installed skills
#[arg(long)]
pub list_skills: bool,
/// Pre-load an existing skill into the session (repeatable). If a single
/// `--skill <NAME>` is given and the skill doesn't exist, opens $EDITOR
/// with a scaffold to create it.
#[arg(long, value_name = "NAME")]
#[arg(long, value_name = "NAME", help_heading = "Session & Memory")]
pub skill: Vec<String>,
/// Input text
#[arg(trailing_var_arg = true)]
text: Vec<String>,
/// Tail logs
#[arg(long)]
pub tail_logs: bool,
/// Disable colored log output
#[arg(long, requires = "tail_logs")]
pub disable_log_colors: bool,
/// Add a secret to the Coyote vault
#[arg(long, value_name = "SECRET_NAME", exclusive = true)]
pub add_secret: Option<String>,
/// Decrypt a secret from the Coyote vault and print the plaintext
#[arg(long, value_name = "SECRET_NAME", exclusive = true, add = ArgValueCompleter::new(secrets_completer))]
pub get_secret: Option<String>,
/// Update an existing secret in the Coyote vault
#[arg(long, value_name = "SECRET_NAME", exclusive = true, add = ArgValueCompleter::new(secrets_completer))]
pub update_secret: Option<String>,
/// Delete a secret from the Coyote vault
#[arg(long, value_name = "SECRET_NAME", exclusive = true, add = ArgValueCompleter::new(secrets_completer))]
pub delete_secret: Option<String>,
/// List all secrets stored in the Coyote vault
#[arg(long, exclusive = true)]
pub list_secrets: bool,
/// Authenticate with an LLM provider using OAuth (e.g., --authenticate client_name)
#[arg(long, exclusive = true, value_name = "CLIENT_NAME")]
pub authenticate: Option<Option<String>>,
/// Authenticate with an OAuth-protected remote MCP server (e.g., --auth-mcp server_name)
#[arg(long, exclusive = true, value_name = "SERVER_NAME", add = ArgValueCompleter::new(mcp_server_completer))]
pub auth_mcp: Option<String>,
/// Generate static shell completion scripts
#[arg(long, value_name = "SHELL", value_enum)]
pub completions: Option<ShellCompletion>,
/// Start an agent
#[arg(short = 'a', long, help_heading = "Agents, RAG & Macros", add = ArgValueCompleter::new(agent_completer))]
pub agent: Option<String>,
/// Set agent variables
#[arg(long, value_names = ["NAME", "VALUE"], num_args = 2, help_heading = "Agents, RAG & Macros")]
pub agent_variable: Vec<String>,
/// Start a RAG
#[arg(long, help_heading = "Agents, RAG & Macros", add = ArgValueCompleter::new(rag_completer))]
pub rag: Option<String>,
/// Rebuild the RAG to sync document changes
#[arg(long, help_heading = "Agents, RAG & Macros")]
pub rebuild_rag: bool,
/// Execute a macro
#[arg(long = "macro", value_name = "MACRO", help_heading = "Agents, RAG & Macros", add = ArgValueCompleter::new(macro_completer))]
pub macro_name: Option<String>,
/// List all available chat models
#[arg(long, help_heading = "List & Discovery")]
pub list_models: bool,
/// List all roles
#[arg(long, help_heading = "List & Discovery")]
pub list_roles: bool,
/// List all sessions
#[arg(long, help_heading = "List & Discovery")]
pub list_sessions: bool,
/// List all agents
#[arg(long, help_heading = "List & Discovery")]
pub list_agents: bool,
/// List all RAGs
#[arg(long, help_heading = "List & Discovery")]
pub list_rags: bool,
/// List all macros
#[arg(long, help_heading = "List & Discovery")]
pub list_macros: bool,
/// List all installed skills
#[arg(long, help_heading = "List & Discovery")]
pub list_skills: bool,
/// Reinstall bundled assets, overwriting any local changes
#[arg(
long,
value_name = "CATEGORY",
value_enum,
help_heading = "Installation & Updates"
)]
pub install: Option<AssetCategory>,
/// Install assets from a remote git repository (URL may be suffixed with #<ref>)
#[arg(long, value_name = "GIT_URL", help_heading = "Installation & Updates")]
pub install_from: Option<String>,
/// Restrict --install-from to a single asset category
#[arg(
long,
value_name = "CATEGORY",
value_enum,
requires = "install_from",
help_heading = "Installation & Updates"
)]
pub filter: Option<InstallFilter>,
/// Overwrite all conflicts without prompting (used with --install-from)
#[arg(
long,
requires = "install_from",
help_heading = "Installation & Updates"
)]
pub install_force: bool,
/// Sync models updates
#[arg(long, help_heading = "Installation & Updates")]
pub sync_models: bool,
/// Update Coyote to the latest release, or to a specific version
#[arg(long, value_name = "VERSION")]
#[arg(long, value_name = "VERSION", help_heading = "Installation & Updates")]
pub update: Option<Option<String>>,
/// With --update, update even if Coyote was installed via a package manager
#[arg(long, requires = "update")]
#[arg(long, requires = "update", help_heading = "Installation & Updates")]
pub force: bool,
/// Add a secret to the Coyote vault
#[arg(
long,
value_name = "SECRET_NAME",
exclusive = true,
help_heading = "Vault & Secrets"
)]
pub add_secret: Option<String>,
/// Decrypt a secret from the Coyote vault and print the plaintext
#[arg(long, value_name = "SECRET_NAME", exclusive = true, help_heading = "Vault & Secrets", add = ArgValueCompleter::new(secrets_completer))]
pub get_secret: Option<String>,
/// Update an existing secret in the Coyote vault
#[arg(long, value_name = "SECRET_NAME", exclusive = true, help_heading = "Vault & Secrets", add = ArgValueCompleter::new(secrets_completer))]
pub update_secret: Option<String>,
/// Delete a secret from the Coyote vault
#[arg(long, value_name = "SECRET_NAME", exclusive = true, help_heading = "Vault & Secrets", add = ArgValueCompleter::new(secrets_completer))]
pub delete_secret: Option<String>,
/// List all secrets stored in the Coyote vault
#[arg(long, exclusive = true, help_heading = "Vault & Secrets")]
pub list_secrets: bool,
/// Authenticate with an LLM provider using OAuth (e.g., --authenticate client_name)
#[arg(
long,
exclusive = true,
value_name = "CLIENT_NAME",
help_heading = "Authentication"
)]
pub authenticate: Option<Option<String>>,
/// Authenticate with an OAuth-protected remote MCP server (e.g., --auth-mcp server_name)
#[arg(long, exclusive = true, value_name = "SERVER_NAME", help_heading = "Authentication", add = ArgValueCompleter::new(mcp_server_completer))]
pub auth_mcp: Option<String>,
/// Launch Coyote inside a Docker sandbox (via `sbx`); name defaults to current directory basename
#[arg(long, value_name = "NAME")]
#[arg(long, value_name = "NAME", help_heading = "Sandbox")]
pub sandbox: Option<Option<String>>,
/// Create the sandbox without bootstrapping the host config or vault password file
#[arg(long, requires = "sandbox")]
#[arg(long, requires = "sandbox", help_heading = "Sandbox")]
pub fresh: bool,
/// Skip discovery and application of all sbx mixins (user and built-in)
#[arg(long, requires = "sandbox")]
#[arg(long, requires = "sandbox", help_heading = "Sandbox")]
pub no_mixins: bool,
/// Display information
#[arg(long, help_heading = "Diagnostics & Tools")]
pub info: bool,
/// Build all configured Bash tool scripts
#[arg(long, help_heading = "Diagnostics & Tools")]
pub build_tools: bool,
/// Tail logs
#[arg(long, help_heading = "Diagnostics & Tools")]
pub tail_logs: bool,
/// Disable colored log output
#[arg(long, requires = "tail_logs", help_heading = "Diagnostics & Tools")]
pub disable_log_colors: bool,
/// Generate static shell completion scripts
#[arg(long, value_name = "SHELL", value_enum, help_heading = "Shell")]
pub completions: Option<ShellCompletion>,
}
impl Cli {
+2 -2
View File
@@ -50,7 +50,7 @@ fn prepare_chat_completions(
let url = format!(
"{}/openai/deployments/{}/chat/completions?api-version=2024-12-01-preview",
&api_base,
api_base,
self_.model.real_name()
);
@@ -69,7 +69,7 @@ fn prepare_embeddings(self_: &AzureOpenAIClient, data: &EmbeddingsData) -> Resul
let url = format!(
"{}/openai/deployments/{}/embeddings?api-version=2024-10-21",
&api_base,
api_base,
self_.model.real_name()
);
+3
View File
@@ -88,6 +88,7 @@ pub struct AppConfig {
pub user_agent: Option<String>,
pub save_shell_history: bool,
pub no_workspace_mcp: bool,
pub sync_models_url: Option<String>,
pub clients: Vec<ClientConfig>,
@@ -162,6 +163,7 @@ impl Default for AppConfig {
user_agent: None,
save_shell_history: true,
no_workspace_mcp: false,
sync_models_url: None,
clients: vec![],
@@ -238,6 +240,7 @@ impl AppConfig {
user_agent: config.user_agent,
save_shell_history: config.save_shell_history,
no_workspace_mcp: false,
sync_models_url: config.sync_models_url,
clients: config.clients,
+10 -10
View File
@@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize};
use crate::config::{
GIT_DIR_NAME, GITIGNORE_FILE_NAME, MEMORY_DIR_NAME, MEMORY_INDEX_FILE_NAME,
WORKSPACE_MEMORY_DIR_NAME, WORKSPACE_MEMORY_FILE_NAME, paths,
WORKSPACE_COYOTE_DIR_NAME, WORKSPACE_MEMORY_FILE_NAME, paths,
};
pub const DEFAULT_MEMORY_CAP_WITH_TOOLS: usize = 6_000;
@@ -27,7 +27,7 @@ pub enum WorkspaceMemory {
pub fn discover_workspace_memory(start: &Path) -> Option<WorkspaceMemory> {
for dir in start.ancestors() {
let structured = dir.join(WORKSPACE_MEMORY_DIR_NAME).join(MEMORY_DIR_NAME);
let structured = dir.join(WORKSPACE_COYOTE_DIR_NAME).join(MEMORY_DIR_NAME);
if structured.join(MEMORY_INDEX_FILE_NAME).exists() {
return Some(WorkspaceMemory::Structured {
workspace_root: dir.to_path_buf(),
@@ -84,8 +84,8 @@ pub fn bootstrap_workspace_memory(git_root: &Path) -> Result<PathBuf> {
fn append_gitignore_entry(git_root: &Path) -> Result<bool> {
let gitignore = git_root.join(GITIGNORE_FILE_NAME);
let entry = format!("{WORKSPACE_MEMORY_DIR_NAME}/{MEMORY_DIR_NAME}/");
let entry_no_slash = format!("{WORKSPACE_MEMORY_DIR_NAME}/{MEMORY_DIR_NAME}");
let entry = format!("{WORKSPACE_COYOTE_DIR_NAME}/{MEMORY_DIR_NAME}/");
let entry_no_slash = format!("{WORKSPACE_COYOTE_DIR_NAME}/{MEMORY_DIR_NAME}");
let existing = fs::read_to_string(&gitignore).unwrap_or_default();
let already_present = existing.lines().any(|line| {
@@ -347,7 +347,7 @@ mod tests {
let root = temp_root("phase1");
let workspace = root.join("workspace");
let workspace_memory_dir = workspace
.join(WORKSPACE_MEMORY_DIR_NAME)
.join(WORKSPACE_COYOTE_DIR_NAME)
.join(MEMORY_DIR_NAME);
fs::create_dir_all(&workspace_memory_dir).unwrap();
fs::write(
@@ -382,7 +382,7 @@ mod tests {
let root = temp_root("prefer");
let workspace = root.join("ws");
let structured = workspace
.join(WORKSPACE_MEMORY_DIR_NAME)
.join(WORKSPACE_COYOTE_DIR_NAME)
.join(MEMORY_DIR_NAME);
fs::create_dir_all(&structured).unwrap();
fs::write(structured.join(MEMORY_INDEX_FILE_NAME), "s").unwrap();
@@ -415,7 +415,7 @@ mod tests {
let root = temp_root("indexes_only");
let workspace = root.join("ws");
let structured = workspace
.join(WORKSPACE_MEMORY_DIR_NAME)
.join(WORKSPACE_COYOTE_DIR_NAME)
.join(MEMORY_DIR_NAME);
fs::create_dir_all(&structured).unwrap();
fs::write(
@@ -450,7 +450,7 @@ mod tests {
let root = temp_root("drill_bodies");
let workspace = root.join("ws");
let structured = workspace
.join(WORKSPACE_MEMORY_DIR_NAME)
.join(WORKSPACE_COYOTE_DIR_NAME)
.join(MEMORY_DIR_NAME);
fs::create_dir_all(&structured).unwrap();
fs::write(structured.join(MEMORY_INDEX_FILE_NAME), "idx").unwrap();
@@ -485,7 +485,7 @@ mod tests {
let root = temp_root("cap");
let workspace = root.join("ws");
let structured = workspace
.join(WORKSPACE_MEMORY_DIR_NAME)
.join(WORKSPACE_COYOTE_DIR_NAME)
.join(MEMORY_DIR_NAME);
fs::create_dir_all(&structured).unwrap();
fs::write(structured.join(MEMORY_INDEX_FILE_NAME), "idx").unwrap();
@@ -575,7 +575,7 @@ mod tests {
let root = temp_root("walk_up");
let workspace = root.join("ws");
let mem_dir = workspace
.join(WORKSPACE_MEMORY_DIR_NAME)
.join(WORKSPACE_COYOTE_DIR_NAME)
.join(MEMORY_DIR_NAME);
fs::create_dir_all(&mem_dir).unwrap();
fs::write(mem_dir.join(MEMORY_INDEX_FILE_NAME), "idx").unwrap();
+1 -1
View File
@@ -143,7 +143,7 @@ const MCP_FILE_NAME: &str = "mcp.json";
const MEMORY_DIR_NAME: &str = "memory";
const MEMORY_INDEX_FILE_NAME: &str = "MEMORY.md";
const WORKSPACE_MEMORY_FILE_NAME: &str = "COYOTE.md";
const WORKSPACE_MEMORY_DIR_NAME: &str = ".coyote";
const WORKSPACE_COYOTE_DIR_NAME: &str = ".coyote";
const SBX_KIT_DIR_NAME: &str = "sbx-kit";
const SBX_KIT_HASH_FILE: &str = "kit.sha256";
const SBX_MIXIN_FILE_NAME: &str = "sbx-mixin.yaml";
+37 -13
View File
@@ -5,7 +5,7 @@ use super::{
GLOBAL_TOOLS_UTILS_DIR_NAME, MACROS_DIR_NAME, MCP_FILE_NAME, MEMORY_DIR_NAME,
MEMORY_INDEX_FILE_NAME, ModelsOverride, RAGS_DIR_NAME, ROLES_DIR_NAME, SBX_KIT_DIR_NAME,
SBX_KIT_HASH_FILE, SBX_MIXIN_FILE_NAME, SBX_MIXIN_KITS_DIR_NAME, SBX_VAULT_MIXINS_DIR_NAME,
SKILLS_DIR_NAME, WORKSPACE_MEMORY_DIR_NAME,
SKILLS_DIR_NAME, WORKSPACE_COYOTE_DIR_NAME,
};
use crate::client::ProviderModels;
use crate::config::REPL_HISTORY_DIR_NAME;
@@ -118,7 +118,7 @@ pub fn global_tools_sbx_mixin_file() -> PathBuf {
pub fn find_workspace_sbx_mixin(start: &Path) -> Option<PathBuf> {
for dir in start.ancestors() {
let candidate = dir
.join(WORKSPACE_MEMORY_DIR_NAME)
.join(WORKSPACE_COYOTE_DIR_NAME)
.join(SBX_MIXIN_FILE_NAME);
if candidate.exists() {
return Some(candidate);
@@ -193,6 +193,24 @@ pub fn skill_file(name: &str) -> PathBuf {
skill_dir(name).join("SKILL.md")
}
pub fn workspace_skills_dir() -> PathBuf {
env::current_dir()
.unwrap_or_default()
.join(WORKSPACE_COYOTE_DIR_NAME)
.join(SKILLS_DIR_NAME)
}
pub fn workspace_skill_file(name: &str) -> PathBuf {
workspace_skills_dir().join(name).join("SKILL.md")
}
pub fn workspace_mcp_config_file() -> PathBuf {
env::current_dir()
.unwrap_or_default()
.join(WORKSPACE_COYOTE_DIR_NAME)
.join(MCP_FILE_NAME)
}
pub fn validate_skill_name(name: &str) -> Result<()> {
if name.is_empty() {
bail!("Skill name cannot be empty");
@@ -318,7 +336,7 @@ pub fn global_memory_index_path() -> PathBuf {
pub fn workspace_memory_dir_for(workspace_root: &Path) -> PathBuf {
workspace_root
.join(WORKSPACE_MEMORY_DIR_NAME)
.join(WORKSPACE_COYOTE_DIR_NAME)
.join(MEMORY_DIR_NAME)
}
@@ -405,15 +423,21 @@ pub fn has_macro(name: &str) -> bool {
pub fn list_skills() -> Vec<String> {
let mut names = Vec::new();
if let Ok(rd) = read_dir(skills_dir()) {
for entry in rd.flatten() {
if let Ok(file_type) = entry.file_type()
&& file_type.is_dir()
&& let Some(name) = entry.file_name().to_str()
&& entry.path().join("SKILL.md").is_file()
&& validate_skill_name(name).is_ok()
{
names.push(name.to_string());
let mut seen = HashSet::new();
for dir in [workspace_skills_dir(), skills_dir()] {
if let Ok(rd) = read_dir(dir) {
for entry in rd.flatten() {
if let Ok(file_type) = entry.file_type()
&& file_type.is_dir()
&& let Some(name) = entry.file_name().to_str()
&& !seen.contains(name)
&& entry.path().join("SKILL.md").is_file()
&& validate_skill_name(name).is_ok()
{
seen.insert(name.to_string());
names.push(name.to_string());
}
}
}
}
@@ -423,7 +447,7 @@ pub fn list_skills() -> Vec<String> {
}
pub fn has_skill(name: &str) -> bool {
skill_file(name).is_file()
workspace_skill_file(name).is_file() || skill_file(name).is_file()
}
pub fn local_models_override() -> Result<Vec<ProviderModels>> {
+1 -1
View File
@@ -261,7 +261,7 @@ impl Session {
data["messages"] = json!(self.messages);
let output = serde_yaml::to_string(&data)
.with_context(|| format!("Unable to show info about session '{}'", &self.name))?;
.with_context(|| format!("Unable to show info about session '{}'", self.name))?;
Ok(output)
}
+5 -1
View File
@@ -117,7 +117,11 @@ impl Skill {
pub fn load(name: &str) -> Result<Self> {
paths::validate_skill_name(name)?;
let path = paths::skill_file(name);
let path = if paths::workspace_skill_file(name).is_file() {
paths::workspace_skill_file(name)
} else {
paths::skill_file(name)
};
let content = read_to_string(&path)
.with_context(|| format!("Failed to read skill '{name}' at {}", path.display()))?;
Ok(Skill::new(name, &content))
+6 -6
View File
@@ -147,10 +147,10 @@ pub async fn eval_tool_calls(
let mut is_all_null = true;
for call in calls {
if let Some(msg) = ctx.tool_scope.tool_tracker.check_loop(&call.clone()) {
let dup_msg = format!("{{\"tool_call_loop_alert\":{}}}", &msg.trim());
let dup_msg = format!("{{\"tool_call_loop_alert\":{}}}", msg.trim());
println!(
"{}",
warning_text(format!("{}: ⚠️ Tool-call loop detected! ⚠️", &call.name).as_str())
warning_text(format!("{}: ⚠️ Tool-call loop detected! ⚠️", call.name).as_str())
);
let val = json!(dup_msg);
output.push(ToolResult::new(call, val));
@@ -730,7 +730,7 @@ impl Functions {
let root_dir = paths::functions_dir();
let tool_path = format!(
"{}/{binary_name}",
&paths::global_tools_dir().to_string_lossy()
paths::global_tools_dir().to_string_lossy()
);
content_template
.replace("{function_name}", binary_name)
@@ -741,7 +741,7 @@ impl Functions {
let root_dir = paths::agent_data_dir(agent_name);
let tool_path = format!(
"{}/{binary_name}",
&paths::global_tools_dir().to_string_lossy()
paths::global_tools_dir().to_string_lossy()
);
content_template
.replace("{function_name}", binary_name)
@@ -870,7 +870,7 @@ impl Functions {
let root_dir = paths::functions_dir();
let tool_path = format!(
"{}/{binary_name}",
&paths::global_tools_dir().to_string_lossy()
paths::global_tools_dir().to_string_lossy()
);
content_template
.replace("{function_name}", binary_name)
@@ -881,7 +881,7 @@ impl Functions {
let root_dir = paths::agent_data_dir(agent_name);
let tool_path = format!(
"{}/{binary_name}",
&paths::global_tools_dir().to_string_lossy()
paths::global_tools_dir().to_string_lossy()
);
content_template
.replace("{function_name}", binary_name)
+6 -2
View File
@@ -187,7 +187,11 @@ async fn main() -> Result<()> {
let abort_signal = create_abort_signal();
let start_mcp_servers = cli.agent.is_none() && cli.role.is_none();
let cfg = Config::load_with_interpolation(info_flag).await?;
let app_config: Arc<AppConfig> = Arc::new(AppConfig::from_config(cfg)?);
let mut app_config = AppConfig::from_config(cfg)?;
if cli.no_workspace_mcp {
app_config.no_workspace_mcp = true;
}
let app_config: Arc<AppConfig> = Arc::new(app_config);
let app_state: Arc<AppState> = Arc::new(
AppState::init(
app_config,
@@ -559,7 +563,7 @@ async fn shell_execute(
match answer_char {
'e' => {
debug!("{} {:?}", shell.cmd, &[&shell.arg, &eval_str]);
debug!("{} {:?}", shell.cmd, [&shell.arg, &eval_str]);
let code = run_command(&shell.cmd, &[&shell.arg, &eval_str], None)?;
if code == 0 && app.save_shell_history {
let _ = append_to_shell_history(&shell.name, &eval_str, code);
+47 -1
View File
@@ -214,7 +214,53 @@ impl McpRegistry {
spec.validate(name)?;
}
registry.config = Some(mcp_servers_config);
let mut merged = mcp_servers_config;
if !app_config.no_workspace_mcp {
let ws_path = paths::workspace_mcp_config_file();
if ws_path.try_exists().unwrap_or(false) {
match tokio::fs::read_to_string(&ws_path).await {
Ok(ws_content) if !ws_content.trim().is_empty() => {
match interpolate_secrets(&ws_content, vault) {
Ok((parsed, missing)) if missing.is_empty() => {
match serde_json::from_str::<McpServersConfig>(&parsed) {
Ok(ws_config) => {
let mut loaded = Vec::new();
for (name, spec) in ws_config.mcp_servers {
match spec.validate(&name) {
Ok(_) => {
loaded.push(name.clone());
merged.mcp_servers.insert(name, spec);
}
Err(e) => warn!(
"Invalid workspace MCP server '{name}': {e}. Skipping."
),
}
}
if !loaded.is_empty() {
eprintln!(
"Loading workspace MCP servers: {}",
loaded.join(", ")
);
}
}
Err(e) => warn!(
"Failed to parse workspace MCP config: {e}. Skipping."
),
}
}
Ok((_, missing)) => warn!(
"Workspace MCP config references missing vault secrets: {missing:?}. Skipping."
),
Err(e) => {
warn!("Failed to process workspace MCP config: {e}. Skipping.")
}
}
}
_ => {}
}
}
}
registry.config = Some(merged);
if start_mcp_servers && app_config.mcp_server_support {
abortable_run_with_spinner(
+4 -5
View File
@@ -358,17 +358,16 @@ mod tests {
use super::*;
use crate::function::JsonSchema;
use std::fs;
use std::time::{SystemTime, UNIX_EPOCH};
use std::sync::atomic::{AtomicU64, Ordering};
static PARSE_COUNTER: AtomicU64 = AtomicU64::new(0);
fn parse_source(
source: &str,
file_name: &str,
parent: &Path,
) -> Result<Vec<FunctionDeclaration>> {
let unique = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("time went backwards")
.as_nanos();
let unique = PARSE_COUNTER.fetch_add(1, Ordering::Relaxed);
let path =
std::env::temp_dir().join(format!("coyote_python_parser_{file_name}_{unique}.py"));
fs::write(&path, source).expect("failed to write temp python source");
+503 -21
View File
@@ -2,12 +2,21 @@ use super::DocumentId;
use crate::client::*;
use anyhow::{Context, Result};
use indexmap::IndexMap;
use indexmap::{IndexMap, IndexSet};
use petgraph::Direction;
use petgraph::graph::NodeIndex;
use petgraph::stable_graph::StableGraph;
use petgraph::visit::EdgeRef;
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
use std::collections::{HashMap, HashSet};
/// Heuristic upper bound on chunk size before warning the user that the
/// extraction LLM call may be truncated. Not a hard limit.
const MAX_CHUNK_CHARS: usize = 24_000;
/// Maximum number of nodes the BFS may visit during a single graph_search.
/// Keeps the synchronous traversal bounded on dense graphs.
pub const MAX_GRAPH_NODES: usize = 500;
const EXTRACTION_PROMPT: &str = r#"Extract entities and relationships from the following text chunk.
@@ -89,16 +98,27 @@ impl Default for KnowledgeGraph {
impl KnowledgeGraph {
pub fn merge(&mut self, doc_id: DocumentId, result: ExtractionResult) {
let mut chunk_nodes: Vec<u32> = vec![];
let mut chunk_nodes: IndexSet<u32> = IndexSet::new();
for extracted in &result.entities {
let key = extracted.name.to_lowercase();
let normalized_type = extracted.entity_type.to_uppercase();
let node_raw = if let Some(&existing) = self.entity_index.get(&key) {
let idx = NodeIndex::new(existing as usize);
if self.graph.contains_node(idx) {
let node = &mut self.graph[idx];
if node.entity_type == "OTHER" && normalized_type != "OTHER" {
node.entity_type = normalized_type;
}
if node.description.is_none() {
node.description = extracted.description.clone();
}
}
existing
} else {
let entity = Entity {
name: extracted.name.clone(),
entity_type: extracted.entity_type.clone(),
entity_type: normalized_type,
description: extracted.description.clone(),
};
let idx = self.graph.add_node(entity);
@@ -106,7 +126,7 @@ impl KnowledgeGraph {
self.entity_index.insert(key, raw);
raw
};
chunk_nodes.push(node_raw);
chunk_nodes.insert(node_raw);
}
for extracted in &result.relationships {
@@ -118,11 +138,14 @@ impl KnowledgeGraph {
) {
let from_idx = NodeIndex::new(from_raw as usize);
let to_idx = NodeIndex::new(to_raw as usize);
// Avoid duplicate edges
if !self.graph.contains_edge(from_idx, to_idx) {
let already_exists = self
.graph
.edges_connecting(from_idx, to_idx)
.any(|e| e.weight().relation_type == extracted.relation_type);
if !already_exists {
let rel = Relationship {
relation_type: extracted.relation_type.clone(),
weight: extracted.weight.unwrap_or(1.0),
weight: extracted.weight.unwrap_or(1.0).clamp(0.0, 1.0),
};
self.graph.add_edge(from_idx, to_idx, rel);
}
@@ -158,6 +181,10 @@ impl KnowledgeGraph {
.filter(|raw| !still_used.contains(raw))
.collect();
if to_remove.is_empty() {
return;
}
for raw in to_remove {
let idx = NodeIndex::new(raw as usize);
if self.graph.contains_node(idx) {
@@ -166,6 +193,57 @@ impl KnowledgeGraph {
self.entity_index.swap_remove(&name);
}
}
self.compact();
}
/// Rebuild the internal graph with consecutive node indices. Eliminates
/// the null tombstone slots that petgraph's StableGraph accumulates after
/// repeated `remove_node` calls, keeping serialized YAML size in check.
fn compact(&mut self) {
let mut new_graph: StableGraph<Entity, Relationship> = StableGraph::new();
let mut old_to_new: HashMap<u32, u32> = HashMap::new();
for &old_raw in self.entity_index.values() {
let old_idx = NodeIndex::new(old_raw as usize);
if self.graph.contains_node(old_idx) {
let entity = self.graph[old_idx].clone();
let new_idx = new_graph.add_node(entity);
old_to_new.insert(old_raw, new_idx.index() as u32);
}
}
for edge_idx in self.graph.edge_indices() {
if let Some((from, to)) = self.graph.edge_endpoints(edge_idx) {
let from_raw = from.index() as u32;
let to_raw = to.index() as u32;
if let (Some(&new_from), Some(&new_to)) =
(old_to_new.get(&from_raw), old_to_new.get(&to_raw))
{
let rel = self.graph[edge_idx].clone();
new_graph.add_edge(
NodeIndex::new(new_from as usize),
NodeIndex::new(new_to as usize),
rel,
);
}
}
}
for raw in self.entity_index.values_mut() {
if let Some(&new_raw) = old_to_new.get(raw) {
*raw = new_raw;
}
}
for node_raws in self.document_entities.values_mut() {
*node_raws = node_raws
.iter()
.filter_map(|raw| old_to_new.get(raw).copied())
.collect();
}
self.graph = new_graph;
}
pub fn build_node_to_docs(&self) -> IndexMap<u32, Vec<DocumentId>> {
@@ -179,30 +257,79 @@ impl KnowledgeGraph {
map
}
pub fn expand_neighbors(&self, seed_nodes: &[u32], hops: usize) -> Vec<u32> {
let mut expanded: indexmap::IndexSet<u32> = seed_nodes.iter().copied().collect();
let mut frontier: Vec<u32> = seed_nodes.to_vec();
/// BFS from seed nodes with weight-decayed scoring.
///
/// Seed node scores are provided by the caller (typically token-overlap
/// ratios). Each neighbor's score is `edge_weight * parent_score`, so
/// strongly-connected neighbors rank higher and weakly-connected ones
/// naturally contribute less. Traversal is capped at `MAX_GRAPH_NODES`
/// total nodes; the highest-scored frontier nodes are expanded first so
/// the budget is spent on the most relevant entities.
///
/// Returns a map of raw node index → score (includes seed nodes).
pub fn expand_neighbors_scored(
&self,
seed_scores: &[(u32, f32)],
hops: usize,
) -> IndexMap<u32, f32> {
let mut node_scores: IndexMap<u32, f32> = IndexMap::new();
for &(raw, score) in seed_scores {
node_scores.insert(raw, score);
}
let mut frontier: Vec<(u32, f32)> = seed_scores.to_vec();
for _ in 0..hops {
let mut next_frontier: Vec<u32> = vec![];
for &raw in &frontier {
let idx = NodeIndex::new(raw as usize);
if self.graph.contains_node(idx) {
for dir in [Direction::Outgoing, Direction::Incoming] {
for neighbor in self.graph.neighbors_directed(idx, dir) {
let n = neighbor.index() as u32;
if expanded.insert(n) {
next_frontier.push(n);
if node_scores.len() >= MAX_GRAPH_NODES {
break;
}
frontier.sort_unstable_by(|a, b| {
b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)
});
let mut next_frontier: Vec<(u32, f32)> = vec![];
'nodes: for (raw, parent_score) in &frontier {
let idx = NodeIndex::new(*raw as usize);
if !self.graph.contains_node(idx) {
continue;
}
for dir in [Direction::Outgoing, Direction::Incoming] {
for edge_ref in self.graph.edges_directed(idx, dir) {
let neighbor_idx = match dir {
Direction::Outgoing => edge_ref.target(),
Direction::Incoming => edge_ref.source(),
};
let neighbor_raw = neighbor_idx.index() as u32;
let candidate = edge_ref.weight().weight * parent_score;
match node_scores.entry(neighbor_raw) {
indexmap::map::Entry::Vacant(e) => {
e.insert(candidate);
next_frontier.push((neighbor_raw, candidate));
}
indexmap::map::Entry::Occupied(mut e) => {
if candidate > *e.get() {
*e.get_mut() = candidate;
}
}
}
if node_scores.len() >= MAX_GRAPH_NODES {
break 'nodes;
}
}
}
}
frontier = next_frontier;
if frontier.is_empty() {
break;
}
}
expanded.into_iter().collect()
node_scores
}
}
@@ -213,6 +340,14 @@ pub async fn extract_entities(
chunk: &str,
prompt_template: Option<&str>,
) -> Result<ExtractionResult> {
if chunk.len() > MAX_CHUNK_CHARS {
warn!(
"Entity extraction chunk is {} chars (heuristic limit: {}); \
the LLM response may be truncated",
chunk.len(),
MAX_CHUNK_CHARS
);
}
let template = prompt_template.unwrap_or(EXTRACTION_PROMPT);
let prompt = template.replace("__CHUNK__", chunk);
let mut messages = vec![Message::new(
@@ -250,3 +385,350 @@ pub async fn extract_entities(
serde_json::from_str::<ExtractionResult>(&json)
.context("Failed to parse entity extraction JSON")
}
#[cfg(test)]
mod tests {
use super::*;
fn entity(name: &str, entity_type: &str) -> ExtractedEntity {
ExtractedEntity {
name: name.to_string(),
entity_type: entity_type.to_string(),
description: None,
}
}
fn rel(from: &str, to: &str, rel_type: &str, weight: f32) -> ExtractedRelationship {
ExtractedRelationship {
from: from.to_string(),
to: to.to_string(),
relation_type: rel_type.to_string(),
weight: Some(weight),
}
}
fn doc(id: usize) -> DocumentId {
DocumentId(id)
}
fn extraction(
entities: Vec<ExtractedEntity>,
rels: Vec<ExtractedRelationship>,
) -> ExtractionResult {
ExtractionResult {
entities,
relationships: rels,
}
}
#[test]
fn merge_deduplicates_by_lowercase_name() {
let mut kg = KnowledgeGraph::default();
kg.merge(
doc(0),
extraction(
vec![
entity("Python", "TECHNOLOGY"),
entity("python", "TECHNOLOGY"),
],
vec![],
),
);
assert_eq!(kg.entity_index.len(), 1);
assert_eq!(kg.graph.node_count(), 1);
}
#[test]
fn merge_chunk_nodes_no_duplicate_doc_entries() {
let mut kg = KnowledgeGraph::default();
kg.merge(
doc(1),
extraction(
vec![
entity("Python", "TECHNOLOGY"),
entity("python", "TECHNOLOGY"),
],
vec![],
),
);
let count = kg.document_entities.get(&1).map(|v| v.len()).unwrap_or(0);
assert_eq!(
count, 1,
"duplicate entity in one chunk should produce one doc_entity entry"
);
}
#[test]
fn merge_normalizes_entity_type_to_uppercase() {
let mut kg = KnowledgeGraph::default();
kg.merge(
doc(0),
extraction(vec![entity("Django", "technology")], vec![]),
);
let raw = kg.entity_index["django"];
assert_eq!(
kg.graph[NodeIndex::new(raw as usize)].entity_type,
"TECHNOLOGY"
);
}
#[test]
fn merge_promotes_type_from_other_to_specific() {
let mut kg = KnowledgeGraph::default();
kg.merge(doc(0), extraction(vec![entity("Python", "OTHER")], vec![]));
kg.merge(
doc(1),
extraction(vec![entity("Python", "TECHNOLOGY")], vec![]),
);
let raw = kg.entity_index["python"];
assert_eq!(
kg.graph[NodeIndex::new(raw as usize)].entity_type,
"TECHNOLOGY"
);
}
#[test]
fn merge_does_not_demote_specific_type_to_other() {
let mut kg = KnowledgeGraph::default();
kg.merge(
doc(0),
extraction(vec![entity("Python", "TECHNOLOGY")], vec![]),
);
kg.merge(doc(1), extraction(vec![entity("Python", "OTHER")], vec![]));
let raw = kg.entity_index["python"];
assert_eq!(
kg.graph[NodeIndex::new(raw as usize)].entity_type,
"TECHNOLOGY"
);
}
#[test]
fn merge_allows_multiple_relation_types_between_same_pair() {
let mut kg = KnowledgeGraph::default();
kg.merge(
doc(0),
extraction(
vec![
entity("Python", "TECHNOLOGY"),
entity("Django", "TECHNOLOGY"),
],
vec![rel("Python", "Django", "implements", 0.9)],
),
);
kg.merge(
doc(1),
extraction(
vec![
entity("Python", "TECHNOLOGY"),
entity("Django", "TECHNOLOGY"),
],
vec![rel("Python", "Django", "uses", 0.8)],
),
);
let from_idx = NodeIndex::new(kg.entity_index["python"] as usize);
let to_idx = NodeIndex::new(kg.entity_index["django"] as usize);
let count = kg.graph.edges_connecting(from_idx, to_idx).count();
assert_eq!(
count, 2,
"two different relation types should produce two edges"
);
}
#[test]
fn merge_deduplicates_same_relation_type() {
let mut kg = KnowledgeGraph::default();
kg.merge(
doc(0),
extraction(
vec![entity("A", "CONCEPT"), entity("B", "CONCEPT")],
vec![rel("A", "B", "uses", 1.0)],
),
);
kg.merge(
doc(1),
extraction(
vec![entity("A", "CONCEPT"), entity("B", "CONCEPT")],
vec![rel("A", "B", "uses", 0.5)],
),
);
let from_idx = NodeIndex::new(kg.entity_index["a"] as usize);
let to_idx = NodeIndex::new(kg.entity_index["b"] as usize);
let count = kg.graph.edges_connecting(from_idx, to_idx).count();
assert_eq!(
count, 1,
"same relation type should not create a duplicate edge"
);
}
#[test]
fn remove_documents_preserves_entity_shared_across_docs() {
let mut kg = KnowledgeGraph::default();
kg.merge(
doc(0),
extraction(
vec![entity("Python", "TECHNOLOGY"), entity("A", "CONCEPT")],
vec![],
),
);
kg.merge(
doc(1),
extraction(
vec![entity("Python", "TECHNOLOGY"), entity("B", "CONCEPT")],
vec![],
),
);
kg.remove_documents(&[doc(0)]);
assert!(
kg.entity_index.contains_key("python"),
"shared entity should survive"
);
assert!(
!kg.entity_index.contains_key("a"),
"exclusive entity should be removed"
);
assert!(
kg.entity_index.contains_key("b"),
"other doc's entity should survive"
);
}
#[test]
fn remove_documents_noop_on_empty_slice() {
let mut kg = KnowledgeGraph::default();
kg.merge(doc(0), extraction(vec![entity("X", "CONCEPT")], vec![]));
kg.remove_documents(&[]);
assert_eq!(kg.entity_index.len(), 1);
}
#[test]
fn remove_documents_compacts_graph() {
let mut kg = KnowledgeGraph::default();
// doc 0: A, B with an edge
kg.merge(
doc(0),
extraction(
vec![entity("A", "CONCEPT"), entity("B", "CONCEPT")],
vec![rel("A", "B", "uses", 1.0)],
),
);
// doc 1: C only
kg.merge(doc(1), extraction(vec![entity("C", "CONCEPT")], vec![]));
kg.remove_documents(&[doc(0)]);
assert_eq!(kg.graph.node_count(), 1);
let c_raw = kg.entity_index["c"];
assert_eq!(
c_raw, 0,
"compacted graph should give surviving node index 0"
);
let refs = kg.document_entities.get(&1).cloned().unwrap_or_default();
assert_eq!(refs, vec![0u32]);
}
#[test]
fn expand_zero_hops_returns_seeds_only() {
let mut kg = KnowledgeGraph::default();
kg.merge(
doc(0),
extraction(
vec![entity("A", "CONCEPT"), entity("B", "CONCEPT")],
vec![rel("A", "B", "uses", 0.9)],
),
);
let a_raw = kg.entity_index["a"];
let result = kg.expand_neighbors_scored(&[(a_raw, 1.0)], 0);
assert_eq!(result.len(), 1);
assert_eq!(result[&a_raw], 1.0);
}
#[test]
fn expand_one_hop_decays_score_by_edge_weight() {
let mut kg = KnowledgeGraph::default();
kg.merge(
doc(0),
extraction(
vec![entity("A", "CONCEPT"), entity("B", "CONCEPT")],
vec![rel("A", "B", "uses", 0.8)],
),
);
let a_raw = kg.entity_index["a"];
let b_raw = kg.entity_index["b"];
let result = kg.expand_neighbors_scored(&[(a_raw, 1.0)], 1);
assert_eq!(result.len(), 2);
assert_eq!(result[&a_raw], 1.0);
let b_score = result[&b_raw];
assert!(
(b_score - 0.8).abs() < 1e-6,
"neighbor score should be edge_weight * parent_score = 0.8, got {b_score}"
);
}
#[test]
fn expand_incoming_edges_also_traversed() {
let mut kg = KnowledgeGraph::default();
// Edge goes B → A; seeding A should still discover B via incoming edge
kg.merge(
doc(0),
extraction(
vec![entity("A", "CONCEPT"), entity("B", "CONCEPT")],
vec![rel("B", "A", "uses", 0.7)],
),
);
let a_raw = kg.entity_index["a"];
let b_raw = kg.entity_index["b"];
let result = kg.expand_neighbors_scored(&[(a_raw, 1.0)], 1);
assert!(
result.contains_key(&b_raw),
"B should be reachable via incoming edge from A"
);
let b_score = result[&b_raw];
assert!((b_score - 0.7).abs() < 1e-6);
}
#[test]
fn expand_picks_best_path_score() {
let mut kg = KnowledgeGraph::default();
// A(0.5) → C(0.9): score 0.45; B(1.0) → C(0.4): score 0.40 — A→C path wins.
kg.merge(
doc(0),
extraction(
vec![
entity("A", "CONCEPT"),
entity("B", "CONCEPT"),
entity("C", "CONCEPT"),
],
vec![rel("A", "C", "uses", 0.9), rel("B", "C", "uses", 0.4)],
),
);
let a_raw = kg.entity_index["a"];
let b_raw = kg.entity_index["b"];
let c_raw = kg.entity_index["c"];
let seeds = vec![(a_raw, 0.5f32), (b_raw, 1.0f32)];
let result = kg.expand_neighbors_scored(&seeds, 1);
let c_score = result[&c_raw];
// Best path: B(1.0) * 0.4 = 0.4, A(0.5) * 0.9 = 0.45 → should be 0.45
assert!(
(c_score - 0.45).abs() < 1e-6,
"C score should reflect best path (0.45), got {c_score}"
);
}
#[test]
fn build_node_to_docs_maps_shared_entity_to_multiple_docs() {
let mut kg = KnowledgeGraph::default();
kg.merge(
doc(0),
extraction(vec![entity("Python", "TECHNOLOGY")], vec![]),
);
kg.merge(
doc(1),
extraction(vec![entity("Python", "TECHNOLOGY")], vec![]),
);
let n2d = kg.build_node_to_docs();
let raw = kg.entity_index["python"];
let docs = &n2d[&raw];
assert!(docs.contains(&DocumentId(0)));
assert!(docs.contains(&DocumentId(1)));
}
}
+51 -41
View File
@@ -25,6 +25,8 @@ use std::{
};
use tokio::time::sleep;
const BM25_SEED_SCORE: f32 = 0.5;
const RAG_TEMPLATE: &str = r#"Answer the query based on the context while respecting the rules. (user query, some textual context and rules, all inside xml tags)
<context>
@@ -752,14 +754,14 @@ impl Rag {
bail!("No RAG files");
}
if self.data.extractor_model.is_some()
&& !new_doc_contents.is_empty()
if !new_doc_contents.is_empty()
&& let Some(extractor_model_id) = self.data.extractor_model.clone()
{
match Model::retrieve_model(&self.app_config, &extractor_model_id, ModelType::Chat) {
Ok(model) => match self.create_embeddings_client(model) {
Ok(client) => {
let total = new_doc_contents.len();
let mut failures = 0usize;
for (i, (doc_id, content)) in new_doc_contents.into_iter().enumerate() {
progress(
&spinner,
@@ -774,14 +776,21 @@ impl Rag {
{
Ok(result) => self.data.knowledge_graph.merge(doc_id, result),
Err(e) => {
debug!("Entity extraction failed for doc {doc_id:?}: {e}")
warn!("Entity extraction failed for doc {doc_id:?}: {e}");
failures += 1;
}
}
}
if failures > 0 {
progress(
&spinner,
format!("Entity extraction: {failures}/{total} chunks failed"),
);
}
}
Err(e) => debug!("Failed to create extractor client: {e}"),
Err(e) => warn!("Failed to create extractor client: {e}"),
},
Err(e) => debug!("Extractor model not found: {e}"),
Err(e) => warn!("Extractor model not found: {e}"),
}
}
@@ -930,9 +939,31 @@ impl Rag {
if kg.entity_index.is_empty() {
return vec![];
}
let query_lower = query.to_lowercase();
let mut seed_nodes: Vec<u32> = kg
let query_lower = query.to_lowercase();
let query_tokens: Vec<&str> = query_lower.split_whitespace().collect();
let token_count = query_tokens.len().max(1);
let score_node = |raw: u32| -> f32 {
let idx = NodeIndex::new(raw as usize);
if !kg.graph.contains_node(idx) {
return 0.0;
}
let entity = &kg.graph[idx];
let combined = format!(
"{} {}",
entity.name,
entity.description.as_deref().unwrap_or("")
)
.to_lowercase();
query_tokens
.iter()
.filter(|t| combined.contains(*t))
.count() as f32
/ token_count as f32
};
let mut seed_scores: Vec<(u32, f32)> = kg
.entity_index
.iter()
.filter(|(name, _)| {
@@ -946,52 +977,31 @@ impl Rag {
.any(|token| token.trim_matches(|c: char| !c.is_alphanumeric()) == name_str)
}
})
.map(|(_, &raw)| raw)
.map(|(_, &raw)| (raw, score_node(raw).max(BM25_SEED_SCORE)))
.collect();
if seed_nodes.is_empty() {
if seed_scores.is_empty() {
let bm25_results = self.bm25.search(query, top_k * 2);
'outer: for result in bm25_results {
if let Some(node_raws) = kg.document_entities.get(&result.document.id.0) {
seed_nodes.extend(node_raws.iter().copied());
if seed_nodes.len() >= top_k {
break 'outer;
for &raw in node_raws {
seed_scores.push((raw, BM25_SEED_SCORE));
if seed_scores.len() >= top_k {
break 'outer;
}
}
}
}
}
if seed_nodes.is_empty() {
if seed_scores.is_empty() {
return vec![];
}
let hops = self.data.graph_hops.unwrap_or(1);
let expanded = kg.expand_neighbors(&seed_nodes, hops);
let query_tokens: Vec<&str> = query_lower.split_whitespace().collect();
let token_count = query_tokens.len().max(1);
let mut scored: Vec<(u32, f32)> = expanded
let mut scored: Vec<(u32, f32)> = kg
.expand_neighbors_scored(&seed_scores, hops)
.into_iter()
.map(|raw| {
let idx = NodeIndex::new(raw as usize);
let score = if kg.graph.contains_node(idx) {
let entity = &kg.graph[idx];
let combined = format!(
"{} {}",
entity.name,
entity.description.as_deref().unwrap_or("")
)
.to_lowercase();
query_tokens
.iter()
.filter(|t| combined.contains(*t))
.count() as f32
/ token_count as f32
} else {
0.0
};
(raw, score)
})
.collect();
scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(Ordering::Equal));
@@ -1349,11 +1359,11 @@ fn set_chunk_size(model: &Model) -> Result<usize> {
fn set_graph_hops(default_value: usize) -> Result<usize> {
let value = Text::new("Set graph expansion hops:")
.with_default(&default_value.to_string())
.with_help_message("Number of hops to expand from matched entities (1 = direct neighbors, 2 = neighbors of neighbors)")
.with_help_message("Number of hops to expand from matched entities (0 = seed nodes only, 1 = direct neighbors, 2 = neighbors of neighbors)")
.with_validator(move |text: &str| {
let out = match text.parse::<usize>() {
Ok(v) if v >= 1 => Validation::Valid,
_ => Validation::Invalid("Must be an integer >= 1".into()),
Ok(_) => Validation::Valid,
_ => Validation::Invalid("Must be a non-negative integer".into()),
};
Ok(out)
})