Compare commits
22
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8bbbd71fec
|
||
|
|
7f89a80f7e
|
||
|
|
19cca06db6
|
||
|
|
e8df9f119c
|
||
|
|
8abe297bfe
|
||
|
|
4ec6daff30
|
||
|
|
9c1067e544
|
||
|
|
2fe6704fbc | ||
|
|
dd40892ad5 | ||
|
|
ed86b7bfc3 | ||
|
|
f32d72a3f2 | ||
|
|
7b00638476 | ||
|
|
6733b3600f
|
||
|
|
de6010d525
|
||
|
|
9b0e26bade
|
||
|
|
ac40043c00
|
||
|
|
d8eec1d427
|
||
|
|
382916c3ee
|
||
|
|
bc3cc10a7b
|
||
|
|
b91f738209
|
||
|
|
4f0dae9b49
|
||
|
|
deb673ebc9
|
@@ -8,9 +8,9 @@ on:
|
|||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
inputs:
|
inputs:
|
||||||
bump_type:
|
bump_type:
|
||||||
description: "Specify the type of version bump"
|
description: 'Specify the type of version bump'
|
||||||
required: true
|
required: true
|
||||||
default: "patch"
|
default: 'patch'
|
||||||
type: choice
|
type: choice
|
||||||
options:
|
options:
|
||||||
- patch
|
- patch
|
||||||
@@ -46,7 +46,7 @@ jobs:
|
|||||||
- name: Set up Python
|
- name: Set up Python
|
||||||
uses: actions/setup-python@v4
|
uses: actions/setup-python@v4
|
||||||
with:
|
with:
|
||||||
python-version: "3.10"
|
python-version: '3.10'
|
||||||
|
|
||||||
- name: Install Commitizen
|
- name: Install Commitizen
|
||||||
run: |
|
run: |
|
||||||
@@ -108,17 +108,19 @@ jobs:
|
|||||||
|
|
||||||
cargo update || true
|
cargo update || true
|
||||||
|
|
||||||
|
sed -i "s|image: 'darkalex17/coyote:v[^']*'|image: 'darkalex17/coyote:v${VERSION}'|" assets/sbx-kit/spec.yaml
|
||||||
|
|
||||||
# Git config that helps in Act
|
# Git config that helps in Act
|
||||||
git config user.name "github-actions[bot]"
|
git config user.name "github-actions[bot]"
|
||||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||||
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
||||||
|
|
||||||
git status --porcelain
|
git status --porcelain
|
||||||
git diff --name-only -- Cargo.toml Cargo.lock || true
|
git diff --name-only -- Cargo.toml Cargo.lock assets/sbx-kit/spec.yaml || true
|
||||||
|
|
||||||
if ! git diff --quiet -- Cargo.toml Cargo.lock; then
|
if ! git diff --quiet -- Cargo.toml Cargo.lock assets/sbx-kit/spec.yaml; then
|
||||||
git add -u -- Cargo.toml Cargo.lock
|
git add -u -- Cargo.toml Cargo.lock assets/sbx-kit/spec.yaml
|
||||||
git commit -m "chore: bump Cargo.toml to $VERSION"
|
git commit -m "chore: bump Cargo.toml and sandbox image to $VERSION"
|
||||||
else
|
else
|
||||||
echo "No changes to commit (already at $VERSION)"
|
echo "No changes to commit (already at $VERSION)"
|
||||||
fi
|
fi
|
||||||
@@ -163,28 +165,28 @@ jobs:
|
|||||||
- target: aarch64-unknown-linux-musl
|
- target: aarch64-unknown-linux-musl
|
||||||
os: ubuntu-latest
|
os: ubuntu-latest
|
||||||
use-cross: true
|
use-cross: true
|
||||||
cargo-flags: ""
|
cargo-flags: ''
|
||||||
- target: aarch64-apple-darwin
|
- target: aarch64-apple-darwin
|
||||||
os: macos-latest
|
os: macos-latest
|
||||||
use-cross: true
|
use-cross: true
|
||||||
cargo-flags: ""
|
cargo-flags: ''
|
||||||
- target: aarch64-pc-windows-msvc
|
- target: aarch64-pc-windows-msvc
|
||||||
os: windows-latest
|
os: windows-latest
|
||||||
use-cross: true
|
use-cross: true
|
||||||
cargo-flags: ""
|
cargo-flags: ''
|
||||||
- target: x86_64-apple-darwin
|
- target: x86_64-apple-darwin
|
||||||
os: macos-latest
|
os: macos-latest
|
||||||
cargo-flags: ""
|
cargo-flags: ''
|
||||||
- target: x86_64-pc-windows-msvc
|
- target: x86_64-pc-windows-msvc
|
||||||
os: windows-latest
|
os: windows-latest
|
||||||
cargo-flags: ""
|
cargo-flags: ''
|
||||||
- target: x86_64-unknown-linux-musl
|
- target: x86_64-unknown-linux-musl
|
||||||
os: ubuntu-latest
|
os: ubuntu-latest
|
||||||
use-cross: true
|
use-cross: true
|
||||||
cargo-flags: ""
|
cargo-flags: ''
|
||||||
- target: x86_64-unknown-linux-gnu
|
- target: x86_64-unknown-linux-gnu
|
||||||
os: ubuntu-latest
|
os: ubuntu-latest
|
||||||
cargo-flags: ""
|
cargo-flags: ''
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Check if actor is repository owner
|
- name: Check if actor is repository owner
|
||||||
@@ -338,7 +340,7 @@ jobs:
|
|||||||
${{ steps.package.outputs.archive }}
|
${{ steps.package.outputs.archive }}
|
||||||
${{ steps.package.outputs.sha }}
|
${{ steps.package.outputs.sha }}
|
||||||
tag_name: v${{ env.RELEASE_VERSION }}
|
tag_name: v${{ env.RELEASE_VERSION }}
|
||||||
name: "v${{ env.RELEASE_VERSION }}"
|
name: 'v${{ env.RELEASE_VERSION }}'
|
||||||
body_path: artifacts/changelog.md
|
body_path: artifacts/changelog.md
|
||||||
prerelease: false
|
prerelease: false
|
||||||
|
|
||||||
@@ -456,3 +458,63 @@ jobs:
|
|||||||
if: env.ACT != 'true'
|
if: env.ACT != 'true'
|
||||||
with:
|
with:
|
||||||
registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
|
registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}
|
||||||
|
|
||||||
|
publish-sandbox-image:
|
||||||
|
needs: [publish-github-release]
|
||||||
|
name: Publish Sandbox Docker Image
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Check if actor is repository owner
|
||||||
|
if: ${{ github.actor != github.repository_owner && env.ACT != 'true' }}
|
||||||
|
run: |
|
||||||
|
echo "You are not authorized to run this workflow."
|
||||||
|
exit 1
|
||||||
|
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
fetch-depth: 1
|
||||||
|
|
||||||
|
- name: Ensure repository is up-to-date
|
||||||
|
if: env.ACT != 'true'
|
||||||
|
run: |
|
||||||
|
git fetch --all
|
||||||
|
git pull
|
||||||
|
|
||||||
|
- name: Get release artifacts
|
||||||
|
uses: actions/download-artifact@v4
|
||||||
|
with:
|
||||||
|
path: artifacts
|
||||||
|
merge-multiple: true
|
||||||
|
|
||||||
|
- name: Set version variable
|
||||||
|
run: |
|
||||||
|
version="$(cat artifacts/release-version)"
|
||||||
|
echo "version=$version" >> $GITHUB_ENV
|
||||||
|
|
||||||
|
- name: Validate release environment variables
|
||||||
|
run: |
|
||||||
|
echo "Release version: ${{ env.version }}"
|
||||||
|
|
||||||
|
- name: Set up QEMU
|
||||||
|
uses: docker/setup-qemu-action@v3
|
||||||
|
|
||||||
|
- name: Set up Docker Buildx
|
||||||
|
uses: docker/setup-buildx-action@v3
|
||||||
|
|
||||||
|
- name: Login to Docker Hub
|
||||||
|
if: env.ACT != 'true'
|
||||||
|
uses: docker/login-action@v3
|
||||||
|
with:
|
||||||
|
username: ${{ secrets.DOCKER_USERNAME }}
|
||||||
|
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||||
|
|
||||||
|
- name: Push to Docker Hub
|
||||||
|
uses: docker/build-push-action@v5
|
||||||
|
with:
|
||||||
|
context: .
|
||||||
|
file: Dockerfile
|
||||||
|
platforms: linux/amd64,linux/arm64
|
||||||
|
push: ${{ env.ACT != 'true' }}
|
||||||
|
tags: darkalex17/coyote:latest, darkalex17/coyote:${{ env.version }}
|
||||||
|
build-args: COYOTE_VERSION=${{ env.version }}
|
||||||
|
|||||||
@@ -1,371 +0,0 @@
|
|||||||
# Graph RAG Design Spec
|
|
||||||
|
|
||||||
## Status: COMPLETE
|
|
||||||
|
|
||||||
### Verified From Code (all claims backed by actual file reads)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Goal
|
|
||||||
|
|
||||||
Extend the existing two-signal hybrid search (vector HNSW + BM25 → RRF) to a three-signal hybrid
|
|
||||||
(vector + BM25 + knowledge graph → RRF). The graph captures entity/relationship knowledge extracted
|
|
||||||
from documents at ingestion time via an LLM call per chunk. At query time, graph traversal expands
|
|
||||||
context beyond semantic similarity.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Verified Current Architecture
|
|
||||||
|
|
||||||
### `Rag` struct (`src/rag/mod.rs:48`)
|
|
||||||
```rust
|
|
||||||
pub struct Rag {
|
|
||||||
app_config: Arc<AppConfig>,
|
|
||||||
name: String,
|
|
||||||
path: String,
|
|
||||||
embedding_model: Model,
|
|
||||||
hnsw: Hnsw<'static, f32, DistCosine>, // ephemeral, rebuilt on load
|
|
||||||
bm25: SearchEngine<DocumentId>, // ephemeral, rebuilt on load
|
|
||||||
data: RagData, // serialized to YAML
|
|
||||||
last_sources: RwLock<Option<String>>,
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### `RagData` struct (`src/rag/mod.rs:892`)
|
|
||||||
```rust
|
|
||||||
pub struct RagData {
|
|
||||||
pub embedding_model: String,
|
|
||||||
pub chunk_size: usize,
|
|
||||||
pub chunk_overlap: usize,
|
|
||||||
pub reranker_model: Option<String>,
|
|
||||||
pub top_k: usize,
|
|
||||||
pub batch_size: Option<usize>,
|
|
||||||
pub next_file_id: FileId,
|
|
||||||
pub document_paths: Vec<String>,
|
|
||||||
pub files: IndexMap<FileId, RagFile>,
|
|
||||||
#[serde(with = "serde_vectors")]
|
|
||||||
pub vectors: IndexMap<DocumentId, Vec<f32>>,
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### `RagData::new` callers (both need updating):
|
|
||||||
1. `Rag::init` (`src/rag/mod.rs:219`) — interactive init path
|
|
||||||
2. `Rag::resolve_init_data` (`src/rag/mod.rs:195`) — config-driven init path
|
|
||||||
|
|
||||||
### `Rag::create` (`src/rag/mod.rs:253`) — all init paths converge here:
|
|
||||||
```rust
|
|
||||||
pub fn create(app: &AppConfig, name: &str, path: &Path, data: RagData) -> Result<Self> {
|
|
||||||
let hnsw = data.build_hnsw();
|
|
||||||
let bm25 = data.build_bm25();
|
|
||||||
let embedding_model = Model::retrieve_model(app, &data.embedding_model, ModelType::Embedding)?;
|
|
||||||
let rag = Rag { app_config: Arc::new(app.clone()), name: name.to_string(),
|
|
||||||
path: path.display().to_string(), data, embedding_model, hnsw, bm25,
|
|
||||||
last_sources: RwLock::new(None) };
|
|
||||||
Ok(rag)
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### `hybrid_search` (`src/rag/mod.rs:710`)
|
|
||||||
```rust
|
|
||||||
async fn hybrid_search(&self, query: &str, top_k: usize, rerank_model: Option<&str>)
|
|
||||||
-> Result<Vec<(DocumentId, String)>>
|
|
||||||
```
|
|
||||||
Runs `vector_search` + `keyword_search` in parallel via `tokio::join!`, then either reranks or
|
|
||||||
applies `reciprocal_rank_fusion(vec![vector_ids, keyword_ids], vec![1.125, 1.0], top_k)`.
|
|
||||||
|
|
||||||
### `reciprocal_rank_fusion` (`src/rag/mod.rs:1186`) — standalone fn, already weight-parameterized:
|
|
||||||
```rust
|
|
||||||
fn reciprocal_rank_fusion(
|
|
||||||
list_of_document_ids: Vec<Vec<DocumentId>>,
|
|
||||||
list_of_weights: Vec<f32>,
|
|
||||||
top_k: usize,
|
|
||||||
) -> Vec<DocumentId>
|
|
||||||
```
|
|
||||||
|
|
||||||
### `RagData::del` (`src/rag/mod.rs:953`):
|
|
||||||
```rust
|
|
||||||
pub fn del(&mut self, file_ids: Vec<FileId>) {
|
|
||||||
for file_id in file_ids {
|
|
||||||
if let Some(file) = self.files.swap_remove(&file_id) {
|
|
||||||
for (document_index, _) in file.documents.iter().enumerate() {
|
|
||||||
let document_id = DocumentId::new(file_id, document_index);
|
|
||||||
self.vectors.swap_remove(&document_id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### `RagNode` (`src/graph/types.rs:331`):
|
|
||||||
```rust
|
|
||||||
pub struct RagNode {
|
|
||||||
pub documents: Vec<String>,
|
|
||||||
pub query: Option<String>,
|
|
||||||
pub top_k: Option<usize>,
|
|
||||||
pub embedding_model: Option<String>,
|
|
||||||
pub chunk_size: Option<usize>,
|
|
||||||
pub chunk_overlap: Option<usize>,
|
|
||||||
pub reranker_model: Option<String>,
|
|
||||||
pub batch_size: Option<usize>,
|
|
||||||
pub state_updates: Option<HashMap<String, String>>,
|
|
||||||
pub timeout: Option<u64>,
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### `Client` trait (`src/client/common.rs:40`):
|
|
||||||
- `async fn chat_completions(&self, input: Input) -> Result<ChatCompletionsOutput>` — needs `Input`
|
|
||||||
- `async fn chat_completions_inner(&self, client: &ReqwestClient, data: ChatCompletionsData) -> Result<ChatCompletionsOutput>` — accessible on `Box<dyn Client>` via vtable
|
|
||||||
- `async fn embeddings(&self, data: &EmbeddingsData) -> Result<Vec<Vec<f32>>>`
|
|
||||||
- `async fn rerank(&self, data: &RerankData) -> Result<RerankOutput>`
|
|
||||||
- `fn build_client(&self) -> Result<ReqwestClient>`
|
|
||||||
- `fn model(&self) -> &Model`
|
|
||||||
|
|
||||||
**Key finding**: `Input` cannot be constructed without `RequestContext` (which `Rag` doesn't have).
|
|
||||||
Instead, `extract_entities` uses `chat_completions_inner` directly with manually built
|
|
||||||
`ChatCompletionsData`. This is accessible via `Box<dyn Client>`.
|
|
||||||
|
|
||||||
### `Message` (`src/client/message.rs:22`):
|
|
||||||
```rust
|
|
||||||
pub fn new(role: MessageRole, content: MessageContent) -> Self
|
|
||||||
```
|
|
||||||
`MessageRole::User`, `MessageContent::Text(String)` — both confirmed.
|
|
||||||
|
|
||||||
### `AppConfig` RAG fields (`src/config/app_config.rs:71`):
|
|
||||||
```rust
|
|
||||||
pub rag_embedding_model: Option<String>,
|
|
||||||
pub rag_reranker_model: Option<String>,
|
|
||||||
pub rag_top_k: usize, // default: 5
|
|
||||||
pub rag_chunk_size: Option<usize>,
|
|
||||||
pub rag_chunk_overlap: Option<usize>,
|
|
||||||
pub rag_template: Option<String>,
|
|
||||||
```
|
|
||||||
|
|
||||||
### `patch_messages` — confirmed exported from `crate::client::*` (used in `input.rs:5`)
|
|
||||||
|
|
||||||
### `init_client(app_config, model)` — works for any `ModelType`, including `Chat`
|
|
||||||
|
|
||||||
### `ModelType` variants: `Chat`, `Embedding`, `Reranker` (confirmed in `model.rs`)
|
|
||||||
|
|
||||||
### petgraph serde: `NodeIndex` serializes as inner `u32`; `StableGraph` preserves index positions
|
|
||||||
through roundtrip. `IndexMap<DocumentId, Vec<NodeIndex>>` safe for YAML (DocumentId is newtype over
|
|
||||||
usize, serializes as integer key).
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## New Dependency
|
|
||||||
|
|
||||||
```toml
|
|
||||||
petgraph = { version = "0.7", features = ["serde-1"] }
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## New File: `src/rag/graph.rs`
|
|
||||||
|
|
||||||
All graph types and extraction logic. Module declared in `mod.rs` as `mod graph; use self::graph::*;`.
|
|
||||||
|
|
||||||
### Types:
|
|
||||||
- `Entity { name: String, entity_type: String, description: Option<String> }`
|
|
||||||
- `Relationship { relation_type: String, weight: f32 }`
|
|
||||||
- `ExtractionResult { entities: Vec<ExtractedEntity>, relationships: Vec<ExtractedRelationship> }`
|
|
||||||
- `ExtractedEntity { name: String, r#type: String, description: Option<String> }`
|
|
||||||
- `ExtractedRelationship { from: String, to: String, r#type: String, weight: Option<f32> }`
|
|
||||||
- `KnowledgeGraph { graph: StableGraph<Entity, Relationship>, entity_index: IndexMap<String, NodeIndex>, document_entities: IndexMap<DocumentId, Vec<NodeIndex>> }`
|
|
||||||
|
|
||||||
### Key methods on `KnowledgeGraph`:
|
|
||||||
- `merge(doc_id: DocumentId, result: ExtractionResult)` — merges extraction into graph
|
|
||||||
- `remove_documents(ids: &[DocumentId])` — removes entities exclusive to deleted documents
|
|
||||||
- `build_node_to_docs(&self) -> IndexMap<NodeIndex, Vec<DocumentId>>` — ephemeral reverse map
|
|
||||||
|
|
||||||
### `extract_entities(client: &dyn Client, chunk: &str) -> Result<ExtractionResult>`:
|
|
||||||
- Builds `ChatCompletionsData` manually (no `Input` needed)
|
|
||||||
- Calls `patch_messages` then `client.chat_completions_inner(&reqwest_client, data).await`
|
|
||||||
- Strips markdown code fences from response before JSON parse
|
|
||||||
- Temperature: `Some(0.0)` for deterministic extraction
|
|
||||||
|
|
||||||
### Extraction prompt: structured JSON output requesting entities + relationships
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Changes to `src/rag/mod.rs`
|
|
||||||
|
|
||||||
### `Rag` struct — add one ephemeral field:
|
|
||||||
```rust
|
|
||||||
node_to_docs: IndexMap<NodeIndex, Vec<DocumentId>>, // ephemeral, rebuilt on load
|
|
||||||
```
|
|
||||||
|
|
||||||
### `Rag::create` — build node_to_docs before moving data:
|
|
||||||
```rust
|
|
||||||
let node_to_docs = data.knowledge_graph.build_node_to_docs();
|
|
||||||
// then add to struct literal
|
|
||||||
```
|
|
||||||
|
|
||||||
### `Rag` Clone impl — add:
|
|
||||||
```rust
|
|
||||||
node_to_docs: self.data.knowledge_graph.build_node_to_docs(),
|
|
||||||
```
|
|
||||||
|
|
||||||
### `RagData` struct — three new fields (all `#[serde(default)]` for backward compat):
|
|
||||||
```rust
|
|
||||||
#[serde(default)]
|
|
||||||
pub graph_enabled: bool,
|
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
||||||
pub extractor_model: Option<String>,
|
|
||||||
#[serde(default)]
|
|
||||||
pub knowledge_graph: KnowledgeGraph,
|
|
||||||
```
|
|
||||||
|
|
||||||
### `RagData::new` — two new params: `graph_enabled: bool, extractor_model: Option<String>`
|
|
||||||
|
|
||||||
### `RagData::del` — collect doc_ids during existing loop, call `remove_documents` at end:
|
|
||||||
```rust
|
|
||||||
let mut doc_ids_to_remove = vec![];
|
|
||||||
for file_id in file_ids {
|
|
||||||
if let Some(file) = self.files.swap_remove(&file_id) {
|
|
||||||
for (document_index, _) in file.documents.iter().enumerate() {
|
|
||||||
let document_id = DocumentId::new(file_id, document_index);
|
|
||||||
self.vectors.swap_remove(&document_id);
|
|
||||||
doc_ids_to_remove.push(document_id);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
self.knowledge_graph.remove_documents(&doc_ids_to_remove);
|
|
||||||
```
|
|
||||||
|
|
||||||
### `Rag::init` (line 219) — add two params to `RagData::new`:
|
|
||||||
```rust
|
|
||||||
app.rag_graph_enabled,
|
|
||||||
app.rag_extractor_model.clone(),
|
|
||||||
```
|
|
||||||
|
|
||||||
### `resolve_init_data` — resolve from config+app, pass to `RagData::new`:
|
|
||||||
```rust
|
|
||||||
let graph_enabled = config.graph_enabled.unwrap_or(app.rag_graph_enabled);
|
|
||||||
let extractor_model = config.extractor_model.clone().or_else(|| app.rag_extractor_model.clone());
|
|
||||||
```
|
|
||||||
|
|
||||||
### `sync_documents` — entity extraction block after `rag_files` built, before embedding:
|
|
||||||
```rust
|
|
||||||
if self.data.graph_enabled {
|
|
||||||
if let Some(extractor_model_id) = self.data.extractor_model.clone() {
|
|
||||||
let model = Model::retrieve_model(&self.app_config, &extractor_model_id, ModelType::Chat)?;
|
|
||||||
let client = self.create_embeddings_client(model)?;
|
|
||||||
let total_chunks: usize = rag_files.iter().map(|f| f.documents.len()).sum();
|
|
||||||
let mut chunk_num = 0;
|
|
||||||
let file_offset = next_file_id;
|
|
||||||
for (batch_file_idx, rag_file) in rag_files.iter().enumerate() {
|
|
||||||
let file_id = file_offset + batch_file_idx;
|
|
||||||
for (doc_idx, doc) in rag_file.documents.iter().enumerate() {
|
|
||||||
chunk_num += 1;
|
|
||||||
progress(&spinner, format!("Extracting entities [{chunk_num}/{total_chunks}]"));
|
|
||||||
let doc_id = DocumentId::new(file_id, doc_idx);
|
|
||||||
match extract_entities(client.as_ref(), &doc.page_content).await {
|
|
||||||
Ok(result) => self.data.knowledge_graph.merge(doc_id, result),
|
|
||||||
Err(e) => debug!("Entity extraction failed for {doc_id:?}: {e}"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### After line 705 (after hnsw/bm25 rebuild in sync_documents):
|
|
||||||
```rust
|
|
||||||
self.node_to_docs = self.data.knowledge_graph.build_node_to_docs();
|
|
||||||
```
|
|
||||||
|
|
||||||
### `hybrid_search` — add third signal:
|
|
||||||
```rust
|
|
||||||
let graph_search_ids: Vec<DocumentId> = if self.data.graph_enabled
|
|
||||||
&& !self.data.knowledge_graph.entity_index.is_empty()
|
|
||||||
{
|
|
||||||
self.graph_search(query, &keyword_search_ids, top_k)
|
|
||||||
} else {
|
|
||||||
vec![]
|
|
||||||
};
|
|
||||||
// RRF: extend to 3-way when graph has results, fall back to 2-way otherwise
|
|
||||||
```
|
|
||||||
|
|
||||||
### New `graph_search` method (sync):
|
|
||||||
```rust
|
|
||||||
fn graph_search(&self, query: &str, bm25_anchor_ids: &[DocumentId], top_k: usize) -> Vec<DocumentId>
|
|
||||||
```
|
|
||||||
Phase 1: entity names from query via substring match in `entity_index`.
|
|
||||||
Phase 2: fallback — entities from top BM25 document chunks.
|
|
||||||
Phase 3: expand 1-hop neighbors in `StableGraph`.
|
|
||||||
Phase 4: score docs by entity overlap ratio, return top_k.
|
|
||||||
|
|
||||||
### `RagInitConfig` — two new fields:
|
|
||||||
```rust
|
|
||||||
pub graph_enabled: Option<bool>,
|
|
||||||
pub extractor_model: Option<String>,
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Changes to `src/config/app_config.rs`
|
|
||||||
|
|
||||||
New fields alongside existing `rag_*` block:
|
|
||||||
```rust
|
|
||||||
pub rag_graph_enabled: bool, // default: false
|
|
||||||
pub rag_extractor_model: Option<String>, // default: None
|
|
||||||
```
|
|
||||||
Defaults, env var overrides, and propagation all follow the same pattern as existing `rag_*` fields.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Changes to `src/graph/types.rs` — `RagNode`
|
|
||||||
|
|
||||||
```rust
|
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
||||||
pub graph_enabled: Option<bool>,
|
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
||||||
pub extractor_model: Option<String>,
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Changes to `src/config/agent.rs`
|
|
||||||
|
|
||||||
Pass new fields through to `RagInitConfig`:
|
|
||||||
```rust
|
|
||||||
graph_enabled: rag_node.graph_enabled,
|
|
||||||
extractor_model: rag_node.extractor_model.clone(),
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Backward Compatibility
|
|
||||||
|
|
||||||
- All new `RagData` fields have `#[serde(default)]` — old YAML files load without migration
|
|
||||||
- `graph_enabled` defaults `false` — existing RAG instances unchanged
|
|
||||||
- `graph_search_ids` empty → 2-way RRF runs (identical to current behavior)
|
|
||||||
- `node_to_docs` rebuild on `create()` is O(n) over empty map for old instances
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## V1 Scope Exclusions
|
|
||||||
|
|
||||||
- LLM entity extraction from query at search time (V1 uses substring match + BM25 anchoring)
|
|
||||||
- Multi-hop traversal (field reserved, 1-hop only in V1)
|
|
||||||
- Entity embeddings / fuzzy entity lookup
|
|
||||||
- Bincode for large-corpus graph storage
|
|
||||||
- Gleaning / multi-pass extraction
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Implementation Progress
|
|
||||||
|
|
||||||
- [x] Cargo.toml — petgraph dependency
|
|
||||||
- [x] src/rag/graph.rs — new file
|
|
||||||
- [x] src/rag/mod.rs — mod/use, Rag struct, create, clone
|
|
||||||
- [x] src/rag/mod.rs — RagData fields, new, del
|
|
||||||
- [x] src/rag/mod.rs — Rag::init, resolve_init_data
|
|
||||||
- [x] src/rag/mod.rs — sync_documents extraction block
|
|
||||||
- [x] src/rag/mod.rs — hybrid_search + graph_search
|
|
||||||
- [x] src/rag/mod.rs — RagInitConfig fields
|
|
||||||
- [x] src/config/app_config.rs — new fields
|
|
||||||
- [x] src/config/mod.rs — propagation
|
|
||||||
- [x] src/graph/types.rs — RagNode fields
|
|
||||||
- [x] src/config/agent.rs — propagation
|
|
||||||
- [x] cargo check — clean (0 warnings, 1065 tests passing)
|
|
||||||
+70
@@ -0,0 +1,70 @@
|
|||||||
|
ARG COYOTE_VERSION
|
||||||
|
FROM docker/sandbox-templates:shell-docker
|
||||||
|
|
||||||
|
ARG COYOTE_VERSION
|
||||||
|
ARG TARGETARCH
|
||||||
|
|
||||||
|
ENV PATH="/home/agent/.cargo/bin:/home/agent/.local/bin:${PATH}"
|
||||||
|
|
||||||
|
USER root
|
||||||
|
|
||||||
|
RUN apt-get update && \
|
||||||
|
apt-get install -y --no-install-recommends \
|
||||||
|
jq curl git \
|
||||||
|
build-essential pkg-config \
|
||||||
|
cmake \
|
||||||
|
clang libclang-dev \
|
||||||
|
musl-tools \
|
||||||
|
libssl-dev \
|
||||||
|
pandoc \
|
||||||
|
bzip2 \
|
||||||
|
nano && \
|
||||||
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
RUN set -euo pipefail; \
|
||||||
|
USQL_VERSION=0.21.4; \
|
||||||
|
case "${TARGETARCH}" in \
|
||||||
|
amd64) USQL_ARCH=amd64 ;; \
|
||||||
|
arm64) USQL_ARCH=arm64 ;; \
|
||||||
|
*) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \
|
||||||
|
esac; \
|
||||||
|
TMPDIR=$(mktemp -d); \
|
||||||
|
curl -fsSL --retry 3 \
|
||||||
|
"https://github.com/xo/usql/releases/download/v${USQL_VERSION}/usql_static-${USQL_VERSION}-linux-${USQL_ARCH}.tar.bz2" \
|
||||||
|
-o "$TMPDIR/usql.tar.bz2"; \
|
||||||
|
tar -xjf "$TMPDIR/usql.tar.bz2" -C "$TMPDIR"; \
|
||||||
|
install -m 0755 "$TMPDIR/usql_static" /usr/local/bin/usql; \
|
||||||
|
rm -rf "$TMPDIR"
|
||||||
|
|
||||||
|
USER 1000
|
||||||
|
|
||||||
|
RUN curl -LsSf https://astral.sh/uv/install.sh | sh && \
|
||||||
|
printf '#!/bin/sh\nexec uv tool run "$@"\n' > "$HOME/.local/bin/uvx" && \
|
||||||
|
chmod +x "$HOME/.local/bin/uvx"
|
||||||
|
|
||||||
|
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
|
||||||
|
sh -s -- -y --default-toolchain stable --profile minimal && \
|
||||||
|
. "$HOME/.cargo/env" && \
|
||||||
|
cargo install --locked iwec && \
|
||||||
|
cargo install --locked ast-grep
|
||||||
|
|
||||||
|
USER root
|
||||||
|
|
||||||
|
RUN set -euo pipefail; \
|
||||||
|
case "${TARGETARCH}" in \
|
||||||
|
amd64) MUSL_TARGET=x86_64-unknown-linux-musl ;; \
|
||||||
|
arm64) MUSL_TARGET=aarch64-unknown-linux-musl ;; \
|
||||||
|
*) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \
|
||||||
|
esac; \
|
||||||
|
TMPDIR=$(mktemp -d); \
|
||||||
|
curl -fsSL --retry 3 \
|
||||||
|
"https://github.com/Dark-Alex-17/coyote/releases/download/v${COYOTE_VERSION}/coyote-${MUSL_TARGET}.tar.gz" \
|
||||||
|
-o "$TMPDIR/coyote.tar.gz"; \
|
||||||
|
tar -xzf "$TMPDIR/coyote.tar.gz" -C "$TMPDIR"; \
|
||||||
|
install -m 0755 "$TMPDIR/coyote" /home/agent/.cargo/bin/coyote; \
|
||||||
|
chown 1000:1000 /home/agent/.cargo/bin/coyote; \
|
||||||
|
rm -rf "$TMPDIR"
|
||||||
|
|
||||||
|
USER 1000
|
||||||
|
|
||||||
|
ENTRYPOINT ["coyote"]
|
||||||
@@ -100,6 +100,32 @@ To upgrade `coyote` using Homebrew:
|
|||||||
brew upgrade coyote
|
brew upgrade coyote
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Docker
|
||||||
|
Coyote is available as a Docker image on Docker Hub (`darkalex17/coyote`) for Linux amd64 and arm64.
|
||||||
|
Useful for CI, ephemeral environments, or anywhere you prefer not to install it natively.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker pull darkalex17/coyote
|
||||||
|
docker run --rm -it darkalex17/coyote
|
||||||
|
```
|
||||||
|
|
||||||
|
To persist your configuration across container runs, mount your existing config directory:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run --rm -it \
|
||||||
|
-v ~/.config/coyote:/home/agent/.config/coyote \
|
||||||
|
darkalex17/coyote
|
||||||
|
```
|
||||||
|
|
||||||
|
If you use the local vault provider and want your vault credentials available in the container, also mount the password file:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker run --rm -it \
|
||||||
|
-v ~/.config/coyote:/home/agent/.config/coyote \
|
||||||
|
-v ~/.coyote_password:/home/agent/.coyote_password:ro \
|
||||||
|
darkalex17/coyote
|
||||||
|
```
|
||||||
|
|
||||||
### Scripts
|
### Scripts
|
||||||
#### Linux/MacOS (`bash`)
|
#### Linux/MacOS (`bash`)
|
||||||
You can use the following command to run a bash script that downloads and installs the latest version of `coyote` for your
|
You can use the following command to run a bash script that downloads and installs the latest version of `coyote` for your
|
||||||
|
|||||||
@@ -10,5 +10,13 @@ set -e
|
|||||||
|
|
||||||
main() {
|
main() {
|
||||||
# shellcheck disable=SC2154
|
# shellcheck disable=SC2154
|
||||||
cat "$argc_path" >> "$LLM_OUTPUT" 2>&1 || echo "No such file or path: $argc_path" >> "$LLM_OUTPUT"
|
local path="$argc_path"
|
||||||
|
|
||||||
|
# An empty result is shown to the model as the opaque literal "DONE"; emit a note instead.
|
||||||
|
if [[ -f "$path" && ! -s "$path" ]]; then
|
||||||
|
echo "(empty file: $path)" >> "$LLM_OUTPUT"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
cat "$path" >> "$LLM_OUTPUT" 2>&1 || echo "No such file or path: $path" >> "$LLM_OUTPUT"
|
||||||
}
|
}
|
||||||
@@ -17,8 +17,8 @@ main() {
|
|||||||
local search_path="${argc_path:-.}"
|
local search_path="${argc_path:-.}"
|
||||||
|
|
||||||
if [[ ! -d "$search_path" ]]; then
|
if [[ ! -d "$search_path" ]]; then
|
||||||
echo "Error: directory not found: $search_path" >> "$LLM_OUTPUT"
|
echo "Error: directory not found: $search_path" >&2
|
||||||
return 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
local results
|
local results
|
||||||
|
|||||||
@@ -21,8 +21,8 @@ main() {
|
|||||||
local include_filter="${argc_include:-}"
|
local include_filter="${argc_include:-}"
|
||||||
|
|
||||||
if [[ ! -e "$search_path" ]]; then
|
if [[ ! -e "$search_path" ]]; then
|
||||||
echo "Error: path not found: $search_path" >> "$LLM_OUTPUT"
|
echo "Error: path not found: $search_path" >&2
|
||||||
return 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
local grep_args=(-nH --color=never)
|
local grep_args=(-nH --color=never)
|
||||||
|
|||||||
@@ -9,5 +9,18 @@ set -e
|
|||||||
|
|
||||||
main() {
|
main() {
|
||||||
# shellcheck disable=SC2154
|
# shellcheck disable=SC2154
|
||||||
ls -1 "$argc_path" >> "$LLM_OUTPUT" 2>&1 || echo "No such path: $argc_path" >> "$LLM_OUTPUT"
|
local path="$argc_path"
|
||||||
|
local output
|
||||||
|
|
||||||
|
if ! output=$(ls -1 "$path" 2>&1); then
|
||||||
|
echo "$output" >> "$LLM_OUTPUT"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# An empty result is shown to the model as the opaque literal "DONE"; emit a note instead.
|
||||||
|
if [[ -z "$output" ]]; then
|
||||||
|
echo "(empty directory: $path)" >> "$LLM_OUTPUT"
|
||||||
|
else
|
||||||
|
echo "$output" >> "$LLM_OUTPUT"
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
@@ -8,8 +8,8 @@ set -e
|
|||||||
# Use the grep tool to find specific content before reading, then read with offset to target the relevant section.
|
# Use the grep tool to find specific content before reading, then read with offset to target the relevant section.
|
||||||
|
|
||||||
# @option --path! The absolute path to the file or directory to read
|
# @option --path! The absolute path to the file or directory to read
|
||||||
# @option --offset The line number to start reading from (1-indexed, default: 1)
|
# @option --offset <INT> The line number to start reading from (1-indexed, default: 1)
|
||||||
# @option --limit The maximum number of lines to read (default: 2000)
|
# @option --limit <INT> The maximum number of lines to read (default: 2000)
|
||||||
|
|
||||||
# @env LLM_OUTPUT=/dev/stdout The output path
|
# @env LLM_OUTPUT=/dev/stdout The output path
|
||||||
|
|
||||||
@@ -23,8 +23,8 @@ main() {
|
|||||||
local limit="${argc_limit:-2000}"
|
local limit="${argc_limit:-2000}"
|
||||||
|
|
||||||
if [[ ! -e "$target" ]]; then
|
if [[ ! -e "$target" ]]; then
|
||||||
echo "Error: path not found: $target" >> "$LLM_OUTPUT"
|
echo "Error: path not found: $target" >&2
|
||||||
return 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ -d "$target" ]]; then
|
if [[ -d "$target" ]]; then
|
||||||
@@ -33,9 +33,20 @@ main() {
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
local total_lines file_bytes
|
local total_lines file_bytes
|
||||||
total_lines=$(wc -l < "$target" 2>/dev/null || echo 0)
|
# awk counts a final line that lacks a trailing newline; wc -l would undercount it by one.
|
||||||
|
total_lines=$(awk 'END { print NR }' "$target" 2>/dev/null || echo 0)
|
||||||
file_bytes=$(wc -c < "$target" 2>/dev/null || echo 0)
|
file_bytes=$(wc -c < "$target" 2>/dev/null || echo 0)
|
||||||
|
|
||||||
|
if [[ "$total_lines" -eq 0 ]]; then
|
||||||
|
echo "(file is empty: $target)" >> "$LLM_OUTPUT"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$offset" -gt "$total_lines" ]]; then
|
||||||
|
echo "(offset $offset is past the end of the file, which has $total_lines lines)" >> "$LLM_OUTPUT"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
if [[ "$file_bytes" -gt "$MAX_BYTES" ]] && [[ "$offset" -eq 1 ]] && [[ "$limit" -ge 2000 ]]; then
|
if [[ "$file_bytes" -gt "$MAX_BYTES" ]] && [[ "$offset" -eq 1 ]] && [[ "$limit" -ge 2000 ]]; then
|
||||||
{
|
{
|
||||||
echo "Warning: Large file (${file_bytes} bytes, ${total_lines} lines). Showing first ${limit} lines."
|
echo "Warning: Large file (${file_bytes} bytes, ${total_lines} lines). Showing first ${limit} lines."
|
||||||
@@ -48,7 +59,8 @@ main() {
|
|||||||
|
|
||||||
sed -n "${offset},${end_line}p" "$target" 2>/dev/null | {
|
sed -n "${offset},${end_line}p" "$target" 2>/dev/null | {
|
||||||
local line_num=$offset
|
local line_num=$offset
|
||||||
while IFS= read -r line; do
|
# `|| [[ -n "$line" ]]` keeps the final line when the file has no trailing newline.
|
||||||
|
while IFS= read -r line || [[ -n "$line" ]]; do
|
||||||
if [[ ${#line} -gt $MAX_LINE_LENGTH ]]; then
|
if [[ ${#line} -gt $MAX_LINE_LENGTH ]]; then
|
||||||
line="${line:0:$MAX_LINE_LENGTH}... (truncated)"
|
line="${line:0:$MAX_LINE_LENGTH}... (truncated)"
|
||||||
fi
|
fi
|
||||||
|
|||||||
@@ -552,7 +552,7 @@ patch_file() {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if (line ~ /^@@ /) {
|
if (line ~ /^@@/) {
|
||||||
mode = "hunk"
|
mode = "hunk"
|
||||||
hunkIndex++
|
hunkIndex++
|
||||||
patchLineIndex++
|
patchLineIndex++
|
||||||
@@ -585,6 +585,10 @@ patch_file() {
|
|||||||
|
|
||||||
if (hunkIndex == 0) {
|
if (hunkIndex == 0) {
|
||||||
print "error: no patch" > "/dev/stderr"
|
print "error: no patch" > "/dev/stderr"
|
||||||
|
print "" > "/dev/stderr"
|
||||||
|
print "No hunk header was found. Each hunk must start with a line beginning \"@@\"" > "/dev/stderr"
|
||||||
|
print "(for example \"@@ ... @@\" or \"@@ -1,4 +1,4 @@\"). Inside a hunk, context lines" > "/dev/stderr"
|
||||||
|
print "start with a single space, removed lines with \"-\", and added lines with \"+\"." > "/dev/stderr"
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -82,6 +82,10 @@ Additional hard rules:
|
|||||||
- If the evidence points to failing hardware or risk of data loss, stop, say so plainly, and present options before
|
- If the evidence points to failing hardware or risk of data loss, stop, say so plainly, and present options before
|
||||||
touching anything else.
|
touching anything else.
|
||||||
|
|
||||||
|
## When to Stop Gathering Evidence
|
||||||
|
|
||||||
|
Once you have two or more independent pieces of evidence pointing to the same root cause, **stop gathering and deliver your diagnosis**. Do not add more verification steps to verify your verification. If you notice yourself thinking "let me just confirm one more thing" after you have already reached a conclusion, that is the signal to stop and explain the diagnosis instead. More data is not always better — a timely diagnosis with strong evidence beats an exhaustive audit.
|
||||||
|
|
||||||
## Communication
|
## Communication
|
||||||
|
|
||||||
- Lead with what you found, not what you did. Then show the key evidence: the command and the relevant lines of its
|
- Lead with what you found, not what you did. Then show the key evidence: the command and the relevant lines of its
|
||||||
|
|||||||
+48
-106
@@ -5,7 +5,7 @@
|
|||||||
# sbx cp $HOME/.config/coyote/ testing:/home/agent/.config/
|
# sbx cp $HOME/.config/coyote/ testing:/home/agent/.config/
|
||||||
# sbx cp $HOME/.coyote_password testing:/home/agent/
|
# sbx cp $HOME/.coyote_password testing:/home/agent/
|
||||||
# sbx run testing --kit ./sbx-kit/
|
# sbx run testing --kit ./sbx-kit/
|
||||||
schemaVersion: "1"
|
schemaVersion: '1'
|
||||||
kind: sandbox
|
kind: sandbox
|
||||||
name: coyote
|
name: coyote
|
||||||
displayName: Coyote
|
displayName: Coyote
|
||||||
@@ -14,10 +14,10 @@ description: >
|
|||||||
CLI & REPL mode, RAG, AI tools & agents, MCP servers, skills, and macros.
|
CLI & REPL mode, RAG, AI tools & agents, MCP servers, skills, and macros.
|
||||||
|
|
||||||
sandbox:
|
sandbox:
|
||||||
image: "docker/sandbox-templates:shell-docker"
|
image: 'darkalex17/coyote:v0.7.4'
|
||||||
aiFilename: COYOTE.md
|
aiFilename: COYOTE.md
|
||||||
entrypoint:
|
entrypoint:
|
||||||
run: ["bash", "-lc", "exec /home/agent/.cargo/bin/coyote"]
|
run: ['bash', '-lc', 'exec /home/agent/.cargo/bin/coyote']
|
||||||
|
|
||||||
network:
|
network:
|
||||||
# Proxy-managed LLM providers: the proxy substitutes `proxy-managed` for
|
# Proxy-managed LLM providers: the proxy substitutes `proxy-managed` for
|
||||||
@@ -50,96 +50,96 @@ network:
|
|||||||
serviceAuth:
|
serviceAuth:
|
||||||
openai:
|
openai:
|
||||||
headerName: Authorization
|
headerName: Authorization
|
||||||
valueFormat: "Bearer %s"
|
valueFormat: 'Bearer %s'
|
||||||
anthropic:
|
anthropic:
|
||||||
headerName: x-api-key
|
headerName: x-api-key
|
||||||
valueFormat: "%s"
|
valueFormat: '%s'
|
||||||
gemini:
|
gemini:
|
||||||
headerName: x-goog-api-key
|
headerName: x-goog-api-key
|
||||||
valueFormat: "%s"
|
valueFormat: '%s'
|
||||||
cohere:
|
cohere:
|
||||||
headerName: Authorization
|
headerName: Authorization
|
||||||
valueFormat: "Bearer %s"
|
valueFormat: 'Bearer %s'
|
||||||
groq:
|
groq:
|
||||||
headerName: Authorization
|
headerName: Authorization
|
||||||
valueFormat: "Bearer %s"
|
valueFormat: 'Bearer %s'
|
||||||
openrouter:
|
openrouter:
|
||||||
headerName: Authorization
|
headerName: Authorization
|
||||||
valueFormat: "Bearer %s"
|
valueFormat: 'Bearer %s'
|
||||||
ai21:
|
ai21:
|
||||||
headerName: Authorization
|
headerName: Authorization
|
||||||
valueFormat: "Bearer %s"
|
valueFormat: 'Bearer %s'
|
||||||
cloudflare:
|
cloudflare:
|
||||||
headerName: Authorization
|
headerName: Authorization
|
||||||
valueFormat: "Bearer %s"
|
valueFormat: 'Bearer %s'
|
||||||
deepinfra:
|
deepinfra:
|
||||||
headerName: Authorization
|
headerName: Authorization
|
||||||
valueFormat: "Bearer %s"
|
valueFormat: 'Bearer %s'
|
||||||
deepseek:
|
deepseek:
|
||||||
headerName: Authorization
|
headerName: Authorization
|
||||||
valueFormat: "Bearer %s"
|
valueFormat: 'Bearer %s'
|
||||||
mistral:
|
mistral:
|
||||||
headerName: Authorization
|
headerName: Authorization
|
||||||
valueFormat: "Bearer %s"
|
valueFormat: 'Bearer %s'
|
||||||
perplexity:
|
perplexity:
|
||||||
headerName: Authorization
|
headerName: Authorization
|
||||||
valueFormat: "Bearer %s"
|
valueFormat: 'Bearer %s'
|
||||||
voyageai:
|
voyageai:
|
||||||
headerName: Authorization
|
headerName: Authorization
|
||||||
valueFormat: "Bearer %s"
|
valueFormat: 'Bearer %s'
|
||||||
xai:
|
xai:
|
||||||
headerName: Authorization
|
headerName: Authorization
|
||||||
valueFormat: "Bearer %s"
|
valueFormat: 'Bearer %s'
|
||||||
jina:
|
jina:
|
||||||
headerName: Authorization
|
headerName: Authorization
|
||||||
valueFormat: "Bearer %s"
|
valueFormat: 'Bearer %s'
|
||||||
ernie:
|
ernie:
|
||||||
headerName: Authorization
|
headerName: Authorization
|
||||||
valueFormat: "Bearer %s"
|
valueFormat: 'Bearer %s'
|
||||||
hunyuan:
|
hunyuan:
|
||||||
headerName: Authorization
|
headerName: Authorization
|
||||||
valueFormat: "Bearer %s"
|
valueFormat: 'Bearer %s'
|
||||||
minimax:
|
minimax:
|
||||||
headerName: Authorization
|
headerName: Authorization
|
||||||
valueFormat: "Bearer %s"
|
valueFormat: 'Bearer %s'
|
||||||
moonshot:
|
moonshot:
|
||||||
headerName: Authorization
|
headerName: Authorization
|
||||||
valueFormat: "Bearer %s"
|
valueFormat: 'Bearer %s'
|
||||||
qianwen:
|
qianwen:
|
||||||
headerName: Authorization
|
headerName: Authorization
|
||||||
valueFormat: "Bearer %s"
|
valueFormat: 'Bearer %s'
|
||||||
zhipuai:
|
zhipuai:
|
||||||
headerName: Authorization
|
headerName: Authorization
|
||||||
valueFormat: "Bearer %s"
|
valueFormat: 'Bearer %s'
|
||||||
allowedDomains:
|
allowedDomains:
|
||||||
# Coyote release + self-update + model-registry sync
|
# Coyote release + self-update + model-registry sync
|
||||||
- "github.com:443"
|
- 'github.com:443'
|
||||||
- "api.github.com:443"
|
- 'api.github.com:443'
|
||||||
- "raw.githubusercontent.com:443"
|
- 'raw.githubusercontent.com:443'
|
||||||
- "objects.githubusercontent.com:443"
|
- 'objects.githubusercontent.com:443'
|
||||||
- "*.githubusercontent.com:443"
|
- '*.githubusercontent.com:443'
|
||||||
# Coyote install paths (cargo install + uv + rustup + Python tool deps at runtime)
|
# Package managers and developer tools (cargo, uv, pip — useful at runtime for user installs)
|
||||||
- "crates.io:443"
|
- 'crates.io:443'
|
||||||
- "static.crates.io:443"
|
- 'static.crates.io:443'
|
||||||
- "pypi.org:443"
|
- 'pypi.org:443'
|
||||||
- "files.pythonhosted.org:443"
|
- 'files.pythonhosted.org:443'
|
||||||
- "astral.sh:443"
|
- 'astral.sh:443'
|
||||||
- "sh.rustup.rs:443"
|
- 'sh.rustup.rs:443'
|
||||||
- "static.rust-lang.org:443"
|
- 'static.rust-lang.org:443'
|
||||||
|
|
||||||
# LLM model OAuth + API endpoints
|
# LLM model OAuth + API endpoints
|
||||||
- "claude.ai:443"
|
- 'claude.ai:443'
|
||||||
- "console.anthropic.com:443"
|
- 'console.anthropic.com:443'
|
||||||
- "accounts.google.com:443"
|
- 'accounts.google.com:443'
|
||||||
# *.googleapis.com covers oauth2 + userinfo + VertexAI regional endpoints
|
# *.googleapis.com covers oauth2 + userinfo + VertexAI regional endpoints
|
||||||
# (*-aiplatform.googleapis.com). Do not narrow without re-checking VertexAI.
|
# (*-aiplatform.googleapis.com). Do not narrow without re-checking VertexAI.
|
||||||
- "*.googleapis.com:443"
|
- '*.googleapis.com:443'
|
||||||
|
|
||||||
# Bedrock and GitHub Models use signed / GitHub-PAT auth that the proxy
|
# Bedrock and GitHub Models use signed / GitHub-PAT auth that the proxy
|
||||||
# cannot rewrite. Domains are allow-listed; credentials must be injected
|
# cannot rewrite. Domains are allow-listed; credentials must be injected
|
||||||
# separately (see README "Extending").
|
# separately (see README "Extending").
|
||||||
- "*.amazonaws.com:443"
|
- '*.amazonaws.com:443'
|
||||||
- "models.inference.ai.azure.com:443"
|
- 'models.inference.ai.azure.com:443'
|
||||||
|
|
||||||
credentials:
|
credentials:
|
||||||
sources:
|
sources:
|
||||||
@@ -210,9 +210,10 @@ credentials:
|
|||||||
|
|
||||||
environment:
|
environment:
|
||||||
variables:
|
variables:
|
||||||
IS_SANDBOX: "1"
|
IS_SANDBOX: '1'
|
||||||
COYOTE_LOG_LEVEL: INFO
|
COYOTE_LOG_LEVEL: INFO
|
||||||
COYOTE_CONFIG_DIR: /home/agent/.config/coyote
|
COYOTE_CONFIG_DIR: /home/agent/.config/coyote
|
||||||
|
EDITOR: nano
|
||||||
proxyManaged:
|
proxyManaged:
|
||||||
- OPENAI_API_KEY
|
- OPENAI_API_KEY
|
||||||
- ANTHROPIC_API_KEY
|
- ANTHROPIC_API_KEY
|
||||||
@@ -238,73 +239,14 @@ environment:
|
|||||||
- ZHIPUAI_API_KEY
|
- ZHIPUAI_API_KEY
|
||||||
|
|
||||||
commands:
|
commands:
|
||||||
install:
|
|
||||||
- command: |
|
|
||||||
sudo apt-get update &&
|
|
||||||
sudo apt-get install -y \
|
|
||||||
jq curl git \
|
|
||||||
build-essential pkg-config \
|
|
||||||
cmake \
|
|
||||||
clang libclang-dev \
|
|
||||||
musl-tools \
|
|
||||||
libssl-dev \
|
|
||||||
pandoc \
|
|
||||||
bzip2
|
|
||||||
user: "1000"
|
|
||||||
description: Install system prerequisites (including pandoc for fetch_url_via_curl)
|
|
||||||
- command: |
|
|
||||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
||||||
if [ -f "$HOME/.local/bin/uv" ]; then
|
|
||||||
printf '#!/bin/sh\nexec uv tool run "$@"\n' > "$HOME/.local/bin/uvx"
|
|
||||||
chmod +x "$HOME/.local/bin/uvx"
|
|
||||||
fi
|
|
||||||
user: "1000"
|
|
||||||
description: Install uv and write a uvx shell wrapper (the installer may place a macOS binary at this path on Docker-for-Mac hosts, which the Linux container cannot execute)
|
|
||||||
- command: |
|
|
||||||
set -euo pipefail
|
|
||||||
USQL_VERSION=0.21.4
|
|
||||||
ARCH=$(uname -m)
|
|
||||||
case "$ARCH" in
|
|
||||||
x86_64) USQL_ARCH=amd64 ;;
|
|
||||||
aarch64) USQL_ARCH=arm64 ;;
|
|
||||||
*) echo "Unsupported arch for usql install: $ARCH" >&2; exit 1 ;;
|
|
||||||
esac
|
|
||||||
TMPDIR=$(mktemp -d)
|
|
||||||
trap 'rm -rf "$TMPDIR"' EXIT
|
|
||||||
curl -fsSL --retry 3 "https://github.com/xo/usql/releases/download/v${USQL_VERSION}/usql_static-${USQL_VERSION}-linux-${USQL_ARCH}.tar.bz2" -o "$TMPDIR/usql.tar.bz2"
|
|
||||||
tar -xjf "$TMPDIR/usql.tar.bz2" -C "$TMPDIR"
|
|
||||||
sudo install -m 0755 "$TMPDIR/usql_static" /usr/local/bin/usql
|
|
||||||
user: "1000"
|
|
||||||
description: Install the usql universal SQL CLI (used by the built-in sql agent and execute_sql_code tool)
|
|
||||||
- command: |
|
|
||||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
|
|
||||||
sh -s -- -y \
|
|
||||||
--default-toolchain stable \
|
|
||||||
--profile minimal \
|
|
||||||
--target x86_64-unknown-linux-musl
|
|
||||||
. "$HOME/.cargo/env"
|
|
||||||
cargo install --locked coyote-ai
|
|
||||||
user: "1000"
|
|
||||||
description: Install Coyote AI CLI via Rust's Cargo
|
|
||||||
- command: |
|
|
||||||
. "$HOME/.cargo/env"
|
|
||||||
cargo install --locked iwec
|
|
||||||
user: "1000"
|
|
||||||
description: Install the IWE MCP server binary (iwec) used by the built-in iwe MCP server and iwe-knowledge-base skill
|
|
||||||
- command: |
|
|
||||||
. "$HOME/.cargo/env"
|
|
||||||
cargo install --locked ast-grep
|
|
||||||
user: "1000"
|
|
||||||
description: Install ast-grep, used by the built-in ast_grep structural code search tool (and the explore agent)
|
|
||||||
|
|
||||||
startup:
|
startup:
|
||||||
- command:
|
- command:
|
||||||
[
|
[
|
||||||
"sh",
|
'sh',
|
||||||
"-c",
|
'-c',
|
||||||
'test -f "$HOME/.config/coyote/config.yaml" || coyote --info >/dev/null 2>&1 || true',
|
'test -f "$HOME/.config/coyote/config.yaml" || coyote --info >/dev/null 2>&1 || true',
|
||||||
]
|
]
|
||||||
user: "1000"
|
user: '1000'
|
||||||
background: false
|
background: false
|
||||||
description: Bootstrap Coyote config directory on first sandbox start
|
description: Bootstrap Coyote config directory on first sandbox start
|
||||||
|
|
||||||
|
|||||||
@@ -16,6 +16,10 @@ evidence yourself — never ask the user to run commands and paste output back.
|
|||||||
5. **State each hypothesis in one line before testing it.** Pivot openly when disproved.
|
5. **State each hypothesis in one line before testing it.** Pivot openly when disproved.
|
||||||
6. **Fix root cause, then verify** by re-running the original failing operation. No verification, no fix.
|
6. **Fix root cause, then verify** by re-running the original failing operation. No verification, no fix.
|
||||||
|
|
||||||
|
## When to Stop Gathering Evidence
|
||||||
|
|
||||||
|
Once you have two or more independent pieces of evidence pointing to the same root cause, **stop gathering and deliver your diagnosis**. Do not add more verification steps to verify your verification. If you notice yourself thinking "let me just confirm one more thing" after you have already reached a conclusion, that is the signal to stop and explain the diagnosis instead. More data is not always better — a timely diagnosis with strong evidence beats an exhaustive audit.
|
||||||
|
|
||||||
## Command Discipline
|
## Command Discipline
|
||||||
|
|
||||||
- Non-interactive and bounded, always: `--no-pager`, `-n`/`--since` on logs, `timeout 10` on anything that might
|
- Non-interactive and bounded, always: `--no-pager`, `-n`/`--since` on logs, `timeout 10` on anything that might
|
||||||
|
|||||||
+5
-1
@@ -134,6 +134,10 @@ enabled_mcp_servers: null # Which MCP servers to enable by default.
|
|||||||
# - slack
|
# - slack
|
||||||
# Example (comma-separated form):
|
# Example (comma-separated form):
|
||||||
# enabled_mcp_servers: github,slack,ddg-search
|
# enabled_mcp_servers: github,slack,ddg-search
|
||||||
|
no_workspace_mcp: false # Disable loading workspace-local MCP servers from .coyote/mcp.json (default: false).
|
||||||
|
# When false (the default), Coyote merges .coyote/mcp.json from the current directory
|
||||||
|
# into the global MCP registry at startup. Workspace entries shadow global ones on
|
||||||
|
# name collision. Set to true (or pass --no-workspace-mcp) to skip this entirely.
|
||||||
|
|
||||||
# ---- Skills ----
|
# ---- Skills ----
|
||||||
# Skills are modular knowledge or capability packs the LLM can load and unload mid-conversation.
|
# Skills are modular knowledge or capability packs the LLM can load and unload mid-conversation.
|
||||||
@@ -199,7 +203,7 @@ rag_chunk_size: null # Defines the size of chunks for document proce
|
|||||||
rag_chunk_overlap: null # Defines the overlap between chunks
|
rag_chunk_overlap: null # Defines the overlap between chunks
|
||||||
rag_extractor_model: null # LLM model for graph-based entity/relationship extraction; when set, enables a graph RAG signal alongside vector and BM25
|
rag_extractor_model: null # LLM model for graph-based entity/relationship extraction; when set, enables a graph RAG signal alongside vector and BM25
|
||||||
rag_extractor_prompt: null # Custom extraction prompt template; must contain __CHUNK__ placeholder; defaults to built-in prompt when null
|
rag_extractor_prompt: null # Custom extraction prompt template; must contain __CHUNK__ placeholder; defaults to built-in prompt when null
|
||||||
rag_graph_hops: 1 # Number of hops to expand from matched entities at query time (1 = direct neighbors; increase for denser graphs)
|
rag_graph_hops: 1 # Number of hops to expand from matched entities at query time (0 = seed nodes only; 1 = direct neighbors; increase for denser graphs)
|
||||||
# Defines the query structure using variables like __CONTEXT__, __SOURCES__, and __INPUT__ to tailor searches to specific needs
|
# Defines the query structure using variables like __CONTEXT__, __SOURCES__, and __INPUT__ to tailor searches to specific needs
|
||||||
rag_template: |
|
rag_template: |
|
||||||
Answer the query based on the context while respecting the rules. (user query, some textual context and rules, all inside xml tags)
|
Answer the query based on the context while respecting the rules. (user query, some textual context and rules, all inside xml tags)
|
||||||
|
|||||||
+1
-1
@@ -227,7 +227,7 @@ nodes:
|
|||||||
reranker_model: null # Optional reranker for hybrid-search results
|
reranker_model: null # Optional reranker for hybrid-search results
|
||||||
extractor_model: null # Optional chat model for graph-based entity/relationship extraction; enables graph RAG signal when set
|
extractor_model: null # Optional chat model for graph-based entity/relationship extraction; enables graph RAG signal when set
|
||||||
extractor_prompt: null # Optional custom extraction prompt; must contain __CHUNK__ placeholder; uses built-in prompt when null
|
extractor_prompt: null # Optional custom extraction prompt; must contain __CHUNK__ placeholder; uses built-in prompt when null
|
||||||
graph_hops: 1 # Graph expansion depth at query time (1 = direct neighbors; increase for denser knowledge graphs)
|
graph_hops: 1 # Graph expansion depth at query time (0 = seed nodes only; 1 = direct neighbors; increase for denser knowledge graphs)
|
||||||
batch_size: 100 # Optional embedding-request batch size
|
batch_size: 100 # Optional embedding-request batch size
|
||||||
state_updates: # {{output}} = { context: <str>, sources: [<path>, ...] }
|
state_updates: # {{output}} = { context: <str>, sources: [<path>, ...] }
|
||||||
context: "{{output.context}}" # writes `context` -> `reducers.context = concat`
|
context: "{{output.context}}" # writes `context` -> `reducers.context = concat`
|
||||||
|
|||||||
+152
-109
@@ -43,6 +43,10 @@ use std::io::{Read, stdin};
|
|||||||
),
|
),
|
||||||
)]
|
)]
|
||||||
pub struct Cli {
|
pub struct Cli {
|
||||||
|
/// Input text
|
||||||
|
#[arg(trailing_var_arg = true)]
|
||||||
|
text: Vec<String>,
|
||||||
|
|
||||||
/// Select a LLM model
|
/// Select a LLM model
|
||||||
#[arg(short, long, add = ArgValueCompleter::new(model_completer))]
|
#[arg(short, long, add = ArgValueCompleter::new(model_completer))]
|
||||||
pub model: Option<String>,
|
pub model: Option<String>,
|
||||||
@@ -52,30 +56,6 @@ pub struct Cli {
|
|||||||
/// Select a role
|
/// Select a role
|
||||||
#[arg(short, long, add = ArgValueCompleter::new(role_completer))]
|
#[arg(short, long, add = ArgValueCompleter::new(role_completer))]
|
||||||
pub role: Option<String>,
|
pub role: Option<String>,
|
||||||
/// Start or join a session
|
|
||||||
#[arg(short = 's', long, add = ArgValueCompleter::new(session_completer))]
|
|
||||||
pub session: Option<Option<String>>,
|
|
||||||
/// Ensure the session is empty
|
|
||||||
#[arg(long)]
|
|
||||||
pub empty_session: bool,
|
|
||||||
/// Ensure the new conversation is saved to the session
|
|
||||||
#[arg(long)]
|
|
||||||
pub save_session: bool,
|
|
||||||
/// Start an agent
|
|
||||||
#[arg(short = 'a', long, add = ArgValueCompleter::new(agent_completer))]
|
|
||||||
pub agent: Option<String>,
|
|
||||||
/// Set agent variables
|
|
||||||
#[arg(long, value_names = ["NAME", "VALUE"], num_args = 2)]
|
|
||||||
pub agent_variable: Vec<String>,
|
|
||||||
/// Start a RAG
|
|
||||||
#[arg(long, add = ArgValueCompleter::new(rag_completer))]
|
|
||||||
pub rag: Option<String>,
|
|
||||||
/// Rebuild the RAG to sync document changes
|
|
||||||
#[arg(long)]
|
|
||||||
pub rebuild_rag: bool,
|
|
||||||
/// Execute a macro
|
|
||||||
#[arg(long = "macro", value_name = "MACRO", add = ArgValueCompleter::new(macro_completer))]
|
|
||||||
pub macro_name: Option<String>,
|
|
||||||
/// Execute commands in natural language
|
/// Execute commands in natural language
|
||||||
#[arg(short = 'e', long)]
|
#[arg(short = 'e', long)]
|
||||||
pub execute: bool,
|
pub execute: bool,
|
||||||
@@ -88,113 +68,176 @@ pub struct Cli {
|
|||||||
/// Turn off stream mode
|
/// Turn off stream mode
|
||||||
#[arg(short = 'S', long)]
|
#[arg(short = 'S', long)]
|
||||||
pub no_stream: bool,
|
pub no_stream: bool,
|
||||||
|
/// Display the message without sending it
|
||||||
|
#[arg(long)]
|
||||||
|
pub dry_run: bool,
|
||||||
|
/// Disable loading workspace MCP servers from .coyote/mcp.json
|
||||||
|
#[arg(long)]
|
||||||
|
pub no_workspace_mcp: bool,
|
||||||
/// Disable memory for this invocation
|
/// Disable memory for this invocation
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
pub no_memory: bool,
|
pub no_memory: bool,
|
||||||
/// Skip permission prompts by setting AUTO_CONFIRM for all tools (dangerous!)
|
/// Skip permission prompts by setting AUTO_CONFIRM for all tools (dangerous!)
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
pub dangerously_skip_permissions: bool,
|
pub dangerously_skip_permissions: bool,
|
||||||
|
|
||||||
|
/// Start or join a session
|
||||||
|
#[arg(short = 's', long, help_heading = "Session & Memory", add = ArgValueCompleter::new(session_completer))]
|
||||||
|
pub session: Option<Option<String>>,
|
||||||
|
/// Ensure the session is empty
|
||||||
|
#[arg(long, help_heading = "Session & Memory")]
|
||||||
|
pub empty_session: bool,
|
||||||
|
/// Ensure the new conversation is saved to the session
|
||||||
|
#[arg(long, help_heading = "Session & Memory")]
|
||||||
|
pub save_session: bool,
|
||||||
/// Bootstrap a memory marker so coyote begins loading memory next run
|
/// Bootstrap a memory marker so coyote begins loading memory next run
|
||||||
#[arg(long, value_name = "SCOPE", value_enum)]
|
#[arg(
|
||||||
|
long,
|
||||||
|
value_name = "SCOPE",
|
||||||
|
value_enum,
|
||||||
|
help_heading = "Session & Memory"
|
||||||
|
)]
|
||||||
pub init_memory: Option<MemoryScope>,
|
pub init_memory: Option<MemoryScope>,
|
||||||
/// Display the message without sending it
|
|
||||||
#[arg(long)]
|
|
||||||
pub dry_run: bool,
|
|
||||||
/// Display information
|
|
||||||
#[arg(long)]
|
|
||||||
pub info: bool,
|
|
||||||
/// Build all configured Bash tool scripts
|
|
||||||
#[arg(long)]
|
|
||||||
pub build_tools: bool,
|
|
||||||
/// Reinstall bundled assets, overwriting any local changes
|
|
||||||
#[arg(long, value_name = "CATEGORY", value_enum)]
|
|
||||||
pub install: Option<AssetCategory>,
|
|
||||||
/// Install assets from a remote git repository (URL may be suffixed with #<ref>)
|
|
||||||
#[arg(long, value_name = "GIT_URL")]
|
|
||||||
pub install_from: Option<String>,
|
|
||||||
/// Restrict --install-from to a single asset category
|
|
||||||
#[arg(long, value_name = "CATEGORY", value_enum, requires = "install_from")]
|
|
||||||
pub filter: Option<InstallFilter>,
|
|
||||||
/// Overwrite all conflicts without prompting (used with --install-from)
|
|
||||||
#[arg(long, requires = "install_from")]
|
|
||||||
pub install_force: bool,
|
|
||||||
/// Sync models updates
|
|
||||||
#[arg(long)]
|
|
||||||
pub sync_models: bool,
|
|
||||||
/// List all available chat models
|
|
||||||
#[arg(long)]
|
|
||||||
pub list_models: bool,
|
|
||||||
/// List all roles
|
|
||||||
#[arg(long)]
|
|
||||||
pub list_roles: bool,
|
|
||||||
/// List all sessions
|
|
||||||
#[arg(long)]
|
|
||||||
pub list_sessions: bool,
|
|
||||||
/// List all agents
|
|
||||||
#[arg(long)]
|
|
||||||
pub list_agents: bool,
|
|
||||||
/// List all RAGs
|
|
||||||
#[arg(long)]
|
|
||||||
pub list_rags: bool,
|
|
||||||
/// List all macros
|
|
||||||
#[arg(long)]
|
|
||||||
pub list_macros: bool,
|
|
||||||
/// List all installed skills
|
|
||||||
#[arg(long)]
|
|
||||||
pub list_skills: bool,
|
|
||||||
/// Pre-load an existing skill into the session (repeatable). If a single
|
/// Pre-load an existing skill into the session (repeatable). If a single
|
||||||
/// `--skill <NAME>` is given and the skill doesn't exist, opens $EDITOR
|
/// `--skill <NAME>` is given and the skill doesn't exist, opens $EDITOR
|
||||||
/// with a scaffold to create it.
|
/// with a scaffold to create it.
|
||||||
#[arg(long, value_name = "NAME")]
|
#[arg(long, value_name = "NAME", help_heading = "Session & Memory")]
|
||||||
pub skill: Vec<String>,
|
pub skill: Vec<String>,
|
||||||
/// Input text
|
|
||||||
#[arg(trailing_var_arg = true)]
|
/// Start an agent
|
||||||
text: Vec<String>,
|
#[arg(short = 'a', long, help_heading = "Agents, RAG & Macros", add = ArgValueCompleter::new(agent_completer))]
|
||||||
/// Tail logs
|
pub agent: Option<String>,
|
||||||
#[arg(long)]
|
/// Set agent variables
|
||||||
pub tail_logs: bool,
|
#[arg(long, value_names = ["NAME", "VALUE"], num_args = 2, help_heading = "Agents, RAG & Macros")]
|
||||||
/// Disable colored log output
|
pub agent_variable: Vec<String>,
|
||||||
#[arg(long, requires = "tail_logs")]
|
/// Start a RAG
|
||||||
pub disable_log_colors: bool,
|
#[arg(long, help_heading = "Agents, RAG & Macros", add = ArgValueCompleter::new(rag_completer))]
|
||||||
/// Add a secret to the Coyote vault
|
pub rag: Option<String>,
|
||||||
#[arg(long, value_name = "SECRET_NAME", exclusive = true)]
|
/// Rebuild the RAG to sync document changes
|
||||||
pub add_secret: Option<String>,
|
#[arg(long, help_heading = "Agents, RAG & Macros")]
|
||||||
/// Decrypt a secret from the Coyote vault and print the plaintext
|
pub rebuild_rag: bool,
|
||||||
#[arg(long, value_name = "SECRET_NAME", exclusive = true, add = ArgValueCompleter::new(secrets_completer))]
|
/// Execute a macro
|
||||||
pub get_secret: Option<String>,
|
#[arg(long = "macro", value_name = "MACRO", help_heading = "Agents, RAG & Macros", add = ArgValueCompleter::new(macro_completer))]
|
||||||
/// Update an existing secret in the Coyote vault
|
pub macro_name: Option<String>,
|
||||||
#[arg(long, value_name = "SECRET_NAME", exclusive = true, add = ArgValueCompleter::new(secrets_completer))]
|
|
||||||
pub update_secret: Option<String>,
|
/// List all available chat models
|
||||||
/// Delete a secret from the Coyote vault
|
#[arg(long, help_heading = "List & Discovery")]
|
||||||
#[arg(long, value_name = "SECRET_NAME", exclusive = true, add = ArgValueCompleter::new(secrets_completer))]
|
pub list_models: bool,
|
||||||
pub delete_secret: Option<String>,
|
/// List all roles
|
||||||
/// List all secrets stored in the Coyote vault
|
#[arg(long, help_heading = "List & Discovery")]
|
||||||
#[arg(long, exclusive = true)]
|
pub list_roles: bool,
|
||||||
pub list_secrets: bool,
|
/// List all sessions
|
||||||
/// Authenticate with an LLM provider using OAuth (e.g., --authenticate client_name)
|
#[arg(long, help_heading = "List & Discovery")]
|
||||||
#[arg(long, exclusive = true, value_name = "CLIENT_NAME")]
|
pub list_sessions: bool,
|
||||||
pub authenticate: Option<Option<String>>,
|
/// List all agents
|
||||||
/// Authenticate with an OAuth-protected remote MCP server (e.g., --auth-mcp server_name)
|
#[arg(long, help_heading = "List & Discovery")]
|
||||||
#[arg(long, exclusive = true, value_name = "SERVER_NAME", add = ArgValueCompleter::new(mcp_server_completer))]
|
pub list_agents: bool,
|
||||||
pub auth_mcp: Option<String>,
|
/// List all RAGs
|
||||||
/// Generate static shell completion scripts
|
#[arg(long, help_heading = "List & Discovery")]
|
||||||
#[arg(long, value_name = "SHELL", value_enum)]
|
pub list_rags: bool,
|
||||||
pub completions: Option<ShellCompletion>,
|
/// List all macros
|
||||||
|
#[arg(long, help_heading = "List & Discovery")]
|
||||||
|
pub list_macros: bool,
|
||||||
|
/// List all installed skills
|
||||||
|
#[arg(long, help_heading = "List & Discovery")]
|
||||||
|
pub list_skills: bool,
|
||||||
|
|
||||||
|
/// Reinstall bundled assets, overwriting any local changes
|
||||||
|
#[arg(
|
||||||
|
long,
|
||||||
|
value_name = "CATEGORY",
|
||||||
|
value_enum,
|
||||||
|
help_heading = "Installation & Updates"
|
||||||
|
)]
|
||||||
|
pub install: Option<AssetCategory>,
|
||||||
|
/// Install assets from a remote git repository (URL may be suffixed with #<ref>)
|
||||||
|
#[arg(long, value_name = "GIT_URL", help_heading = "Installation & Updates")]
|
||||||
|
pub install_from: Option<String>,
|
||||||
|
/// Restrict --install-from to a single asset category
|
||||||
|
#[arg(
|
||||||
|
long,
|
||||||
|
value_name = "CATEGORY",
|
||||||
|
value_enum,
|
||||||
|
requires = "install_from",
|
||||||
|
help_heading = "Installation & Updates"
|
||||||
|
)]
|
||||||
|
pub filter: Option<InstallFilter>,
|
||||||
|
/// Overwrite all conflicts without prompting (used with --install-from)
|
||||||
|
#[arg(
|
||||||
|
long,
|
||||||
|
requires = "install_from",
|
||||||
|
help_heading = "Installation & Updates"
|
||||||
|
)]
|
||||||
|
pub install_force: bool,
|
||||||
|
/// Sync models updates
|
||||||
|
#[arg(long, help_heading = "Installation & Updates")]
|
||||||
|
pub sync_models: bool,
|
||||||
/// Update Coyote to the latest release, or to a specific version
|
/// Update Coyote to the latest release, or to a specific version
|
||||||
#[arg(long, value_name = "VERSION")]
|
#[arg(long, value_name = "VERSION", help_heading = "Installation & Updates")]
|
||||||
pub update: Option<Option<String>>,
|
pub update: Option<Option<String>>,
|
||||||
/// With --update, update even if Coyote was installed via a package manager
|
/// With --update, update even if Coyote was installed via a package manager
|
||||||
#[arg(long, requires = "update")]
|
#[arg(long, requires = "update", help_heading = "Installation & Updates")]
|
||||||
pub force: bool,
|
pub force: bool,
|
||||||
|
|
||||||
|
/// Add a secret to the Coyote vault
|
||||||
|
#[arg(
|
||||||
|
long,
|
||||||
|
value_name = "SECRET_NAME",
|
||||||
|
exclusive = true,
|
||||||
|
help_heading = "Vault & Secrets"
|
||||||
|
)]
|
||||||
|
pub add_secret: Option<String>,
|
||||||
|
/// Decrypt a secret from the Coyote vault and print the plaintext
|
||||||
|
#[arg(long, value_name = "SECRET_NAME", exclusive = true, help_heading = "Vault & Secrets", add = ArgValueCompleter::new(secrets_completer))]
|
||||||
|
pub get_secret: Option<String>,
|
||||||
|
/// Update an existing secret in the Coyote vault
|
||||||
|
#[arg(long, value_name = "SECRET_NAME", exclusive = true, help_heading = "Vault & Secrets", add = ArgValueCompleter::new(secrets_completer))]
|
||||||
|
pub update_secret: Option<String>,
|
||||||
|
/// Delete a secret from the Coyote vault
|
||||||
|
#[arg(long, value_name = "SECRET_NAME", exclusive = true, help_heading = "Vault & Secrets", add = ArgValueCompleter::new(secrets_completer))]
|
||||||
|
pub delete_secret: Option<String>,
|
||||||
|
/// List all secrets stored in the Coyote vault
|
||||||
|
#[arg(long, exclusive = true, help_heading = "Vault & Secrets")]
|
||||||
|
pub list_secrets: bool,
|
||||||
|
|
||||||
|
/// Authenticate with an LLM provider using OAuth (e.g., --authenticate client_name)
|
||||||
|
#[arg(
|
||||||
|
long,
|
||||||
|
exclusive = true,
|
||||||
|
value_name = "CLIENT_NAME",
|
||||||
|
help_heading = "Authentication"
|
||||||
|
)]
|
||||||
|
pub authenticate: Option<Option<String>>,
|
||||||
|
/// Authenticate with an OAuth-protected remote MCP server (e.g., --auth-mcp server_name)
|
||||||
|
#[arg(long, exclusive = true, value_name = "SERVER_NAME", help_heading = "Authentication", add = ArgValueCompleter::new(mcp_server_completer))]
|
||||||
|
pub auth_mcp: Option<String>,
|
||||||
|
|
||||||
/// Launch Coyote inside a Docker sandbox (via `sbx`); name defaults to current directory basename
|
/// Launch Coyote inside a Docker sandbox (via `sbx`); name defaults to current directory basename
|
||||||
#[arg(long, value_name = "NAME")]
|
#[arg(long, value_name = "NAME", help_heading = "Sandbox")]
|
||||||
pub sandbox: Option<Option<String>>,
|
pub sandbox: Option<Option<String>>,
|
||||||
/// Create the sandbox without bootstrapping the host config or vault password file
|
/// Create the sandbox without bootstrapping the host config or vault password file
|
||||||
#[arg(long, requires = "sandbox")]
|
#[arg(long, requires = "sandbox", help_heading = "Sandbox")]
|
||||||
pub fresh: bool,
|
pub fresh: bool,
|
||||||
/// Skip discovery and application of all sbx mixins (user and built-in)
|
/// Skip discovery and application of all sbx mixins (user and built-in)
|
||||||
#[arg(long, requires = "sandbox")]
|
#[arg(long, requires = "sandbox", help_heading = "Sandbox")]
|
||||||
pub no_mixins: bool,
|
pub no_mixins: bool,
|
||||||
|
|
||||||
|
/// Display information
|
||||||
|
#[arg(long, help_heading = "Diagnostics & Tools")]
|
||||||
|
pub info: bool,
|
||||||
|
/// Build all configured Bash tool scripts
|
||||||
|
#[arg(long, help_heading = "Diagnostics & Tools")]
|
||||||
|
pub build_tools: bool,
|
||||||
|
/// Tail logs
|
||||||
|
#[arg(long, help_heading = "Diagnostics & Tools")]
|
||||||
|
pub tail_logs: bool,
|
||||||
|
/// Disable colored log output
|
||||||
|
#[arg(long, requires = "tail_logs", help_heading = "Diagnostics & Tools")]
|
||||||
|
pub disable_log_colors: bool,
|
||||||
|
|
||||||
|
/// Generate static shell completion scripts
|
||||||
|
#[arg(long, value_name = "SHELL", value_enum, help_heading = "Shell")]
|
||||||
|
pub completions: Option<ShellCompletion>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Cli {
|
impl Cli {
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ fn prepare_chat_completions(
|
|||||||
|
|
||||||
let url = format!(
|
let url = format!(
|
||||||
"{}/openai/deployments/{}/chat/completions?api-version=2024-12-01-preview",
|
"{}/openai/deployments/{}/chat/completions?api-version=2024-12-01-preview",
|
||||||
&api_base,
|
api_base,
|
||||||
self_.model.real_name()
|
self_.model.real_name()
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -69,7 +69,7 @@ fn prepare_embeddings(self_: &AzureOpenAIClient, data: &EmbeddingsData) -> Resul
|
|||||||
|
|
||||||
let url = format!(
|
let url = format!(
|
||||||
"{}/openai/deployments/{}/embeddings?api-version=2024-10-21",
|
"{}/openai/deployments/{}/embeddings?api-version=2024-10-21",
|
||||||
&api_base,
|
api_base,
|
||||||
self_.model.real_name()
|
self_.model.real_name()
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|||||||
@@ -88,6 +88,7 @@ pub struct AppConfig {
|
|||||||
|
|
||||||
pub user_agent: Option<String>,
|
pub user_agent: Option<String>,
|
||||||
pub save_shell_history: bool,
|
pub save_shell_history: bool,
|
||||||
|
pub no_workspace_mcp: bool,
|
||||||
pub sync_models_url: Option<String>,
|
pub sync_models_url: Option<String>,
|
||||||
|
|
||||||
pub clients: Vec<ClientConfig>,
|
pub clients: Vec<ClientConfig>,
|
||||||
@@ -162,6 +163,7 @@ impl Default for AppConfig {
|
|||||||
|
|
||||||
user_agent: None,
|
user_agent: None,
|
||||||
save_shell_history: true,
|
save_shell_history: true,
|
||||||
|
no_workspace_mcp: false,
|
||||||
sync_models_url: None,
|
sync_models_url: None,
|
||||||
|
|
||||||
clients: vec![],
|
clients: vec![],
|
||||||
@@ -238,6 +240,7 @@ impl AppConfig {
|
|||||||
|
|
||||||
user_agent: config.user_agent,
|
user_agent: config.user_agent,
|
||||||
save_shell_history: config.save_shell_history,
|
save_shell_history: config.save_shell_history,
|
||||||
|
no_workspace_mcp: false,
|
||||||
sync_models_url: config.sync_models_url,
|
sync_models_url: config.sync_models_url,
|
||||||
|
|
||||||
clients: config.clients,
|
clients: config.clients,
|
||||||
@@ -308,16 +311,18 @@ impl AppConfig {
|
|||||||
|
|
||||||
pub fn editor(&self) -> Result<String> {
|
pub fn editor(&self) -> Result<String> {
|
||||||
super::EDITOR.get_or_init(move || {
|
super::EDITOR.get_or_init(move || {
|
||||||
let editor = self.editor.clone()
|
if let Some(editor) = self.editor.clone()
|
||||||
.or_else(|| env::var("VISUAL").ok().or_else(|| env::var("EDITOR").ok()))
|
.or_else(|| env::var("VISUAL").ok().or_else(|| env::var("EDITOR").ok()))
|
||||||
.unwrap_or_else(|| {
|
&& which::which(&editor).is_ok()
|
||||||
if cfg!(windows) {
|
{
|
||||||
|
return Some(editor);
|
||||||
|
}
|
||||||
|
let default = if cfg!(windows) {
|
||||||
"notepad".to_string()
|
"notepad".to_string()
|
||||||
} else {
|
} else {
|
||||||
"nano".to_string()
|
"nano".to_string()
|
||||||
}
|
};
|
||||||
});
|
which::which(&default).ok().map(|_| default)
|
||||||
which::which(&editor).ok().map(|_| editor)
|
|
||||||
})
|
})
|
||||||
.clone()
|
.clone()
|
||||||
.ok_or_else(|| anyhow!("Editor not found. Please add the `editor` configuration or set the $EDITOR or $VISUAL environment variable."))
|
.ok_or_else(|| anyhow!("Editor not found. Please add the `editor` configuration or set the $EDITOR or $VISUAL environment variable."))
|
||||||
|
|||||||
+10
-10
@@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize};
|
|||||||
|
|
||||||
use crate::config::{
|
use crate::config::{
|
||||||
GIT_DIR_NAME, GITIGNORE_FILE_NAME, MEMORY_DIR_NAME, MEMORY_INDEX_FILE_NAME,
|
GIT_DIR_NAME, GITIGNORE_FILE_NAME, MEMORY_DIR_NAME, MEMORY_INDEX_FILE_NAME,
|
||||||
WORKSPACE_MEMORY_DIR_NAME, WORKSPACE_MEMORY_FILE_NAME, paths,
|
WORKSPACE_COYOTE_DIR_NAME, WORKSPACE_MEMORY_FILE_NAME, paths,
|
||||||
};
|
};
|
||||||
|
|
||||||
pub const DEFAULT_MEMORY_CAP_WITH_TOOLS: usize = 6_000;
|
pub const DEFAULT_MEMORY_CAP_WITH_TOOLS: usize = 6_000;
|
||||||
@@ -27,7 +27,7 @@ pub enum WorkspaceMemory {
|
|||||||
|
|
||||||
pub fn discover_workspace_memory(start: &Path) -> Option<WorkspaceMemory> {
|
pub fn discover_workspace_memory(start: &Path) -> Option<WorkspaceMemory> {
|
||||||
for dir in start.ancestors() {
|
for dir in start.ancestors() {
|
||||||
let structured = dir.join(WORKSPACE_MEMORY_DIR_NAME).join(MEMORY_DIR_NAME);
|
let structured = dir.join(WORKSPACE_COYOTE_DIR_NAME).join(MEMORY_DIR_NAME);
|
||||||
if structured.join(MEMORY_INDEX_FILE_NAME).exists() {
|
if structured.join(MEMORY_INDEX_FILE_NAME).exists() {
|
||||||
return Some(WorkspaceMemory::Structured {
|
return Some(WorkspaceMemory::Structured {
|
||||||
workspace_root: dir.to_path_buf(),
|
workspace_root: dir.to_path_buf(),
|
||||||
@@ -84,8 +84,8 @@ pub fn bootstrap_workspace_memory(git_root: &Path) -> Result<PathBuf> {
|
|||||||
|
|
||||||
fn append_gitignore_entry(git_root: &Path) -> Result<bool> {
|
fn append_gitignore_entry(git_root: &Path) -> Result<bool> {
|
||||||
let gitignore = git_root.join(GITIGNORE_FILE_NAME);
|
let gitignore = git_root.join(GITIGNORE_FILE_NAME);
|
||||||
let entry = format!("{WORKSPACE_MEMORY_DIR_NAME}/{MEMORY_DIR_NAME}/");
|
let entry = format!("{WORKSPACE_COYOTE_DIR_NAME}/{MEMORY_DIR_NAME}/");
|
||||||
let entry_no_slash = format!("{WORKSPACE_MEMORY_DIR_NAME}/{MEMORY_DIR_NAME}");
|
let entry_no_slash = format!("{WORKSPACE_COYOTE_DIR_NAME}/{MEMORY_DIR_NAME}");
|
||||||
|
|
||||||
let existing = fs::read_to_string(&gitignore).unwrap_or_default();
|
let existing = fs::read_to_string(&gitignore).unwrap_or_default();
|
||||||
let already_present = existing.lines().any(|line| {
|
let already_present = existing.lines().any(|line| {
|
||||||
@@ -347,7 +347,7 @@ mod tests {
|
|||||||
let root = temp_root("phase1");
|
let root = temp_root("phase1");
|
||||||
let workspace = root.join("workspace");
|
let workspace = root.join("workspace");
|
||||||
let workspace_memory_dir = workspace
|
let workspace_memory_dir = workspace
|
||||||
.join(WORKSPACE_MEMORY_DIR_NAME)
|
.join(WORKSPACE_COYOTE_DIR_NAME)
|
||||||
.join(MEMORY_DIR_NAME);
|
.join(MEMORY_DIR_NAME);
|
||||||
fs::create_dir_all(&workspace_memory_dir).unwrap();
|
fs::create_dir_all(&workspace_memory_dir).unwrap();
|
||||||
fs::write(
|
fs::write(
|
||||||
@@ -382,7 +382,7 @@ mod tests {
|
|||||||
let root = temp_root("prefer");
|
let root = temp_root("prefer");
|
||||||
let workspace = root.join("ws");
|
let workspace = root.join("ws");
|
||||||
let structured = workspace
|
let structured = workspace
|
||||||
.join(WORKSPACE_MEMORY_DIR_NAME)
|
.join(WORKSPACE_COYOTE_DIR_NAME)
|
||||||
.join(MEMORY_DIR_NAME);
|
.join(MEMORY_DIR_NAME);
|
||||||
fs::create_dir_all(&structured).unwrap();
|
fs::create_dir_all(&structured).unwrap();
|
||||||
fs::write(structured.join(MEMORY_INDEX_FILE_NAME), "s").unwrap();
|
fs::write(structured.join(MEMORY_INDEX_FILE_NAME), "s").unwrap();
|
||||||
@@ -415,7 +415,7 @@ mod tests {
|
|||||||
let root = temp_root("indexes_only");
|
let root = temp_root("indexes_only");
|
||||||
let workspace = root.join("ws");
|
let workspace = root.join("ws");
|
||||||
let structured = workspace
|
let structured = workspace
|
||||||
.join(WORKSPACE_MEMORY_DIR_NAME)
|
.join(WORKSPACE_COYOTE_DIR_NAME)
|
||||||
.join(MEMORY_DIR_NAME);
|
.join(MEMORY_DIR_NAME);
|
||||||
fs::create_dir_all(&structured).unwrap();
|
fs::create_dir_all(&structured).unwrap();
|
||||||
fs::write(
|
fs::write(
|
||||||
@@ -450,7 +450,7 @@ mod tests {
|
|||||||
let root = temp_root("drill_bodies");
|
let root = temp_root("drill_bodies");
|
||||||
let workspace = root.join("ws");
|
let workspace = root.join("ws");
|
||||||
let structured = workspace
|
let structured = workspace
|
||||||
.join(WORKSPACE_MEMORY_DIR_NAME)
|
.join(WORKSPACE_COYOTE_DIR_NAME)
|
||||||
.join(MEMORY_DIR_NAME);
|
.join(MEMORY_DIR_NAME);
|
||||||
fs::create_dir_all(&structured).unwrap();
|
fs::create_dir_all(&structured).unwrap();
|
||||||
fs::write(structured.join(MEMORY_INDEX_FILE_NAME), "idx").unwrap();
|
fs::write(structured.join(MEMORY_INDEX_FILE_NAME), "idx").unwrap();
|
||||||
@@ -485,7 +485,7 @@ mod tests {
|
|||||||
let root = temp_root("cap");
|
let root = temp_root("cap");
|
||||||
let workspace = root.join("ws");
|
let workspace = root.join("ws");
|
||||||
let structured = workspace
|
let structured = workspace
|
||||||
.join(WORKSPACE_MEMORY_DIR_NAME)
|
.join(WORKSPACE_COYOTE_DIR_NAME)
|
||||||
.join(MEMORY_DIR_NAME);
|
.join(MEMORY_DIR_NAME);
|
||||||
fs::create_dir_all(&structured).unwrap();
|
fs::create_dir_all(&structured).unwrap();
|
||||||
fs::write(structured.join(MEMORY_INDEX_FILE_NAME), "idx").unwrap();
|
fs::write(structured.join(MEMORY_INDEX_FILE_NAME), "idx").unwrap();
|
||||||
@@ -575,7 +575,7 @@ mod tests {
|
|||||||
let root = temp_root("walk_up");
|
let root = temp_root("walk_up");
|
||||||
let workspace = root.join("ws");
|
let workspace = root.join("ws");
|
||||||
let mem_dir = workspace
|
let mem_dir = workspace
|
||||||
.join(WORKSPACE_MEMORY_DIR_NAME)
|
.join(WORKSPACE_COYOTE_DIR_NAME)
|
||||||
.join(MEMORY_DIR_NAME);
|
.join(MEMORY_DIR_NAME);
|
||||||
fs::create_dir_all(&mem_dir).unwrap();
|
fs::create_dir_all(&mem_dir).unwrap();
|
||||||
fs::write(mem_dir.join(MEMORY_INDEX_FILE_NAME), "idx").unwrap();
|
fs::write(mem_dir.join(MEMORY_INDEX_FILE_NAME), "idx").unwrap();
|
||||||
|
|||||||
+1
-1
@@ -143,7 +143,7 @@ const MCP_FILE_NAME: &str = "mcp.json";
|
|||||||
const MEMORY_DIR_NAME: &str = "memory";
|
const MEMORY_DIR_NAME: &str = "memory";
|
||||||
const MEMORY_INDEX_FILE_NAME: &str = "MEMORY.md";
|
const MEMORY_INDEX_FILE_NAME: &str = "MEMORY.md";
|
||||||
const WORKSPACE_MEMORY_FILE_NAME: &str = "COYOTE.md";
|
const WORKSPACE_MEMORY_FILE_NAME: &str = "COYOTE.md";
|
||||||
const WORKSPACE_MEMORY_DIR_NAME: &str = ".coyote";
|
const WORKSPACE_COYOTE_DIR_NAME: &str = ".coyote";
|
||||||
const SBX_KIT_DIR_NAME: &str = "sbx-kit";
|
const SBX_KIT_DIR_NAME: &str = "sbx-kit";
|
||||||
const SBX_KIT_HASH_FILE: &str = "kit.sha256";
|
const SBX_KIT_HASH_FILE: &str = "kit.sha256";
|
||||||
const SBX_MIXIN_FILE_NAME: &str = "sbx-mixin.yaml";
|
const SBX_MIXIN_FILE_NAME: &str = "sbx-mixin.yaml";
|
||||||
|
|||||||
+29
-5
@@ -5,7 +5,7 @@ use super::{
|
|||||||
GLOBAL_TOOLS_UTILS_DIR_NAME, MACROS_DIR_NAME, MCP_FILE_NAME, MEMORY_DIR_NAME,
|
GLOBAL_TOOLS_UTILS_DIR_NAME, MACROS_DIR_NAME, MCP_FILE_NAME, MEMORY_DIR_NAME,
|
||||||
MEMORY_INDEX_FILE_NAME, ModelsOverride, RAGS_DIR_NAME, ROLES_DIR_NAME, SBX_KIT_DIR_NAME,
|
MEMORY_INDEX_FILE_NAME, ModelsOverride, RAGS_DIR_NAME, ROLES_DIR_NAME, SBX_KIT_DIR_NAME,
|
||||||
SBX_KIT_HASH_FILE, SBX_MIXIN_FILE_NAME, SBX_MIXIN_KITS_DIR_NAME, SBX_VAULT_MIXINS_DIR_NAME,
|
SBX_KIT_HASH_FILE, SBX_MIXIN_FILE_NAME, SBX_MIXIN_KITS_DIR_NAME, SBX_VAULT_MIXINS_DIR_NAME,
|
||||||
SKILLS_DIR_NAME, WORKSPACE_MEMORY_DIR_NAME,
|
SKILLS_DIR_NAME, WORKSPACE_COYOTE_DIR_NAME,
|
||||||
};
|
};
|
||||||
use crate::client::ProviderModels;
|
use crate::client::ProviderModels;
|
||||||
use crate::config::REPL_HISTORY_DIR_NAME;
|
use crate::config::REPL_HISTORY_DIR_NAME;
|
||||||
@@ -118,7 +118,7 @@ pub fn global_tools_sbx_mixin_file() -> PathBuf {
|
|||||||
pub fn find_workspace_sbx_mixin(start: &Path) -> Option<PathBuf> {
|
pub fn find_workspace_sbx_mixin(start: &Path) -> Option<PathBuf> {
|
||||||
for dir in start.ancestors() {
|
for dir in start.ancestors() {
|
||||||
let candidate = dir
|
let candidate = dir
|
||||||
.join(WORKSPACE_MEMORY_DIR_NAME)
|
.join(WORKSPACE_COYOTE_DIR_NAME)
|
||||||
.join(SBX_MIXIN_FILE_NAME);
|
.join(SBX_MIXIN_FILE_NAME);
|
||||||
if candidate.exists() {
|
if candidate.exists() {
|
||||||
return Some(candidate);
|
return Some(candidate);
|
||||||
@@ -193,6 +193,24 @@ pub fn skill_file(name: &str) -> PathBuf {
|
|||||||
skill_dir(name).join("SKILL.md")
|
skill_dir(name).join("SKILL.md")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn workspace_skills_dir() -> PathBuf {
|
||||||
|
env::current_dir()
|
||||||
|
.unwrap_or_default()
|
||||||
|
.join(WORKSPACE_COYOTE_DIR_NAME)
|
||||||
|
.join(SKILLS_DIR_NAME)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn workspace_skill_file(name: &str) -> PathBuf {
|
||||||
|
workspace_skills_dir().join(name).join("SKILL.md")
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn workspace_mcp_config_file() -> PathBuf {
|
||||||
|
env::current_dir()
|
||||||
|
.unwrap_or_default()
|
||||||
|
.join(WORKSPACE_COYOTE_DIR_NAME)
|
||||||
|
.join(MCP_FILE_NAME)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn validate_skill_name(name: &str) -> Result<()> {
|
pub fn validate_skill_name(name: &str) -> Result<()> {
|
||||||
if name.is_empty() {
|
if name.is_empty() {
|
||||||
bail!("Skill name cannot be empty");
|
bail!("Skill name cannot be empty");
|
||||||
@@ -318,7 +336,7 @@ pub fn global_memory_index_path() -> PathBuf {
|
|||||||
|
|
||||||
pub fn workspace_memory_dir_for(workspace_root: &Path) -> PathBuf {
|
pub fn workspace_memory_dir_for(workspace_root: &Path) -> PathBuf {
|
||||||
workspace_root
|
workspace_root
|
||||||
.join(WORKSPACE_MEMORY_DIR_NAME)
|
.join(WORKSPACE_COYOTE_DIR_NAME)
|
||||||
.join(MEMORY_DIR_NAME)
|
.join(MEMORY_DIR_NAME)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -405,25 +423,31 @@ pub fn has_macro(name: &str) -> bool {
|
|||||||
|
|
||||||
pub fn list_skills() -> Vec<String> {
|
pub fn list_skills() -> Vec<String> {
|
||||||
let mut names = Vec::new();
|
let mut names = Vec::new();
|
||||||
if let Ok(rd) = read_dir(skills_dir()) {
|
let mut seen = HashSet::new();
|
||||||
|
|
||||||
|
for dir in [workspace_skills_dir(), skills_dir()] {
|
||||||
|
if let Ok(rd) = read_dir(dir) {
|
||||||
for entry in rd.flatten() {
|
for entry in rd.flatten() {
|
||||||
if let Ok(file_type) = entry.file_type()
|
if let Ok(file_type) = entry.file_type()
|
||||||
&& file_type.is_dir()
|
&& file_type.is_dir()
|
||||||
&& let Some(name) = entry.file_name().to_str()
|
&& let Some(name) = entry.file_name().to_str()
|
||||||
|
&& !seen.contains(name)
|
||||||
&& entry.path().join("SKILL.md").is_file()
|
&& entry.path().join("SKILL.md").is_file()
|
||||||
&& validate_skill_name(name).is_ok()
|
&& validate_skill_name(name).is_ok()
|
||||||
{
|
{
|
||||||
|
seen.insert(name.to_string());
|
||||||
names.push(name.to_string());
|
names.push(name.to_string());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
names.sort_unstable();
|
names.sort_unstable();
|
||||||
names
|
names
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn has_skill(name: &str) -> bool {
|
pub fn has_skill(name: &str) -> bool {
|
||||||
skill_file(name).is_file()
|
workspace_skill_file(name).is_file() || skill_file(name).is_file()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn local_models_override() -> Result<Vec<ProviderModels>> {
|
pub fn local_models_override() -> Result<Vec<ProviderModels>> {
|
||||||
|
|||||||
@@ -261,7 +261,7 @@ impl Session {
|
|||||||
data["messages"] = json!(self.messages);
|
data["messages"] = json!(self.messages);
|
||||||
|
|
||||||
let output = serde_yaml::to_string(&data)
|
let output = serde_yaml::to_string(&data)
|
||||||
.with_context(|| format!("Unable to show info about session '{}'", &self.name))?;
|
.with_context(|| format!("Unable to show info about session '{}'", self.name))?;
|
||||||
Ok(output)
|
Ok(output)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+5
-1
@@ -117,7 +117,11 @@ impl Skill {
|
|||||||
|
|
||||||
pub fn load(name: &str) -> Result<Self> {
|
pub fn load(name: &str) -> Result<Self> {
|
||||||
paths::validate_skill_name(name)?;
|
paths::validate_skill_name(name)?;
|
||||||
let path = paths::skill_file(name);
|
let path = if paths::workspace_skill_file(name).is_file() {
|
||||||
|
paths::workspace_skill_file(name)
|
||||||
|
} else {
|
||||||
|
paths::skill_file(name)
|
||||||
|
};
|
||||||
let content = read_to_string(&path)
|
let content = read_to_string(&path)
|
||||||
.with_context(|| format!("Failed to read skill '{name}' at {}", path.display()))?;
|
.with_context(|| format!("Failed to read skill '{name}' at {}", path.display()))?;
|
||||||
Ok(Skill::new(name, &content))
|
Ok(Skill::new(name, &content))
|
||||||
|
|||||||
+6
-6
@@ -147,10 +147,10 @@ pub async fn eval_tool_calls(
|
|||||||
let mut is_all_null = true;
|
let mut is_all_null = true;
|
||||||
for call in calls {
|
for call in calls {
|
||||||
if let Some(msg) = ctx.tool_scope.tool_tracker.check_loop(&call.clone()) {
|
if let Some(msg) = ctx.tool_scope.tool_tracker.check_loop(&call.clone()) {
|
||||||
let dup_msg = format!("{{\"tool_call_loop_alert\":{}}}", &msg.trim());
|
let dup_msg = format!("{{\"tool_call_loop_alert\":{}}}", msg.trim());
|
||||||
println!(
|
println!(
|
||||||
"{}",
|
"{}",
|
||||||
warning_text(format!("{}: ⚠️ Tool-call loop detected! ⚠️", &call.name).as_str())
|
warning_text(format!("{}: ⚠️ Tool-call loop detected! ⚠️", call.name).as_str())
|
||||||
);
|
);
|
||||||
let val = json!(dup_msg);
|
let val = json!(dup_msg);
|
||||||
output.push(ToolResult::new(call, val));
|
output.push(ToolResult::new(call, val));
|
||||||
@@ -730,7 +730,7 @@ impl Functions {
|
|||||||
let root_dir = paths::functions_dir();
|
let root_dir = paths::functions_dir();
|
||||||
let tool_path = format!(
|
let tool_path = format!(
|
||||||
"{}/{binary_name}",
|
"{}/{binary_name}",
|
||||||
&paths::global_tools_dir().to_string_lossy()
|
paths::global_tools_dir().to_string_lossy()
|
||||||
);
|
);
|
||||||
content_template
|
content_template
|
||||||
.replace("{function_name}", binary_name)
|
.replace("{function_name}", binary_name)
|
||||||
@@ -741,7 +741,7 @@ impl Functions {
|
|||||||
let root_dir = paths::agent_data_dir(agent_name);
|
let root_dir = paths::agent_data_dir(agent_name);
|
||||||
let tool_path = format!(
|
let tool_path = format!(
|
||||||
"{}/{binary_name}",
|
"{}/{binary_name}",
|
||||||
&paths::global_tools_dir().to_string_lossy()
|
paths::global_tools_dir().to_string_lossy()
|
||||||
);
|
);
|
||||||
content_template
|
content_template
|
||||||
.replace("{function_name}", binary_name)
|
.replace("{function_name}", binary_name)
|
||||||
@@ -870,7 +870,7 @@ impl Functions {
|
|||||||
let root_dir = paths::functions_dir();
|
let root_dir = paths::functions_dir();
|
||||||
let tool_path = format!(
|
let tool_path = format!(
|
||||||
"{}/{binary_name}",
|
"{}/{binary_name}",
|
||||||
&paths::global_tools_dir().to_string_lossy()
|
paths::global_tools_dir().to_string_lossy()
|
||||||
);
|
);
|
||||||
content_template
|
content_template
|
||||||
.replace("{function_name}", binary_name)
|
.replace("{function_name}", binary_name)
|
||||||
@@ -881,7 +881,7 @@ impl Functions {
|
|||||||
let root_dir = paths::agent_data_dir(agent_name);
|
let root_dir = paths::agent_data_dir(agent_name);
|
||||||
let tool_path = format!(
|
let tool_path = format!(
|
||||||
"{}/{binary_name}",
|
"{}/{binary_name}",
|
||||||
&paths::global_tools_dir().to_string_lossy()
|
paths::global_tools_dir().to_string_lossy()
|
||||||
);
|
);
|
||||||
content_template
|
content_template
|
||||||
.replace("{function_name}", binary_name)
|
.replace("{function_name}", binary_name)
|
||||||
|
|||||||
+6
-2
@@ -187,7 +187,11 @@ async fn main() -> Result<()> {
|
|||||||
let abort_signal = create_abort_signal();
|
let abort_signal = create_abort_signal();
|
||||||
let start_mcp_servers = cli.agent.is_none() && cli.role.is_none();
|
let start_mcp_servers = cli.agent.is_none() && cli.role.is_none();
|
||||||
let cfg = Config::load_with_interpolation(info_flag).await?;
|
let cfg = Config::load_with_interpolation(info_flag).await?;
|
||||||
let app_config: Arc<AppConfig> = Arc::new(AppConfig::from_config(cfg)?);
|
let mut app_config = AppConfig::from_config(cfg)?;
|
||||||
|
if cli.no_workspace_mcp {
|
||||||
|
app_config.no_workspace_mcp = true;
|
||||||
|
}
|
||||||
|
let app_config: Arc<AppConfig> = Arc::new(app_config);
|
||||||
let app_state: Arc<AppState> = Arc::new(
|
let app_state: Arc<AppState> = Arc::new(
|
||||||
AppState::init(
|
AppState::init(
|
||||||
app_config,
|
app_config,
|
||||||
@@ -559,7 +563,7 @@ async fn shell_execute(
|
|||||||
|
|
||||||
match answer_char {
|
match answer_char {
|
||||||
'e' => {
|
'e' => {
|
||||||
debug!("{} {:?}", shell.cmd, &[&shell.arg, &eval_str]);
|
debug!("{} {:?}", shell.cmd, [&shell.arg, &eval_str]);
|
||||||
let code = run_command(&shell.cmd, &[&shell.arg, &eval_str], None)?;
|
let code = run_command(&shell.cmd, &[&shell.arg, &eval_str], None)?;
|
||||||
if code == 0 && app.save_shell_history {
|
if code == 0 && app.save_shell_history {
|
||||||
let _ = append_to_shell_history(&shell.name, &eval_str, code);
|
let _ = append_to_shell_history(&shell.name, &eval_str, code);
|
||||||
|
|||||||
+47
-1
@@ -214,7 +214,53 @@ impl McpRegistry {
|
|||||||
spec.validate(name)?;
|
spec.validate(name)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
registry.config = Some(mcp_servers_config);
|
let mut merged = mcp_servers_config;
|
||||||
|
if !app_config.no_workspace_mcp {
|
||||||
|
let ws_path = paths::workspace_mcp_config_file();
|
||||||
|
if ws_path.try_exists().unwrap_or(false) {
|
||||||
|
match tokio::fs::read_to_string(&ws_path).await {
|
||||||
|
Ok(ws_content) if !ws_content.trim().is_empty() => {
|
||||||
|
match interpolate_secrets(&ws_content, vault) {
|
||||||
|
Ok((parsed, missing)) if missing.is_empty() => {
|
||||||
|
match serde_json::from_str::<McpServersConfig>(&parsed) {
|
||||||
|
Ok(ws_config) => {
|
||||||
|
let mut loaded = Vec::new();
|
||||||
|
for (name, spec) in ws_config.mcp_servers {
|
||||||
|
match spec.validate(&name) {
|
||||||
|
Ok(_) => {
|
||||||
|
loaded.push(name.clone());
|
||||||
|
merged.mcp_servers.insert(name, spec);
|
||||||
|
}
|
||||||
|
Err(e) => warn!(
|
||||||
|
"Invalid workspace MCP server '{name}': {e}. Skipping."
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !loaded.is_empty() {
|
||||||
|
eprintln!(
|
||||||
|
"Loading workspace MCP servers: {}",
|
||||||
|
loaded.join(", ")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => warn!(
|
||||||
|
"Failed to parse workspace MCP config: {e}. Skipping."
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok((_, missing)) => warn!(
|
||||||
|
"Workspace MCP config references missing vault secrets: {missing:?}. Skipping."
|
||||||
|
),
|
||||||
|
Err(e) => {
|
||||||
|
warn!("Failed to process workspace MCP config: {e}. Skipping.")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
registry.config = Some(merged);
|
||||||
|
|
||||||
if start_mcp_servers && app_config.mcp_server_support {
|
if start_mcp_servers && app_config.mcp_server_support {
|
||||||
abortable_run_with_spinner(
|
abortable_run_with_spinner(
|
||||||
|
|||||||
@@ -358,17 +358,16 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use crate::function::JsonSchema;
|
use crate::function::JsonSchema;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::time::{SystemTime, UNIX_EPOCH};
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
|
||||||
|
static PARSE_COUNTER: AtomicU64 = AtomicU64::new(0);
|
||||||
|
|
||||||
fn parse_source(
|
fn parse_source(
|
||||||
source: &str,
|
source: &str,
|
||||||
file_name: &str,
|
file_name: &str,
|
||||||
parent: &Path,
|
parent: &Path,
|
||||||
) -> Result<Vec<FunctionDeclaration>> {
|
) -> Result<Vec<FunctionDeclaration>> {
|
||||||
let unique = SystemTime::now()
|
let unique = PARSE_COUNTER.fetch_add(1, Ordering::Relaxed);
|
||||||
.duration_since(UNIX_EPOCH)
|
|
||||||
.expect("time went backwards")
|
|
||||||
.as_nanos();
|
|
||||||
let path =
|
let path =
|
||||||
std::env::temp_dir().join(format!("coyote_python_parser_{file_name}_{unique}.py"));
|
std::env::temp_dir().join(format!("coyote_python_parser_{file_name}_{unique}.py"));
|
||||||
fs::write(&path, source).expect("failed to write temp python source");
|
fs::write(&path, source).expect("failed to write temp python source");
|
||||||
|
|||||||
+502
-20
@@ -2,12 +2,21 @@ use super::DocumentId;
|
|||||||
use crate::client::*;
|
use crate::client::*;
|
||||||
|
|
||||||
use anyhow::{Context, Result};
|
use anyhow::{Context, Result};
|
||||||
use indexmap::IndexMap;
|
use indexmap::{IndexMap, IndexSet};
|
||||||
use petgraph::Direction;
|
use petgraph::Direction;
|
||||||
use petgraph::graph::NodeIndex;
|
use petgraph::graph::NodeIndex;
|
||||||
use petgraph::stable_graph::StableGraph;
|
use petgraph::stable_graph::StableGraph;
|
||||||
|
use petgraph::visit::EdgeRef;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use std::collections::HashSet;
|
use std::collections::{HashMap, HashSet};
|
||||||
|
|
||||||
|
/// Heuristic upper bound on chunk size before warning the user that the
|
||||||
|
/// extraction LLM call may be truncated. Not a hard limit.
|
||||||
|
const MAX_CHUNK_CHARS: usize = 24_000;
|
||||||
|
|
||||||
|
/// Maximum number of nodes the BFS may visit during a single graph_search.
|
||||||
|
/// Keeps the synchronous traversal bounded on dense graphs.
|
||||||
|
pub const MAX_GRAPH_NODES: usize = 500;
|
||||||
|
|
||||||
const EXTRACTION_PROMPT: &str = r#"Extract entities and relationships from the following text chunk.
|
const EXTRACTION_PROMPT: &str = r#"Extract entities and relationships from the following text chunk.
|
||||||
|
|
||||||
@@ -89,16 +98,27 @@ impl Default for KnowledgeGraph {
|
|||||||
|
|
||||||
impl KnowledgeGraph {
|
impl KnowledgeGraph {
|
||||||
pub fn merge(&mut self, doc_id: DocumentId, result: ExtractionResult) {
|
pub fn merge(&mut self, doc_id: DocumentId, result: ExtractionResult) {
|
||||||
let mut chunk_nodes: Vec<u32> = vec![];
|
let mut chunk_nodes: IndexSet<u32> = IndexSet::new();
|
||||||
|
|
||||||
for extracted in &result.entities {
|
for extracted in &result.entities {
|
||||||
let key = extracted.name.to_lowercase();
|
let key = extracted.name.to_lowercase();
|
||||||
|
let normalized_type = extracted.entity_type.to_uppercase();
|
||||||
let node_raw = if let Some(&existing) = self.entity_index.get(&key) {
|
let node_raw = if let Some(&existing) = self.entity_index.get(&key) {
|
||||||
|
let idx = NodeIndex::new(existing as usize);
|
||||||
|
if self.graph.contains_node(idx) {
|
||||||
|
let node = &mut self.graph[idx];
|
||||||
|
if node.entity_type == "OTHER" && normalized_type != "OTHER" {
|
||||||
|
node.entity_type = normalized_type;
|
||||||
|
}
|
||||||
|
if node.description.is_none() {
|
||||||
|
node.description = extracted.description.clone();
|
||||||
|
}
|
||||||
|
}
|
||||||
existing
|
existing
|
||||||
} else {
|
} else {
|
||||||
let entity = Entity {
|
let entity = Entity {
|
||||||
name: extracted.name.clone(),
|
name: extracted.name.clone(),
|
||||||
entity_type: extracted.entity_type.clone(),
|
entity_type: normalized_type,
|
||||||
description: extracted.description.clone(),
|
description: extracted.description.clone(),
|
||||||
};
|
};
|
||||||
let idx = self.graph.add_node(entity);
|
let idx = self.graph.add_node(entity);
|
||||||
@@ -106,7 +126,7 @@ impl KnowledgeGraph {
|
|||||||
self.entity_index.insert(key, raw);
|
self.entity_index.insert(key, raw);
|
||||||
raw
|
raw
|
||||||
};
|
};
|
||||||
chunk_nodes.push(node_raw);
|
chunk_nodes.insert(node_raw);
|
||||||
}
|
}
|
||||||
|
|
||||||
for extracted in &result.relationships {
|
for extracted in &result.relationships {
|
||||||
@@ -118,11 +138,14 @@ impl KnowledgeGraph {
|
|||||||
) {
|
) {
|
||||||
let from_idx = NodeIndex::new(from_raw as usize);
|
let from_idx = NodeIndex::new(from_raw as usize);
|
||||||
let to_idx = NodeIndex::new(to_raw as usize);
|
let to_idx = NodeIndex::new(to_raw as usize);
|
||||||
// Avoid duplicate edges
|
let already_exists = self
|
||||||
if !self.graph.contains_edge(from_idx, to_idx) {
|
.graph
|
||||||
|
.edges_connecting(from_idx, to_idx)
|
||||||
|
.any(|e| e.weight().relation_type == extracted.relation_type);
|
||||||
|
if !already_exists {
|
||||||
let rel = Relationship {
|
let rel = Relationship {
|
||||||
relation_type: extracted.relation_type.clone(),
|
relation_type: extracted.relation_type.clone(),
|
||||||
weight: extracted.weight.unwrap_or(1.0),
|
weight: extracted.weight.unwrap_or(1.0).clamp(0.0, 1.0),
|
||||||
};
|
};
|
||||||
self.graph.add_edge(from_idx, to_idx, rel);
|
self.graph.add_edge(from_idx, to_idx, rel);
|
||||||
}
|
}
|
||||||
@@ -158,6 +181,10 @@ impl KnowledgeGraph {
|
|||||||
.filter(|raw| !still_used.contains(raw))
|
.filter(|raw| !still_used.contains(raw))
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
|
if to_remove.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
for raw in to_remove {
|
for raw in to_remove {
|
||||||
let idx = NodeIndex::new(raw as usize);
|
let idx = NodeIndex::new(raw as usize);
|
||||||
if self.graph.contains_node(idx) {
|
if self.graph.contains_node(idx) {
|
||||||
@@ -166,6 +193,57 @@ impl KnowledgeGraph {
|
|||||||
self.entity_index.swap_remove(&name);
|
self.entity_index.swap_remove(&name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
self.compact();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Rebuild the internal graph with consecutive node indices. Eliminates
|
||||||
|
/// the null tombstone slots that petgraph's StableGraph accumulates after
|
||||||
|
/// repeated `remove_node` calls, keeping serialized YAML size in check.
|
||||||
|
fn compact(&mut self) {
|
||||||
|
let mut new_graph: StableGraph<Entity, Relationship> = StableGraph::new();
|
||||||
|
let mut old_to_new: HashMap<u32, u32> = HashMap::new();
|
||||||
|
|
||||||
|
for &old_raw in self.entity_index.values() {
|
||||||
|
let old_idx = NodeIndex::new(old_raw as usize);
|
||||||
|
if self.graph.contains_node(old_idx) {
|
||||||
|
let entity = self.graph[old_idx].clone();
|
||||||
|
let new_idx = new_graph.add_node(entity);
|
||||||
|
old_to_new.insert(old_raw, new_idx.index() as u32);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for edge_idx in self.graph.edge_indices() {
|
||||||
|
if let Some((from, to)) = self.graph.edge_endpoints(edge_idx) {
|
||||||
|
let from_raw = from.index() as u32;
|
||||||
|
let to_raw = to.index() as u32;
|
||||||
|
if let (Some(&new_from), Some(&new_to)) =
|
||||||
|
(old_to_new.get(&from_raw), old_to_new.get(&to_raw))
|
||||||
|
{
|
||||||
|
let rel = self.graph[edge_idx].clone();
|
||||||
|
new_graph.add_edge(
|
||||||
|
NodeIndex::new(new_from as usize),
|
||||||
|
NodeIndex::new(new_to as usize),
|
||||||
|
rel,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for raw in self.entity_index.values_mut() {
|
||||||
|
if let Some(&new_raw) = old_to_new.get(raw) {
|
||||||
|
*raw = new_raw;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for node_raws in self.document_entities.values_mut() {
|
||||||
|
*node_raws = node_raws
|
||||||
|
.iter()
|
||||||
|
.filter_map(|raw| old_to_new.get(raw).copied())
|
||||||
|
.collect();
|
||||||
|
}
|
||||||
|
|
||||||
|
self.graph = new_graph;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn build_node_to_docs(&self) -> IndexMap<u32, Vec<DocumentId>> {
|
pub fn build_node_to_docs(&self) -> IndexMap<u32, Vec<DocumentId>> {
|
||||||
@@ -179,30 +257,79 @@ impl KnowledgeGraph {
|
|||||||
map
|
map
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn expand_neighbors(&self, seed_nodes: &[u32], hops: usize) -> Vec<u32> {
|
/// BFS from seed nodes with weight-decayed scoring.
|
||||||
let mut expanded: indexmap::IndexSet<u32> = seed_nodes.iter().copied().collect();
|
///
|
||||||
let mut frontier: Vec<u32> = seed_nodes.to_vec();
|
/// Seed node scores are provided by the caller (typically token-overlap
|
||||||
|
/// ratios). Each neighbor's score is `edge_weight * parent_score`, so
|
||||||
|
/// strongly-connected neighbors rank higher and weakly-connected ones
|
||||||
|
/// naturally contribute less. Traversal is capped at `MAX_GRAPH_NODES`
|
||||||
|
/// total nodes; the highest-scored frontier nodes are expanded first so
|
||||||
|
/// the budget is spent on the most relevant entities.
|
||||||
|
///
|
||||||
|
/// Returns a map of raw node index → score (includes seed nodes).
|
||||||
|
pub fn expand_neighbors_scored(
|
||||||
|
&self,
|
||||||
|
seed_scores: &[(u32, f32)],
|
||||||
|
hops: usize,
|
||||||
|
) -> IndexMap<u32, f32> {
|
||||||
|
let mut node_scores: IndexMap<u32, f32> = IndexMap::new();
|
||||||
|
for &(raw, score) in seed_scores {
|
||||||
|
node_scores.insert(raw, score);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut frontier: Vec<(u32, f32)> = seed_scores.to_vec();
|
||||||
|
|
||||||
for _ in 0..hops {
|
for _ in 0..hops {
|
||||||
let mut next_frontier: Vec<u32> = vec![];
|
if node_scores.len() >= MAX_GRAPH_NODES {
|
||||||
for &raw in &frontier {
|
break;
|
||||||
let idx = NodeIndex::new(raw as usize);
|
}
|
||||||
if self.graph.contains_node(idx) {
|
|
||||||
|
frontier.sort_unstable_by(|a, b| {
|
||||||
|
b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal)
|
||||||
|
});
|
||||||
|
|
||||||
|
let mut next_frontier: Vec<(u32, f32)> = vec![];
|
||||||
|
|
||||||
|
'nodes: for (raw, parent_score) in &frontier {
|
||||||
|
let idx = NodeIndex::new(*raw as usize);
|
||||||
|
if !self.graph.contains_node(idx) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
for dir in [Direction::Outgoing, Direction::Incoming] {
|
for dir in [Direction::Outgoing, Direction::Incoming] {
|
||||||
for neighbor in self.graph.neighbors_directed(idx, dir) {
|
for edge_ref in self.graph.edges_directed(idx, dir) {
|
||||||
let n = neighbor.index() as u32;
|
let neighbor_idx = match dir {
|
||||||
if expanded.insert(n) {
|
Direction::Outgoing => edge_ref.target(),
|
||||||
next_frontier.push(n);
|
Direction::Incoming => edge_ref.source(),
|
||||||
|
};
|
||||||
|
let neighbor_raw = neighbor_idx.index() as u32;
|
||||||
|
let candidate = edge_ref.weight().weight * parent_score;
|
||||||
|
|
||||||
|
match node_scores.entry(neighbor_raw) {
|
||||||
|
indexmap::map::Entry::Vacant(e) => {
|
||||||
|
e.insert(candidate);
|
||||||
|
next_frontier.push((neighbor_raw, candidate));
|
||||||
}
|
}
|
||||||
|
indexmap::map::Entry::Occupied(mut e) => {
|
||||||
|
if candidate > *e.get() {
|
||||||
|
*e.get_mut() = candidate;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if node_scores.len() >= MAX_GRAPH_NODES {
|
||||||
|
break 'nodes;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
frontier = next_frontier;
|
frontier = next_frontier;
|
||||||
if frontier.is_empty() {
|
if frontier.is_empty() {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
expanded.into_iter().collect()
|
|
||||||
|
node_scores
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -213,6 +340,14 @@ pub async fn extract_entities(
|
|||||||
chunk: &str,
|
chunk: &str,
|
||||||
prompt_template: Option<&str>,
|
prompt_template: Option<&str>,
|
||||||
) -> Result<ExtractionResult> {
|
) -> Result<ExtractionResult> {
|
||||||
|
if chunk.len() > MAX_CHUNK_CHARS {
|
||||||
|
warn!(
|
||||||
|
"Entity extraction chunk is {} chars (heuristic limit: {}); \
|
||||||
|
the LLM response may be truncated",
|
||||||
|
chunk.len(),
|
||||||
|
MAX_CHUNK_CHARS
|
||||||
|
);
|
||||||
|
}
|
||||||
let template = prompt_template.unwrap_or(EXTRACTION_PROMPT);
|
let template = prompt_template.unwrap_or(EXTRACTION_PROMPT);
|
||||||
let prompt = template.replace("__CHUNK__", chunk);
|
let prompt = template.replace("__CHUNK__", chunk);
|
||||||
let mut messages = vec![Message::new(
|
let mut messages = vec![Message::new(
|
||||||
@@ -250,3 +385,350 @@ pub async fn extract_entities(
|
|||||||
serde_json::from_str::<ExtractionResult>(&json)
|
serde_json::from_str::<ExtractionResult>(&json)
|
||||||
.context("Failed to parse entity extraction JSON")
|
.context("Failed to parse entity extraction JSON")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
fn entity(name: &str, entity_type: &str) -> ExtractedEntity {
|
||||||
|
ExtractedEntity {
|
||||||
|
name: name.to_string(),
|
||||||
|
entity_type: entity_type.to_string(),
|
||||||
|
description: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rel(from: &str, to: &str, rel_type: &str, weight: f32) -> ExtractedRelationship {
|
||||||
|
ExtractedRelationship {
|
||||||
|
from: from.to_string(),
|
||||||
|
to: to.to_string(),
|
||||||
|
relation_type: rel_type.to_string(),
|
||||||
|
weight: Some(weight),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn doc(id: usize) -> DocumentId {
|
||||||
|
DocumentId(id)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn extraction(
|
||||||
|
entities: Vec<ExtractedEntity>,
|
||||||
|
rels: Vec<ExtractedRelationship>,
|
||||||
|
) -> ExtractionResult {
|
||||||
|
ExtractionResult {
|
||||||
|
entities,
|
||||||
|
relationships: rels,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn merge_deduplicates_by_lowercase_name() {
|
||||||
|
let mut kg = KnowledgeGraph::default();
|
||||||
|
kg.merge(
|
||||||
|
doc(0),
|
||||||
|
extraction(
|
||||||
|
vec![
|
||||||
|
entity("Python", "TECHNOLOGY"),
|
||||||
|
entity("python", "TECHNOLOGY"),
|
||||||
|
],
|
||||||
|
vec![],
|
||||||
|
),
|
||||||
|
);
|
||||||
|
assert_eq!(kg.entity_index.len(), 1);
|
||||||
|
assert_eq!(kg.graph.node_count(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn merge_chunk_nodes_no_duplicate_doc_entries() {
|
||||||
|
let mut kg = KnowledgeGraph::default();
|
||||||
|
kg.merge(
|
||||||
|
doc(1),
|
||||||
|
extraction(
|
||||||
|
vec![
|
||||||
|
entity("Python", "TECHNOLOGY"),
|
||||||
|
entity("python", "TECHNOLOGY"),
|
||||||
|
],
|
||||||
|
vec![],
|
||||||
|
),
|
||||||
|
);
|
||||||
|
let count = kg.document_entities.get(&1).map(|v| v.len()).unwrap_or(0);
|
||||||
|
assert_eq!(
|
||||||
|
count, 1,
|
||||||
|
"duplicate entity in one chunk should produce one doc_entity entry"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn merge_normalizes_entity_type_to_uppercase() {
|
||||||
|
let mut kg = KnowledgeGraph::default();
|
||||||
|
kg.merge(
|
||||||
|
doc(0),
|
||||||
|
extraction(vec![entity("Django", "technology")], vec![]),
|
||||||
|
);
|
||||||
|
let raw = kg.entity_index["django"];
|
||||||
|
assert_eq!(
|
||||||
|
kg.graph[NodeIndex::new(raw as usize)].entity_type,
|
||||||
|
"TECHNOLOGY"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn merge_promotes_type_from_other_to_specific() {
|
||||||
|
let mut kg = KnowledgeGraph::default();
|
||||||
|
kg.merge(doc(0), extraction(vec![entity("Python", "OTHER")], vec![]));
|
||||||
|
kg.merge(
|
||||||
|
doc(1),
|
||||||
|
extraction(vec![entity("Python", "TECHNOLOGY")], vec![]),
|
||||||
|
);
|
||||||
|
let raw = kg.entity_index["python"];
|
||||||
|
assert_eq!(
|
||||||
|
kg.graph[NodeIndex::new(raw as usize)].entity_type,
|
||||||
|
"TECHNOLOGY"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn merge_does_not_demote_specific_type_to_other() {
|
||||||
|
let mut kg = KnowledgeGraph::default();
|
||||||
|
kg.merge(
|
||||||
|
doc(0),
|
||||||
|
extraction(vec![entity("Python", "TECHNOLOGY")], vec![]),
|
||||||
|
);
|
||||||
|
kg.merge(doc(1), extraction(vec![entity("Python", "OTHER")], vec![]));
|
||||||
|
let raw = kg.entity_index["python"];
|
||||||
|
assert_eq!(
|
||||||
|
kg.graph[NodeIndex::new(raw as usize)].entity_type,
|
||||||
|
"TECHNOLOGY"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn merge_allows_multiple_relation_types_between_same_pair() {
|
||||||
|
let mut kg = KnowledgeGraph::default();
|
||||||
|
kg.merge(
|
||||||
|
doc(0),
|
||||||
|
extraction(
|
||||||
|
vec![
|
||||||
|
entity("Python", "TECHNOLOGY"),
|
||||||
|
entity("Django", "TECHNOLOGY"),
|
||||||
|
],
|
||||||
|
vec![rel("Python", "Django", "implements", 0.9)],
|
||||||
|
),
|
||||||
|
);
|
||||||
|
kg.merge(
|
||||||
|
doc(1),
|
||||||
|
extraction(
|
||||||
|
vec![
|
||||||
|
entity("Python", "TECHNOLOGY"),
|
||||||
|
entity("Django", "TECHNOLOGY"),
|
||||||
|
],
|
||||||
|
vec![rel("Python", "Django", "uses", 0.8)],
|
||||||
|
),
|
||||||
|
);
|
||||||
|
let from_idx = NodeIndex::new(kg.entity_index["python"] as usize);
|
||||||
|
let to_idx = NodeIndex::new(kg.entity_index["django"] as usize);
|
||||||
|
let count = kg.graph.edges_connecting(from_idx, to_idx).count();
|
||||||
|
assert_eq!(
|
||||||
|
count, 2,
|
||||||
|
"two different relation types should produce two edges"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn merge_deduplicates_same_relation_type() {
|
||||||
|
let mut kg = KnowledgeGraph::default();
|
||||||
|
kg.merge(
|
||||||
|
doc(0),
|
||||||
|
extraction(
|
||||||
|
vec![entity("A", "CONCEPT"), entity("B", "CONCEPT")],
|
||||||
|
vec![rel("A", "B", "uses", 1.0)],
|
||||||
|
),
|
||||||
|
);
|
||||||
|
kg.merge(
|
||||||
|
doc(1),
|
||||||
|
extraction(
|
||||||
|
vec![entity("A", "CONCEPT"), entity("B", "CONCEPT")],
|
||||||
|
vec![rel("A", "B", "uses", 0.5)],
|
||||||
|
),
|
||||||
|
);
|
||||||
|
let from_idx = NodeIndex::new(kg.entity_index["a"] as usize);
|
||||||
|
let to_idx = NodeIndex::new(kg.entity_index["b"] as usize);
|
||||||
|
let count = kg.graph.edges_connecting(from_idx, to_idx).count();
|
||||||
|
assert_eq!(
|
||||||
|
count, 1,
|
||||||
|
"same relation type should not create a duplicate edge"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn remove_documents_preserves_entity_shared_across_docs() {
|
||||||
|
let mut kg = KnowledgeGraph::default();
|
||||||
|
kg.merge(
|
||||||
|
doc(0),
|
||||||
|
extraction(
|
||||||
|
vec![entity("Python", "TECHNOLOGY"), entity("A", "CONCEPT")],
|
||||||
|
vec![],
|
||||||
|
),
|
||||||
|
);
|
||||||
|
kg.merge(
|
||||||
|
doc(1),
|
||||||
|
extraction(
|
||||||
|
vec![entity("Python", "TECHNOLOGY"), entity("B", "CONCEPT")],
|
||||||
|
vec![],
|
||||||
|
),
|
||||||
|
);
|
||||||
|
kg.remove_documents(&[doc(0)]);
|
||||||
|
assert!(
|
||||||
|
kg.entity_index.contains_key("python"),
|
||||||
|
"shared entity should survive"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!kg.entity_index.contains_key("a"),
|
||||||
|
"exclusive entity should be removed"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
kg.entity_index.contains_key("b"),
|
||||||
|
"other doc's entity should survive"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn remove_documents_noop_on_empty_slice() {
|
||||||
|
let mut kg = KnowledgeGraph::default();
|
||||||
|
kg.merge(doc(0), extraction(vec![entity("X", "CONCEPT")], vec![]));
|
||||||
|
kg.remove_documents(&[]);
|
||||||
|
assert_eq!(kg.entity_index.len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn remove_documents_compacts_graph() {
|
||||||
|
let mut kg = KnowledgeGraph::default();
|
||||||
|
// doc 0: A, B with an edge
|
||||||
|
kg.merge(
|
||||||
|
doc(0),
|
||||||
|
extraction(
|
||||||
|
vec![entity("A", "CONCEPT"), entity("B", "CONCEPT")],
|
||||||
|
vec![rel("A", "B", "uses", 1.0)],
|
||||||
|
),
|
||||||
|
);
|
||||||
|
// doc 1: C only
|
||||||
|
kg.merge(doc(1), extraction(vec![entity("C", "CONCEPT")], vec![]));
|
||||||
|
|
||||||
|
kg.remove_documents(&[doc(0)]);
|
||||||
|
|
||||||
|
assert_eq!(kg.graph.node_count(), 1);
|
||||||
|
let c_raw = kg.entity_index["c"];
|
||||||
|
assert_eq!(
|
||||||
|
c_raw, 0,
|
||||||
|
"compacted graph should give surviving node index 0"
|
||||||
|
);
|
||||||
|
let refs = kg.document_entities.get(&1).cloned().unwrap_or_default();
|
||||||
|
assert_eq!(refs, vec![0u32]);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn expand_zero_hops_returns_seeds_only() {
|
||||||
|
let mut kg = KnowledgeGraph::default();
|
||||||
|
kg.merge(
|
||||||
|
doc(0),
|
||||||
|
extraction(
|
||||||
|
vec![entity("A", "CONCEPT"), entity("B", "CONCEPT")],
|
||||||
|
vec![rel("A", "B", "uses", 0.9)],
|
||||||
|
),
|
||||||
|
);
|
||||||
|
let a_raw = kg.entity_index["a"];
|
||||||
|
let result = kg.expand_neighbors_scored(&[(a_raw, 1.0)], 0);
|
||||||
|
assert_eq!(result.len(), 1);
|
||||||
|
assert_eq!(result[&a_raw], 1.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn expand_one_hop_decays_score_by_edge_weight() {
|
||||||
|
let mut kg = KnowledgeGraph::default();
|
||||||
|
kg.merge(
|
||||||
|
doc(0),
|
||||||
|
extraction(
|
||||||
|
vec![entity("A", "CONCEPT"), entity("B", "CONCEPT")],
|
||||||
|
vec![rel("A", "B", "uses", 0.8)],
|
||||||
|
),
|
||||||
|
);
|
||||||
|
let a_raw = kg.entity_index["a"];
|
||||||
|
let b_raw = kg.entity_index["b"];
|
||||||
|
let result = kg.expand_neighbors_scored(&[(a_raw, 1.0)], 1);
|
||||||
|
assert_eq!(result.len(), 2);
|
||||||
|
assert_eq!(result[&a_raw], 1.0);
|
||||||
|
let b_score = result[&b_raw];
|
||||||
|
assert!(
|
||||||
|
(b_score - 0.8).abs() < 1e-6,
|
||||||
|
"neighbor score should be edge_weight * parent_score = 0.8, got {b_score}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn expand_incoming_edges_also_traversed() {
|
||||||
|
let mut kg = KnowledgeGraph::default();
|
||||||
|
// Edge goes B → A; seeding A should still discover B via incoming edge
|
||||||
|
kg.merge(
|
||||||
|
doc(0),
|
||||||
|
extraction(
|
||||||
|
vec![entity("A", "CONCEPT"), entity("B", "CONCEPT")],
|
||||||
|
vec![rel("B", "A", "uses", 0.7)],
|
||||||
|
),
|
||||||
|
);
|
||||||
|
let a_raw = kg.entity_index["a"];
|
||||||
|
let b_raw = kg.entity_index["b"];
|
||||||
|
let result = kg.expand_neighbors_scored(&[(a_raw, 1.0)], 1);
|
||||||
|
assert!(
|
||||||
|
result.contains_key(&b_raw),
|
||||||
|
"B should be reachable via incoming edge from A"
|
||||||
|
);
|
||||||
|
let b_score = result[&b_raw];
|
||||||
|
assert!((b_score - 0.7).abs() < 1e-6);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn expand_picks_best_path_score() {
|
||||||
|
let mut kg = KnowledgeGraph::default();
|
||||||
|
// A(0.5) → C(0.9): score 0.45; B(1.0) → C(0.4): score 0.40 — A→C path wins.
|
||||||
|
kg.merge(
|
||||||
|
doc(0),
|
||||||
|
extraction(
|
||||||
|
vec![
|
||||||
|
entity("A", "CONCEPT"),
|
||||||
|
entity("B", "CONCEPT"),
|
||||||
|
entity("C", "CONCEPT"),
|
||||||
|
],
|
||||||
|
vec![rel("A", "C", "uses", 0.9), rel("B", "C", "uses", 0.4)],
|
||||||
|
),
|
||||||
|
);
|
||||||
|
let a_raw = kg.entity_index["a"];
|
||||||
|
let b_raw = kg.entity_index["b"];
|
||||||
|
let c_raw = kg.entity_index["c"];
|
||||||
|
let seeds = vec![(a_raw, 0.5f32), (b_raw, 1.0f32)];
|
||||||
|
let result = kg.expand_neighbors_scored(&seeds, 1);
|
||||||
|
let c_score = result[&c_raw];
|
||||||
|
// Best path: B(1.0) * 0.4 = 0.4, A(0.5) * 0.9 = 0.45 → should be 0.45
|
||||||
|
assert!(
|
||||||
|
(c_score - 0.45).abs() < 1e-6,
|
||||||
|
"C score should reflect best path (0.45), got {c_score}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn build_node_to_docs_maps_shared_entity_to_multiple_docs() {
|
||||||
|
let mut kg = KnowledgeGraph::default();
|
||||||
|
kg.merge(
|
||||||
|
doc(0),
|
||||||
|
extraction(vec![entity("Python", "TECHNOLOGY")], vec![]),
|
||||||
|
);
|
||||||
|
kg.merge(
|
||||||
|
doc(1),
|
||||||
|
extraction(vec![entity("Python", "TECHNOLOGY")], vec![]),
|
||||||
|
);
|
||||||
|
let n2d = kg.build_node_to_docs();
|
||||||
|
let raw = kg.entity_index["python"];
|
||||||
|
let docs = &n2d[&raw];
|
||||||
|
assert!(docs.contains(&DocumentId(0)));
|
||||||
|
assert!(docs.contains(&DocumentId(1)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+50
-40
@@ -25,6 +25,8 @@ use std::{
|
|||||||
};
|
};
|
||||||
use tokio::time::sleep;
|
use tokio::time::sleep;
|
||||||
|
|
||||||
|
const BM25_SEED_SCORE: f32 = 0.5;
|
||||||
|
|
||||||
const RAG_TEMPLATE: &str = r#"Answer the query based on the context while respecting the rules. (user query, some textual context and rules, all inside xml tags)
|
const RAG_TEMPLATE: &str = r#"Answer the query based on the context while respecting the rules. (user query, some textual context and rules, all inside xml tags)
|
||||||
|
|
||||||
<context>
|
<context>
|
||||||
@@ -752,14 +754,14 @@ impl Rag {
|
|||||||
bail!("No RAG files");
|
bail!("No RAG files");
|
||||||
}
|
}
|
||||||
|
|
||||||
if self.data.extractor_model.is_some()
|
if !new_doc_contents.is_empty()
|
||||||
&& !new_doc_contents.is_empty()
|
|
||||||
&& let Some(extractor_model_id) = self.data.extractor_model.clone()
|
&& let Some(extractor_model_id) = self.data.extractor_model.clone()
|
||||||
{
|
{
|
||||||
match Model::retrieve_model(&self.app_config, &extractor_model_id, ModelType::Chat) {
|
match Model::retrieve_model(&self.app_config, &extractor_model_id, ModelType::Chat) {
|
||||||
Ok(model) => match self.create_embeddings_client(model) {
|
Ok(model) => match self.create_embeddings_client(model) {
|
||||||
Ok(client) => {
|
Ok(client) => {
|
||||||
let total = new_doc_contents.len();
|
let total = new_doc_contents.len();
|
||||||
|
let mut failures = 0usize;
|
||||||
for (i, (doc_id, content)) in new_doc_contents.into_iter().enumerate() {
|
for (i, (doc_id, content)) in new_doc_contents.into_iter().enumerate() {
|
||||||
progress(
|
progress(
|
||||||
&spinner,
|
&spinner,
|
||||||
@@ -774,14 +776,21 @@ impl Rag {
|
|||||||
{
|
{
|
||||||
Ok(result) => self.data.knowledge_graph.merge(doc_id, result),
|
Ok(result) => self.data.knowledge_graph.merge(doc_id, result),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
debug!("Entity extraction failed for doc {doc_id:?}: {e}")
|
warn!("Entity extraction failed for doc {doc_id:?}: {e}");
|
||||||
|
failures += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if failures > 0 {
|
||||||
|
progress(
|
||||||
|
&spinner,
|
||||||
|
format!("Entity extraction: {failures}/{total} chunks failed"),
|
||||||
|
);
|
||||||
}
|
}
|
||||||
Err(e) => debug!("Failed to create extractor client: {e}"),
|
}
|
||||||
|
Err(e) => warn!("Failed to create extractor client: {e}"),
|
||||||
},
|
},
|
||||||
Err(e) => debug!("Extractor model not found: {e}"),
|
Err(e) => warn!("Extractor model not found: {e}"),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -930,9 +939,31 @@ impl Rag {
|
|||||||
if kg.entity_index.is_empty() {
|
if kg.entity_index.is_empty() {
|
||||||
return vec![];
|
return vec![];
|
||||||
}
|
}
|
||||||
let query_lower = query.to_lowercase();
|
|
||||||
|
|
||||||
let mut seed_nodes: Vec<u32> = kg
|
let query_lower = query.to_lowercase();
|
||||||
|
let query_tokens: Vec<&str> = query_lower.split_whitespace().collect();
|
||||||
|
let token_count = query_tokens.len().max(1);
|
||||||
|
|
||||||
|
let score_node = |raw: u32| -> f32 {
|
||||||
|
let idx = NodeIndex::new(raw as usize);
|
||||||
|
if !kg.graph.contains_node(idx) {
|
||||||
|
return 0.0;
|
||||||
|
}
|
||||||
|
let entity = &kg.graph[idx];
|
||||||
|
let combined = format!(
|
||||||
|
"{} {}",
|
||||||
|
entity.name,
|
||||||
|
entity.description.as_deref().unwrap_or("")
|
||||||
|
)
|
||||||
|
.to_lowercase();
|
||||||
|
query_tokens
|
||||||
|
.iter()
|
||||||
|
.filter(|t| combined.contains(*t))
|
||||||
|
.count() as f32
|
||||||
|
/ token_count as f32
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut seed_scores: Vec<(u32, f32)> = kg
|
||||||
.entity_index
|
.entity_index
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|(name, _)| {
|
.filter(|(name, _)| {
|
||||||
@@ -946,52 +977,31 @@ impl Rag {
|
|||||||
.any(|token| token.trim_matches(|c: char| !c.is_alphanumeric()) == name_str)
|
.any(|token| token.trim_matches(|c: char| !c.is_alphanumeric()) == name_str)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
.map(|(_, &raw)| raw)
|
.map(|(_, &raw)| (raw, score_node(raw).max(BM25_SEED_SCORE)))
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
if seed_nodes.is_empty() {
|
if seed_scores.is_empty() {
|
||||||
let bm25_results = self.bm25.search(query, top_k * 2);
|
let bm25_results = self.bm25.search(query, top_k * 2);
|
||||||
'outer: for result in bm25_results {
|
'outer: for result in bm25_results {
|
||||||
if let Some(node_raws) = kg.document_entities.get(&result.document.id.0) {
|
if let Some(node_raws) = kg.document_entities.get(&result.document.id.0) {
|
||||||
seed_nodes.extend(node_raws.iter().copied());
|
for &raw in node_raws {
|
||||||
if seed_nodes.len() >= top_k {
|
seed_scores.push((raw, BM25_SEED_SCORE));
|
||||||
|
if seed_scores.len() >= top_k {
|
||||||
break 'outer;
|
break 'outer;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if seed_nodes.is_empty() {
|
if seed_scores.is_empty() {
|
||||||
return vec![];
|
return vec![];
|
||||||
}
|
}
|
||||||
|
|
||||||
let hops = self.data.graph_hops.unwrap_or(1);
|
let hops = self.data.graph_hops.unwrap_or(1);
|
||||||
let expanded = kg.expand_neighbors(&seed_nodes, hops);
|
let mut scored: Vec<(u32, f32)> = kg
|
||||||
|
.expand_neighbors_scored(&seed_scores, hops)
|
||||||
let query_tokens: Vec<&str> = query_lower.split_whitespace().collect();
|
|
||||||
let token_count = query_tokens.len().max(1);
|
|
||||||
let mut scored: Vec<(u32, f32)> = expanded
|
|
||||||
.into_iter()
|
.into_iter()
|
||||||
.map(|raw| {
|
|
||||||
let idx = NodeIndex::new(raw as usize);
|
|
||||||
let score = if kg.graph.contains_node(idx) {
|
|
||||||
let entity = &kg.graph[idx];
|
|
||||||
let combined = format!(
|
|
||||||
"{} {}",
|
|
||||||
entity.name,
|
|
||||||
entity.description.as_deref().unwrap_or("")
|
|
||||||
)
|
|
||||||
.to_lowercase();
|
|
||||||
query_tokens
|
|
||||||
.iter()
|
|
||||||
.filter(|t| combined.contains(*t))
|
|
||||||
.count() as f32
|
|
||||||
/ token_count as f32
|
|
||||||
} else {
|
|
||||||
0.0
|
|
||||||
};
|
|
||||||
(raw, score)
|
|
||||||
})
|
|
||||||
.collect();
|
.collect();
|
||||||
scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(Ordering::Equal));
|
scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(Ordering::Equal));
|
||||||
|
|
||||||
@@ -1349,11 +1359,11 @@ fn set_chunk_size(model: &Model) -> Result<usize> {
|
|||||||
fn set_graph_hops(default_value: usize) -> Result<usize> {
|
fn set_graph_hops(default_value: usize) -> Result<usize> {
|
||||||
let value = Text::new("Set graph expansion hops:")
|
let value = Text::new("Set graph expansion hops:")
|
||||||
.with_default(&default_value.to_string())
|
.with_default(&default_value.to_string())
|
||||||
.with_help_message("Number of hops to expand from matched entities (1 = direct neighbors, 2 = neighbors of neighbors)")
|
.with_help_message("Number of hops to expand from matched entities (0 = seed nodes only, 1 = direct neighbors, 2 = neighbors of neighbors)")
|
||||||
.with_validator(move |text: &str| {
|
.with_validator(move |text: &str| {
|
||||||
let out = match text.parse::<usize>() {
|
let out = match text.parse::<usize>() {
|
||||||
Ok(v) if v >= 1 => Validation::Valid,
|
Ok(_) => Validation::Valid,
|
||||||
_ => Validation::Invalid("Must be an integer >= 1".into()),
|
_ => Validation::Invalid("Must be a non-negative integer".into()),
|
||||||
};
|
};
|
||||||
Ok(out)
|
Ok(out)
|
||||||
})
|
})
|
||||||
|
|||||||
Reference in New Issue
Block a user