Compare commits
11
Commits
ebe7816600
...
92c8ff8934
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
92c8ff8934 | ||
|
|
af697d129b | ||
|
|
895ecc812f | ||
|
|
93e90106a8 | ||
|
|
35e4c82f27 | ||
|
|
ee45e42013 | ||
|
|
3d640b9efa | ||
|
|
730f942bab | ||
|
|
2efeda0ba7 | ||
|
|
df62c32822 | ||
|
|
3762bbe09f |
+9
-10
@@ -3,7 +3,7 @@ name = "coyote-ai"
|
||||
version = "0.8.3"
|
||||
edition = "2024"
|
||||
authors = ["Alex Clarke <alex.j.tusa@gmail.com>"]
|
||||
description = "An all-in-one, batteries included LLM CLI Tool"
|
||||
description = "An all-in-one, batteries-included LLM runtime"
|
||||
keywords = ["chatgpt", "llm", "cli", "ai", "repl"]
|
||||
homepage = "https://github.com/Dark-Alex-17/coyote"
|
||||
repository = "https://github.com/Dark-Alex-17/coyote"
|
||||
@@ -51,7 +51,13 @@ textwrap = "0.16.0"
|
||||
ansi_colours = "1.2.2"
|
||||
eventsource-stream = "0.2.3"
|
||||
log = "0.4.28"
|
||||
log4rs = { version = "1.4.0", features = ["file_appender", "rolling_file_appender", "compound_policy", "fixed_window_roller", "size_trigger"] }
|
||||
log4rs = { version = "1.4.0", features = [
|
||||
"file_appender",
|
||||
"rolling_file_appender",
|
||||
"compound_policy",
|
||||
"fixed_window_roller",
|
||||
"size_trigger",
|
||||
] }
|
||||
shell-words = "1.1.0"
|
||||
sha2 = "0.10.8"
|
||||
unicode-width = "0.2.0"
|
||||
@@ -113,14 +119,7 @@ qrcode = "0.14"
|
||||
|
||||
[dependencies.reqwest]
|
||||
version = "0.13.3"
|
||||
features = [
|
||||
"json",
|
||||
"multipart",
|
||||
"stream",
|
||||
"form",
|
||||
"socks",
|
||||
"rustls",
|
||||
]
|
||||
features = ["json", "multipart", "stream", "form", "socks", "rustls"]
|
||||
default-features = false
|
||||
|
||||
[dependencies.syntect]
|
||||
|
||||
+1
-1
@@ -99,7 +99,7 @@ LABEL com.docker.sandboxes="templates" \
|
||||
com.docker.sandboxes.flavor="shell-docker" \
|
||||
com.docker.sandboxes.start-docker="true" \
|
||||
org.opencontainers.image.title="coyote" \
|
||||
org.opencontainers.image.description="An all-in-one, batteries-included LLM CLI tool: Shell Assistant, CLI & REPL mode, RAG, AI tools & agents, MCP servers, skills, and macros." \
|
||||
org.opencontainers.image.description="An all-in-one, batteries-included LLM runtime: Shell Assistant, CLI & REPL mode, RAG, AI tools & agents, MCP servers, skills, and macros." \
|
||||
org.opencontainers.image.source="https://github.com/Dark-Alex-17/coyote" \
|
||||
org.opencontainers.image.version="${COYOTE_VERSION}"
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Coyote: All-in-one, batteries-included LLM CLI Tool
|
||||
# Coyote: All-in-one, batteries-included LLM runtime
|
||||
|
||||

|
||||
[](https://crates.io/crates/coyote-ai)
|
||||
@@ -8,8 +8,8 @@
|
||||

|
||||
[](./LICENSE)
|
||||
|
||||
Coyote is an all-in-one, batteries-included, LLM CLI tool featuring Shell Assistant, CLI & REPL Mode, RAG, AI Tools &
|
||||
Agents, and More.
|
||||
Coyote is an all-in-one, batteries-included, LLM runtime featuring Shell Assistant, CLI & REPL Mode, RAG, AI Tools &
|
||||
Agents, and more.
|
||||
|
||||
It is designed to include a number of useful agents, roles, macros, and more so users can get up and running with Coyote
|
||||
in as little time as possible. You can also install entire bundles of agents, roles, macros, tools, and MCP servers from
|
||||
|
||||
@@ -3,8 +3,9 @@ description: |
|
||||
Design-doc orchestrator for any project. Consumes a high-level design doc, decomposes it into a
|
||||
gated plan (gatekeeper self-containedness + oracle plan-review) and ~1-engineer-day task files,
|
||||
spawns one Sisyphus per task on a single run branch, verifies each with an adversarial
|
||||
plan-conformance check, and finishes with ONE draft PR (CI checks watched to green) plus tracked
|
||||
follow-up tasks. Task state lives on disk in a plans directory, so runs survive context compression.
|
||||
plan-conformance check (plus a black-box usage-pattern probe for consumer-facing surface), and
|
||||
finishes with ONE draft PR (CI checks watched to green) plus tracked follow-up tasks. Task state
|
||||
lives on disk in a plans directory, so runs survive context compression.
|
||||
version: 2.1.0
|
||||
agent_session: temp
|
||||
auto_continue: true
|
||||
@@ -18,6 +19,7 @@ spawnable_agents:
|
||||
- explore
|
||||
- librarian
|
||||
- adversary
|
||||
- probe
|
||||
- gatekeeper
|
||||
max_concurrent_agents: 10
|
||||
max_agent_depth: 10
|
||||
@@ -149,7 +151,8 @@ instructions: |
|
||||
|
||||
**Gate 1 — Self-containedness (`gatekeeper`).** The plan must pass the "docker container" test:
|
||||
every question a context-free implementer will hit is answered inline or delegated via a verified
|
||||
pointer to code/docs (where infra code goes, DB tech/target, layout to mirror, test commands, ...).
|
||||
pointer to code/docs (where infra code goes, DB tech/target, layout to mirror, test commands,
|
||||
local-run recipe for any consumer-facing surface the plan creates, ...).
|
||||
|
||||
> `agent__spawn --agent gatekeeper --prompt "Audit this plan for self-containedness. Return
|
||||
> SEALED/LEAKY. Plan: {{plans_dir}}/PLAN-<slug>.md. Target project: {{project_dir}}."`
|
||||
@@ -286,6 +289,39 @@ instructions: |
|
||||
escalate — do not silently change scope.
|
||||
- **`ADVERSARIAL_REVIEW: CONFORMS`** → conformance satisfied. Also confirm the stated test
|
||||
commands pass (run them if feasible) before closing.
|
||||
- **Usage-pattern probe (consumer-facing tasks).** If the task added or changed consumer-facing
|
||||
surface (endpoints/RPCs/CLI commands, request/response shapes, contract semantics like
|
||||
patch-vs-replace, idempotency, auth on routes), ALSO spawn `probe` for an independent
|
||||
black-box behavioral verdict — it boots the code locally from a clean state, runs existing
|
||||
usage suites for regressions, and spec-first-tests the changed surface with the repo's
|
||||
existing suite tooling or whatever is available (e.g. Hurl/curl, grpcurl, direct CLI
|
||||
invocation). Skip it (one-line note) for tasks with no consumer-visible surface.
|
||||
|
||||
```
|
||||
agent__spawn --agent probe --prompt "Probe TASK-NNN's changed surface from the consumer's
|
||||
perspective. Return PASS/FAIL/INCONCLUSIVE.
|
||||
|
||||
CHANGE: run get_diff --base <the task's BASE SHA recorded at claim time> in {{project_dir}}.
|
||||
|
||||
SPEC — expected behavior to verify against:
|
||||
<paste the task's acceptance criteria + relevant API contract sections VERBATIM>
|
||||
|
||||
LOCAL-RUN RECIPE: <paste the plan's local-run recipe verbatim — Gate 1 requires one for
|
||||
consumer-facing tasks>
|
||||
|
||||
EXISTING SUITES: <paths + run commands from the plan, or 'discover them'>"
|
||||
```
|
||||
|
||||
Set the probe's `project_dir` to {{project_dir}}. Verdict handling:
|
||||
- **`USAGE_PROBE: FAIL`** → blocker, same loop as DIVERGES: resume the SAME Sisyphus session
|
||||
with the behavioral findings (including repros) verbatim; re-run `probe` ONCE (resume ITS
|
||||
session so it reuses its environment and tests); still FAILing on the same findings →
|
||||
STOP and escalate.
|
||||
- **`USAGE_PROBE: PASS`** → have Sisyphus adopt probe's new test files (paths are in its
|
||||
report) as a commit on the run branch so they ship as permanent regression coverage.
|
||||
- **`USAGE_PROBE: INCONCLUSIVE`** → the local-run recipe is missing or broken — a PLAN gap,
|
||||
not a code failure. Fix the recipe (amend the plan) or escalate, re-run once; NEVER count
|
||||
INCONCLUSIVE as PASS or FAIL.
|
||||
- If Sisyphus reports failure after its own recovery, surface the evidence and consult `oracle`
|
||||
for diagnosis before deciding whether to retry, re-scope, or escalate.
|
||||
|
||||
@@ -328,8 +364,8 @@ instructions: |
|
||||
({{project_dir}}/.worktrees/task-NNN) and the branch with the task branch. Sisyphus commits and
|
||||
pushes the TASK branch. All other prompt sections unchanged — still no PRs, still no
|
||||
creating/switching branches (the worktree arrives already on its branch).
|
||||
- Run the adversary check in the worktree: `get_diff --base <BASE SHA>` — identical semantics to
|
||||
sequential mode.
|
||||
- Run the adversary check (and, for consumer-facing tasks, the probe check) in the worktree:
|
||||
`get_diff --base <BASE SHA>` — identical semantics to sequential mode.
|
||||
|
||||
**Integration (architect is the integrator; merges are ALWAYS one at a time):**
|
||||
1. When a task's Sisyphus finishes AND its adversary check CONFORMS, merge in the PRIMARY checkout:
|
||||
@@ -404,6 +440,7 @@ instructions: |
|
||||
| `oracle` | Plan review (`plan-review`), and diagnosis when a task fails after Sisyphus recovery. |
|
||||
| `gatekeeper` | Plan self-containedness gate (Phase C Gate 1): audits the PLAN for the "docker container" standard, returns SEALED/LEAKY with the missing implementer questions. |
|
||||
| `adversary` | Post-implementation plan-conformance verdict per task (CONFORMS/DIVERGES). |
|
||||
| `probe` | Black-box behavioral verdict on a task's consumer-facing surface: boots the code locally from clean state, runs existing usage suites + spec-first tests. Returns USAGE_PROBE PASS/FAIL/INCONCLUSIVE. |
|
||||
|
||||
## Escalation handling
|
||||
|
||||
@@ -424,6 +461,8 @@ instructions: |
|
||||
- Materializing tasks from a plan the gatekeeper marked LEAKY (or never audited), or that Oracle
|
||||
rejected (or never reviewed).
|
||||
- Marking a task complete without the adversary's CONFORMS verdict and verified acceptance criteria.
|
||||
- Closing a consumer-facing task without a `probe` verdict, or treating `INCONCLUSIVE` as PASS —
|
||||
an unprobeable consumer-facing change is a plan gap to fix, not a checkbox to skip.
|
||||
- Code comments referencing the plan/design doc/phases/steps/TASK numbers → docs drift, comments
|
||||
rot; plan references live in commit messages only.
|
||||
- A `.env` file landing in any repo → only `.env.example` with placeholder keys is committable;
|
||||
|
||||
@@ -17,6 +17,7 @@ variables:
|
||||
mcp_servers:
|
||||
- ddg-search
|
||||
global_tools:
|
||||
- web_search_coyote.sh
|
||||
- ast_grep.sh
|
||||
- fs_read.sh
|
||||
- fs_cat.sh
|
||||
|
||||
@@ -8,9 +8,10 @@ description: |
|
||||
sisyphus alongside explore when unfamiliar libraries/APIs/frameworks are
|
||||
involved.
|
||||
|
||||
version: "1.0"
|
||||
version: '1.0'
|
||||
|
||||
global_tools:
|
||||
- web_search_coyote.sh
|
||||
- fetch_url_via_curl.sh
|
||||
|
||||
mcp_servers:
|
||||
@@ -35,13 +36,13 @@ reducers:
|
||||
output: overwrite
|
||||
|
||||
initial_state:
|
||||
language_ecosystem: "general"
|
||||
doc_domain_hints: ""
|
||||
refined_search_query: ""
|
||||
question_type: "concept"
|
||||
search_output: ""
|
||||
oss_output: ""
|
||||
findings: ""
|
||||
language_ecosystem: 'general'
|
||||
doc_domain_hints: ''
|
||||
refined_search_query: ''
|
||||
question_type: 'concept'
|
||||
search_output: ''
|
||||
oss_output: ''
|
||||
findings: ''
|
||||
|
||||
start: triage
|
||||
|
||||
@@ -104,9 +105,15 @@ nodes:
|
||||
type: string
|
||||
enum: [api_reference, best_practice, debugging, concept]
|
||||
description: The kind of question being asked.
|
||||
required: [language_ecosystem, doc_domain_hints, refined_search_query, question_type]
|
||||
required:
|
||||
[
|
||||
language_ecosystem,
|
||||
doc_domain_hints,
|
||||
refined_search_query,
|
||||
question_type,
|
||||
]
|
||||
state_updates:
|
||||
last_node_output: "{{output}}"
|
||||
last_node_output: '{{output}}'
|
||||
fallback: end_failure
|
||||
next: [search, search_oss]
|
||||
|
||||
@@ -174,13 +181,15 @@ nodes:
|
||||
- Refined query: {{refined_search_query}}
|
||||
- Question type: {{question_type}}
|
||||
|
||||
Use the ddg-search tool. Prioritize the hinted doc domains when present
|
||||
(e.g., search with `site:docs.python.org pathlib` style queries).
|
||||
Use the ddg-search tool or the web_search_coyote tool. Prioritize the
|
||||
hinted doc domains when present (e.g., search with `site:docs.python.org
|
||||
pathlib` style queries).
|
||||
tools:
|
||||
- mcp:ddg-search
|
||||
- web_search_coyote
|
||||
max_iterations: 15
|
||||
state_updates:
|
||||
search_output: "{{output}}"
|
||||
search_output: '{{output}}'
|
||||
fallback: synthesize
|
||||
next: synthesize
|
||||
|
||||
@@ -250,7 +259,7 @@ nodes:
|
||||
- mcp:personal-github
|
||||
max_iterations: 15
|
||||
state_updates:
|
||||
oss_output: "{{output}}"
|
||||
oss_output: '{{output}}'
|
||||
fallback: synthesize
|
||||
next: synthesize
|
||||
|
||||
@@ -336,7 +345,7 @@ nodes:
|
||||
- fetch_url_via_curl
|
||||
max_iterations: 20
|
||||
state_updates:
|
||||
findings: "{{output}}"
|
||||
findings: '{{output}}'
|
||||
fallback: final_format
|
||||
next: final_format
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ variables:
|
||||
mcp_servers:
|
||||
- ddg-search
|
||||
global_tools:
|
||||
- web_search_coyote.sh
|
||||
- ast_grep.sh
|
||||
- fs_read.sh
|
||||
- fs_cat.sh
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
# Probe
|
||||
|
||||
A **black-box usage-pattern verifier**. Where every other reviewer reads *text* — the diff
|
||||
([`code-reviewer`](../code-reviewer/README.md)), the plan ([`adversary`](../adversary/README.md)),
|
||||
the attack surface ([`security-reviewer`](../security-reviewer/README.md)) — `probe` asks the one
|
||||
question none of them can answer without running the thing:
|
||||
|
||||
> **"Does the changed consumer-facing surface actually behave as the spec promises when used,
|
||||
> starting from nothing?"**
|
||||
|
||||
It boots the system locally from a clean slate, runs any existing usage suites first (regression
|
||||
check), derives expected behaviors from the **spec** — never the implementation — and authors
|
||||
tests for the uncovered usage patterns: cold-start/empty-state calls, idempotent re-calls, invalid
|
||||
input, auth on new routes, partial-update (patch-vs-replace) semantics, serialization edges,
|
||||
pagination limits, error-shape consistency. These are exactly the defects invisible to static
|
||||
review.
|
||||
|
||||
## Why it's separate from the other reviewers
|
||||
|
||||
| | `code-reviewer` | `adversary` | `security-reviewer` | `probe` |
|
||||
|---|---|---|---|---|
|
||||
| Question | Is the code good? | Does it match the plan? | Can it be abused? | Does it *work* when used? |
|
||||
| Method | Reads the diff | Diff vs. criteria | Source→sink tracing | **Runs the system**, black-box |
|
||||
| Blind spot it covers | slop, bugs, coupling | skipped criteria, drift | injection, authz gaps | behavioral quirks, regressions, contract surprises |
|
||||
| Output | severity findings | `CONFORMS`/`DIVERGES` | `PASS`/`FAIL` | `PASS`/`FAIL`/`INCONCLUSIVE` |
|
||||
|
||||
The independence is behavioral: expectations are written from the spec/contract **before** reading
|
||||
handler code, so the implementer's misreadings can't become the probe's assertions — the same
|
||||
principle that makes `adversary` valuable, applied to runtime behavior.
|
||||
|
||||
## Verdict (blocking, three-way)
|
||||
|
||||
```
|
||||
USAGE_PROBE: PASS
|
||||
Surface: <...>. Existing suites: <N run, all green | none found>. New tests: <M authored at <path>, all green>.
|
||||
```
|
||||
|
||||
```
|
||||
USAGE_PROBE: FAIL
|
||||
Behavioral findings:
|
||||
1. <surface + case> — <spec'd behavior> — <observed behavior> — REPRO: <exact request + response> — <test file>
|
||||
```
|
||||
|
||||
```
|
||||
USAGE_PROBE: INCONCLUSIVE
|
||||
Could not establish a clean local environment: <verbatim error>. Missing: <the recipe/fixture that would unblock>.
|
||||
```
|
||||
|
||||
- **`FAIL` blocks completion** — the caller resumes the SAME implementer session with the findings
|
||||
pasted verbatim, then re-runs `probe` once to confirm.
|
||||
- **`INCONCLUSIVE` is the honest third state**: the environment, not the code, is the blocker. It
|
||||
routes the fix to the local-run recipe (often a plan gap the `gatekeeper` should have caught) and
|
||||
is never disguised as `PASS` or `FAIL`.
|
||||
|
||||
Every `FAIL` finding carries an exact reproduction (request/command + response received) and the
|
||||
test file that proves it.
|
||||
|
||||
## How it probes
|
||||
|
||||
Driven by the [`usage-pattern-testing`](../../skills/usage-pattern-testing/SKILL.md) skill:
|
||||
|
||||
1. **Spec first** — expected behaviors written from acceptance criteria + API contract before any
|
||||
implementation reads.
|
||||
2. **Regression first** — discover and run existing usage suites; every failure classified as
|
||||
BUG / EXPECTED-CHANGE / ENV before anything new is authored.
|
||||
3. **Delta only** — new tests cover only the usage patterns existing suites miss, written in the
|
||||
repo's suite conventions so they're adoptable as permanent regression coverage.
|
||||
4. **Clean, local, isolated** — ephemeral state, mocked externals, full teardown; bounded retries
|
||||
for startup only, never to mask flakiness.
|
||||
|
||||
Toolbox by surface — the repo's existing suite format always comes first, and these are examples,
|
||||
not requirements: [Hurl](https://hurl.dev) or `curl` scripts for HTTP/REST/JSON (Hurl files double
|
||||
as committed suites), `grpcurl` for pure gRPC, direct invocation for CLIs.
|
||||
|
||||
Unlike the read-only reviewers, `probe` **writes test files** (and only test files) — the tests
|
||||
are a deliverable alongside the verdict. It never modifies implementation code.
|
||||
|
||||
## Usage
|
||||
|
||||
Spawned by `sisyphus` (post-coder, when the change touches consumer-facing surface) or `architect`
|
||||
(Phase E, alongside `adversary`). The spawn prompt IS its entire context — include the change, the
|
||||
spec, and the local-run recipe:
|
||||
|
||||
```sh
|
||||
agent__spawn --agent probe --prompt "
|
||||
## TASK
|
||||
Probe the changed API surface for TASK-NNN from the consumer's perspective. Return PASS/FAIL/INCONCLUSIVE.
|
||||
|
||||
## CHANGE
|
||||
Run get_diff --base <ref>, or: <paste the changed-surface summary>
|
||||
|
||||
## SPEC — expected behavior to verify against
|
||||
<paste acceptance criteria + API contract sections (or contract file paths) VERBATIM>
|
||||
|
||||
## LOCAL-RUN RECIPE
|
||||
<how to boot the stack clean: build, deps/stubs, ports, migrations, teardown — or the doc that has it>
|
||||
|
||||
## EXISTING SUITES
|
||||
<paths + run commands, or 'discover them'>
|
||||
"
|
||||
```
|
||||
|
||||
Direct invocation for ad-hoc use:
|
||||
|
||||
```sh
|
||||
coyote -a probe --agent-variable project_dir /path/to/repo \
|
||||
"Probe the /widgets endpoints changed in the last commit against this spec: <paste spec>"
|
||||
```
|
||||
|
||||
### Tools
|
||||
|
||||
- `get_diff [--base <ref>]` — staged → unstaged → `HEAD~1` fallback (or an explicit base SHA/branch) to locate the changed surface.
|
||||
- `get_changed_files [--base <ref>]` — quick changed-file map.
|
||||
- Plus `fs_*`/`ast_grep` for suite discovery and contract reads, `fs_write`/`fs_patch` for authoring test files, and `execute_command` for booting the stack and running suites.
|
||||
- Probing tools (`curl`, Hurl, grpcurl, the repo's own harness) are invoked via `execute_command`
|
||||
(no wrapper tool — probing needs their full CLI surface), and none is a hard requirement: the
|
||||
[`usage-pattern-testing`](../../skills/usage-pattern-testing/SKILL.md) skill has probe reuse the repo's existing suite tooling first and fall back to what's available.
|
||||
The optional [`sbx-mixin.yaml`](sbx-mixin.yaml) preinstalls Hurl + grpcurl for sandbox runs.
|
||||
|
||||
## Related
|
||||
|
||||
- [`usage-pattern-testing`](../../skills/usage-pattern-testing/SKILL.md) — the methodology it runs on.
|
||||
- [`adversary`](../adversary/README.md) — static plan-conformance counterpart (text), where `probe` is dynamic (behavior).
|
||||
- [`gatekeeper`](../gatekeeper/README.md) — ensures plans ship the local-run recipe `probe` consumes.
|
||||
@@ -0,0 +1,129 @@
|
||||
name: probe
|
||||
description: Black-box usage-pattern verifier - exercises a change's consumer-facing surface (HTTP APIs, RPCs, CLIs) as a real cold-start consumer against a locally running instance with clean, isolated state. Runs existing usage suites first for regressions (whatever format the repo uses - Hurl files, curl scripts, collections), authors spec-first tests for uncovered patterns in the repo's suite conventions (tools like Hurl and grpcurl are examples, not requirements), and returns a blocking USAGE_PROBE PASS/FAIL/INCONCLUSIVE verdict. Complements code-reviewer (quality), adversary (plan conformance), and security-reviewer (abuse). Designed to be delegated to by sisyphus and architect.
|
||||
version: 1.0.0
|
||||
|
||||
auto_continue: true
|
||||
max_auto_continues: 25
|
||||
inject_todo_instructions: true
|
||||
|
||||
skills_enabled: true
|
||||
enabled_skills:
|
||||
- usage-pattern-testing
|
||||
|
||||
variables:
|
||||
- name: project_dir
|
||||
description: Project directory containing the change under test - where suites are discovered, the stack is booted, and new tests are written
|
||||
default: '.'
|
||||
- name: auto_confirm
|
||||
description: Auto-confirm command execution
|
||||
default: '1'
|
||||
|
||||
global_tools:
|
||||
- ast_grep.sh
|
||||
- fs_read.sh
|
||||
- fs_cat.sh
|
||||
- fs_grep.sh
|
||||
- fs_glob.sh
|
||||
- fs_ls.sh
|
||||
- fs_write.sh
|
||||
- fs_patch.sh
|
||||
- execute_command.sh
|
||||
|
||||
instructions: |
|
||||
You are the usage-pattern probe. You answer ONE question: **does the changed consumer-facing
|
||||
surface actually behave as the spec promises when used, starting from a clean slate?** Every
|
||||
other reviewer reads text — the diff, the plan, the code. You are the only gate that BOOTS the
|
||||
system locally and exercises it the way a consumer will: cold, black-box, spec-first.
|
||||
|
||||
You are NOT the code-quality reviewer (`code-reviewer`), NOT the plan-conformance reviewer
|
||||
(`adversary`), and NOT the security reviewer (`security-reviewer`). You judge observable
|
||||
behavior. Your value is behavioral independence: expectations derived from the spec BEFORE
|
||||
reading the implementation, so the implementer's misreadings cannot become your assertions.
|
||||
|
||||
## Step 0: Load the skill
|
||||
|
||||
Before anything else, `skill__load` `usage-pattern-testing`. It carries your methodology: the
|
||||
spec-first independence rule, the regression-first protocol (find and run existing suites before
|
||||
authoring anything), the usage-pattern checklist (cold start, idempotency, invalid input, auth,
|
||||
partial-update semantics, serialization edges, pagination, error shapes), the clean-environment
|
||||
discipline, the failure-classification table (BUG / EXPECTED-CHANGE / ENV), the per-surface
|
||||
toolbox (the repo's existing suite tooling comes first; Hurl/curl for HTTP, grpcurl for gRPC,
|
||||
and direct invocation for CLIs are examples, not requirements), and the exact verdict format.
|
||||
The skill body is your source of truth for HOW to probe; these instructions handle
|
||||
workflow and I/O.
|
||||
|
||||
## Input (the spawn prompt IS your entire context)
|
||||
|
||||
You are given:
|
||||
1. **The change** — a diff pasted inline, a summary of the changed surface, or an instruction to
|
||||
run `git diff`/`get_diff` (optionally against a base ref) in {{project_dir}}.
|
||||
2. **The spec** — acceptance criteria, plan section, or API contract (or paths to the contract
|
||||
files: IDL/schema/OpenAPI/proto). This is what you derive expected behaviors FROM.
|
||||
3. **A local-run recipe** (strongly preferred) — how to boot the system locally from a clean
|
||||
state: build command, dependencies to start/stub, ports, migration/seed steps, teardown. If
|
||||
absent, look for one in the repo's contributor docs and dev scripts before inventing your own.
|
||||
4. **Pointers to existing usage suites** (optional) — where black-box tests already live and how
|
||||
to run them. If absent, discover them per the skill.
|
||||
|
||||
If the spec is missing, STOP and say so: behavior cannot be judged without a promise to judge
|
||||
against. Do not infer the spec from the implementation.
|
||||
|
||||
## Workflow
|
||||
|
||||
1. Load `usage-pattern-testing`.
|
||||
2. Identify the changed consumer-facing surface from the diff/summary. No consumer-facing surface
|
||||
→ return PASS with a one-line "no probeable surface" note; do not boot anything.
|
||||
3. **Spec first:** write down expected behaviors as concrete request→response pairs from the
|
||||
spec/contract, BEFORE reading handler code (implementation reads are for ports/config/startup
|
||||
wiring only).
|
||||
4. Discover existing usage suites; bring up the clean local environment per the recipe; run the
|
||||
existing suites FIRST and classify every failure (regression vs expected contract change vs
|
||||
environment).
|
||||
5. Map existing coverage against your expected behaviors; author tests for the uncovered
|
||||
patterns only, in the repo's suite location and conventions, walking the skill's
|
||||
usage-pattern checklist.
|
||||
6. Run the new tests. Classify every failure. Reproduce non-deterministic results twice and read
|
||||
the server logs before classifying.
|
||||
7. Tear the environment down. Emit the verdict in the skill's exact format.
|
||||
|
||||
## Output — verdict (MANDATORY, exact format)
|
||||
|
||||
End with EXACTLY one of the skill's three sentinels so the caller can route on it:
|
||||
|
||||
- `USAGE_PROBE: PASS` — existing suites green (or none), new spec-first tests green. List
|
||||
surface probed, suites run, and tests authored (with paths, so the caller can adopt them).
|
||||
- `USAGE_PROBE: FAIL` — behavioral findings, each with the spec'd behavior quoted, the observed
|
||||
behavior, the EXACT reproduction (request/command + response received), and the test file.
|
||||
- `USAGE_PROBE: INCONCLUSIVE` — a clean local environment could not be established. State what
|
||||
failed verbatim and EXACTLY what recipe/fixture/mock would unblock. Include any partial
|
||||
results. INCONCLUSIVE is honest and routes the fix to the environment recipe — NEVER disguise
|
||||
it as PASS or FAIL.
|
||||
|
||||
## Rules
|
||||
|
||||
1. **Never modify implementation code.** Your only writes are new/updated TEST files (in the
|
||||
repo's suite conventions) and throwaway environment scaffolding you tear down. The
|
||||
implementer owns all fixes.
|
||||
2. **Spec-first or nothing.** Expectations written from the spec before implementation reads.
|
||||
If the spec and the contract files disagree, that is a finding — report it, don't pick one
|
||||
silently.
|
||||
3. **Regressions before new coverage.** Existing suites run first; a regression is only
|
||||
acceptable when the spec explicitly changed that contract (then flag the stale test for
|
||||
update — never delete or silence it).
|
||||
4. **Clean, local, isolated.** Fresh ephemeral state, mocked externals, no dependence on
|
||||
pre-existing data or running services, full teardown. Bounded retries for startup only —
|
||||
never to mask a flaky assertion.
|
||||
5. **Classify every failure** as BUG / EXPECTED-CHANGE / ENV per the skill table. The verdict
|
||||
depends on the classification being honest.
|
||||
6. **Committed tests are the deliverable** alongside the verdict: write them where the repo's
|
||||
suites live so the caller can adopt them as permanent regression coverage. Report their paths.
|
||||
7. Be terse and decisive. Three reproducible behavioral findings beat fifteen speculative ones.
|
||||
If everything works as spec'd, it PASSes — say so.
|
||||
|
||||
## Context
|
||||
- Project: {{project_dir}}
|
||||
- CWD: {{__cwd__}}
|
||||
- Shell: {{__shell__}}
|
||||
|
||||
## Available Tools
|
||||
{{__tools__}}
|
||||
@@ -0,0 +1,76 @@
|
||||
schemaVersion: '1'
|
||||
kind: mixin
|
||||
name: agent-probe
|
||||
description: >
|
||||
Optional convenience for the probe agent: preinstalls Hurl (HTTP
|
||||
usage-pattern tests) and grpcurl (gRPC probing) — the example tools its
|
||||
skill reaches for — and allows the GitHub release endpoints the fallback
|
||||
installers download from. Neither tool is required: probe reuses the repo's
|
||||
existing suite tooling first and falls back to what's available. Hurl
|
||||
prefers the distro package: the prebuilt GitHub tarball dynamically links
|
||||
libxml2.so.2, which newer distros no longer ship (e.g. Ubuntu 26.04 moved
|
||||
to libxml2.so.16). The services under probe run on localhost, which needs
|
||||
no network allowance. POSIX-only: sbx runs these commands with /bin/sh (dash).
|
||||
|
||||
network:
|
||||
allowedDomains:
|
||||
# Latest-release lookup + tarball downloads (GitHub redirects release
|
||||
# assets to *.githubusercontent.com object hosts)
|
||||
- 'api.github.com:443'
|
||||
- 'github.com:443'
|
||||
- 'objects.githubusercontent.com:443'
|
||||
- 'release-assets.githubusercontent.com:443'
|
||||
|
||||
commands:
|
||||
install:
|
||||
- command: |
|
||||
set -eu
|
||||
if command -v hurl >/dev/null 2>&1; then
|
||||
hurl --version
|
||||
exit 0
|
||||
fi
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
sudo apt-get update
|
||||
if apt-cache policy hurl 2>/dev/null | grep -q 'Candidate: [0-9]'; then
|
||||
sudo apt-get install -y --no-install-recommends hurl
|
||||
hurl --version
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
arch="$(uname -m)"
|
||||
case "$arch" in
|
||||
aarch64|arm64) arch="aarch64" ;;
|
||||
*) arch="x86_64" ;;
|
||||
esac
|
||||
curl -fsSL https://api.github.com/repos/Orange-OpenSource/hurl/releases/latest -o /tmp/hurl-release.json
|
||||
ver="$(sed -n 's/.*"tag_name": *"\([^"]*\)".*/\1/p' /tmp/hurl-release.json | head -1)"
|
||||
curl -fsSL "https://github.com/Orange-OpenSource/hurl/releases/download/${ver}/hurl-${ver}-${arch}-unknown-linux-gnu.tar.gz" -o /tmp/hurl.tgz
|
||||
mkdir -p /tmp/hurl-extract
|
||||
tar -xzf /tmp/hurl.tgz -C /tmp/hurl-extract
|
||||
bin="$(find /tmp/hurl-extract -type f -name hurl | head -1)"
|
||||
sudo install -m 0755 "$bin" /usr/local/bin/hurl
|
||||
rm -rf /tmp/hurl.tgz /tmp/hurl-extract /tmp/hurl-release.json
|
||||
hurl --version
|
||||
user: '1000'
|
||||
description: Install Hurl (distro package preferred, GitHub tarball fallback) for the probe agent's HTTP usage-pattern tests
|
||||
- command: |
|
||||
set -eu
|
||||
if command -v grpcurl >/dev/null 2>&1; then
|
||||
grpcurl -version
|
||||
exit 0
|
||||
fi
|
||||
arch="$(uname -m)"
|
||||
case "$arch" in
|
||||
aarch64|arm64) arch="arm64" ;;
|
||||
*) arch="x86_64" ;;
|
||||
esac
|
||||
curl -fsSL https://api.github.com/repos/fullstorydev/grpcurl/releases/latest -o /tmp/grpcurl-release.json
|
||||
ver="$(sed -n 's/.*"tag_name": *"v\([^"]*\)".*/\1/p' /tmp/grpcurl-release.json | head -1)"
|
||||
curl -fsSL "https://github.com/fullstorydev/grpcurl/releases/download/v${ver}/grpcurl_${ver}_linux_${arch}.tar.gz" -o /tmp/grpcurl.tgz
|
||||
mkdir -p /tmp/grpcurl-extract
|
||||
tar -xzf /tmp/grpcurl.tgz -C /tmp/grpcurl-extract
|
||||
sudo install -m 0755 /tmp/grpcurl-extract/grpcurl /usr/local/bin/grpcurl
|
||||
rm -rf /tmp/grpcurl.tgz /tmp/grpcurl-extract /tmp/grpcurl-release.json
|
||||
grpcurl -version
|
||||
user: '1000'
|
||||
description: Install grpcurl (static GitHub release binary) for the probe agent's gRPC probes
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eo pipefail
|
||||
|
||||
# @env LLM_OUTPUT=/dev/stdout
|
||||
# @env LLM_AGENT_VAR_PROJECT_DIR=.
|
||||
# @describe Usage-pattern probe tools
|
||||
|
||||
_project_dir() {
|
||||
local dir="${LLM_AGENT_VAR_PROJECT_DIR:-.}"
|
||||
(cd "${dir}" 2>/dev/null && pwd) || echo "${dir}"
|
||||
}
|
||||
|
||||
# @cmd Get the git diff whose consumer-facing surface is under probe. Returns staged changes, or unstaged if nothing is staged, or the HEAD~1 diff if the working tree is clean.
|
||||
# @option --base Optional base ref to diff against (e.g., "main", "HEAD~3", a commit SHA, or a task's base SHA)
|
||||
get_diff() {
|
||||
local project_dir
|
||||
project_dir=$(_project_dir)
|
||||
# shellcheck disable=SC2154
|
||||
local base="${argc_base:-}"
|
||||
|
||||
local diff_output=""
|
||||
if [[ -n "${base}" ]]; then
|
||||
diff_output=$(cd "${project_dir}" && git diff "${base}" 2>&1) || true
|
||||
else
|
||||
diff_output=$(cd "${project_dir}" && git diff --cached 2>&1) || true
|
||||
if [[ -z "${diff_output}" ]]; then
|
||||
diff_output=$(cd "${project_dir}" && git diff 2>&1) || true
|
||||
fi
|
||||
if [[ -z "${diff_output}" ]]; then
|
||||
diff_output=$(cd "${project_dir}" && git diff HEAD~1 2>&1) || true
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "${diff_output}" ]]; then
|
||||
echo "No changes found to probe in ${project_dir}." >> "$LLM_OUTPUT"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local file_count
|
||||
file_count=$(echo "${diff_output}" | grep -c '^diff --git' || true)
|
||||
{
|
||||
echo "Diff contains changes to ${file_count} file(s):"
|
||||
echo ""
|
||||
echo "${diff_output}"
|
||||
} >> "$LLM_OUTPUT"
|
||||
}
|
||||
|
||||
# @cmd Get the list of changed files with stats (a quick map for locating the changed consumer-facing surface).
|
||||
# @option --base Optional base ref to diff against
|
||||
get_changed_files() {
|
||||
local project_dir
|
||||
project_dir=$(_project_dir)
|
||||
local base="${argc_base:-}"
|
||||
|
||||
local stat_output=""
|
||||
if [[ -n "${base}" ]]; then
|
||||
stat_output=$(cd "${project_dir}" && git diff --stat "${base}" 2>&1) || true
|
||||
else
|
||||
stat_output=$(cd "${project_dir}" && git diff --cached --stat 2>&1) || true
|
||||
if [[ -z "${stat_output}" ]]; then
|
||||
stat_output=$(cd "${project_dir}" && git diff --stat 2>&1) || true
|
||||
fi
|
||||
if [[ -z "${stat_output}" ]]; then
|
||||
stat_output=$(cd "${project_dir}" && git diff --stat HEAD~1 2>&1) || true
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "${stat_output}" ]]; then
|
||||
echo "No changes found in ${project_dir}." >> "$LLM_OUTPUT"
|
||||
return 0
|
||||
fi
|
||||
|
||||
{
|
||||
echo "Changed files:"
|
||||
echo ""
|
||||
echo "${stat_output}"
|
||||
} >> "$LLM_OUTPUT"
|
||||
}
|
||||
@@ -16,6 +16,7 @@ spawnable_agents:
|
||||
- code-reviewer
|
||||
- adversary
|
||||
- security-reviewer
|
||||
- probe
|
||||
- architecture-reviewer
|
||||
- step-runner
|
||||
max_concurrent_agents: 40
|
||||
@@ -65,6 +66,7 @@ global_tools:
|
||||
- fs_write.sh
|
||||
- fs_patch.sh
|
||||
- execute_command.sh
|
||||
- web_search_coyote.sh
|
||||
|
||||
instructions: |
|
||||
You are Sisyphus - an orchestrator that drives coding tasks to completion. You do NOT work alone when specialists are available. You classify, delegate, verify, complete.
|
||||
@@ -389,7 +391,66 @@ instructions: |
|
||||
- **`Pre-existing, out of scope:` findings** — surface to the user but do not act on them. They predate this work and aren't the current task's responsibility.
|
||||
- **Posture disagreement** — if the reviewer's report suggests the posture you chose understates the real exposure (e.g. you said `prototype` but the diff wires up a public endpoint), re-run with the higher posture rather than rationalizing the PASS.
|
||||
|
||||
Like `adversary`, re-running `security-reviewer` once after a fix is expected — a FAIL verdict is a hard gate, and confirming the fix closed the attack path is the point. Run all applicable reviewers (`code-reviewer`, `adversary`, `security-reviewer`) — they cover disjoint failure modes; one passing says nothing about the others.
|
||||
Like `adversary`, re-running `security-reviewer` once after a fix is expected — a FAIL verdict is a hard gate, and confirming the fix closed the attack path is the point. Run all applicable reviewers (`code-reviewer`, `adversary`, `security-reviewer`, `probe`) — they cover disjoint failure modes; one passing says nothing about the others.
|
||||
|
||||
### Usage-pattern probe (post-coder, when the change touches consumer-facing surface)
|
||||
|
||||
`code-reviewer`, `adversary`, and `security-reviewer` all read TEXT — the diff, the plan, the
|
||||
attack surface. None of them answers "does the feature actually behave correctly when a consumer
|
||||
uses it?" Spawn `probe` when the change touches consumer-facing surface. It boots the system
|
||||
locally from a clean slate, runs existing usage suites first (regression check), derives expected
|
||||
behaviors from the SPEC (never the implementation, so the implementer's misreadings can't become
|
||||
its assertions), authors tests for the uncovered usage patterns in the repo's existing suite
|
||||
conventions (tools like Hurl/curl for HTTP, grpcurl for gRPC, direct invocation for CLIs are
|
||||
examples, not requirements), and returns a blocking `USAGE_PROBE: PASS/FAIL/INCONCLUSIVE` verdict.
|
||||
|
||||
**When to spawn it** — ANY of these:
|
||||
|
||||
1. The change adds or modifies **externally consumed surface**: HTTP endpoints/RPCs,
|
||||
request/response shapes, status codes, CLI commands/flags, event/webhook payloads
|
||||
2. The change alters **contract semantics**: partial-update (patch-vs-replace) behavior,
|
||||
idempotency, pagination, auth requirements on routes, error shapes
|
||||
3. **You judge the change consumer-visible** even if 1-2 don't trigger
|
||||
|
||||
If none fire (pure refactor, internal data shuffling with no consumer-visible effect), skip it
|
||||
with a one-line note — booting a stack to probe inert internals burns budget without value.
|
||||
|
||||
**Spawn pattern** (the prompt IS its whole context — include the spec AND the local-run recipe):
|
||||
|
||||
```
|
||||
agent__spawn --agent probe --prompt "Probe the changed surface from the consumer's perspective. Return PASS/FAIL/INCONCLUSIVE.
|
||||
|
||||
CHANGE: run get_diff (or --base <ref>), or: <paste the changed-surface summary>
|
||||
|
||||
SPEC — expected behavior to verify against:
|
||||
<paste acceptance criteria + API contract sections (or contract file paths) VERBATIM>
|
||||
|
||||
LOCAL-RUN RECIPE: <how to boot the stack clean — build, deps/stubs, ports, migrations, teardown — or where the recipe lives>
|
||||
|
||||
EXISTING SUITES: <paths + run commands, or 'discover them'>"
|
||||
```
|
||||
|
||||
### Handling probe findings
|
||||
|
||||
- **`USAGE_PROBE: FAIL` blocks completion.** Do not mark the task done. Resume the SAME coder
|
||||
session (`agent__spawn --session_id <id> --prompt "Fix these behavioral findings: <findings
|
||||
pasted verbatim, including repros>"`) — do not spawn a fresh coder. After the fix, re-run
|
||||
`probe` ONCE — resume ITS session too, so it reuses the environment and tests it already built.
|
||||
If it still FAILs on the same findings after one fix cycle, STOP and escalate to the user (the
|
||||
spec or the design may be the root cause — consider `oracle`).
|
||||
- **`USAGE_PROBE: PASS`** — proceed. Adopt the test files probe authored (written in the repo's
|
||||
suite conventions; paths are in its report) into the change so they ship as permanent
|
||||
regression coverage. Surface any stale-test or recipe observations to the user.
|
||||
- **`USAGE_PROBE: INCONCLUSIVE`** — the ENVIRONMENT, not the code, is the blocker. Never treat it
|
||||
as PASS or FAIL. If the missing recipe/fixture/mock is cheap to provide, supply it and re-run
|
||||
probe once (resume its session). Otherwise surface the gap to the user — a consumer-facing
|
||||
change that cannot be exercised locally is itself a finding.
|
||||
- **Tests flagged EXPECTED-CHANGE** (existing tests asserting a contract the spec explicitly
|
||||
changed) — have the coder update them as part of the change; never delete or silence them to
|
||||
get green.
|
||||
|
||||
Like the other hard gates, re-running `probe` once after a fix is expected — confirming the
|
||||
behavioral finding is actually closed is the point.
|
||||
|
||||
### Observability pass (post-coder, advisory — when the change adds operational surface)
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ kind: sandbox
|
||||
name: coyote
|
||||
displayName: Coyote
|
||||
description: >
|
||||
An all-in-one, batteries-included LLM CLI tool featuring Shell Assistant,
|
||||
An all-in-one, batteries-included LLM runtime featuring Shell Assistant,
|
||||
CLI & REPL mode, RAG, AI tools & agents, MCP servers, skills, and macros.
|
||||
|
||||
sandbox:
|
||||
|
||||
@@ -26,7 +26,7 @@ Walk EVERY category. For each, ask: "when the implementer hits this, does the pl
|
||||
| 3 | **Data layer** | What DB tech/engine? What migration tool and directory? What naming conventions for tables/columns? Which existing tables does this touch or reference? |
|
||||
| 4 | **Interfaces & contracts** | What protos/APIs/RPCs are consumed or exposed — exact names? Where do proto definitions live and how are they regenerated? What downstream consumers depend on the shapes this plan creates? |
|
||||
| 5 | **Conventions & tooling** | Which language/framework versions? Error-handling and logging patterns — which file shows the canon? Lint/format/build commands? Where is the repo's own CLAUDE.md / contributor doc and does the plan tell the implementer to read it? |
|
||||
| 6 | **Testing & verification** | Test framework and directory conventions? EXACT commands to run tests/build from the repo root? What proves each acceptance criterion? |
|
||||
| 6 | **Testing & verification** | Test framework and directory conventions? EXACT commands to run tests/build from the repo root? What proves each acceptance criterion? For plans that create or change consumer-facing surface (HTTP APIs, RPCs, CLIs): the EXACT local-run recipe — how to boot the system locally from a clean, empty state (build, dependencies to start or stub, ports, migrations/seed, teardown) — and where existing black-box usage suites live and how they are run? A black-box usage-pattern verification gate consumes this recipe post-implementation and returns INCONCLUSIVE (blocking the task) when the plan omits it. |
|
||||
| 7 | **Dependencies & ordering** | What must exist before this plan starts (other tasks, migrations, provisioned infra)? What does this plan produce that later work depends on? |
|
||||
| 8 | **Config, secrets & environments** | New env vars/config keys — where are they declared and injected? Secrets — vault/parameter store conventions? Staging vs production differences that affect implementation? |
|
||||
| 9 | **Scope boundaries** | Is Out of scope present and specific? Are "tempting adjacent fixes" explicitly deferred? |
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
---
|
||||
description: Verify a change from the consumer's perspective - exercise the changed surface (HTTP API, RPC, CLI) black-box against a locally running instance with clean, isolated state. Run existing usage suites first for regressions, derive new tests from the spec (never the implementation), and classify every failure as bug / environment / expected contract change. Produces a USAGE_PROBE PASS/FAIL/INCONCLUSIVE verdict. Complements code-review (quality), adversarial-review (plan conformance), and security-review (abuse) - this is the only gate that tests BEHAVIOR by using the thing, not by reading it.
|
||||
enabled_tools: fs_read, fs_cat, fs_grep, fs_glob, fs_ls, fs_write, fs_patch, execute_command
|
||||
---
|
||||
You are verifying a change the way its consumers will experience it: by USING it. Every other
|
||||
review gate reads text — the diff, the plan, the code. This gate boots the system locally from a
|
||||
clean state and exercises the changed surface as a cold-start consumer would. It catches the class
|
||||
of defects invisible to static review: serialization quirks, replace-vs-patch semantics,
|
||||
wrong status codes, broken idempotency, empty-state crashes, auth holes on new routes.
|
||||
|
||||
## The one question
|
||||
|
||||
**Does the changed consumer-facing surface behave as the spec promises when actually used,
|
||||
starting from nothing?** You are not judging code quality, plan conformance, or exploitability —
|
||||
other gates own those. You judge observable behavior.
|
||||
|
||||
## The independence rule (spec-first, or the gate is worthless)
|
||||
|
||||
Derive expected behaviors from the **spec** — the plan/task acceptance criteria, the API contract
|
||||
(IDL/schema/OpenAPI/proto definitions), the documented CLI help — **BEFORE reading the
|
||||
implementation**. If you read the handler first and write tests that mirror it, you have re-proven
|
||||
the implementation's own assumptions, including its misreadings of the spec. Order of operations:
|
||||
|
||||
1. Read the spec + contract. Write down the expected behaviors as concrete request→response pairs.
|
||||
2. Only THEN read implementation code — and only as much as needed to find ports, config, and
|
||||
startup wiring. Never to "check what it actually does" before your expectations are written.
|
||||
|
||||
## Phase order
|
||||
|
||||
### 1. Identify the surface under test
|
||||
|
||||
From the diff (or the caller's summary): which endpoints/RPCs/commands were added or changed?
|
||||
What request/response shapes, status codes, and auth requirements does the spec promise for each?
|
||||
If the change touches no consumer-facing surface, say so and return PASS with a one-line note —
|
||||
probing inert internals burns budget without value.
|
||||
|
||||
### 2. Regression-first: find and run existing usage suites
|
||||
|
||||
Discover what already exists before authoring anything:
|
||||
|
||||
- `fs_glob` for suite files in ANY format the repo uses: `**/*.hurl`, `**/*.http`, `**/*.rest`,
|
||||
`**/*.postman_collection.json`, `**/*.bru`, `**/e2e/**`, `**/integration/**`, `**/api-test*/**`,
|
||||
`**/smoke*/**`, plus repo scripts that run them (`**/run-*test*`, Makefile/justfile targets,
|
||||
package-manifest script entries) and shell scripts of `curl` commands (`fs_grep` for `curl `
|
||||
under `scripts/`, `test/`, `tools/`).
|
||||
- Read the repo's contributor docs for the sanctioned way to run them.
|
||||
|
||||
Run the existing suites against the changed code FIRST. Every failure here is a candidate
|
||||
regression. Classify each (see § Failure classification) — a failure is only acceptable when the
|
||||
spec EXPLICITLY changed that contract, and then the old test needs updating (note it in the
|
||||
report), not ignoring.
|
||||
|
||||
### 3. Map coverage, author the delta
|
||||
|
||||
List which of your expected behaviors from step 1 the existing suites already prove. Author new
|
||||
tests ONLY for the uncovered ones. Walk this usage-pattern checklist for each changed surface —
|
||||
these are the cases implementers systematically forget:
|
||||
|
||||
| Pattern | What to probe |
|
||||
|---------|---------------|
|
||||
| Cold start / empty state | First-ever call with no pre-existing data: list → empty (not 500), get → not-found (not panic) |
|
||||
| Happy path | The spec's primary flow, end to end, asserting the full response shape — not just the status code |
|
||||
| Idempotency / re-call | Same create/update twice: duplicate error or no-op, per the spec — never silent double-write |
|
||||
| Invalid input | Missing required fields, wrong types, out-of-range values, malformed body → the spec's error shape and code, not a 500 |
|
||||
| Auth on the new surface | Missing/expired/insufficient credentials → the correct 401/403 (a new route with no auth check is a common miss) |
|
||||
| Not-found and stale references | Operations on IDs that don't exist or were deleted |
|
||||
| Partial update semantics | Does omitting a field preserve it (patch) or delete it (replace)? Assert whichever the spec promises — this is a classic silent-data-loss bug |
|
||||
| Serialization edges | Zero values, empty lists, unset optionals: encoders that omit zero values make `== false`/`== null` asserts lie — assert existence/absence per the actual encoding |
|
||||
| Pagination / limits | Page past the end, limit 0/1/max, stable ordering if promised |
|
||||
| Error shape consistency | New errors follow the same envelope as the rest of the surface |
|
||||
| State transitions | Illegal transitions rejected; legal ones observable via subsequent reads |
|
||||
|
||||
Write the new tests where the repo's existing suites live, following their naming and layout
|
||||
conventions, so they are adoptable as permanent regression tests. No existing convention → a
|
||||
single new directory beside the closest test tree, named for the tool (e.g. `tests/usage/`).
|
||||
|
||||
### 4. Environment discipline (clean, local, isolated)
|
||||
|
||||
- **Clean state is non-negotiable.** Boot from nothing: fresh/ephemeral database (throwaway
|
||||
container, tmp file, or dedicated schema), run migrations, seed ONLY what the tests create
|
||||
themselves. Tests that depend on pre-existing data are not cold-start tests.
|
||||
- **Fully local.** Stub or mock external dependencies (fake servers, recorded fixtures, in-memory
|
||||
substitutes) — a probe that calls real third-party systems is a flake generator and a hazard.
|
||||
- **Prefer the repo's own recipe.** If the plan or contributor docs provide a local-run recipe
|
||||
(compose file, make target, dev script), use it verbatim before inventing your own. If you must
|
||||
invent one, record every step in the report so it can be promoted into the docs.
|
||||
- **Teardown.** Leave no running processes, containers, or dirty state behind.
|
||||
- **Bounded startup retries only.** Retry/poll while the stack boots (bounded attempts, short
|
||||
interval). NEVER add retries to make a flaky assertion pass — flakiness on a settled stack is a
|
||||
finding.
|
||||
|
||||
### 5. Failure classification (every failure gets exactly one)
|
||||
|
||||
| Class | Meaning | Effect on verdict |
|
||||
|-------|---------|-------------------|
|
||||
| **BUG** | The running system violates the spec | FAIL — report with repro |
|
||||
| **EXPECTED-CHANGE** | An existing test asserts a contract the spec explicitly changed | Does not fail the verdict; the stale test is flagged for update |
|
||||
| **ENV** | The failure is in bringing the stack up or reaching it, not in behavior | Does not count as a bug; if it prevents meaningful probing → INCONCLUSIVE |
|
||||
|
||||
Misclassifying ENV as BUG sends the implementer chasing ghosts; misclassifying BUG as ENV ships
|
||||
the defect. When unsure, reproduce twice and read the server logs before deciding.
|
||||
|
||||
## Toolbox (repo conventions first; these are examples, not requirements)
|
||||
|
||||
No specific tool is required. Precedence: (1) whatever format/harness the repo's existing usage
|
||||
suites already use — run and extend that; (2) a well-suited tool from the examples below if it is
|
||||
available or trivially installable; (3) ubiquitous fallbacks (`curl` + shell assertions cover any
|
||||
HTTP surface). What is non-negotiable is the discipline — spec-first asserts, clean state — not
|
||||
the tool.
|
||||
|
||||
| Surface | Example tools | Notes |
|
||||
|---------|--------------|-------|
|
||||
| HTTP/REST/JSON (incl. gRPC-over-HTTP with JSON encoding) | [Hurl](https://hurl.dev) `.hurl` files; `curl` scripts | Hurl: plain-text request/assert format, capturable variables, `retry` for eventual consistency; files double as committed regression suites |
|
||||
| Pure gRPC/protobuf | `grpcurl` (scripted) | Use server reflection or point at the proto files |
|
||||
| CLI | Direct invocation via `execute_command` | Assert exit codes AND output; probe stdin/args edge cases |
|
||||
| Anything else | `curl`/scripts/the repo's own test harness | Same discipline: spec-first asserts, clean state |
|
||||
|
||||
Optional niceties like Hurl and grpcurl may already be preinstalled (e.g. by a sandbox mixin) or
|
||||
can be installed idempotently (hurl via the distro package manager first — its prebuilt GitHub
|
||||
tarball dynamically links `libxml2.so.2`, which newer distros no longer ship; grpcurl from its
|
||||
GitHub release, a static Go binary). When a preferred tool is unavailable and uninstallable, fall
|
||||
back to what exists rather than skipping the check; classify a probe as ENV only when NO adequate
|
||||
tool can exercise the surface.
|
||||
|
||||
If you use Hurl, gotchas that produce false results if unknown:
|
||||
|
||||
- `[Captures]` run BEFORE `[Asserts]` in the same entry — capture a replaced value under a NEW
|
||||
variable name, or your inequality asserts compare a value to itself.
|
||||
- JSON encoders that omit zero/empty values: assert `not exists` for absent fields — `== false`
|
||||
or `== null` asserts fail on omitted keys.
|
||||
- Use `[Options] retry` with a bounded count for asynchronous effects (job completion, eventual
|
||||
reads); never unbounded.
|
||||
|
||||
## Verdict format
|
||||
|
||||
End with EXACTLY one of:
|
||||
|
||||
```
|
||||
USAGE_PROBE: PASS
|
||||
Surface: <endpoints/RPCs/commands probed>. Existing suites: <N run, all green | none found>.
|
||||
New tests: <M authored at <path>, all green>.
|
||||
<optional: 1-3 non-blocking observations (stale tests to update, recipe gaps)>
|
||||
```
|
||||
|
||||
```
|
||||
USAGE_PROBE: FAIL
|
||||
Surface: <...>. Existing suites: <N run, X failed (Y regressions, Z expected-change)>. New tests: <M authored, W failed>.
|
||||
Behavioral findings:
|
||||
1. <surface + case> — <spec'd behavior, quoting the spec> — <observed behavior> — REPRO: <exact request/command + response received> — <test file:entry>
|
||||
Stale tests needing update (expected-change): <list or none>
|
||||
```
|
||||
|
||||
```
|
||||
USAGE_PROBE: INCONCLUSIVE
|
||||
Could not establish a clean local environment: <what failed, verbatim error>.
|
||||
Missing: <the exact recipe/fixture/mock that would unblock — phrased so the plan author can add it>.
|
||||
Partial results (if any): <what did run and what it showed>
|
||||
```
|
||||
|
||||
Every FAIL finding MUST include the exact reproduction (request/command and the response
|
||||
received) and cite the test file — a behavioral complaint without a repro is noise. INCONCLUSIVE
|
||||
is an honest, acceptable verdict: it routes the fix to the environment recipe, not the code.
|
||||
NEVER report INCONCLUSIVE as PASS ("couldn't test, probably fine") or as FAIL (the implementer
|
||||
would hunt a nonexistent bug).
|
||||
|
||||
## Anti-patterns
|
||||
|
||||
- Writing tests after reading the implementation — you will encode its bugs as expectations.
|
||||
- Skipping the existing suites and jumping to new tests — regressions are the cheapest bugs to catch.
|
||||
- Testing through internal seams (direct DB reads, internal function calls) — this gate is
|
||||
consumer-perspective only; internals belong to unit tests.
|
||||
- Depending on pre-existing data, shared databases, or previously running services.
|
||||
- Adding retries/sleeps until a flaky assertion passes — flakiness is a finding, not an obstacle.
|
||||
- Reporting an environment failure as a behavioral FAIL (or burying it in a PASS).
|
||||
- Throwaway tests in /tmp — tests that don't land in the repo's suite location die with the run.
|
||||
- Asserting only status codes — shape and content are where the quirks live.
|
||||
@@ -48,7 +48,7 @@ summarization_model: null # Model to use for summarizing sub-agent output
|
||||
summarization_threshold: 4000 # Character threshold above which sub-agent output is summarized before returning to parent
|
||||
escalation_timeout: 300 # Seconds a sub-agent waits for a user interaction response before timing out (default: 5 minutes)
|
||||
mcp_servers: # Optional list of MCP servers that the agent utilizes
|
||||
- github # Corresponds to the name of an MCP server in the `<coyote-config-dir>/functions/mcp.json` file
|
||||
- github # Corresponds to the name of an MCP server in the `<coyote-config-dir>/mcp.json` file
|
||||
global_tools: # Optional list of additional global tools to enable for the agent; i.e. not tools specific to the agent
|
||||
- web_search
|
||||
- fs
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Documentation: https://docs.brew.sh/Formula-Cookbook
|
||||
# https://rubydoc.brew.sh/Formula
|
||||
class Coyote < Formula
|
||||
desc "All-in-one, batteries included LLM CLI tool"
|
||||
desc "All-in-one, batteries-included LLM runtime"
|
||||
homepage "https://github.com/Dark-Alex-17/coyote"
|
||||
if OS.mac? and Hardware::CPU.arm?
|
||||
url "https://github.com/Dark-Alex-17/coyote/releases/download/v$version/coyote-aarch64-apple-darwin.tar.gz"
|
||||
|
||||
+1
-1
@@ -315,7 +315,7 @@ pub struct Cli {
|
||||
/// URL for http/sse MCP server (used with --mcp-add)
|
||||
#[arg(long, value_name = "URL", help_heading = "MCP Servers")]
|
||||
pub url: Option<String>,
|
||||
/// Scope for MCP config: user (~/.config/coyote/functions/mcp.json) or workspace (./.coyote/mcp.json). Default: user
|
||||
/// Scope for MCP config: user (~/.config/coyote/mcp.json) or workspace (./.coyote/mcp.json). Default: user
|
||||
#[arg(long, value_enum, value_name = "SCOPE", help_heading = "MCP Servers")]
|
||||
pub scope: Option<McpScopeArg>,
|
||||
/// Environment variable for stdio MCP server (repeatable): --env KEY=VALUE
|
||||
|
||||
@@ -32,7 +32,7 @@ pub fn install_remote(git_url: &str, filter: Option<InstallFilter>, force: bool)
|
||||
if layout.is_empty() {
|
||||
println!(
|
||||
"No recognized assets found in {git_url}. Expected one or more of: \
|
||||
agents/, roles/, skills/, macros/, functions/tools/, functions/mcp.json"
|
||||
agents/, roles/, skills/, macros/, functions/tools/, mcp.json"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
@@ -1263,6 +1263,11 @@ fn scan_remote_layout(root: &Path) -> Result<RemoteLayout> {
|
||||
layout.macros = Some(macros);
|
||||
}
|
||||
|
||||
let root_mcp = root.join("mcp.json");
|
||||
if root_mcp.is_file() {
|
||||
layout.mcp_json = Some(root_mcp);
|
||||
}
|
||||
|
||||
let functions = root.join("functions");
|
||||
if functions.is_dir() {
|
||||
let tools = functions.join("tools");
|
||||
@@ -1270,8 +1275,9 @@ fn scan_remote_layout(root: &Path) -> Result<RemoteLayout> {
|
||||
layout.functions_tools = Some(tools);
|
||||
}
|
||||
|
||||
// Legacy bundle layout; a root-level mcp.json wins when both exist.
|
||||
let mcp = functions.join("mcp.json");
|
||||
if mcp.is_file() {
|
||||
if layout.mcp_json.is_none() && mcp.is_file() {
|
||||
layout.mcp_json = Some(mcp);
|
||||
}
|
||||
}
|
||||
@@ -2642,6 +2648,30 @@ mod tests {
|
||||
let _ = fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scan_remote_layout_finds_root_mcp_json() {
|
||||
let root = fresh_temp_dir("scan-root-mcp-test-");
|
||||
touch(&root.join("mcp.json"));
|
||||
|
||||
let layout = scan_remote_layout(&root).unwrap();
|
||||
|
||||
assert_eq!(layout.mcp_json, Some(root.join("mcp.json")));
|
||||
let _ = fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scan_remote_layout_prefers_root_mcp_json_over_functions() {
|
||||
let root = fresh_temp_dir("scan-mcp-precedence-test-");
|
||||
touch(&root.join("mcp.json"));
|
||||
fs::create_dir_all(root.join("functions")).unwrap();
|
||||
touch(&root.join("functions/mcp.json"));
|
||||
|
||||
let layout = scan_remote_layout(&root).unwrap();
|
||||
|
||||
assert_eq!(layout.mcp_json, Some(root.join("mcp.json")));
|
||||
let _ = fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scan_remote_layout_finds_skills_only() {
|
||||
let root = fresh_temp_dir("scan-skills-only-");
|
||||
|
||||
@@ -287,6 +287,18 @@ pub fn functions_bin_dir() -> PathBuf {
|
||||
}
|
||||
|
||||
pub fn mcp_config_file() -> PathBuf {
|
||||
let preferred = local_dir(MCP_FILE_NAME);
|
||||
if preferred.exists() {
|
||||
return preferred;
|
||||
}
|
||||
let legacy = legacy_mcp_config_file();
|
||||
if legacy.exists() {
|
||||
return legacy;
|
||||
}
|
||||
preferred
|
||||
}
|
||||
|
||||
pub fn legacy_mcp_config_file() -> PathBuf {
|
||||
functions_dir().join(MCP_FILE_NAME)
|
||||
}
|
||||
|
||||
@@ -837,6 +849,80 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
mod user_mcp_resolution {
|
||||
use super::*;
|
||||
use serial_test::serial;
|
||||
|
||||
fn with_config_dir<F: FnOnce(&Path)>(f: F) {
|
||||
let unique = time::SystemTime::now()
|
||||
.duration_since(time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos();
|
||||
let root = env::temp_dir().join(format!("coyote-user-mcp-test-{unique}"));
|
||||
fs::create_dir_all(root.join(FUNCTIONS_DIR_NAME)).unwrap();
|
||||
let config_env = get_env_name("config_dir");
|
||||
let functions_env = get_env_name("functions_dir");
|
||||
let prev_config = env::var_os(&config_env);
|
||||
let prev_functions = env::var_os(&functions_env);
|
||||
unsafe {
|
||||
env::set_var(&config_env, &root);
|
||||
env::set_var(&functions_env, root.join(FUNCTIONS_DIR_NAME));
|
||||
}
|
||||
f(&root);
|
||||
unsafe {
|
||||
match prev_config {
|
||||
Some(v) => env::set_var(&config_env, v),
|
||||
None => env::remove_var(&config_env),
|
||||
}
|
||||
match prev_functions {
|
||||
Some(v) => env::set_var(&functions_env, v),
|
||||
None => env::remove_var(&functions_env),
|
||||
}
|
||||
}
|
||||
let _ = fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn defaults_to_preferred_location_when_neither_exists() {
|
||||
with_config_dir(|root| {
|
||||
assert_eq!(mcp_config_file(), root.join(MCP_FILE_NAME));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn falls_back_to_legacy_location_when_only_it_exists() {
|
||||
with_config_dir(|root| {
|
||||
let legacy = root.join(FUNCTIONS_DIR_NAME).join(MCP_FILE_NAME);
|
||||
fs::write(&legacy, "{}").unwrap();
|
||||
assert_eq!(mcp_config_file(), legacy);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn prefers_new_location_when_both_exist() {
|
||||
with_config_dir(|root| {
|
||||
let preferred = root.join(MCP_FILE_NAME);
|
||||
let legacy = root.join(FUNCTIONS_DIR_NAME).join(MCP_FILE_NAME);
|
||||
fs::write(&preferred, "{}").unwrap();
|
||||
fs::write(&legacy, "{}").unwrap();
|
||||
assert_eq!(mcp_config_file(), preferred);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn uses_preferred_location_when_only_it_exists() {
|
||||
with_config_dir(|root| {
|
||||
let preferred = root.join(MCP_FILE_NAME);
|
||||
fs::write(&preferred, "{}").unwrap();
|
||||
assert_eq!(mcp_config_file(), preferred);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sandbox_kit_override_reflects_env_var_state() {
|
||||
let env_name = get_env_name("sandbox_kit");
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use super::bundles::BundleStore;
|
||||
use super::rag_cache::{RagCache, RagKey};
|
||||
use super::session::Session;
|
||||
use super::session::{INTERRUPTED_RESPONSE_TEXT, Session};
|
||||
use super::skill::{SKILL_SCAFFOLD, Skill};
|
||||
use super::skill_policy::SkillPolicy;
|
||||
use super::skill_registry::SkillRegistry;
|
||||
@@ -958,13 +958,25 @@ impl RequestContext {
|
||||
let mut i = input.clone();
|
||||
i.clear_patch();
|
||||
if let Some(session) = i.session_mut(&mut self.session) {
|
||||
let _ = session.add_message(&i, "[Response interrupted due to error]");
|
||||
let _ = session.add_message(&i, INTERRUPTED_RESPONSE_TEXT);
|
||||
if !app.dry_run && session.save_session() == Some(true) {
|
||||
let _ = session.flush();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn has_recoverable_interruption(&self) -> bool {
|
||||
let live = self
|
||||
.last_message
|
||||
.as_ref()
|
||||
.map(|v| v.continuous && v.input.with_session())
|
||||
.unwrap_or(false);
|
||||
live || self
|
||||
.session
|
||||
.as_ref()
|
||||
.is_some_and(Session::has_interrupted_error_checkpoint)
|
||||
}
|
||||
|
||||
pub fn discontinuous_last_message(&mut self) {
|
||||
if let Some(last_message) = self.last_message.as_mut() {
|
||||
last_message.continuous = false;
|
||||
@@ -1865,6 +1877,7 @@ impl RequestContext {
|
||||
("rags_dir", display_path(&paths::rags_dir())),
|
||||
("macros_dir", display_path(&paths::macros_dir())),
|
||||
("functions_dir", display_path(&paths::functions_dir())),
|
||||
("mcp_config_file", display_path(&paths::mcp_config_file())),
|
||||
("sbx_kit_dir", display_path(&paths::sbx_kit_dir())),
|
||||
("messages_file", display_path(&self.messages_file())),
|
||||
];
|
||||
@@ -6738,6 +6751,63 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_recoverable_interruption_false_by_default() {
|
||||
let ctx = create_test_ctx();
|
||||
assert!(!ctx.has_recoverable_interruption());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_recoverable_interruption_true_for_live_interrupted_turn() {
|
||||
let mut ctx = create_test_ctx();
|
||||
ctx.session = Some(Session::default());
|
||||
let app = Arc::clone(&ctx.app.config);
|
||||
let input = Input::from_str(&ctx, "hello", None).unwrap();
|
||||
|
||||
ctx.on_chat_completion_error(app.as_ref(), &input);
|
||||
|
||||
assert!(ctx.has_recoverable_interruption());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_recoverable_interruption_false_after_normal_exchange() {
|
||||
let mut ctx = create_test_ctx();
|
||||
ctx.session = Some(Session::default());
|
||||
let input = Input::from_str(&ctx, "hello", None).unwrap();
|
||||
ctx.session
|
||||
.as_mut()
|
||||
.unwrap()
|
||||
.add_message(&input, "all done")
|
||||
.unwrap();
|
||||
|
||||
assert!(!ctx.has_recoverable_interruption());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn has_recoverable_interruption_survives_session_save_and_reload() {
|
||||
// Simulates the full user flow: a turn crashes mid-tool-loop (the
|
||||
// checkpoint lands in the session), the user runs `.save session`,
|
||||
// exits, and a fresh process resumes the session. The in-memory
|
||||
// last_message is gone; only the persisted checkpoint can mark the
|
||||
// session recoverable for `.recover`.
|
||||
let mut ctx = create_test_ctx();
|
||||
ctx.session = Some(Session::default());
|
||||
let app = Arc::clone(&ctx.app.config);
|
||||
let input = Input::from_str(&ctx, "hello", None).unwrap();
|
||||
ctx.on_chat_completion_error(app.as_ref(), &input);
|
||||
|
||||
let yaml = serde_yaml::to_string(ctx.session.as_ref().unwrap()).unwrap();
|
||||
let reloaded: Session = serde_yaml::from_str(&yaml).unwrap();
|
||||
|
||||
let mut resumed_ctx = create_test_ctx();
|
||||
resumed_ctx.session = Some(reloaded);
|
||||
assert!(resumed_ctx.last_message.is_none());
|
||||
assert!(
|
||||
resumed_ctx.has_recoverable_interruption(),
|
||||
".recover must work on a session resumed after an interrupted turn"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn after_chat_completion_sweeps_auto_unload_skills_at_turn_end() {
|
||||
let mut ctx = create_test_ctx();
|
||||
|
||||
+90
-2
@@ -16,6 +16,8 @@ use std::sync::LazyLock;
|
||||
|
||||
static RE_AUTONAME_PREFIX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\d{8}T\d{6}-").unwrap());
|
||||
|
||||
pub const INTERRUPTED_RESPONSE_TEXT: &str = "[Response interrupted due to error]";
|
||||
|
||||
#[derive(Debug, Clone, Default, Deserialize, Serialize)]
|
||||
pub struct Session {
|
||||
#[serde(rename(serialize = "model", deserialize = "model"))]
|
||||
@@ -175,6 +177,13 @@ impl Session {
|
||||
self.messages.is_empty() && self.compressed_messages.is_empty()
|
||||
}
|
||||
|
||||
pub fn has_interrupted_error_checkpoint(&self) -> bool {
|
||||
self.messages.last().is_some_and(|message| {
|
||||
message.role.is_assistant()
|
||||
&& matches!(&message.content, MessageContent::Text(text) if text.ends_with(INTERRUPTED_RESPONSE_TEXT))
|
||||
})
|
||||
}
|
||||
|
||||
pub fn messages(&self) -> &[Message] {
|
||||
&self.messages
|
||||
}
|
||||
@@ -934,9 +943,9 @@ impl AutoName {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::client::{Message, MessageContent, MessageRole, Model};
|
||||
use crate::client::{Message, MessageContent, MessageContentToolCalls, MessageRole, Model};
|
||||
use crate::config::{AppConfig, AppState, RequestContext, WorkingMode};
|
||||
use crate::function::Functions;
|
||||
use crate::function::{Functions, ToolCall, ToolResult};
|
||||
use std::sync::Arc;
|
||||
|
||||
#[test]
|
||||
@@ -949,6 +958,85 @@ mod tests {
|
||||
assert!(!session.dirty());
|
||||
}
|
||||
|
||||
fn push_interrupted_turn(session: &mut Session) {
|
||||
session.messages.push(Message::new(
|
||||
MessageRole::User,
|
||||
MessageContent::Text("do things".to_string()),
|
||||
));
|
||||
session.messages.push(Message::new(
|
||||
MessageRole::Tool,
|
||||
MessageContent::ToolCalls(MessageContentToolCalls::new(
|
||||
vec![ToolResult::new(ToolCall::default(), json!("ok"))],
|
||||
String::new(),
|
||||
)),
|
||||
));
|
||||
session.messages.push(Message::new(
|
||||
MessageRole::Assistant,
|
||||
MessageContent::Text(INTERRUPTED_RESPONSE_TEXT.to_string()),
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_has_interrupted_error_checkpoint_detects_sentinel() {
|
||||
let mut session = Session::default();
|
||||
assert!(!session.has_interrupted_error_checkpoint());
|
||||
|
||||
session.messages.push(Message::new(
|
||||
MessageRole::User,
|
||||
MessageContent::Text("hi".to_string()),
|
||||
));
|
||||
session.messages.push(Message::new(
|
||||
MessageRole::Assistant,
|
||||
MessageContent::Text("hello".to_string()),
|
||||
));
|
||||
assert!(
|
||||
!session.has_interrupted_error_checkpoint(),
|
||||
"a normal completed exchange is not an interruption"
|
||||
);
|
||||
|
||||
push_interrupted_turn(&mut session);
|
||||
assert!(session.has_interrupted_error_checkpoint());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_detects_checkpoint_appended_by_crashed_continue_turn() {
|
||||
// A crash during a `.continue` turn appends the sentinel to the
|
||||
// previous assistant text (add_message's continue_output branch)
|
||||
// instead of pushing a standalone checkpoint message.
|
||||
let mut session = Session::default();
|
||||
session.messages.push(Message::new(
|
||||
MessageRole::User,
|
||||
MessageContent::Text("hi".to_string()),
|
||||
));
|
||||
session.messages.push(Message::new(
|
||||
MessageRole::Assistant,
|
||||
MessageContent::Text(format!("partial answer{INTERRUPTED_RESPONSE_TEXT}")),
|
||||
));
|
||||
|
||||
assert!(session.has_interrupted_error_checkpoint());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_interrupted_checkpoint_with_tool_calls_survives_yaml_round_trip() {
|
||||
let mut session = Session::default();
|
||||
push_interrupted_turn(&mut session);
|
||||
|
||||
let yaml = serde_yaml::to_string(&session).unwrap();
|
||||
let reloaded: Session = serde_yaml::from_str(&yaml).unwrap();
|
||||
|
||||
assert!(
|
||||
reloaded.has_interrupted_error_checkpoint(),
|
||||
"interruption checkpoint must survive save/reload"
|
||||
);
|
||||
assert!(
|
||||
reloaded.messages.iter().any(|m| matches!(
|
||||
&m.content,
|
||||
MessageContent::ToolCalls(tc) if tc.tool_results.len() == 1
|
||||
)),
|
||||
"tool calls made before the crash must survive save/reload"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn session_enabled_macros_absent_is_none() {
|
||||
let session: Session = serde_yaml::from_str("model: provider:test\nmessages: []").unwrap();
|
||||
|
||||
+109
-2
@@ -23,7 +23,8 @@ use indexmap::IndexMap;
|
||||
use parking_lot::{Mutex, RwLock};
|
||||
use serde_json::{Value, json};
|
||||
use std::collections::HashMap;
|
||||
use std::path::PathBuf;
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Stdio;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
@@ -42,6 +43,8 @@ const JOB_KILL_GRACE: Duration = Duration::from_secs(5);
|
||||
|
||||
const JOB_PUMP_DRAIN_GRACE: Duration = Duration::from_secs(2);
|
||||
|
||||
const JOB_FILE_TAIL_POLL: Duration = Duration::from_millis(300);
|
||||
|
||||
pub fn is_agent_task(supervisor: Option<&Arc<RwLock<Supervisor>>>, id: &str) -> bool {
|
||||
id.starts_with("agent_")
|
||||
|| id.starts_with("graph_agent_")
|
||||
@@ -966,6 +969,34 @@ async fn pump_into_ring(mut reader: impl AsyncReadExt + Unpin, output_buf: Arc<M
|
||||
}
|
||||
}
|
||||
|
||||
fn drain_output_file(path: &Path, offset: &Mutex<u64>, output_buf: &Mutex<RingBuf>) {
|
||||
let mut offset = offset.lock();
|
||||
let Ok(mut file) = fs::File::open(path) else {
|
||||
return;
|
||||
};
|
||||
if file.seek(SeekFrom::Start(*offset)).is_err() {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut appended = Vec::new();
|
||||
let Ok(n) = file.read_to_end(&mut appended) else {
|
||||
return;
|
||||
};
|
||||
|
||||
if n > 0 {
|
||||
*offset += n as u64;
|
||||
output_buf.lock().push(&appended);
|
||||
}
|
||||
}
|
||||
|
||||
struct AbortOnDrop(tokio::task::JoinHandle<()>);
|
||||
|
||||
impl Drop for AbortOnDrop {
|
||||
fn drop(&mut self) {
|
||||
self.0.abort();
|
||||
}
|
||||
}
|
||||
|
||||
/// Deletes the env snapshot's temp files however the job task exits,
|
||||
/// including a cancel/abort dropping the future mid-await.
|
||||
struct TempFileGuard(Vec<PathBuf>);
|
||||
@@ -1030,6 +1061,18 @@ async fn run_process_job(
|
||||
.ok_or_else(|| anyhow!("Failed to capture stderr"))?;
|
||||
let stdout_pump = tokio::spawn(pump_into_ring(stdout, Arc::clone(&output_buf)));
|
||||
let stderr_pump = tokio::spawn(pump_into_ring(stderr, Arc::clone(&output_buf)));
|
||||
let file_offset = Arc::new(Mutex::new(0u64));
|
||||
let file_tailer = AbortOnDrop(tokio::spawn({
|
||||
let path = snapshot.output_file.clone();
|
||||
let offset = Arc::clone(&file_offset);
|
||||
let output_buf = Arc::clone(&output_buf);
|
||||
async move {
|
||||
loop {
|
||||
time::sleep(JOB_FILE_TAIL_POLL).await;
|
||||
drain_output_file(&path, &offset, &output_buf);
|
||||
}
|
||||
}
|
||||
}));
|
||||
|
||||
let wait_result = if snapshot.timeout_secs > 0 {
|
||||
match time::timeout(Duration::from_secs(snapshot.timeout_secs), child.wait()).await {
|
||||
@@ -1039,6 +1082,8 @@ async fn run_process_job(
|
||||
state.lock().pgid = None;
|
||||
drain_pump(stdout_pump).await;
|
||||
drain_pump(stderr_pump).await;
|
||||
drop(file_tailer);
|
||||
drain_output_file(&snapshot.output_file, &file_offset, &output_buf);
|
||||
let output_bytes_captured = output_buf.lock().total_written();
|
||||
let message = format!(
|
||||
"Tool call '{}' timed out after {}s and was killed (set COYOTE_TOOL_TIMEOUT to adjust; 0 = unlimited)",
|
||||
@@ -1068,6 +1113,8 @@ async fn run_process_job(
|
||||
state.lock().pgid = None;
|
||||
drain_pump(stdout_pump).await;
|
||||
drain_pump(stderr_pump).await;
|
||||
drop(file_tailer);
|
||||
drain_output_file(&snapshot.output_file, &file_offset, &output_buf);
|
||||
let output_bytes_captured = output_buf.lock().total_written();
|
||||
|
||||
let exit_code = status.code();
|
||||
@@ -2370,7 +2417,11 @@ mod tests {
|
||||
assert_eq!(result.output, json!({"output": "hi"}));
|
||||
let tail = String::from_utf8_lossy(&output_buf.lock().tail()).to_string();
|
||||
assert!(tail.contains("captured"));
|
||||
assert_eq!(result.output_bytes_captured, 9);
|
||||
assert!(
|
||||
tail.contains("hi"),
|
||||
"file bytes must reach the ring: {tail}"
|
||||
);
|
||||
assert_eq!(result.output_bytes_captured, 11);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -2442,6 +2493,62 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn run_process_job_counts_output_file_bytes() {
|
||||
run_async(async {
|
||||
let state = Arc::new(Mutex::new(JobState {
|
||||
status: JobStatus::Running,
|
||||
pgid: None,
|
||||
}));
|
||||
let output_buf = Arc::new(Mutex::new(RingBuf::default()));
|
||||
let snapshot = test_snapshot("sh", &["-c", "printf hello >> \"$LLM_OUTPUT\""], 0);
|
||||
|
||||
let result = run_process_job(snapshot, state, Arc::clone(&output_buf))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result.exit_code, Some(0));
|
||||
assert_eq!(result.output_bytes_captured, 5);
|
||||
let tail = output_buf.lock().tail();
|
||||
assert_eq!(String::from_utf8_lossy(&tail), "hello");
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn run_process_job_streams_output_file_bytes_mid_run() {
|
||||
run_async(async {
|
||||
let state = Arc::new(Mutex::new(JobState {
|
||||
status: JobStatus::Running,
|
||||
pgid: None,
|
||||
}));
|
||||
let output_buf = Arc::new(Mutex::new(RingBuf::default()));
|
||||
let snapshot =
|
||||
test_snapshot("sh", &["-c", "printf early >> \"$LLM_OUTPUT\"; sleep 5"], 0);
|
||||
let task_state = Arc::clone(&state);
|
||||
let task_buf = Arc::clone(&output_buf);
|
||||
let task =
|
||||
tokio::spawn(async move { run_process_job(snapshot, task_state, task_buf).await });
|
||||
|
||||
let mut saw_bytes_mid_run = false;
|
||||
for _ in 0..45 {
|
||||
time::sleep(Duration::from_millis(100)).await;
|
||||
if !task.is_finished() && output_buf.lock().total_written() >= 5 {
|
||||
saw_bytes_mid_run = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
let result = task.await.unwrap().unwrap();
|
||||
|
||||
assert!(
|
||||
saw_bytes_mid_run,
|
||||
"ring buffer never saw $LLM_OUTPUT bytes while the job was still running"
|
||||
);
|
||||
assert_eq!(result.exit_code, Some(0));
|
||||
});
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn handle_collect_returns_after_term_ignoring_job_is_aborted() {
|
||||
|
||||
+5
-1
@@ -517,7 +517,11 @@ impl Functions {
|
||||
anyhow!("Failed to load embedded function file: {}", file.as_ref())
|
||||
})?;
|
||||
let content = unsafe { std::str::from_utf8_unchecked(&embedded_file.data) };
|
||||
let file_path = paths::functions_dir().join(file.as_ref());
|
||||
let file_path = if file.as_ref() == "mcp.json" {
|
||||
paths::mcp_config_file()
|
||||
} else {
|
||||
paths::functions_dir().join(file.as_ref())
|
||||
};
|
||||
let is_script = file_path
|
||||
.extension()
|
||||
.and_then(OsStr::to_str)
|
||||
|
||||
+1
-6
@@ -1191,12 +1191,7 @@ pub async fn run_repl_command(
|
||||
ask(ctx, abort_signal.clone(), input, true).await?;
|
||||
}
|
||||
".recover" => {
|
||||
let has_recoverable = ctx
|
||||
.last_message
|
||||
.as_ref()
|
||||
.map(|v| v.continuous && v.input.with_session())
|
||||
.unwrap_or(false);
|
||||
if !has_recoverable {
|
||||
if !ctx.has_recoverable_interruption() {
|
||||
bail!("Unable to recover: no interrupted session response to recover from");
|
||||
}
|
||||
let recovery_text = args
|
||||
|
||||
Reference in New Issue
Block a user