Merge remote-tracking branch 'origin/main' into feat/mcp-tool-whitelist
This commit is contained in:
+9
-10
@@ -3,7 +3,7 @@ name = "coyote-ai"
|
||||
version = "0.8.3"
|
||||
edition = "2024"
|
||||
authors = ["Alex Clarke <alex.j.tusa@gmail.com>"]
|
||||
description = "An all-in-one, batteries included LLM CLI Tool"
|
||||
description = "The batteries-included runtime for LLMs"
|
||||
keywords = ["chatgpt", "llm", "cli", "ai", "repl"]
|
||||
homepage = "https://github.com/Dark-Alex-17/coyote"
|
||||
repository = "https://github.com/Dark-Alex-17/coyote"
|
||||
@@ -51,7 +51,13 @@ textwrap = "0.16.0"
|
||||
ansi_colours = "1.2.2"
|
||||
eventsource-stream = "0.2.3"
|
||||
log = "0.4.28"
|
||||
log4rs = { version = "1.4.0", features = ["file_appender", "rolling_file_appender", "compound_policy", "fixed_window_roller", "size_trigger"] }
|
||||
log4rs = { version = "1.4.0", features = [
|
||||
"file_appender",
|
||||
"rolling_file_appender",
|
||||
"compound_policy",
|
||||
"fixed_window_roller",
|
||||
"size_trigger",
|
||||
] }
|
||||
shell-words = "1.1.0"
|
||||
sha2 = "0.10.8"
|
||||
unicode-width = "0.2.0"
|
||||
@@ -113,14 +119,7 @@ qrcode = "0.14"
|
||||
|
||||
[dependencies.reqwest]
|
||||
version = "0.13.3"
|
||||
features = [
|
||||
"json",
|
||||
"multipart",
|
||||
"stream",
|
||||
"form",
|
||||
"socks",
|
||||
"rustls",
|
||||
]
|
||||
features = ["json", "multipart", "stream", "form", "socks", "rustls"]
|
||||
default-features = false
|
||||
|
||||
[dependencies.syntect]
|
||||
|
||||
+1
-1
@@ -99,7 +99,7 @@ LABEL com.docker.sandboxes="templates" \
|
||||
com.docker.sandboxes.flavor="shell-docker" \
|
||||
com.docker.sandboxes.start-docker="true" \
|
||||
org.opencontainers.image.title="coyote" \
|
||||
org.opencontainers.image.description="An all-in-one, batteries-included LLM CLI tool: Shell Assistant, CLI & REPL mode, RAG, AI tools & agents, MCP servers, skills, and macros." \
|
||||
org.opencontainers.image.description="The batteries-included runtime for LLMs: Shell Assistant, CLI & REPL mode, RAG, AI tools & agents, MCP servers, skills, and macros." \
|
||||
org.opencontainers.image.source="https://github.com/Dark-Alex-17/coyote" \
|
||||
org.opencontainers.image.version="${COYOTE_VERSION}"
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# Coyote: All-in-one, batteries-included LLM CLI Tool
|
||||
# Coyote: The batteries-included runtime for LLMs
|
||||
|
||||
|
||||
[](https://crates.io/crates/coyote-ai)
|
||||
@@ -6,14 +6,18 @@
|
||||
|
||||
[](https://github.com/Dark-Alex-17/coyote/releases)
|
||||
|
||||
[](./LICENSE)
|
||||
[](https://github.com/Dark-Alex-17/coyote/blob/main/LICENSE)
|
||||
|
||||
Coyote is an all-in-one, batteries-included, LLM CLI tool featuring Shell Assistant, CLI & REPL Mode, RAG, AI Tools &
|
||||
Agents, and More.
|
||||
Coyote is an **all-in-one, batteries-included LLM runtime** for building, running, and interacting with AI from your terminal.
|
||||
It brings together a Shell Assistant, CLI & REPL modes, RAG, tools, agents, MCP, skills, sandboxes, multi-agent workflows, and
|
||||
more in a single runtime.
|
||||
|
||||
Coyote comes ready to use with built-in agents, roles, macros, and tools, so you can get started without assembling an AI
|
||||
stack from scratch. When you want to extend it, entire bundles of agents, roles, macros, tools, MCP servers, and other
|
||||
configurations can be installed directly from any Git repository.
|
||||
|
||||
See [Bundles](https://github.com/Dark-Alex-17/coyote/wiki/Bundles) to learn how to create, install, and share Coyote bundles.
|
||||
|
||||
It is designed to include a number of useful agents, roles, macros, and more so users can get up and running with Coyote
|
||||
in as little time as possible. You can also install entire bundles of agents, roles, macros, tools, and MCP servers from
|
||||
any git repository. See [Sharing Configurations](https://github.com/Dark-Alex-17/coyote/wiki/Sharing-Configurations) for more information.
|
||||
|
||||
|
||||
|
||||
@@ -263,7 +267,7 @@ coyote | Out-String | Invoke-Expression
|
||||
|
||||
### Shell Integration
|
||||
You can integrate Coyote's Shell Assistant into your shell for enhanced command-line assistance. Add the code in the
|
||||
corresponding [shell integration script](./scripts/shell-integration) to your shell. Then, you can invoke Coyote to convert natural language to
|
||||
corresponding [shell integration script](https://github.com/Dark-Alex-17/coyote/tree/main/scripts/shell-integration) to your shell. Then, you can invoke Coyote to convert natural language to
|
||||
shell commands by pressing `Alt-e`. For example:
|
||||
|
||||
```shell
|
||||
@@ -281,7 +285,7 @@ coyote --info | grep 'config_file' | awk '{print $2}'
|
||||
```
|
||||
|
||||
The configuration file consists of a number of settings. To see a full example configuration file with every setting
|
||||
defined, refer to the [example configuration file](./config.example.yaml).
|
||||
defined, refer to the [example configuration file](https://github.com/Dark-Alex-17/coyote/blob/main/config.example.yaml).
|
||||
|
||||
### Default LLM
|
||||
The following settings are available to configure the default LLM that is used when you start Coyote, and its
|
||||
@@ -335,7 +339,7 @@ The appearance of Coyote can be modified using the following settings:
|
||||
|
||||
Coyote began as a fork of [AIChat CLI](https://github.com/sigoden/aichat) and has since evolved into an independent project.
|
||||
|
||||
See [CREDITS.md](./CREDITS.md) for full attribution and background.
|
||||
See [CREDITS.md](https://github.com/Dark-Alex-17/coyote/blob/main/CREDITS.md) for full attribution and background.
|
||||
|
||||
---
|
||||
|
||||
@@ -346,11 +350,11 @@ See [CREDITS.md](./CREDITS.md) for full attribution and background.
|
||||
|
||||
## License
|
||||
|
||||
Coyote is licensed under the [GNU Affero General Public License v3.0](./LICENSE)
|
||||
Coyote is licensed under the [GNU Affero General Public License v3.0](https://github.com/Dark-Alex-17/coyote/blob/main/LICENSE)
|
||||
(AGPL-3.0-only).
|
||||
|
||||
Coyote began as a fork of [AIChat](https://github.com/sigoden/aichat)
|
||||
(Copyright (c) sigoden), which is licensed under the MIT License. Substantial
|
||||
portions of Coyote are derived from AIChat and remain available under the MIT
|
||||
License, preserved in [LICENSE-MIT](./LICENSE-MIT). See [NOTICE](./NOTICE) and
|
||||
[CREDITS.md](./CREDITS.md) for details.
|
||||
License, preserved in [LICENSE-MIT](https://github.com/Dark-Alex-17/coyote/blob/main/LICENSE-MIT). See [NOTICE](https://github.com/Dark-Alex-17/coyote/blob/main/NOTICE) and
|
||||
[CREDITS.md](https://github.com/Dark-Alex-17/coyote/blob/main/CREDITS.md) for details.
|
||||
|
||||
@@ -3,8 +3,9 @@ description: |
|
||||
Design-doc orchestrator for any project. Consumes a high-level design doc, decomposes it into a
|
||||
gated plan (gatekeeper self-containedness + oracle plan-review) and ~1-engineer-day task files,
|
||||
spawns one Sisyphus per task on a single run branch, verifies each with an adversarial
|
||||
plan-conformance check, and finishes with ONE draft PR (CI checks watched to green) plus tracked
|
||||
follow-up tasks. Task state lives on disk in a plans directory, so runs survive context compression.
|
||||
plan-conformance check (plus a black-box usage-pattern probe for consumer-facing surface), and
|
||||
finishes with ONE draft PR (CI checks watched to green) plus tracked follow-up tasks. Task state
|
||||
lives on disk in a plans directory, so runs survive context compression.
|
||||
version: 2.1.0
|
||||
agent_session: temp
|
||||
auto_continue: true
|
||||
@@ -18,6 +19,7 @@ spawnable_agents:
|
||||
- explore
|
||||
- librarian
|
||||
- adversary
|
||||
- probe
|
||||
- gatekeeper
|
||||
max_concurrent_agents: 10
|
||||
max_agent_depth: 10
|
||||
@@ -149,7 +151,8 @@ instructions: |
|
||||
|
||||
**Gate 1 — Self-containedness (`gatekeeper`).** The plan must pass the "docker container" test:
|
||||
every question a context-free implementer will hit is answered inline or delegated via a verified
|
||||
pointer to code/docs (where infra code goes, DB tech/target, layout to mirror, test commands, ...).
|
||||
pointer to code/docs (where infra code goes, DB tech/target, layout to mirror, test commands,
|
||||
local-run recipe for any consumer-facing surface the plan creates, ...).
|
||||
|
||||
> `agent__spawn --agent gatekeeper --prompt "Audit this plan for self-containedness. Return
|
||||
> SEALED/LEAKY. Plan: {{plans_dir}}/PLAN-<slug>.md. Target project: {{project_dir}}."`
|
||||
@@ -286,6 +289,39 @@ instructions: |
|
||||
escalate — do not silently change scope.
|
||||
- **`ADVERSARIAL_REVIEW: CONFORMS`** → conformance satisfied. Also confirm the stated test
|
||||
commands pass (run them if feasible) before closing.
|
||||
- **Usage-pattern probe (consumer-facing tasks).** If the task added or changed consumer-facing
|
||||
surface (endpoints/RPCs/CLI commands, request/response shapes, contract semantics like
|
||||
patch-vs-replace, idempotency, auth on routes), ALSO spawn `probe` for an independent
|
||||
black-box behavioral verdict — it boots the code locally from a clean state, runs existing
|
||||
usage suites for regressions, and spec-first-tests the changed surface with the repo's
|
||||
existing suite tooling or whatever is available (e.g. Hurl/curl, grpcurl, direct CLI
|
||||
invocation). Skip it (one-line note) for tasks with no consumer-visible surface.
|
||||
|
||||
```
|
||||
agent__spawn --agent probe --prompt "Probe TASK-NNN's changed surface from the consumer's
|
||||
perspective. Return PASS/FAIL/INCONCLUSIVE.
|
||||
|
||||
CHANGE: run get_diff --base <the task's BASE SHA recorded at claim time> in {{project_dir}}.
|
||||
|
||||
SPEC — expected behavior to verify against:
|
||||
<paste the task's acceptance criteria + relevant API contract sections VERBATIM>
|
||||
|
||||
LOCAL-RUN RECIPE: <paste the plan's local-run recipe verbatim — Gate 1 requires one for
|
||||
consumer-facing tasks>
|
||||
|
||||
EXISTING SUITES: <paths + run commands from the plan, or 'discover them'>"
|
||||
```
|
||||
|
||||
Set the probe's `project_dir` to {{project_dir}}. Verdict handling:
|
||||
- **`USAGE_PROBE: FAIL`** → blocker, same loop as DIVERGES: resume the SAME Sisyphus session
|
||||
with the behavioral findings (including repros) verbatim; re-run `probe` ONCE (resume ITS
|
||||
session so it reuses its environment and tests); still FAILing on the same findings →
|
||||
STOP and escalate.
|
||||
- **`USAGE_PROBE: PASS`** → have Sisyphus adopt probe's new test files (paths are in its
|
||||
report) as a commit on the run branch so they ship as permanent regression coverage.
|
||||
- **`USAGE_PROBE: INCONCLUSIVE`** → the local-run recipe is missing or broken — a PLAN gap,
|
||||
not a code failure. Fix the recipe (amend the plan) or escalate, re-run once; NEVER count
|
||||
INCONCLUSIVE as PASS or FAIL.
|
||||
- If Sisyphus reports failure after its own recovery, surface the evidence and consult `oracle`
|
||||
for diagnosis before deciding whether to retry, re-scope, or escalate.
|
||||
|
||||
@@ -328,8 +364,8 @@ instructions: |
|
||||
({{project_dir}}/.worktrees/task-NNN) and the branch with the task branch. Sisyphus commits and
|
||||
pushes the TASK branch. All other prompt sections unchanged — still no PRs, still no
|
||||
creating/switching branches (the worktree arrives already on its branch).
|
||||
- Run the adversary check in the worktree: `get_diff --base <BASE SHA>` — identical semantics to
|
||||
sequential mode.
|
||||
- Run the adversary check (and, for consumer-facing tasks, the probe check) in the worktree:
|
||||
`get_diff --base <BASE SHA>` — identical semantics to sequential mode.
|
||||
|
||||
**Integration (architect is the integrator; merges are ALWAYS one at a time):**
|
||||
1. When a task's Sisyphus finishes AND its adversary check CONFORMS, merge in the PRIMARY checkout:
|
||||
@@ -404,6 +440,7 @@ instructions: |
|
||||
| `oracle` | Plan review (`plan-review`), and diagnosis when a task fails after Sisyphus recovery. |
|
||||
| `gatekeeper` | Plan self-containedness gate (Phase C Gate 1): audits the PLAN for the "docker container" standard, returns SEALED/LEAKY with the missing implementer questions. |
|
||||
| `adversary` | Post-implementation plan-conformance verdict per task (CONFORMS/DIVERGES). |
|
||||
| `probe` | Black-box behavioral verdict on a task's consumer-facing surface: boots the code locally from clean state, runs existing usage suites + spec-first tests. Returns USAGE_PROBE PASS/FAIL/INCONCLUSIVE. |
|
||||
|
||||
## Escalation handling
|
||||
|
||||
@@ -424,6 +461,8 @@ instructions: |
|
||||
- Materializing tasks from a plan the gatekeeper marked LEAKY (or never audited), or that Oracle
|
||||
rejected (or never reviewed).
|
||||
- Marking a task complete without the adversary's CONFORMS verdict and verified acceptance criteria.
|
||||
- Closing a consumer-facing task without a `probe` verdict, or treating `INCONCLUSIVE` as PASS —
|
||||
an unprobeable consumer-facing change is a plan gap to fix, not a checkbox to skip.
|
||||
- Code comments referencing the plan/design doc/phases/steps/TASK numbers → docs drift, comments
|
||||
rot; plan references live in commit messages only.
|
||||
- A `.env` file landing in any repo → only `.env.example` with placeholder keys is committable;
|
||||
|
||||
@@ -17,6 +17,7 @@ variables:
|
||||
mcp_servers:
|
||||
- ddg-search
|
||||
global_tools:
|
||||
- web_search_coyote.sh
|
||||
- ast_grep.sh
|
||||
- fs_read.sh
|
||||
- fs_cat.sh
|
||||
|
||||
@@ -8,9 +8,10 @@ description: |
|
||||
sisyphus alongside explore when unfamiliar libraries/APIs/frameworks are
|
||||
involved.
|
||||
|
||||
version: "1.0"
|
||||
version: '1.0'
|
||||
|
||||
global_tools:
|
||||
- web_search_coyote.sh
|
||||
- fetch_url_via_curl.sh
|
||||
|
||||
mcp_servers:
|
||||
@@ -35,13 +36,13 @@ reducers:
|
||||
output: overwrite
|
||||
|
||||
initial_state:
|
||||
language_ecosystem: "general"
|
||||
doc_domain_hints: ""
|
||||
refined_search_query: ""
|
||||
question_type: "concept"
|
||||
search_output: ""
|
||||
oss_output: ""
|
||||
findings: ""
|
||||
language_ecosystem: 'general'
|
||||
doc_domain_hints: ''
|
||||
refined_search_query: ''
|
||||
question_type: 'concept'
|
||||
search_output: ''
|
||||
oss_output: ''
|
||||
findings: ''
|
||||
|
||||
start: triage
|
||||
|
||||
@@ -104,9 +105,15 @@ nodes:
|
||||
type: string
|
||||
enum: [api_reference, best_practice, debugging, concept]
|
||||
description: The kind of question being asked.
|
||||
required: [language_ecosystem, doc_domain_hints, refined_search_query, question_type]
|
||||
required:
|
||||
[
|
||||
language_ecosystem,
|
||||
doc_domain_hints,
|
||||
refined_search_query,
|
||||
question_type,
|
||||
]
|
||||
state_updates:
|
||||
last_node_output: "{{output}}"
|
||||
last_node_output: '{{output}}'
|
||||
fallback: end_failure
|
||||
next: [search, search_oss]
|
||||
|
||||
@@ -174,13 +181,15 @@ nodes:
|
||||
- Refined query: {{refined_search_query}}
|
||||
- Question type: {{question_type}}
|
||||
|
||||
Use the ddg-search tool. Prioritize the hinted doc domains when present
|
||||
(e.g., search with `site:docs.python.org pathlib` style queries).
|
||||
Use the ddg-search tool or the web_search_coyote tool. Prioritize the
|
||||
hinted doc domains when present (e.g., search with `site:docs.python.org
|
||||
pathlib` style queries).
|
||||
tools:
|
||||
- mcp:ddg-search
|
||||
- web_search_coyote
|
||||
max_iterations: 15
|
||||
state_updates:
|
||||
search_output: "{{output}}"
|
||||
search_output: '{{output}}'
|
||||
fallback: synthesize
|
||||
next: synthesize
|
||||
|
||||
@@ -250,7 +259,7 @@ nodes:
|
||||
- mcp:personal-github
|
||||
max_iterations: 15
|
||||
state_updates:
|
||||
oss_output: "{{output}}"
|
||||
oss_output: '{{output}}'
|
||||
fallback: synthesize
|
||||
next: synthesize
|
||||
|
||||
@@ -336,7 +345,7 @@ nodes:
|
||||
- fetch_url_via_curl
|
||||
max_iterations: 20
|
||||
state_updates:
|
||||
findings: "{{output}}"
|
||||
findings: '{{output}}'
|
||||
fallback: final_format
|
||||
next: final_format
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ variables:
|
||||
mcp_servers:
|
||||
- ddg-search
|
||||
global_tools:
|
||||
- web_search_coyote.sh
|
||||
- ast_grep.sh
|
||||
- fs_read.sh
|
||||
- fs_cat.sh
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
# Probe
|
||||
|
||||
A **black-box usage-pattern verifier**. Where every other reviewer reads *text* — the diff
|
||||
([`code-reviewer`](../code-reviewer/README.md)), the plan ([`adversary`](../adversary/README.md)),
|
||||
the attack surface ([`security-reviewer`](../security-reviewer/README.md)) — `probe` asks the one
|
||||
question none of them can answer without running the thing:
|
||||
|
||||
> **"Does the changed consumer-facing surface actually behave as the spec promises when used,
|
||||
> starting from nothing?"**
|
||||
|
||||
It boots the system locally from a clean slate, runs any existing usage suites first (regression
|
||||
check), derives expected behaviors from the **spec** — never the implementation — and authors
|
||||
tests for the uncovered usage patterns: cold-start/empty-state calls, idempotent re-calls, invalid
|
||||
input, auth on new routes, partial-update (patch-vs-replace) semantics, serialization edges,
|
||||
pagination limits, error-shape consistency. These are exactly the defects invisible to static
|
||||
review.
|
||||
|
||||
## Why it's separate from the other reviewers
|
||||
|
||||
| | `code-reviewer` | `adversary` | `security-reviewer` | `probe` |
|
||||
|---|---|---|---|---|
|
||||
| Question | Is the code good? | Does it match the plan? | Can it be abused? | Does it *work* when used? |
|
||||
| Method | Reads the diff | Diff vs. criteria | Source→sink tracing | **Runs the system**, black-box |
|
||||
| Blind spot it covers | slop, bugs, coupling | skipped criteria, drift | injection, authz gaps | behavioral quirks, regressions, contract surprises |
|
||||
| Output | severity findings | `CONFORMS`/`DIVERGES` | `PASS`/`FAIL` | `PASS`/`FAIL`/`INCONCLUSIVE` |
|
||||
|
||||
The independence is behavioral: expectations are written from the spec/contract **before** reading
|
||||
handler code, so the implementer's misreadings can't become the probe's assertions — the same
|
||||
principle that makes `adversary` valuable, applied to runtime behavior.
|
||||
|
||||
## Verdict (blocking, three-way)
|
||||
|
||||
```
|
||||
USAGE_PROBE: PASS
|
||||
Surface: <...>. Existing suites: <N run, all green | none found>. New tests: <M authored at <path>, all green>.
|
||||
```
|
||||
|
||||
```
|
||||
USAGE_PROBE: FAIL
|
||||
Behavioral findings:
|
||||
1. <surface + case> — <spec'd behavior> — <observed behavior> — REPRO: <exact request + response> — <test file>
|
||||
```
|
||||
|
||||
```
|
||||
USAGE_PROBE: INCONCLUSIVE
|
||||
Could not establish a clean local environment: <verbatim error>. Missing: <the recipe/fixture that would unblock>.
|
||||
```
|
||||
|
||||
- **`FAIL` blocks completion** — the caller resumes the SAME implementer session with the findings
|
||||
pasted verbatim, then re-runs `probe` once to confirm.
|
||||
- **`INCONCLUSIVE` is the honest third state**: the environment, not the code, is the blocker. It
|
||||
routes the fix to the local-run recipe (often a plan gap the `gatekeeper` should have caught) and
|
||||
is never disguised as `PASS` or `FAIL`.
|
||||
|
||||
Every `FAIL` finding carries an exact reproduction (request/command + response received) and the
|
||||
test file that proves it.
|
||||
|
||||
## How it probes
|
||||
|
||||
Driven by the [`usage-pattern-testing`](../../skills/usage-pattern-testing/SKILL.md) skill:
|
||||
|
||||
1. **Spec first** — expected behaviors written from acceptance criteria + API contract before any
|
||||
implementation reads.
|
||||
2. **Regression first** — discover and run existing usage suites; every failure classified as
|
||||
BUG / EXPECTED-CHANGE / ENV before anything new is authored.
|
||||
3. **Delta only** — new tests cover only the usage patterns existing suites miss, written in the
|
||||
repo's suite conventions so they're adoptable as permanent regression coverage.
|
||||
4. **Clean, local, isolated** — ephemeral state, mocked externals, full teardown; bounded retries
|
||||
for startup only, never to mask flakiness.
|
||||
|
||||
Toolbox by surface — the repo's existing suite format always comes first, and these are examples,
|
||||
not requirements: [Hurl](https://hurl.dev) or `curl` scripts for HTTP/REST/JSON (Hurl files double
|
||||
as committed suites), `grpcurl` for pure gRPC, direct invocation for CLIs.
|
||||
|
||||
Unlike the read-only reviewers, `probe` **writes test files** (and only test files) — the tests
|
||||
are a deliverable alongside the verdict. It never modifies implementation code.
|
||||
|
||||
## Usage
|
||||
|
||||
Spawned by `sisyphus` (post-coder, when the change touches consumer-facing surface) or `architect`
|
||||
(Phase E, alongside `adversary`). The spawn prompt IS its entire context — include the change, the
|
||||
spec, and the local-run recipe:
|
||||
|
||||
```sh
|
||||
agent__spawn --agent probe --prompt "
|
||||
## TASK
|
||||
Probe the changed API surface for TASK-NNN from the consumer's perspective. Return PASS/FAIL/INCONCLUSIVE.
|
||||
|
||||
## CHANGE
|
||||
Run get_diff --base <ref>, or: <paste the changed-surface summary>
|
||||
|
||||
## SPEC — expected behavior to verify against
|
||||
<paste acceptance criteria + API contract sections (or contract file paths) VERBATIM>
|
||||
|
||||
## LOCAL-RUN RECIPE
|
||||
<how to boot the stack clean: build, deps/stubs, ports, migrations, teardown — or the doc that has it>
|
||||
|
||||
## EXISTING SUITES
|
||||
<paths + run commands, or 'discover them'>
|
||||
"
|
||||
```
|
||||
|
||||
Direct invocation for ad-hoc use:
|
||||
|
||||
```sh
|
||||
coyote -a probe --agent-variable project_dir /path/to/repo \
|
||||
"Probe the /widgets endpoints changed in the last commit against this spec: <paste spec>"
|
||||
```
|
||||
|
||||
### Tools
|
||||
|
||||
- `get_diff [--base <ref>]` — staged → unstaged → `HEAD~1` fallback (or an explicit base SHA/branch) to locate the changed surface.
|
||||
- `get_changed_files [--base <ref>]` — quick changed-file map.
|
||||
- Plus `fs_*`/`ast_grep` for suite discovery and contract reads, `fs_write`/`fs_patch` for authoring test files, and `execute_command` for booting the stack and running suites.
|
||||
- Probing tools (`curl`, Hurl, grpcurl, the repo's own harness) are invoked via `execute_command`
|
||||
(no wrapper tool — probing needs their full CLI surface), and none is a hard requirement: the
|
||||
[`usage-pattern-testing`](../../skills/usage-pattern-testing/SKILL.md) skill has probe reuse the repo's existing suite tooling first and fall back to what's available.
|
||||
The optional [`sbx-mixin.yaml`](sbx-mixin.yaml) preinstalls Hurl + grpcurl for sandbox runs.
|
||||
|
||||
## Related
|
||||
|
||||
- [`usage-pattern-testing`](../../skills/usage-pattern-testing/SKILL.md) — the methodology it runs on.
|
||||
- [`adversary`](../adversary/README.md) — static plan-conformance counterpart (text), where `probe` is dynamic (behavior).
|
||||
- [`gatekeeper`](../gatekeeper/README.md) — ensures plans ship the local-run recipe `probe` consumes.
|
||||
@@ -0,0 +1,129 @@
|
||||
name: probe
|
||||
description: Black-box usage-pattern verifier - exercises a change's consumer-facing surface (HTTP APIs, RPCs, CLIs) as a real cold-start consumer against a locally running instance with clean, isolated state. Runs existing usage suites first for regressions (whatever format the repo uses - Hurl files, curl scripts, collections), authors spec-first tests for uncovered patterns in the repo's suite conventions (tools like Hurl and grpcurl are examples, not requirements), and returns a blocking USAGE_PROBE PASS/FAIL/INCONCLUSIVE verdict. Complements code-reviewer (quality), adversary (plan conformance), and security-reviewer (abuse). Designed to be delegated to by sisyphus and architect.
|
||||
version: 1.0.0
|
||||
|
||||
auto_continue: true
|
||||
max_auto_continues: 25
|
||||
inject_todo_instructions: true
|
||||
|
||||
skills_enabled: true
|
||||
enabled_skills:
|
||||
- usage-pattern-testing
|
||||
|
||||
variables:
|
||||
- name: project_dir
|
||||
description: Project directory containing the change under test - where suites are discovered, the stack is booted, and new tests are written
|
||||
default: '.'
|
||||
- name: auto_confirm
|
||||
description: Auto-confirm command execution
|
||||
default: '1'
|
||||
|
||||
global_tools:
|
||||
- ast_grep.sh
|
||||
- fs_read.sh
|
||||
- fs_cat.sh
|
||||
- fs_grep.sh
|
||||
- fs_glob.sh
|
||||
- fs_ls.sh
|
||||
- fs_write.sh
|
||||
- fs_patch.sh
|
||||
- execute_command.sh
|
||||
|
||||
instructions: |
|
||||
You are the usage-pattern probe. You answer ONE question: **does the changed consumer-facing
|
||||
surface actually behave as the spec promises when used, starting from a clean slate?** Every
|
||||
other reviewer reads text — the diff, the plan, the code. You are the only gate that BOOTS the
|
||||
system locally and exercises it the way a consumer will: cold, black-box, spec-first.
|
||||
|
||||
You are NOT the code-quality reviewer (`code-reviewer`), NOT the plan-conformance reviewer
|
||||
(`adversary`), and NOT the security reviewer (`security-reviewer`). You judge observable
|
||||
behavior. Your value is behavioral independence: expectations derived from the spec BEFORE
|
||||
reading the implementation, so the implementer's misreadings cannot become your assertions.
|
||||
|
||||
## Step 0: Load the skill
|
||||
|
||||
Before anything else, `skill__load` `usage-pattern-testing`. It carries your methodology: the
|
||||
spec-first independence rule, the regression-first protocol (find and run existing suites before
|
||||
authoring anything), the usage-pattern checklist (cold start, idempotency, invalid input, auth,
|
||||
partial-update semantics, serialization edges, pagination, error shapes), the clean-environment
|
||||
discipline, the failure-classification table (BUG / EXPECTED-CHANGE / ENV), the per-surface
|
||||
toolbox (the repo's existing suite tooling comes first; Hurl/curl for HTTP, grpcurl for gRPC,
|
||||
and direct invocation for CLIs are examples, not requirements), and the exact verdict format.
|
||||
The skill body is your source of truth for HOW to probe; these instructions handle
|
||||
workflow and I/O.
|
||||
|
||||
## Input (the spawn prompt IS your entire context)
|
||||
|
||||
You are given:
|
||||
1. **The change** — a diff pasted inline, a summary of the changed surface, or an instruction to
|
||||
run `git diff`/`get_diff` (optionally against a base ref) in {{project_dir}}.
|
||||
2. **The spec** — acceptance criteria, plan section, or API contract (or paths to the contract
|
||||
files: IDL/schema/OpenAPI/proto). This is what you derive expected behaviors FROM.
|
||||
3. **A local-run recipe** (strongly preferred) — how to boot the system locally from a clean
|
||||
state: build command, dependencies to start/stub, ports, migration/seed steps, teardown. If
|
||||
absent, look for one in the repo's contributor docs and dev scripts before inventing your own.
|
||||
4. **Pointers to existing usage suites** (optional) — where black-box tests already live and how
|
||||
to run them. If absent, discover them per the skill.
|
||||
|
||||
If the spec is missing, STOP and say so: behavior cannot be judged without a promise to judge
|
||||
against. Do not infer the spec from the implementation.
|
||||
|
||||
## Workflow
|
||||
|
||||
1. Load `usage-pattern-testing`.
|
||||
2. Identify the changed consumer-facing surface from the diff/summary. No consumer-facing surface
|
||||
→ return PASS with a one-line "no probeable surface" note; do not boot anything.
|
||||
3. **Spec first:** write down expected behaviors as concrete request→response pairs from the
|
||||
spec/contract, BEFORE reading handler code (implementation reads are for ports/config/startup
|
||||
wiring only).
|
||||
4. Discover existing usage suites; bring up the clean local environment per the recipe; run the
|
||||
existing suites FIRST and classify every failure (regression vs expected contract change vs
|
||||
environment).
|
||||
5. Map existing coverage against your expected behaviors; author tests for the uncovered
|
||||
patterns only, in the repo's suite location and conventions, walking the skill's
|
||||
usage-pattern checklist.
|
||||
6. Run the new tests. Classify every failure. Reproduce non-deterministic results twice and read
|
||||
the server logs before classifying.
|
||||
7. Tear the environment down. Emit the verdict in the skill's exact format.
|
||||
|
||||
## Output — verdict (MANDATORY, exact format)
|
||||
|
||||
End with EXACTLY one of the skill's three sentinels so the caller can route on it:
|
||||
|
||||
- `USAGE_PROBE: PASS` — existing suites green (or none), new spec-first tests green. List
|
||||
surface probed, suites run, and tests authored (with paths, so the caller can adopt them).
|
||||
- `USAGE_PROBE: FAIL` — behavioral findings, each with the spec'd behavior quoted, the observed
|
||||
behavior, the EXACT reproduction (request/command + response received), and the test file.
|
||||
- `USAGE_PROBE: INCONCLUSIVE` — a clean local environment could not be established. State what
|
||||
failed verbatim and EXACTLY what recipe/fixture/mock would unblock. Include any partial
|
||||
results. INCONCLUSIVE is honest and routes the fix to the environment recipe — NEVER disguise
|
||||
it as PASS or FAIL.
|
||||
|
||||
## Rules
|
||||
|
||||
1. **Never modify implementation code.** Your only writes are new/updated TEST files (in the
|
||||
repo's suite conventions) and throwaway environment scaffolding you tear down. The
|
||||
implementer owns all fixes.
|
||||
2. **Spec-first or nothing.** Expectations written from the spec before implementation reads.
|
||||
If the spec and the contract files disagree, that is a finding — report it, don't pick one
|
||||
silently.
|
||||
3. **Regressions before new coverage.** Existing suites run first; a regression is only
|
||||
acceptable when the spec explicitly changed that contract (then flag the stale test for
|
||||
update — never delete or silence it).
|
||||
4. **Clean, local, isolated.** Fresh ephemeral state, mocked externals, no dependence on
|
||||
pre-existing data or running services, full teardown. Bounded retries for startup only —
|
||||
never to mask a flaky assertion.
|
||||
5. **Classify every failure** as BUG / EXPECTED-CHANGE / ENV per the skill table. The verdict
|
||||
depends on the classification being honest.
|
||||
6. **Committed tests are the deliverable** alongside the verdict: write them where the repo's
|
||||
suites live so the caller can adopt them as permanent regression coverage. Report their paths.
|
||||
7. Be terse and decisive. Three reproducible behavioral findings beat fifteen speculative ones.
|
||||
If everything works as spec'd, it PASSes — say so.
|
||||
|
||||
## Context
|
||||
- Project: {{project_dir}}
|
||||
- CWD: {{__cwd__}}
|
||||
- Shell: {{__shell__}}
|
||||
|
||||
## Available Tools
|
||||
{{__tools__}}
|
||||
@@ -0,0 +1,76 @@
|
||||
schemaVersion: '1'
|
||||
kind: mixin
|
||||
name: agent-probe
|
||||
description: >
|
||||
Optional convenience for the probe agent: preinstalls Hurl (HTTP
|
||||
usage-pattern tests) and grpcurl (gRPC probing) — the example tools its
|
||||
skill reaches for — and allows the GitHub release endpoints the fallback
|
||||
installers download from. Neither tool is required: probe reuses the repo's
|
||||
existing suite tooling first and falls back to what's available. Hurl
|
||||
prefers the distro package: the prebuilt GitHub tarball dynamically links
|
||||
libxml2.so.2, which newer distros no longer ship (e.g. Ubuntu 26.04 moved
|
||||
to libxml2.so.16). The services under probe run on localhost, which needs
|
||||
no network allowance. POSIX-only: sbx runs these commands with /bin/sh (dash).
|
||||
|
||||
network:
|
||||
allowedDomains:
|
||||
# Latest-release lookup + tarball downloads (GitHub redirects release
|
||||
# assets to *.githubusercontent.com object hosts)
|
||||
- 'api.github.com:443'
|
||||
- 'github.com:443'
|
||||
- 'objects.githubusercontent.com:443'
|
||||
- 'release-assets.githubusercontent.com:443'
|
||||
|
||||
commands:
|
||||
install:
|
||||
- command: |
|
||||
set -eu
|
||||
if command -v hurl >/dev/null 2>&1; then
|
||||
hurl --version
|
||||
exit 0
|
||||
fi
|
||||
if command -v apt-get >/dev/null 2>&1; then
|
||||
sudo apt-get update
|
||||
if apt-cache policy hurl 2>/dev/null | grep -q 'Candidate: [0-9]'; then
|
||||
sudo apt-get install -y --no-install-recommends hurl
|
||||
hurl --version
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
arch="$(uname -m)"
|
||||
case "$arch" in
|
||||
aarch64|arm64) arch="aarch64" ;;
|
||||
*) arch="x86_64" ;;
|
||||
esac
|
||||
curl -fsSL https://api.github.com/repos/Orange-OpenSource/hurl/releases/latest -o /tmp/hurl-release.json
|
||||
ver="$(sed -n 's/.*"tag_name": *"\([^"]*\)".*/\1/p' /tmp/hurl-release.json | head -1)"
|
||||
curl -fsSL "https://github.com/Orange-OpenSource/hurl/releases/download/${ver}/hurl-${ver}-${arch}-unknown-linux-gnu.tar.gz" -o /tmp/hurl.tgz
|
||||
mkdir -p /tmp/hurl-extract
|
||||
tar -xzf /tmp/hurl.tgz -C /tmp/hurl-extract
|
||||
bin="$(find /tmp/hurl-extract -type f -name hurl | head -1)"
|
||||
sudo install -m 0755 "$bin" /usr/local/bin/hurl
|
||||
rm -rf /tmp/hurl.tgz /tmp/hurl-extract /tmp/hurl-release.json
|
||||
hurl --version
|
||||
user: '1000'
|
||||
description: Install Hurl (distro package preferred, GitHub tarball fallback) for the probe agent's HTTP usage-pattern tests
|
||||
- command: |
|
||||
set -eu
|
||||
if command -v grpcurl >/dev/null 2>&1; then
|
||||
grpcurl -version
|
||||
exit 0
|
||||
fi
|
||||
arch="$(uname -m)"
|
||||
case "$arch" in
|
||||
aarch64|arm64) arch="arm64" ;;
|
||||
*) arch="x86_64" ;;
|
||||
esac
|
||||
curl -fsSL https://api.github.com/repos/fullstorydev/grpcurl/releases/latest -o /tmp/grpcurl-release.json
|
||||
ver="$(sed -n 's/.*"tag_name": *"v\([^"]*\)".*/\1/p' /tmp/grpcurl-release.json | head -1)"
|
||||
curl -fsSL "https://github.com/fullstorydev/grpcurl/releases/download/v${ver}/grpcurl_${ver}_linux_${arch}.tar.gz" -o /tmp/grpcurl.tgz
|
||||
mkdir -p /tmp/grpcurl-extract
|
||||
tar -xzf /tmp/grpcurl.tgz -C /tmp/grpcurl-extract
|
||||
sudo install -m 0755 /tmp/grpcurl-extract/grpcurl /usr/local/bin/grpcurl
|
||||
rm -rf /tmp/grpcurl.tgz /tmp/grpcurl-extract /tmp/grpcurl-release.json
|
||||
grpcurl -version
|
||||
user: '1000'
|
||||
description: Install grpcurl (static GitHub release binary) for the probe agent's gRPC probes
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eo pipefail
|
||||
|
||||
# @env LLM_OUTPUT=/dev/stdout
|
||||
# @env LLM_AGENT_VAR_PROJECT_DIR=.
|
||||
# @describe Usage-pattern probe tools
|
||||
|
||||
_project_dir() {
|
||||
local dir="${LLM_AGENT_VAR_PROJECT_DIR:-.}"
|
||||
(cd "${dir}" 2>/dev/null && pwd) || echo "${dir}"
|
||||
}
|
||||
|
||||
# @cmd Get the git diff whose consumer-facing surface is under probe. Returns staged changes, or unstaged if nothing is staged, or the HEAD~1 diff if the working tree is clean.
|
||||
# @option --base Optional base ref to diff against (e.g., "main", "HEAD~3", a commit SHA, or a task's base SHA)
|
||||
get_diff() {
|
||||
local project_dir
|
||||
project_dir=$(_project_dir)
|
||||
# shellcheck disable=SC2154
|
||||
local base="${argc_base:-}"
|
||||
|
||||
local diff_output=""
|
||||
if [[ -n "${base}" ]]; then
|
||||
diff_output=$(cd "${project_dir}" && git diff "${base}" 2>&1) || true
|
||||
else
|
||||
diff_output=$(cd "${project_dir}" && git diff --cached 2>&1) || true
|
||||
if [[ -z "${diff_output}" ]]; then
|
||||
diff_output=$(cd "${project_dir}" && git diff 2>&1) || true
|
||||
fi
|
||||
if [[ -z "${diff_output}" ]]; then
|
||||
diff_output=$(cd "${project_dir}" && git diff HEAD~1 2>&1) || true
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "${diff_output}" ]]; then
|
||||
echo "No changes found to probe in ${project_dir}." >> "$LLM_OUTPUT"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local file_count
|
||||
file_count=$(echo "${diff_output}" | grep -c '^diff --git' || true)
|
||||
{
|
||||
echo "Diff contains changes to ${file_count} file(s):"
|
||||
echo ""
|
||||
echo "${diff_output}"
|
||||
} >> "$LLM_OUTPUT"
|
||||
}
|
||||
|
||||
# @cmd Get the list of changed files with stats (a quick map for locating the changed consumer-facing surface).
|
||||
# @option --base Optional base ref to diff against
|
||||
get_changed_files() {
|
||||
local project_dir
|
||||
project_dir=$(_project_dir)
|
||||
local base="${argc_base:-}"
|
||||
|
||||
local stat_output=""
|
||||
if [[ -n "${base}" ]]; then
|
||||
stat_output=$(cd "${project_dir}" && git diff --stat "${base}" 2>&1) || true
|
||||
else
|
||||
stat_output=$(cd "${project_dir}" && git diff --cached --stat 2>&1) || true
|
||||
if [[ -z "${stat_output}" ]]; then
|
||||
stat_output=$(cd "${project_dir}" && git diff --stat 2>&1) || true
|
||||
fi
|
||||
if [[ -z "${stat_output}" ]]; then
|
||||
stat_output=$(cd "${project_dir}" && git diff --stat HEAD~1 2>&1) || true
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -z "${stat_output}" ]]; then
|
||||
echo "No changes found in ${project_dir}." >> "$LLM_OUTPUT"
|
||||
return 0
|
||||
fi
|
||||
|
||||
{
|
||||
echo "Changed files:"
|
||||
echo ""
|
||||
echo "${stat_output}"
|
||||
} >> "$LLM_OUTPUT"
|
||||
}
|
||||
@@ -16,6 +16,7 @@ spawnable_agents:
|
||||
- code-reviewer
|
||||
- adversary
|
||||
- security-reviewer
|
||||
- probe
|
||||
- architecture-reviewer
|
||||
- step-runner
|
||||
max_concurrent_agents: 40
|
||||
@@ -65,6 +66,7 @@ global_tools:
|
||||
- fs_write.sh
|
||||
- fs_patch.sh
|
||||
- execute_command.sh
|
||||
- web_search_coyote.sh
|
||||
|
||||
instructions: |
|
||||
You are Sisyphus - an orchestrator that drives coding tasks to completion. You do NOT work alone when specialists are available. You classify, delegate, verify, complete.
|
||||
@@ -389,7 +391,66 @@ instructions: |
|
||||
- **`Pre-existing, out of scope:` findings** — surface to the user but do not act on them. They predate this work and aren't the current task's responsibility.
|
||||
- **Posture disagreement** — if the reviewer's report suggests the posture you chose understates the real exposure (e.g. you said `prototype` but the diff wires up a public endpoint), re-run with the higher posture rather than rationalizing the PASS.
|
||||
|
||||
Like `adversary`, re-running `security-reviewer` once after a fix is expected — a FAIL verdict is a hard gate, and confirming the fix closed the attack path is the point. Run all applicable reviewers (`code-reviewer`, `adversary`, `security-reviewer`) — they cover disjoint failure modes; one passing says nothing about the others.
|
||||
Like `adversary`, re-running `security-reviewer` once after a fix is expected — a FAIL verdict is a hard gate, and confirming the fix closed the attack path is the point. Run all applicable reviewers (`code-reviewer`, `adversary`, `security-reviewer`, `probe`) — they cover disjoint failure modes; one passing says nothing about the others.
|
||||
|
||||
### Usage-pattern probe (post-coder, when the change touches consumer-facing surface)
|
||||
|
||||
`code-reviewer`, `adversary`, and `security-reviewer` all read TEXT — the diff, the plan, the
|
||||
attack surface. None of them answers "does the feature actually behave correctly when a consumer
|
||||
uses it?" Spawn `probe` when the change touches consumer-facing surface. It boots the system
|
||||
locally from a clean slate, runs existing usage suites first (regression check), derives expected
|
||||
behaviors from the SPEC (never the implementation, so the implementer's misreadings can't become
|
||||
its assertions), authors tests for the uncovered usage patterns in the repo's existing suite
|
||||
conventions (tools like Hurl/curl for HTTP, grpcurl for gRPC, direct invocation for CLIs are
|
||||
examples, not requirements), and returns a blocking `USAGE_PROBE: PASS/FAIL/INCONCLUSIVE` verdict.
|
||||
|
||||
**When to spawn it** — ANY of these:
|
||||
|
||||
1. The change adds or modifies **externally consumed surface**: HTTP endpoints/RPCs,
|
||||
request/response shapes, status codes, CLI commands/flags, event/webhook payloads
|
||||
2. The change alters **contract semantics**: partial-update (patch-vs-replace) behavior,
|
||||
idempotency, pagination, auth requirements on routes, error shapes
|
||||
3. **You judge the change consumer-visible** even if 1-2 don't trigger
|
||||
|
||||
If none fire (pure refactor, internal data shuffling with no consumer-visible effect), skip it
|
||||
with a one-line note — booting a stack to probe inert internals burns budget without value.
|
||||
|
||||
**Spawn pattern** (the prompt IS its whole context — include the spec AND the local-run recipe):
|
||||
|
||||
```
|
||||
agent__spawn --agent probe --prompt "Probe the changed surface from the consumer's perspective. Return PASS/FAIL/INCONCLUSIVE.
|
||||
|
||||
CHANGE: run get_diff (or --base <ref>), or: <paste the changed-surface summary>
|
||||
|
||||
SPEC — expected behavior to verify against:
|
||||
<paste acceptance criteria + API contract sections (or contract file paths) VERBATIM>
|
||||
|
||||
LOCAL-RUN RECIPE: <how to boot the stack clean — build, deps/stubs, ports, migrations, teardown — or where the recipe lives>
|
||||
|
||||
EXISTING SUITES: <paths + run commands, or 'discover them'>"
|
||||
```
|
||||
|
||||
### Handling probe findings
|
||||
|
||||
- **`USAGE_PROBE: FAIL` blocks completion.** Do not mark the task done. Resume the SAME coder
|
||||
session (`agent__spawn --session_id <id> --prompt "Fix these behavioral findings: <findings
|
||||
pasted verbatim, including repros>"`) — do not spawn a fresh coder. After the fix, re-run
|
||||
`probe` ONCE — resume ITS session too, so it reuses the environment and tests it already built.
|
||||
If it still FAILs on the same findings after one fix cycle, STOP and escalate to the user (the
|
||||
spec or the design may be the root cause — consider `oracle`).
|
||||
- **`USAGE_PROBE: PASS`** — proceed. Adopt the test files probe authored (written in the repo's
|
||||
suite conventions; paths are in its report) into the change so they ship as permanent
|
||||
regression coverage. Surface any stale-test or recipe observations to the user.
|
||||
- **`USAGE_PROBE: INCONCLUSIVE`** — the ENVIRONMENT, not the code, is the blocker. Never treat it
|
||||
as PASS or FAIL. If the missing recipe/fixture/mock is cheap to provide, supply it and re-run
|
||||
probe once (resume its session). Otherwise surface the gap to the user — a consumer-facing
|
||||
change that cannot be exercised locally is itself a finding.
|
||||
- **Tests flagged EXPECTED-CHANGE** (existing tests asserting a contract the spec explicitly
|
||||
changed) — have the coder update them as part of the change; never delete or silence them to
|
||||
get green.
|
||||
|
||||
Like the other hard gates, re-running `probe` once after a fix is expected — confirming the
|
||||
behavioral finding is actually closed is the point.
|
||||
|
||||
### Observability pass (post-coder, advisory — when the change adds operational surface)
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ kind: sandbox
|
||||
name: coyote
|
||||
displayName: Coyote
|
||||
description: >
|
||||
An all-in-one, batteries-included LLM CLI tool featuring Shell Assistant,
|
||||
The batteries-included runtime for LLMs, featuring Shell Assistant,
|
||||
CLI & REPL mode, RAG, AI tools & agents, MCP servers, skills, and macros.
|
||||
|
||||
sandbox:
|
||||
|
||||
@@ -26,7 +26,7 @@ Walk EVERY category. For each, ask: "when the implementer hits this, does the pl
|
||||
| 3 | **Data layer** | What DB tech/engine? What migration tool and directory? What naming conventions for tables/columns? Which existing tables does this touch or reference? |
|
||||
| 4 | **Interfaces & contracts** | What protos/APIs/RPCs are consumed or exposed — exact names? Where do proto definitions live and how are they regenerated? What downstream consumers depend on the shapes this plan creates? |
|
||||
| 5 | **Conventions & tooling** | Which language/framework versions? Error-handling and logging patterns — which file shows the canon? Lint/format/build commands? Where is the repo's own CLAUDE.md / contributor doc and does the plan tell the implementer to read it? |
|
||||
| 6 | **Testing & verification** | Test framework and directory conventions? EXACT commands to run tests/build from the repo root? What proves each acceptance criterion? |
|
||||
| 6 | **Testing & verification** | Test framework and directory conventions? EXACT commands to run tests/build from the repo root? What proves each acceptance criterion? For plans that create or change consumer-facing surface (HTTP APIs, RPCs, CLIs): the EXACT local-run recipe — how to boot the system locally from a clean, empty state (build, dependencies to start or stub, ports, migrations/seed, teardown) — and where existing black-box usage suites live and how they are run? A black-box usage-pattern verification gate consumes this recipe post-implementation and returns INCONCLUSIVE (blocking the task) when the plan omits it. |
|
||||
| 7 | **Dependencies & ordering** | What must exist before this plan starts (other tasks, migrations, provisioned infra)? What does this plan produce that later work depends on? |
|
||||
| 8 | **Config, secrets & environments** | New env vars/config keys — where are they declared and injected? Secrets — vault/parameter store conventions? Staging vs production differences that affect implementation? |
|
||||
| 9 | **Scope boundaries** | Is Out of scope present and specific? Are "tempting adjacent fixes" explicitly deferred? |
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
---
|
||||
description: Verify a change from the consumer's perspective - exercise the changed surface (HTTP API, RPC, CLI) black-box against a locally running instance with clean, isolated state. Run existing usage suites first for regressions, derive new tests from the spec (never the implementation), and classify every failure as bug / environment / expected contract change. Produces a USAGE_PROBE PASS/FAIL/INCONCLUSIVE verdict. Complements code-review (quality), adversarial-review (plan conformance), and security-review (abuse) - this is the only gate that tests BEHAVIOR by using the thing, not by reading it.
|
||||
enabled_tools: fs_read, fs_cat, fs_grep, fs_glob, fs_ls, fs_write, fs_patch, execute_command
|
||||
---
|
||||
You are verifying a change the way its consumers will experience it: by USING it. Every other
|
||||
review gate reads text — the diff, the plan, the code. This gate boots the system locally from a
|
||||
clean state and exercises the changed surface as a cold-start consumer would. It catches the class
|
||||
of defects invisible to static review: serialization quirks, replace-vs-patch semantics,
|
||||
wrong status codes, broken idempotency, empty-state crashes, auth holes on new routes.
|
||||
|
||||
## The one question
|
||||
|
||||
**Does the changed consumer-facing surface behave as the spec promises when actually used,
|
||||
starting from nothing?** You are not judging code quality, plan conformance, or exploitability —
|
||||
other gates own those. You judge observable behavior.
|
||||
|
||||
## The independence rule (spec-first, or the gate is worthless)
|
||||
|
||||
Derive expected behaviors from the **spec** — the plan/task acceptance criteria, the API contract
|
||||
(IDL/schema/OpenAPI/proto definitions), the documented CLI help — **BEFORE reading the
|
||||
implementation**. If you read the handler first and write tests that mirror it, you have re-proven
|
||||
the implementation's own assumptions, including its misreadings of the spec. Order of operations:
|
||||
|
||||
1. Read the spec + contract. Write down the expected behaviors as concrete request→response pairs.
|
||||
2. Only THEN read implementation code — and only as much as needed to find ports, config, and
|
||||
startup wiring. Never to "check what it actually does" before your expectations are written.
|
||||
|
||||
## Phase order
|
||||
|
||||
### 1. Identify the surface under test
|
||||
|
||||
From the diff (or the caller's summary): which endpoints/RPCs/commands were added or changed?
|
||||
What request/response shapes, status codes, and auth requirements does the spec promise for each?
|
||||
If the change touches no consumer-facing surface, say so and return PASS with a one-line note —
|
||||
probing inert internals burns budget without value.
|
||||
|
||||
### 2. Regression-first: find and run existing usage suites
|
||||
|
||||
Discover what already exists before authoring anything:
|
||||
|
||||
- `fs_glob` for suite files in ANY format the repo uses: `**/*.hurl`, `**/*.http`, `**/*.rest`,
|
||||
`**/*.postman_collection.json`, `**/*.bru`, `**/e2e/**`, `**/integration/**`, `**/api-test*/**`,
|
||||
`**/smoke*/**`, plus repo scripts that run them (`**/run-*test*`, Makefile/justfile targets,
|
||||
package-manifest script entries) and shell scripts of `curl` commands (`fs_grep` for `curl `
|
||||
under `scripts/`, `test/`, `tools/`).
|
||||
- Read the repo's contributor docs for the sanctioned way to run them.
|
||||
|
||||
Run the existing suites against the changed code FIRST. Every failure here is a candidate
|
||||
regression. Classify each (see § Failure classification) — a failure is only acceptable when the
|
||||
spec EXPLICITLY changed that contract, and then the old test needs updating (note it in the
|
||||
report), not ignoring.
|
||||
|
||||
### 3. Map coverage, author the delta
|
||||
|
||||
List which of your expected behaviors from step 1 the existing suites already prove. Author new
|
||||
tests ONLY for the uncovered ones. Walk this usage-pattern checklist for each changed surface —
|
||||
these are the cases implementers systematically forget:
|
||||
|
||||
| Pattern | What to probe |
|
||||
|---------|---------------|
|
||||
| Cold start / empty state | First-ever call with no pre-existing data: list → empty (not 500), get → not-found (not panic) |
|
||||
| Happy path | The spec's primary flow, end to end, asserting the full response shape — not just the status code |
|
||||
| Idempotency / re-call | Same create/update twice: duplicate error or no-op, per the spec — never silent double-write |
|
||||
| Invalid input | Missing required fields, wrong types, out-of-range values, malformed body → the spec's error shape and code, not a 500 |
|
||||
| Auth on the new surface | Missing/expired/insufficient credentials → the correct 401/403 (a new route with no auth check is a common miss) |
|
||||
| Not-found and stale references | Operations on IDs that don't exist or were deleted |
|
||||
| Partial update semantics | Does omitting a field preserve it (patch) or delete it (replace)? Assert whichever the spec promises — this is a classic silent-data-loss bug |
|
||||
| Serialization edges | Zero values, empty lists, unset optionals: encoders that omit zero values make `== false`/`== null` asserts lie — assert existence/absence per the actual encoding |
|
||||
| Pagination / limits | Page past the end, limit 0/1/max, stable ordering if promised |
|
||||
| Error shape consistency | New errors follow the same envelope as the rest of the surface |
|
||||
| State transitions | Illegal transitions rejected; legal ones observable via subsequent reads |
|
||||
|
||||
Write the new tests where the repo's existing suites live, following their naming and layout
|
||||
conventions, so they are adoptable as permanent regression tests. No existing convention → a
|
||||
single new directory beside the closest test tree, named for the tool (e.g. `tests/usage/`).
|
||||
|
||||
### 4. Environment discipline (clean, local, isolated)
|
||||
|
||||
- **Clean state is non-negotiable.** Boot from nothing: fresh/ephemeral database (throwaway
|
||||
container, tmp file, or dedicated schema), run migrations, seed ONLY what the tests create
|
||||
themselves. Tests that depend on pre-existing data are not cold-start tests.
|
||||
- **Fully local.** Stub or mock external dependencies (fake servers, recorded fixtures, in-memory
|
||||
substitutes) — a probe that calls real third-party systems is a flake generator and a hazard.
|
||||
- **Prefer the repo's own recipe.** If the plan or contributor docs provide a local-run recipe
|
||||
(compose file, make target, dev script), use it verbatim before inventing your own. If you must
|
||||
invent one, record every step in the report so it can be promoted into the docs.
|
||||
- **Teardown.** Leave no running processes, containers, or dirty state behind.
|
||||
- **Bounded startup retries only.** Retry/poll while the stack boots (bounded attempts, short
|
||||
interval). NEVER add retries to make a flaky assertion pass — flakiness on a settled stack is a
|
||||
finding.
|
||||
|
||||
### 5. Failure classification (every failure gets exactly one)
|
||||
|
||||
| Class | Meaning | Effect on verdict |
|
||||
|-------|---------|-------------------|
|
||||
| **BUG** | The running system violates the spec | FAIL — report with repro |
|
||||
| **EXPECTED-CHANGE** | An existing test asserts a contract the spec explicitly changed | Does not fail the verdict; the stale test is flagged for update |
|
||||
| **ENV** | The failure is in bringing the stack up or reaching it, not in behavior | Does not count as a bug; if it prevents meaningful probing → INCONCLUSIVE |
|
||||
|
||||
Misclassifying ENV as BUG sends the implementer chasing ghosts; misclassifying BUG as ENV ships
|
||||
the defect. When unsure, reproduce twice and read the server logs before deciding.
|
||||
|
||||
## Toolbox (repo conventions first; these are examples, not requirements)
|
||||
|
||||
No specific tool is required. Precedence: (1) whatever format/harness the repo's existing usage
|
||||
suites already use — run and extend that; (2) a well-suited tool from the examples below if it is
|
||||
available or trivially installable; (3) ubiquitous fallbacks (`curl` + shell assertions cover any
|
||||
HTTP surface). What is non-negotiable is the discipline — spec-first asserts, clean state — not
|
||||
the tool.
|
||||
|
||||
| Surface | Example tools | Notes |
|
||||
|---------|--------------|-------|
|
||||
| HTTP/REST/JSON (incl. gRPC-over-HTTP with JSON encoding) | [Hurl](https://hurl.dev) `.hurl` files; `curl` scripts | Hurl: plain-text request/assert format, capturable variables, `retry` for eventual consistency; files double as committed regression suites |
|
||||
| Pure gRPC/protobuf | `grpcurl` (scripted) | Use server reflection or point at the proto files |
|
||||
| CLI | Direct invocation via `execute_command` | Assert exit codes AND output; probe stdin/args edge cases |
|
||||
| Anything else | `curl`/scripts/the repo's own test harness | Same discipline: spec-first asserts, clean state |
|
||||
|
||||
Optional niceties like Hurl and grpcurl may already be preinstalled (e.g. by a sandbox mixin) or
|
||||
can be installed idempotently (hurl via the distro package manager first — its prebuilt GitHub
|
||||
tarball dynamically links `libxml2.so.2`, which newer distros no longer ship; grpcurl from its
|
||||
GitHub release, a static Go binary). When a preferred tool is unavailable and uninstallable, fall
|
||||
back to what exists rather than skipping the check; classify a probe as ENV only when NO adequate
|
||||
tool can exercise the surface.
|
||||
|
||||
If you use Hurl, gotchas that produce false results if unknown:
|
||||
|
||||
- `[Captures]` run BEFORE `[Asserts]` in the same entry — capture a replaced value under a NEW
|
||||
variable name, or your inequality asserts compare a value to itself.
|
||||
- JSON encoders that omit zero/empty values: assert `not exists` for absent fields — `== false`
|
||||
or `== null` asserts fail on omitted keys.
|
||||
- Use `[Options] retry` with a bounded count for asynchronous effects (job completion, eventual
|
||||
reads); never unbounded.
|
||||
|
||||
## Verdict format
|
||||
|
||||
End with EXACTLY one of:
|
||||
|
||||
```
|
||||
USAGE_PROBE: PASS
|
||||
Surface: <endpoints/RPCs/commands probed>. Existing suites: <N run, all green | none found>.
|
||||
New tests: <M authored at <path>, all green>.
|
||||
<optional: 1-3 non-blocking observations (stale tests to update, recipe gaps)>
|
||||
```
|
||||
|
||||
```
|
||||
USAGE_PROBE: FAIL
|
||||
Surface: <...>. Existing suites: <N run, X failed (Y regressions, Z expected-change)>. New tests: <M authored, W failed>.
|
||||
Behavioral findings:
|
||||
1. <surface + case> — <spec'd behavior, quoting the spec> — <observed behavior> — REPRO: <exact request/command + response received> — <test file:entry>
|
||||
Stale tests needing update (expected-change): <list or none>
|
||||
```
|
||||
|
||||
```
|
||||
USAGE_PROBE: INCONCLUSIVE
|
||||
Could not establish a clean local environment: <what failed, verbatim error>.
|
||||
Missing: <the exact recipe/fixture/mock that would unblock — phrased so the plan author can add it>.
|
||||
Partial results (if any): <what did run and what it showed>
|
||||
```
|
||||
|
||||
Every FAIL finding MUST include the exact reproduction (request/command and the response
|
||||
received) and cite the test file — a behavioral complaint without a repro is noise. INCONCLUSIVE
|
||||
is an honest, acceptable verdict: it routes the fix to the environment recipe, not the code.
|
||||
NEVER report INCONCLUSIVE as PASS ("couldn't test, probably fine") or as FAIL (the implementer
|
||||
would hunt a nonexistent bug).
|
||||
|
||||
## Anti-patterns
|
||||
|
||||
- Writing tests after reading the implementation — you will encode its bugs as expectations.
|
||||
- Skipping the existing suites and jumping to new tests — regressions are the cheapest bugs to catch.
|
||||
- Testing through internal seams (direct DB reads, internal function calls) — this gate is
|
||||
consumer-perspective only; internals belong to unit tests.
|
||||
- Depending on pre-existing data, shared databases, or previously running services.
|
||||
- Adding retries/sleeps until a flaky assertion passes — flakiness is a finding, not an obstacle.
|
||||
- Reporting an environment failure as a behavioral FAIL (or burying it in a PASS).
|
||||
- Throwaway tests in /tmp — tests that don't land in the repo's suite location die with the run.
|
||||
- Asserting only status codes — shape and content are where the quirks live.
|
||||
@@ -1,7 +1,7 @@
|
||||
# Documentation: https://docs.brew.sh/Formula-Cookbook
|
||||
# https://rubydoc.brew.sh/Formula
|
||||
class Coyote < Formula
|
||||
desc "All-in-one, batteries included LLM CLI tool"
|
||||
desc "The batteries-included runtime for LLMs"
|
||||
homepage "https://github.com/Dark-Alex-17/coyote"
|
||||
if OS.mac? and Hardware::CPU.arm?
|
||||
url "https://github.com/Dark-Alex-17/coyote/releases/download/v$version/coyote-aarch64-apple-darwin.tar.gz"
|
||||
|
||||
Reference in New Issue
Block a user