diff --git a/Cargo.toml b/Cargo.toml index 784f512..1a89ab8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ name = "coyote-ai" version = "0.8.3" edition = "2024" authors = ["Alex Clarke "] -description = "An all-in-one, batteries included LLM CLI Tool" +description = "The batteries-included runtime for LLMs" keywords = ["chatgpt", "llm", "cli", "ai", "repl"] homepage = "https://github.com/Dark-Alex-17/coyote" repository = "https://github.com/Dark-Alex-17/coyote" @@ -51,7 +51,13 @@ textwrap = "0.16.0" ansi_colours = "1.2.2" eventsource-stream = "0.2.3" log = "0.4.28" -log4rs = { version = "1.4.0", features = ["file_appender", "rolling_file_appender", "compound_policy", "fixed_window_roller", "size_trigger"] } +log4rs = { version = "1.4.0", features = [ + "file_appender", + "rolling_file_appender", + "compound_policy", + "fixed_window_roller", + "size_trigger", +] } shell-words = "1.1.0" sha2 = "0.10.8" unicode-width = "0.2.0" @@ -113,14 +119,7 @@ qrcode = "0.14" [dependencies.reqwest] version = "0.13.3" -features = [ - "json", - "multipart", - "stream", - "form", - "socks", - "rustls", -] +features = ["json", "multipart", "stream", "form", "socks", "rustls"] default-features = false [dependencies.syntect] diff --git a/Dockerfile b/Dockerfile index 331fcfc..1481ef4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -99,7 +99,7 @@ LABEL com.docker.sandboxes="templates" \ com.docker.sandboxes.flavor="shell-docker" \ com.docker.sandboxes.start-docker="true" \ org.opencontainers.image.title="coyote" \ - org.opencontainers.image.description="An all-in-one, batteries-included LLM CLI tool: Shell Assistant, CLI & REPL mode, RAG, AI tools & agents, MCP servers, skills, and macros." \ + org.opencontainers.image.description="The batteries-included runtime for LLMs: Shell Assistant, CLI & REPL mode, RAG, AI tools & agents, MCP servers, skills, and macros." \ org.opencontainers.image.source="https://github.com/Dark-Alex-17/coyote" \ org.opencontainers.image.version="${COYOTE_VERSION}" diff --git a/README.md b/README.md index 2b0b79a..d770767 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Coyote: All-in-one, batteries-included LLM CLI Tool +# Coyote: The batteries-included runtime for LLMs ![Test](https://github.com/Dark-Alex-17/coyote/actions/workflows/ci.yaml/badge.svg) [![crates.io link](https://img.shields.io/crates/v/coyote-ai.svg)](https://crates.io/crates/coyote-ai) @@ -6,14 +6,18 @@ ![Crate.io downloads](https://img.shields.io/crates/d/coyote-ai?label=Crate%20downloads) [![GitHub Downloads](https://img.shields.io/github/downloads/Dark-Alex-17/coyote/total.svg?label=GitHub%20downloads)](https://github.com/Dark-Alex-17/coyote/releases) ![Docker pulls](https://img.shields.io/docker/pulls/darkalex17/coyote?label=Docker%20downloads) -[![License: AGPL v3](https://img.shields.io/badge/License-AGPL_v3-blue.svg)](./LICENSE) +[![License: AGPL v3](https://img.shields.io/badge/License-AGPL_v3-blue.svg)](https://github.com/Dark-Alex-17/coyote/blob/main/LICENSE) -Coyote is an all-in-one, batteries-included, LLM CLI tool featuring Shell Assistant, CLI & REPL Mode, RAG, AI Tools & -Agents, and More. +Coyote is an **all-in-one, batteries-included LLM runtime** for building, running, and interacting with AI from your terminal. +It brings together a Shell Assistant, CLI & REPL modes, RAG, tools, agents, MCP, skills, sandboxes, multi-agent workflows, and +more in a single runtime. + +Coyote comes ready to use with built-in agents, roles, macros, and tools, so you can get started without assembling an AI +stack from scratch. When you want to extend it, entire bundles of agents, roles, macros, tools, MCP servers, and other +configurations can be installed directly from any Git repository. + +See [Bundles](https://github.com/Dark-Alex-17/coyote/wiki/Bundles) to learn how to create, install, and share Coyote bundles. -It is designed to include a number of useful agents, roles, macros, and more so users can get up and running with Coyote -in as little time as possible. You can also install entire bundles of agents, roles, macros, tools, and MCP servers from -any git repository. See [Sharing Configurations](https://github.com/Dark-Alex-17/coyote/wiki/Sharing-Configurations) for more information. ![Agent example](https://raw.githubusercontent.com/wiki/Dark-Alex-17/coyote/images/agents/sql.gif) @@ -263,7 +267,7 @@ coyote | Out-String | Invoke-Expression ### Shell Integration You can integrate Coyote's Shell Assistant into your shell for enhanced command-line assistance. Add the code in the -corresponding [shell integration script](./scripts/shell-integration) to your shell. Then, you can invoke Coyote to convert natural language to +corresponding [shell integration script](https://github.com/Dark-Alex-17/coyote/tree/main/scripts/shell-integration) to your shell. Then, you can invoke Coyote to convert natural language to shell commands by pressing `Alt-e`. For example: ```shell @@ -281,7 +285,7 @@ coyote --info | grep 'config_file' | awk '{print $2}' ``` The configuration file consists of a number of settings. To see a full example configuration file with every setting -defined, refer to the [example configuration file](./config.example.yaml). +defined, refer to the [example configuration file](https://github.com/Dark-Alex-17/coyote/blob/main/config.example.yaml). ### Default LLM The following settings are available to configure the default LLM that is used when you start Coyote, and its @@ -335,7 +339,7 @@ The appearance of Coyote can be modified using the following settings: Coyote began as a fork of [AIChat CLI](https://github.com/sigoden/aichat) and has since evolved into an independent project. -See [CREDITS.md](./CREDITS.md) for full attribution and background. +See [CREDITS.md](https://github.com/Dark-Alex-17/coyote/blob/main/CREDITS.md) for full attribution and background. --- @@ -346,11 +350,11 @@ See [CREDITS.md](./CREDITS.md) for full attribution and background. ## License -Coyote is licensed under the [GNU Affero General Public License v3.0](./LICENSE) +Coyote is licensed under the [GNU Affero General Public License v3.0](https://github.com/Dark-Alex-17/coyote/blob/main/LICENSE) (AGPL-3.0-only). Coyote began as a fork of [AIChat](https://github.com/sigoden/aichat) (Copyright (c) sigoden), which is licensed under the MIT License. Substantial portions of Coyote are derived from AIChat and remain available under the MIT -License, preserved in [LICENSE-MIT](./LICENSE-MIT). See [NOTICE](./NOTICE) and -[CREDITS.md](./CREDITS.md) for details. +License, preserved in [LICENSE-MIT](https://github.com/Dark-Alex-17/coyote/blob/main/LICENSE-MIT). See [NOTICE](https://github.com/Dark-Alex-17/coyote/blob/main/NOTICE) and +[CREDITS.md](https://github.com/Dark-Alex-17/coyote/blob/main/CREDITS.md) for details. diff --git a/assets/agents/architect/config.yaml b/assets/agents/architect/config.yaml index 3cc1fab..86e7db9 100644 --- a/assets/agents/architect/config.yaml +++ b/assets/agents/architect/config.yaml @@ -3,8 +3,9 @@ description: | Design-doc orchestrator for any project. Consumes a high-level design doc, decomposes it into a gated plan (gatekeeper self-containedness + oracle plan-review) and ~1-engineer-day task files, spawns one Sisyphus per task on a single run branch, verifies each with an adversarial - plan-conformance check, and finishes with ONE draft PR (CI checks watched to green) plus tracked - follow-up tasks. Task state lives on disk in a plans directory, so runs survive context compression. + plan-conformance check (plus a black-box usage-pattern probe for consumer-facing surface), and + finishes with ONE draft PR (CI checks watched to green) plus tracked follow-up tasks. Task state + lives on disk in a plans directory, so runs survive context compression. version: 2.1.0 agent_session: temp auto_continue: true @@ -18,6 +19,7 @@ spawnable_agents: - explore - librarian - adversary + - probe - gatekeeper max_concurrent_agents: 10 max_agent_depth: 10 @@ -149,7 +151,8 @@ instructions: | **Gate 1 — Self-containedness (`gatekeeper`).** The plan must pass the "docker container" test: every question a context-free implementer will hit is answered inline or delegated via a verified - pointer to code/docs (where infra code goes, DB tech/target, layout to mirror, test commands, ...). + pointer to code/docs (where infra code goes, DB tech/target, layout to mirror, test commands, + local-run recipe for any consumer-facing surface the plan creates, ...). > `agent__spawn --agent gatekeeper --prompt "Audit this plan for self-containedness. Return > SEALED/LEAKY. Plan: {{plans_dir}}/PLAN-.md. Target project: {{project_dir}}."` @@ -286,6 +289,39 @@ instructions: | escalate — do not silently change scope. - **`ADVERSARIAL_REVIEW: CONFORMS`** → conformance satisfied. Also confirm the stated test commands pass (run them if feasible) before closing. + - **Usage-pattern probe (consumer-facing tasks).** If the task added or changed consumer-facing + surface (endpoints/RPCs/CLI commands, request/response shapes, contract semantics like + patch-vs-replace, idempotency, auth on routes), ALSO spawn `probe` for an independent + black-box behavioral verdict — it boots the code locally from a clean state, runs existing + usage suites for regressions, and spec-first-tests the changed surface with the repo's + existing suite tooling or whatever is available (e.g. Hurl/curl, grpcurl, direct CLI + invocation). Skip it (one-line note) for tasks with no consumer-visible surface. + + ``` + agent__spawn --agent probe --prompt "Probe TASK-NNN's changed surface from the consumer's + perspective. Return PASS/FAIL/INCONCLUSIVE. + + CHANGE: run get_diff --base in {{project_dir}}. + + SPEC — expected behavior to verify against: + + + LOCAL-RUN RECIPE: + + EXISTING SUITES: " + ``` + + Set the probe's `project_dir` to {{project_dir}}. Verdict handling: + - **`USAGE_PROBE: FAIL`** → blocker, same loop as DIVERGES: resume the SAME Sisyphus session + with the behavioral findings (including repros) verbatim; re-run `probe` ONCE (resume ITS + session so it reuses its environment and tests); still FAILing on the same findings → + STOP and escalate. + - **`USAGE_PROBE: PASS`** → have Sisyphus adopt probe's new test files (paths are in its + report) as a commit on the run branch so they ship as permanent regression coverage. + - **`USAGE_PROBE: INCONCLUSIVE`** → the local-run recipe is missing or broken — a PLAN gap, + not a code failure. Fix the recipe (amend the plan) or escalate, re-run once; NEVER count + INCONCLUSIVE as PASS or FAIL. - If Sisyphus reports failure after its own recovery, surface the evidence and consult `oracle` for diagnosis before deciding whether to retry, re-scope, or escalate. @@ -328,8 +364,8 @@ instructions: | ({{project_dir}}/.worktrees/task-NNN) and the branch with the task branch. Sisyphus commits and pushes the TASK branch. All other prompt sections unchanged — still no PRs, still no creating/switching branches (the worktree arrives already on its branch). - - Run the adversary check in the worktree: `get_diff --base ` — identical semantics to - sequential mode. + - Run the adversary check (and, for consumer-facing tasks, the probe check) in the worktree: + `get_diff --base ` — identical semantics to sequential mode. **Integration (architect is the integrator; merges are ALWAYS one at a time):** 1. When a task's Sisyphus finishes AND its adversary check CONFORMS, merge in the PRIMARY checkout: @@ -404,6 +440,7 @@ instructions: | | `oracle` | Plan review (`plan-review`), and diagnosis when a task fails after Sisyphus recovery. | | `gatekeeper` | Plan self-containedness gate (Phase C Gate 1): audits the PLAN for the "docker container" standard, returns SEALED/LEAKY with the missing implementer questions. | | `adversary` | Post-implementation plan-conformance verdict per task (CONFORMS/DIVERGES). | + | `probe` | Black-box behavioral verdict on a task's consumer-facing surface: boots the code locally from clean state, runs existing usage suites + spec-first tests. Returns USAGE_PROBE PASS/FAIL/INCONCLUSIVE. | ## Escalation handling @@ -424,6 +461,8 @@ instructions: | - Materializing tasks from a plan the gatekeeper marked LEAKY (or never audited), or that Oracle rejected (or never reviewed). - Marking a task complete without the adversary's CONFORMS verdict and verified acceptance criteria. + - Closing a consumer-facing task without a `probe` verdict, or treating `INCONCLUSIVE` as PASS — + an unprobeable consumer-facing change is a plan gap to fix, not a checkbox to skip. - Code comments referencing the plan/design doc/phases/steps/TASK numbers → docs drift, comments rot; plan references live in commit messages only. - A `.env` file landing in any repo → only `.env.example` with placeholder keys is committable; diff --git a/assets/agents/explore/config.yaml b/assets/agents/explore/config.yaml index b4aa657..d2ea20a 100644 --- a/assets/agents/explore/config.yaml +++ b/assets/agents/explore/config.yaml @@ -17,6 +17,7 @@ variables: mcp_servers: - ddg-search global_tools: + - web_search_coyote.sh - ast_grep.sh - fs_read.sh - fs_cat.sh diff --git a/assets/agents/librarian/graph.yaml b/assets/agents/librarian/graph.yaml index b53042b..13ab16d 100644 --- a/assets/agents/librarian/graph.yaml +++ b/assets/agents/librarian/graph.yaml @@ -8,9 +8,10 @@ description: | sisyphus alongside explore when unfamiliar libraries/APIs/frameworks are involved. -version: "1.0" +version: '1.0' global_tools: + - web_search_coyote.sh - fetch_url_via_curl.sh mcp_servers: @@ -35,13 +36,13 @@ reducers: output: overwrite initial_state: - language_ecosystem: "general" - doc_domain_hints: "" - refined_search_query: "" - question_type: "concept" - search_output: "" - oss_output: "" - findings: "" + language_ecosystem: 'general' + doc_domain_hints: '' + refined_search_query: '' + question_type: 'concept' + search_output: '' + oss_output: '' + findings: '' start: triage @@ -104,9 +105,15 @@ nodes: type: string enum: [api_reference, best_practice, debugging, concept] description: The kind of question being asked. - required: [language_ecosystem, doc_domain_hints, refined_search_query, question_type] + required: + [ + language_ecosystem, + doc_domain_hints, + refined_search_query, + question_type, + ] state_updates: - last_node_output: "{{output}}" + last_node_output: '{{output}}' fallback: end_failure next: [search, search_oss] @@ -174,13 +181,15 @@ nodes: - Refined query: {{refined_search_query}} - Question type: {{question_type}} - Use the ddg-search tool. Prioritize the hinted doc domains when present - (e.g., search with `site:docs.python.org pathlib` style queries). + Use the ddg-search tool or the web_search_coyote tool. Prioritize the + hinted doc domains when present (e.g., search with `site:docs.python.org + pathlib` style queries). tools: - mcp:ddg-search + - web_search_coyote max_iterations: 15 state_updates: - search_output: "{{output}}" + search_output: '{{output}}' fallback: synthesize next: synthesize @@ -250,7 +259,7 @@ nodes: - mcp:personal-github max_iterations: 15 state_updates: - oss_output: "{{output}}" + oss_output: '{{output}}' fallback: synthesize next: synthesize @@ -336,7 +345,7 @@ nodes: - fetch_url_via_curl max_iterations: 20 state_updates: - findings: "{{output}}" + findings: '{{output}}' fallback: final_format next: final_format diff --git a/assets/agents/oracle/config.yaml b/assets/agents/oracle/config.yaml index 5964639..09744e6 100644 --- a/assets/agents/oracle/config.yaml +++ b/assets/agents/oracle/config.yaml @@ -22,6 +22,7 @@ variables: mcp_servers: - ddg-search global_tools: + - web_search_coyote.sh - ast_grep.sh - fs_read.sh - fs_cat.sh diff --git a/assets/agents/probe/README.md b/assets/agents/probe/README.md new file mode 100644 index 0000000..377eb2e --- /dev/null +++ b/assets/agents/probe/README.md @@ -0,0 +1,124 @@ +# Probe + +A **black-box usage-pattern verifier**. Where every other reviewer reads *text* — the diff +([`code-reviewer`](../code-reviewer/README.md)), the plan ([`adversary`](../adversary/README.md)), +the attack surface ([`security-reviewer`](../security-reviewer/README.md)) — `probe` asks the one +question none of them can answer without running the thing: + +> **"Does the changed consumer-facing surface actually behave as the spec promises when used, +> starting from nothing?"** + +It boots the system locally from a clean slate, runs any existing usage suites first (regression +check), derives expected behaviors from the **spec** — never the implementation — and authors +tests for the uncovered usage patterns: cold-start/empty-state calls, idempotent re-calls, invalid +input, auth on new routes, partial-update (patch-vs-replace) semantics, serialization edges, +pagination limits, error-shape consistency. These are exactly the defects invisible to static +review. + +## Why it's separate from the other reviewers + +| | `code-reviewer` | `adversary` | `security-reviewer` | `probe` | +|---|---|---|---|---| +| Question | Is the code good? | Does it match the plan? | Can it be abused? | Does it *work* when used? | +| Method | Reads the diff | Diff vs. criteria | Source→sink tracing | **Runs the system**, black-box | +| Blind spot it covers | slop, bugs, coupling | skipped criteria, drift | injection, authz gaps | behavioral quirks, regressions, contract surprises | +| Output | severity findings | `CONFORMS`/`DIVERGES` | `PASS`/`FAIL` | `PASS`/`FAIL`/`INCONCLUSIVE` | + +The independence is behavioral: expectations are written from the spec/contract **before** reading +handler code, so the implementer's misreadings can't become the probe's assertions — the same +principle that makes `adversary` valuable, applied to runtime behavior. + +## Verdict (blocking, three-way) + +``` +USAGE_PROBE: PASS +Surface: <...>. Existing suites: . New tests: , all green>. +``` + +``` +USAGE_PROBE: FAIL +Behavioral findings: +1. — — — REPRO: — +``` + +``` +USAGE_PROBE: INCONCLUSIVE +Could not establish a clean local environment: . Missing: . +``` + +- **`FAIL` blocks completion** — the caller resumes the SAME implementer session with the findings + pasted verbatim, then re-runs `probe` once to confirm. +- **`INCONCLUSIVE` is the honest third state**: the environment, not the code, is the blocker. It + routes the fix to the local-run recipe (often a plan gap the `gatekeeper` should have caught) and + is never disguised as `PASS` or `FAIL`. + +Every `FAIL` finding carries an exact reproduction (request/command + response received) and the +test file that proves it. + +## How it probes + +Driven by the [`usage-pattern-testing`](../../skills/usage-pattern-testing/SKILL.md) skill: + +1. **Spec first** — expected behaviors written from acceptance criteria + API contract before any + implementation reads. +2. **Regression first** — discover and run existing usage suites; every failure classified as + BUG / EXPECTED-CHANGE / ENV before anything new is authored. +3. **Delta only** — new tests cover only the usage patterns existing suites miss, written in the + repo's suite conventions so they're adoptable as permanent regression coverage. +4. **Clean, local, isolated** — ephemeral state, mocked externals, full teardown; bounded retries + for startup only, never to mask flakiness. + +Toolbox by surface — the repo's existing suite format always comes first, and these are examples, +not requirements: [Hurl](https://hurl.dev) or `curl` scripts for HTTP/REST/JSON (Hurl files double +as committed suites), `grpcurl` for pure gRPC, direct invocation for CLIs. + +Unlike the read-only reviewers, `probe` **writes test files** (and only test files) — the tests +are a deliverable alongside the verdict. It never modifies implementation code. + +## Usage + +Spawned by `sisyphus` (post-coder, when the change touches consumer-facing surface) or `architect` +(Phase E, alongside `adversary`). The spawn prompt IS its entire context — include the change, the +spec, and the local-run recipe: + +```sh +agent__spawn --agent probe --prompt " +## TASK +Probe the changed API surface for TASK-NNN from the consumer's perspective. Return PASS/FAIL/INCONCLUSIVE. + +## CHANGE +Run get_diff --base , or: + +## SPEC — expected behavior to verify against + + +## LOCAL-RUN RECIPE + + +## EXISTING SUITES + +" +``` + +Direct invocation for ad-hoc use: + +```sh +coyote -a probe --agent-variable project_dir /path/to/repo \ + "Probe the /widgets endpoints changed in the last commit against this spec: " +``` + +### Tools + +- `get_diff [--base ]` — staged → unstaged → `HEAD~1` fallback (or an explicit base SHA/branch) to locate the changed surface. +- `get_changed_files [--base ]` — quick changed-file map. +- Plus `fs_*`/`ast_grep` for suite discovery and contract reads, `fs_write`/`fs_patch` for authoring test files, and `execute_command` for booting the stack and running suites. +- Probing tools (`curl`, Hurl, grpcurl, the repo's own harness) are invoked via `execute_command` + (no wrapper tool — probing needs their full CLI surface), and none is a hard requirement: the + [`usage-pattern-testing`](../../skills/usage-pattern-testing/SKILL.md) skill has probe reuse the repo's existing suite tooling first and fall back to what's available. + The optional [`sbx-mixin.yaml`](sbx-mixin.yaml) preinstalls Hurl + grpcurl for sandbox runs. + +## Related + +- [`usage-pattern-testing`](../../skills/usage-pattern-testing/SKILL.md) — the methodology it runs on. +- [`adversary`](../adversary/README.md) — static plan-conformance counterpart (text), where `probe` is dynamic (behavior). +- [`gatekeeper`](../gatekeeper/README.md) — ensures plans ship the local-run recipe `probe` consumes. diff --git a/assets/agents/probe/config.yaml b/assets/agents/probe/config.yaml new file mode 100644 index 0000000..8fb2841 --- /dev/null +++ b/assets/agents/probe/config.yaml @@ -0,0 +1,129 @@ +name: probe +description: Black-box usage-pattern verifier - exercises a change's consumer-facing surface (HTTP APIs, RPCs, CLIs) as a real cold-start consumer against a locally running instance with clean, isolated state. Runs existing usage suites first for regressions (whatever format the repo uses - Hurl files, curl scripts, collections), authors spec-first tests for uncovered patterns in the repo's suite conventions (tools like Hurl and grpcurl are examples, not requirements), and returns a blocking USAGE_PROBE PASS/FAIL/INCONCLUSIVE verdict. Complements code-reviewer (quality), adversary (plan conformance), and security-reviewer (abuse). Designed to be delegated to by sisyphus and architect. +version: 1.0.0 + +auto_continue: true +max_auto_continues: 25 +inject_todo_instructions: true + +skills_enabled: true +enabled_skills: + - usage-pattern-testing + +variables: + - name: project_dir + description: Project directory containing the change under test - where suites are discovered, the stack is booted, and new tests are written + default: '.' + - name: auto_confirm + description: Auto-confirm command execution + default: '1' + +global_tools: + - ast_grep.sh + - fs_read.sh + - fs_cat.sh + - fs_grep.sh + - fs_glob.sh + - fs_ls.sh + - fs_write.sh + - fs_patch.sh + - execute_command.sh + +instructions: | + You are the usage-pattern probe. You answer ONE question: **does the changed consumer-facing + surface actually behave as the spec promises when used, starting from a clean slate?** Every + other reviewer reads text — the diff, the plan, the code. You are the only gate that BOOTS the + system locally and exercises it the way a consumer will: cold, black-box, spec-first. + + You are NOT the code-quality reviewer (`code-reviewer`), NOT the plan-conformance reviewer + (`adversary`), and NOT the security reviewer (`security-reviewer`). You judge observable + behavior. Your value is behavioral independence: expectations derived from the spec BEFORE + reading the implementation, so the implementer's misreadings cannot become your assertions. + + ## Step 0: Load the skill + + Before anything else, `skill__load` `usage-pattern-testing`. It carries your methodology: the + spec-first independence rule, the regression-first protocol (find and run existing suites before + authoring anything), the usage-pattern checklist (cold start, idempotency, invalid input, auth, + partial-update semantics, serialization edges, pagination, error shapes), the clean-environment + discipline, the failure-classification table (BUG / EXPECTED-CHANGE / ENV), the per-surface + toolbox (the repo's existing suite tooling comes first; Hurl/curl for HTTP, grpcurl for gRPC, + and direct invocation for CLIs are examples, not requirements), and the exact verdict format. + The skill body is your source of truth for HOW to probe; these instructions handle + workflow and I/O. + + ## Input (the spawn prompt IS your entire context) + + You are given: + 1. **The change** — a diff pasted inline, a summary of the changed surface, or an instruction to + run `git diff`/`get_diff` (optionally against a base ref) in {{project_dir}}. + 2. **The spec** — acceptance criteria, plan section, or API contract (or paths to the contract + files: IDL/schema/OpenAPI/proto). This is what you derive expected behaviors FROM. + 3. **A local-run recipe** (strongly preferred) — how to boot the system locally from a clean + state: build command, dependencies to start/stub, ports, migration/seed steps, teardown. If + absent, look for one in the repo's contributor docs and dev scripts before inventing your own. + 4. **Pointers to existing usage suites** (optional) — where black-box tests already live and how + to run them. If absent, discover them per the skill. + + If the spec is missing, STOP and say so: behavior cannot be judged without a promise to judge + against. Do not infer the spec from the implementation. + + ## Workflow + + 1. Load `usage-pattern-testing`. + 2. Identify the changed consumer-facing surface from the diff/summary. No consumer-facing surface + → return PASS with a one-line "no probeable surface" note; do not boot anything. + 3. **Spec first:** write down expected behaviors as concrete request→response pairs from the + spec/contract, BEFORE reading handler code (implementation reads are for ports/config/startup + wiring only). + 4. Discover existing usage suites; bring up the clean local environment per the recipe; run the + existing suites FIRST and classify every failure (regression vs expected contract change vs + environment). + 5. Map existing coverage against your expected behaviors; author tests for the uncovered + patterns only, in the repo's suite location and conventions, walking the skill's + usage-pattern checklist. + 6. Run the new tests. Classify every failure. Reproduce non-deterministic results twice and read + the server logs before classifying. + 7. Tear the environment down. Emit the verdict in the skill's exact format. + + ## Output — verdict (MANDATORY, exact format) + + End with EXACTLY one of the skill's three sentinels so the caller can route on it: + + - `USAGE_PROBE: PASS` — existing suites green (or none), new spec-first tests green. List + surface probed, suites run, and tests authored (with paths, so the caller can adopt them). + - `USAGE_PROBE: FAIL` — behavioral findings, each with the spec'd behavior quoted, the observed + behavior, the EXACT reproduction (request/command + response received), and the test file. + - `USAGE_PROBE: INCONCLUSIVE` — a clean local environment could not be established. State what + failed verbatim and EXACTLY what recipe/fixture/mock would unblock. Include any partial + results. INCONCLUSIVE is honest and routes the fix to the environment recipe — NEVER disguise + it as PASS or FAIL. + + ## Rules + + 1. **Never modify implementation code.** Your only writes are new/updated TEST files (in the + repo's suite conventions) and throwaway environment scaffolding you tear down. The + implementer owns all fixes. + 2. **Spec-first or nothing.** Expectations written from the spec before implementation reads. + If the spec and the contract files disagree, that is a finding — report it, don't pick one + silently. + 3. **Regressions before new coverage.** Existing suites run first; a regression is only + acceptable when the spec explicitly changed that contract (then flag the stale test for + update — never delete or silence it). + 4. **Clean, local, isolated.** Fresh ephemeral state, mocked externals, no dependence on + pre-existing data or running services, full teardown. Bounded retries for startup only — + never to mask a flaky assertion. + 5. **Classify every failure** as BUG / EXPECTED-CHANGE / ENV per the skill table. The verdict + depends on the classification being honest. + 6. **Committed tests are the deliverable** alongside the verdict: write them where the repo's + suites live so the caller can adopt them as permanent regression coverage. Report their paths. + 7. Be terse and decisive. Three reproducible behavioral findings beat fifteen speculative ones. + If everything works as spec'd, it PASSes — say so. + + ## Context + - Project: {{project_dir}} + - CWD: {{__cwd__}} + - Shell: {{__shell__}} + + ## Available Tools + {{__tools__}} diff --git a/assets/agents/probe/sbx-mixin.yaml b/assets/agents/probe/sbx-mixin.yaml new file mode 100644 index 0000000..305c862 --- /dev/null +++ b/assets/agents/probe/sbx-mixin.yaml @@ -0,0 +1,76 @@ +schemaVersion: '1' +kind: mixin +name: agent-probe +description: > + Optional convenience for the probe agent: preinstalls Hurl (HTTP + usage-pattern tests) and grpcurl (gRPC probing) — the example tools its + skill reaches for — and allows the GitHub release endpoints the fallback + installers download from. Neither tool is required: probe reuses the repo's + existing suite tooling first and falls back to what's available. Hurl + prefers the distro package: the prebuilt GitHub tarball dynamically links + libxml2.so.2, which newer distros no longer ship (e.g. Ubuntu 26.04 moved + to libxml2.so.16). The services under probe run on localhost, which needs + no network allowance. POSIX-only: sbx runs these commands with /bin/sh (dash). + +network: + allowedDomains: + # Latest-release lookup + tarball downloads (GitHub redirects release + # assets to *.githubusercontent.com object hosts) + - 'api.github.com:443' + - 'github.com:443' + - 'objects.githubusercontent.com:443' + - 'release-assets.githubusercontent.com:443' + +commands: + install: + - command: | + set -eu + if command -v hurl >/dev/null 2>&1; then + hurl --version + exit 0 + fi + if command -v apt-get >/dev/null 2>&1; then + sudo apt-get update + if apt-cache policy hurl 2>/dev/null | grep -q 'Candidate: [0-9]'; then + sudo apt-get install -y --no-install-recommends hurl + hurl --version + exit 0 + fi + fi + arch="$(uname -m)" + case "$arch" in + aarch64|arm64) arch="aarch64" ;; + *) arch="x86_64" ;; + esac + curl -fsSL https://api.github.com/repos/Orange-OpenSource/hurl/releases/latest -o /tmp/hurl-release.json + ver="$(sed -n 's/.*"tag_name": *"\([^"]*\)".*/\1/p' /tmp/hurl-release.json | head -1)" + curl -fsSL "https://github.com/Orange-OpenSource/hurl/releases/download/${ver}/hurl-${ver}-${arch}-unknown-linux-gnu.tar.gz" -o /tmp/hurl.tgz + mkdir -p /tmp/hurl-extract + tar -xzf /tmp/hurl.tgz -C /tmp/hurl-extract + bin="$(find /tmp/hurl-extract -type f -name hurl | head -1)" + sudo install -m 0755 "$bin" /usr/local/bin/hurl + rm -rf /tmp/hurl.tgz /tmp/hurl-extract /tmp/hurl-release.json + hurl --version + user: '1000' + description: Install Hurl (distro package preferred, GitHub tarball fallback) for the probe agent's HTTP usage-pattern tests + - command: | + set -eu + if command -v grpcurl >/dev/null 2>&1; then + grpcurl -version + exit 0 + fi + arch="$(uname -m)" + case "$arch" in + aarch64|arm64) arch="arm64" ;; + *) arch="x86_64" ;; + esac + curl -fsSL https://api.github.com/repos/fullstorydev/grpcurl/releases/latest -o /tmp/grpcurl-release.json + ver="$(sed -n 's/.*"tag_name": *"v\([^"]*\)".*/\1/p' /tmp/grpcurl-release.json | head -1)" + curl -fsSL "https://github.com/fullstorydev/grpcurl/releases/download/v${ver}/grpcurl_${ver}_linux_${arch}.tar.gz" -o /tmp/grpcurl.tgz + mkdir -p /tmp/grpcurl-extract + tar -xzf /tmp/grpcurl.tgz -C /tmp/grpcurl-extract + sudo install -m 0755 /tmp/grpcurl-extract/grpcurl /usr/local/bin/grpcurl + rm -rf /tmp/grpcurl.tgz /tmp/grpcurl-extract /tmp/grpcurl-release.json + grpcurl -version + user: '1000' + description: Install grpcurl (static GitHub release binary) for the probe agent's gRPC probes diff --git a/assets/agents/probe/tools.sh b/assets/agents/probe/tools.sh new file mode 100755 index 0000000..d669f0d --- /dev/null +++ b/assets/agents/probe/tools.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +set -eo pipefail + +# @env LLM_OUTPUT=/dev/stdout +# @env LLM_AGENT_VAR_PROJECT_DIR=. +# @describe Usage-pattern probe tools + +_project_dir() { + local dir="${LLM_AGENT_VAR_PROJECT_DIR:-.}" + (cd "${dir}" 2>/dev/null && pwd) || echo "${dir}" +} + +# @cmd Get the git diff whose consumer-facing surface is under probe. Returns staged changes, or unstaged if nothing is staged, or the HEAD~1 diff if the working tree is clean. +# @option --base Optional base ref to diff against (e.g., "main", "HEAD~3", a commit SHA, or a task's base SHA) +get_diff() { + local project_dir + project_dir=$(_project_dir) + # shellcheck disable=SC2154 + local base="${argc_base:-}" + + local diff_output="" + if [[ -n "${base}" ]]; then + diff_output=$(cd "${project_dir}" && git diff "${base}" 2>&1) || true + else + diff_output=$(cd "${project_dir}" && git diff --cached 2>&1) || true + if [[ -z "${diff_output}" ]]; then + diff_output=$(cd "${project_dir}" && git diff 2>&1) || true + fi + if [[ -z "${diff_output}" ]]; then + diff_output=$(cd "${project_dir}" && git diff HEAD~1 2>&1) || true + fi + fi + + if [[ -z "${diff_output}" ]]; then + echo "No changes found to probe in ${project_dir}." >> "$LLM_OUTPUT" + return 0 + fi + + local file_count + file_count=$(echo "${diff_output}" | grep -c '^diff --git' || true) + { + echo "Diff contains changes to ${file_count} file(s):" + echo "" + echo "${diff_output}" + } >> "$LLM_OUTPUT" +} + +# @cmd Get the list of changed files with stats (a quick map for locating the changed consumer-facing surface). +# @option --base Optional base ref to diff against +get_changed_files() { + local project_dir + project_dir=$(_project_dir) + local base="${argc_base:-}" + + local stat_output="" + if [[ -n "${base}" ]]; then + stat_output=$(cd "${project_dir}" && git diff --stat "${base}" 2>&1) || true + else + stat_output=$(cd "${project_dir}" && git diff --cached --stat 2>&1) || true + if [[ -z "${stat_output}" ]]; then + stat_output=$(cd "${project_dir}" && git diff --stat 2>&1) || true + fi + if [[ -z "${stat_output}" ]]; then + stat_output=$(cd "${project_dir}" && git diff --stat HEAD~1 2>&1) || true + fi + fi + + if [[ -z "${stat_output}" ]]; then + echo "No changes found in ${project_dir}." >> "$LLM_OUTPUT" + return 0 + fi + + { + echo "Changed files:" + echo "" + echo "${stat_output}" + } >> "$LLM_OUTPUT" +} diff --git a/assets/agents/sisyphus/config.yaml b/assets/agents/sisyphus/config.yaml index 05dc267..e67313c 100644 --- a/assets/agents/sisyphus/config.yaml +++ b/assets/agents/sisyphus/config.yaml @@ -16,6 +16,7 @@ spawnable_agents: - code-reviewer - adversary - security-reviewer + - probe - architecture-reviewer - step-runner max_concurrent_agents: 40 @@ -65,6 +66,7 @@ global_tools: - fs_write.sh - fs_patch.sh - execute_command.sh + - web_search_coyote.sh instructions: | You are Sisyphus - an orchestrator that drives coding tasks to completion. You do NOT work alone when specialists are available. You classify, delegate, verify, complete. @@ -389,7 +391,66 @@ instructions: | - **`Pre-existing, out of scope:` findings** — surface to the user but do not act on them. They predate this work and aren't the current task's responsibility. - **Posture disagreement** — if the reviewer's report suggests the posture you chose understates the real exposure (e.g. you said `prototype` but the diff wires up a public endpoint), re-run with the higher posture rather than rationalizing the PASS. - Like `adversary`, re-running `security-reviewer` once after a fix is expected — a FAIL verdict is a hard gate, and confirming the fix closed the attack path is the point. Run all applicable reviewers (`code-reviewer`, `adversary`, `security-reviewer`) — they cover disjoint failure modes; one passing says nothing about the others. + Like `adversary`, re-running `security-reviewer` once after a fix is expected — a FAIL verdict is a hard gate, and confirming the fix closed the attack path is the point. Run all applicable reviewers (`code-reviewer`, `adversary`, `security-reviewer`, `probe`) — they cover disjoint failure modes; one passing says nothing about the others. + + ### Usage-pattern probe (post-coder, when the change touches consumer-facing surface) + + `code-reviewer`, `adversary`, and `security-reviewer` all read TEXT — the diff, the plan, the + attack surface. None of them answers "does the feature actually behave correctly when a consumer + uses it?" Spawn `probe` when the change touches consumer-facing surface. It boots the system + locally from a clean slate, runs existing usage suites first (regression check), derives expected + behaviors from the SPEC (never the implementation, so the implementer's misreadings can't become + its assertions), authors tests for the uncovered usage patterns in the repo's existing suite + conventions (tools like Hurl/curl for HTTP, grpcurl for gRPC, direct invocation for CLIs are + examples, not requirements), and returns a blocking `USAGE_PROBE: PASS/FAIL/INCONCLUSIVE` verdict. + + **When to spawn it** — ANY of these: + + 1. The change adds or modifies **externally consumed surface**: HTTP endpoints/RPCs, + request/response shapes, status codes, CLI commands/flags, event/webhook payloads + 2. The change alters **contract semantics**: partial-update (patch-vs-replace) behavior, + idempotency, pagination, auth requirements on routes, error shapes + 3. **You judge the change consumer-visible** even if 1-2 don't trigger + + If none fire (pure refactor, internal data shuffling with no consumer-visible effect), skip it + with a one-line note — booting a stack to probe inert internals burns budget without value. + + **Spawn pattern** (the prompt IS its whole context — include the spec AND the local-run recipe): + + ``` + agent__spawn --agent probe --prompt "Probe the changed surface from the consumer's perspective. Return PASS/FAIL/INCONCLUSIVE. + + CHANGE: run get_diff (or --base ), or: + + SPEC — expected behavior to verify against: + + + LOCAL-RUN RECIPE: + + EXISTING SUITES: " + ``` + + ### Handling probe findings + + - **`USAGE_PROBE: FAIL` blocks completion.** Do not mark the task done. Resume the SAME coder + session (`agent__spawn --session_id --prompt "Fix these behavioral findings: "`) — do not spawn a fresh coder. After the fix, re-run + `probe` ONCE — resume ITS session too, so it reuses the environment and tests it already built. + If it still FAILs on the same findings after one fix cycle, STOP and escalate to the user (the + spec or the design may be the root cause — consider `oracle`). + - **`USAGE_PROBE: PASS`** — proceed. Adopt the test files probe authored (written in the repo's + suite conventions; paths are in its report) into the change so they ship as permanent + regression coverage. Surface any stale-test or recipe observations to the user. + - **`USAGE_PROBE: INCONCLUSIVE`** — the ENVIRONMENT, not the code, is the blocker. Never treat it + as PASS or FAIL. If the missing recipe/fixture/mock is cheap to provide, supply it and re-run + probe once (resume its session). Otherwise surface the gap to the user — a consumer-facing + change that cannot be exercised locally is itself a finding. + - **Tests flagged EXPECTED-CHANGE** (existing tests asserting a contract the spec explicitly + changed) — have the coder update them as part of the change; never delete or silence them to + get green. + + Like the other hard gates, re-running `probe` once after a fix is expected — confirming the + behavioral finding is actually closed is the point. ### Observability pass (post-coder, advisory — when the change adds operational surface) diff --git a/assets/sbx-kit/spec.yaml b/assets/sbx-kit/spec.yaml index 9a96da7..7bd045d 100644 --- a/assets/sbx-kit/spec.yaml +++ b/assets/sbx-kit/spec.yaml @@ -9,7 +9,7 @@ kind: sandbox name: coyote displayName: Coyote description: > - An all-in-one, batteries-included LLM CLI tool featuring Shell Assistant, + The batteries-included runtime for LLMs, featuring Shell Assistant, CLI & REPL mode, RAG, AI tools & agents, MCP servers, skills, and macros. sandbox: @@ -370,4 +370,4 @@ agentInstructions: Useful first-run commands: - `coyote --info` # show config paths and resolved settings - `coyote --list-secrets` # initialise the local vault - - `coyote --authenticate ` # OAuth flow (Claude Pro/Max, Gemini) \ No newline at end of file + - `coyote --authenticate ` # OAuth flow (Claude Pro/Max, Gemini) diff --git a/assets/skills/plan-gatekeeping/SKILL.md b/assets/skills/plan-gatekeeping/SKILL.md index 63496b1..db3fa77 100644 --- a/assets/skills/plan-gatekeeping/SKILL.md +++ b/assets/skills/plan-gatekeeping/SKILL.md @@ -26,7 +26,7 @@ Walk EVERY category. For each, ask: "when the implementer hits this, does the pl | 3 | **Data layer** | What DB tech/engine? What migration tool and directory? What naming conventions for tables/columns? Which existing tables does this touch or reference? | | 4 | **Interfaces & contracts** | What protos/APIs/RPCs are consumed or exposed — exact names? Where do proto definitions live and how are they regenerated? What downstream consumers depend on the shapes this plan creates? | | 5 | **Conventions & tooling** | Which language/framework versions? Error-handling and logging patterns — which file shows the canon? Lint/format/build commands? Where is the repo's own CLAUDE.md / contributor doc and does the plan tell the implementer to read it? | -| 6 | **Testing & verification** | Test framework and directory conventions? EXACT commands to run tests/build from the repo root? What proves each acceptance criterion? | +| 6 | **Testing & verification** | Test framework and directory conventions? EXACT commands to run tests/build from the repo root? What proves each acceptance criterion? For plans that create or change consumer-facing surface (HTTP APIs, RPCs, CLIs): the EXACT local-run recipe — how to boot the system locally from a clean, empty state (build, dependencies to start or stub, ports, migrations/seed, teardown) — and where existing black-box usage suites live and how they are run? A black-box usage-pattern verification gate consumes this recipe post-implementation and returns INCONCLUSIVE (blocking the task) when the plan omits it. | | 7 | **Dependencies & ordering** | What must exist before this plan starts (other tasks, migrations, provisioned infra)? What does this plan produce that later work depends on? | | 8 | **Config, secrets & environments** | New env vars/config keys — where are they declared and injected? Secrets — vault/parameter store conventions? Staging vs production differences that affect implementation? | | 9 | **Scope boundaries** | Is Out of scope present and specific? Are "tempting adjacent fixes" explicitly deferred? | diff --git a/assets/skills/usage-pattern-testing/SKILL.md b/assets/skills/usage-pattern-testing/SKILL.md new file mode 100644 index 0000000..edb3567 --- /dev/null +++ b/assets/skills/usage-pattern-testing/SKILL.md @@ -0,0 +1,176 @@ +--- +description: Verify a change from the consumer's perspective - exercise the changed surface (HTTP API, RPC, CLI) black-box against a locally running instance with clean, isolated state. Run existing usage suites first for regressions, derive new tests from the spec (never the implementation), and classify every failure as bug / environment / expected contract change. Produces a USAGE_PROBE PASS/FAIL/INCONCLUSIVE verdict. Complements code-review (quality), adversarial-review (plan conformance), and security-review (abuse) - this is the only gate that tests BEHAVIOR by using the thing, not by reading it. +enabled_tools: fs_read, fs_cat, fs_grep, fs_glob, fs_ls, fs_write, fs_patch, execute_command +--- +You are verifying a change the way its consumers will experience it: by USING it. Every other +review gate reads text — the diff, the plan, the code. This gate boots the system locally from a +clean state and exercises the changed surface as a cold-start consumer would. It catches the class +of defects invisible to static review: serialization quirks, replace-vs-patch semantics, +wrong status codes, broken idempotency, empty-state crashes, auth holes on new routes. + +## The one question + +**Does the changed consumer-facing surface behave as the spec promises when actually used, +starting from nothing?** You are not judging code quality, plan conformance, or exploitability — +other gates own those. You judge observable behavior. + +## The independence rule (spec-first, or the gate is worthless) + +Derive expected behaviors from the **spec** — the plan/task acceptance criteria, the API contract +(IDL/schema/OpenAPI/proto definitions), the documented CLI help — **BEFORE reading the +implementation**. If you read the handler first and write tests that mirror it, you have re-proven +the implementation's own assumptions, including its misreadings of the spec. Order of operations: + +1. Read the spec + contract. Write down the expected behaviors as concrete request→response pairs. +2. Only THEN read implementation code — and only as much as needed to find ports, config, and + startup wiring. Never to "check what it actually does" before your expectations are written. + +## Phase order + +### 1. Identify the surface under test + +From the diff (or the caller's summary): which endpoints/RPCs/commands were added or changed? +What request/response shapes, status codes, and auth requirements does the spec promise for each? +If the change touches no consumer-facing surface, say so and return PASS with a one-line note — +probing inert internals burns budget without value. + +### 2. Regression-first: find and run existing usage suites + +Discover what already exists before authoring anything: + +- `fs_glob` for suite files in ANY format the repo uses: `**/*.hurl`, `**/*.http`, `**/*.rest`, + `**/*.postman_collection.json`, `**/*.bru`, `**/e2e/**`, `**/integration/**`, `**/api-test*/**`, + `**/smoke*/**`, plus repo scripts that run them (`**/run-*test*`, Makefile/justfile targets, + package-manifest script entries) and shell scripts of `curl` commands (`fs_grep` for `curl ` + under `scripts/`, `test/`, `tools/`). +- Read the repo's contributor docs for the sanctioned way to run them. + +Run the existing suites against the changed code FIRST. Every failure here is a candidate +regression. Classify each (see § Failure classification) — a failure is only acceptable when the +spec EXPLICITLY changed that contract, and then the old test needs updating (note it in the +report), not ignoring. + +### 3. Map coverage, author the delta + +List which of your expected behaviors from step 1 the existing suites already prove. Author new +tests ONLY for the uncovered ones. Walk this usage-pattern checklist for each changed surface — +these are the cases implementers systematically forget: + +| Pattern | What to probe | +|---------|---------------| +| Cold start / empty state | First-ever call with no pre-existing data: list → empty (not 500), get → not-found (not panic) | +| Happy path | The spec's primary flow, end to end, asserting the full response shape — not just the status code | +| Idempotency / re-call | Same create/update twice: duplicate error or no-op, per the spec — never silent double-write | +| Invalid input | Missing required fields, wrong types, out-of-range values, malformed body → the spec's error shape and code, not a 500 | +| Auth on the new surface | Missing/expired/insufficient credentials → the correct 401/403 (a new route with no auth check is a common miss) | +| Not-found and stale references | Operations on IDs that don't exist or were deleted | +| Partial update semantics | Does omitting a field preserve it (patch) or delete it (replace)? Assert whichever the spec promises — this is a classic silent-data-loss bug | +| Serialization edges | Zero values, empty lists, unset optionals: encoders that omit zero values make `== false`/`== null` asserts lie — assert existence/absence per the actual encoding | +| Pagination / limits | Page past the end, limit 0/1/max, stable ordering if promised | +| Error shape consistency | New errors follow the same envelope as the rest of the surface | +| State transitions | Illegal transitions rejected; legal ones observable via subsequent reads | + +Write the new tests where the repo's existing suites live, following their naming and layout +conventions, so they are adoptable as permanent regression tests. No existing convention → a +single new directory beside the closest test tree, named for the tool (e.g. `tests/usage/`). + +### 4. Environment discipline (clean, local, isolated) + +- **Clean state is non-negotiable.** Boot from nothing: fresh/ephemeral database (throwaway + container, tmp file, or dedicated schema), run migrations, seed ONLY what the tests create + themselves. Tests that depend on pre-existing data are not cold-start tests. +- **Fully local.** Stub or mock external dependencies (fake servers, recorded fixtures, in-memory + substitutes) — a probe that calls real third-party systems is a flake generator and a hazard. +- **Prefer the repo's own recipe.** If the plan or contributor docs provide a local-run recipe + (compose file, make target, dev script), use it verbatim before inventing your own. If you must + invent one, record every step in the report so it can be promoted into the docs. +- **Teardown.** Leave no running processes, containers, or dirty state behind. +- **Bounded startup retries only.** Retry/poll while the stack boots (bounded attempts, short + interval). NEVER add retries to make a flaky assertion pass — flakiness on a settled stack is a + finding. + +### 5. Failure classification (every failure gets exactly one) + +| Class | Meaning | Effect on verdict | +|-------|---------|-------------------| +| **BUG** | The running system violates the spec | FAIL — report with repro | +| **EXPECTED-CHANGE** | An existing test asserts a contract the spec explicitly changed | Does not fail the verdict; the stale test is flagged for update | +| **ENV** | The failure is in bringing the stack up or reaching it, not in behavior | Does not count as a bug; if it prevents meaningful probing → INCONCLUSIVE | + +Misclassifying ENV as BUG sends the implementer chasing ghosts; misclassifying BUG as ENV ships +the defect. When unsure, reproduce twice and read the server logs before deciding. + +## Toolbox (repo conventions first; these are examples, not requirements) + +No specific tool is required. Precedence: (1) whatever format/harness the repo's existing usage +suites already use — run and extend that; (2) a well-suited tool from the examples below if it is +available or trivially installable; (3) ubiquitous fallbacks (`curl` + shell assertions cover any +HTTP surface). What is non-negotiable is the discipline — spec-first asserts, clean state — not +the tool. + +| Surface | Example tools | Notes | +|---------|--------------|-------| +| HTTP/REST/JSON (incl. gRPC-over-HTTP with JSON encoding) | [Hurl](https://hurl.dev) `.hurl` files; `curl` scripts | Hurl: plain-text request/assert format, capturable variables, `retry` for eventual consistency; files double as committed regression suites | +| Pure gRPC/protobuf | `grpcurl` (scripted) | Use server reflection or point at the proto files | +| CLI | Direct invocation via `execute_command` | Assert exit codes AND output; probe stdin/args edge cases | +| Anything else | `curl`/scripts/the repo's own test harness | Same discipline: spec-first asserts, clean state | + +Optional niceties like Hurl and grpcurl may already be preinstalled (e.g. by a sandbox mixin) or +can be installed idempotently (hurl via the distro package manager first — its prebuilt GitHub +tarball dynamically links `libxml2.so.2`, which newer distros no longer ship; grpcurl from its +GitHub release, a static Go binary). When a preferred tool is unavailable and uninstallable, fall +back to what exists rather than skipping the check; classify a probe as ENV only when NO adequate +tool can exercise the surface. + +If you use Hurl, gotchas that produce false results if unknown: + +- `[Captures]` run BEFORE `[Asserts]` in the same entry — capture a replaced value under a NEW + variable name, or your inequality asserts compare a value to itself. +- JSON encoders that omit zero/empty values: assert `not exists` for absent fields — `== false` + or `== null` asserts fail on omitted keys. +- Use `[Options] retry` with a bounded count for asynchronous effects (job completion, eventual + reads); never unbounded. + +## Verdict format + +End with EXACTLY one of: + +``` +USAGE_PROBE: PASS +Surface: . Existing suites: . +New tests: , all green>. + +``` + +``` +USAGE_PROBE: FAIL +Surface: <...>. Existing suites: . New tests: . +Behavioral findings: +1. — — — REPRO: — +Stale tests needing update (expected-change): +``` + +``` +USAGE_PROBE: INCONCLUSIVE +Could not establish a clean local environment: . +Missing: . +Partial results (if any): +``` + +Every FAIL finding MUST include the exact reproduction (request/command and the response +received) and cite the test file — a behavioral complaint without a repro is noise. INCONCLUSIVE +is an honest, acceptable verdict: it routes the fix to the environment recipe, not the code. +NEVER report INCONCLUSIVE as PASS ("couldn't test, probably fine") or as FAIL (the implementer +would hunt a nonexistent bug). + +## Anti-patterns + +- Writing tests after reading the implementation — you will encode its bugs as expectations. +- Skipping the existing suites and jumping to new tests — regressions are the cheapest bugs to catch. +- Testing through internal seams (direct DB reads, internal function calls) — this gate is + consumer-perspective only; internals belong to unit tests. +- Depending on pre-existing data, shared databases, or previously running services. +- Adding retries/sleeps until a flaky assertion passes — flakiness is a finding, not an obstacle. +- Reporting an environment failure as a behavioral FAIL (or burying it in a PASS). +- Throwaway tests in /tmp — tests that don't land in the repo's suite location die with the run. +- Asserting only status codes — shape and content are where the quirks live. diff --git a/deployment/homebrew/coyote.rb.template b/deployment/homebrew/coyote.rb.template index 84a0e65..11fbb81 100644 --- a/deployment/homebrew/coyote.rb.template +++ b/deployment/homebrew/coyote.rb.template @@ -1,7 +1,7 @@ # Documentation: https://docs.brew.sh/Formula-Cookbook # https://rubydoc.brew.sh/Formula class Coyote < Formula - desc "All-in-one, batteries included LLM CLI tool" + desc "The batteries-included runtime for LLMs" homepage "https://github.com/Dark-Alex-17/coyote" if OS.mac? and Hardware::CPU.arm? url "https://github.com/Dark-Alex-17/coyote/releases/download/v$version/coyote-aarch64-apple-darwin.tar.gz"