From 9a505f951a649424030d44ecb58fd5fba0576827 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 14 Sep 2026 00:22:28 -0600 Subject: [PATCH 01/15] feat(TASK-028): escalation_timeout 0 waits indefinitely and is the default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Child escalations (coder plan approvals, deviation gates, user__* calls from subagents) previously expired after the escalating agent's escalation_timeout, defaulting to 300 s, and 0 timed out instantly. In autonomous runs the orchestrator often could not reply within that window, so approvals silently fell through to the best-judgment fallback. - Extract await_escalation_reply as the wait seam: 0 awaits the oneshot with no deadline; nonzero keeps tokio::time::timeout with the same timeout message. A dropped sender still resolves to the existing cancelled reply either way. - DEFAULT_ESCALATION_TIMEOUT_SECS and default_escalation_timeout() -> 0, so graph agents (which inherit the default via from_graph) are covered. - Document the new semantics in config.agent.example.yaml and drop the 5-minute-timeout claim from the default spawn instructions. - Tests: paused-clock unlimited wait (3600 s), nonzero expiry message, dropped-sender completion, default pin, and a grep pin that the stale timeout wording is gone from src/, assets/, and the example config. Plan: PLAN-review-retry §C (D3). --- config.agent.example.yaml | 2 +- src/config/agent.rs | 2 +- src/config/prompts.rs | 2 +- src/function/user_interaction.rs | 129 ++++++++++++++++++++++++++++--- 4 files changed, 121 insertions(+), 14 deletions(-) diff --git a/config.agent.example.yaml b/config.agent.example.yaml index 039171e..bcdd79a 100644 --- a/config.agent.example.yaml +++ b/config.agent.example.yaml @@ -48,7 +48,7 @@ max_concurrent_jobs: inject_spawn_instructions: true # Inject the default agent spawning instructions into the agent's system prompt summarization_model: null # Model to use for summarizing sub-agent output (e.g. 'openai:gpt-4o-mini'); defaults to current model summarization_threshold: 4000 # Character threshold above which sub-agent output is summarized before returning to parent -escalation_timeout: 300 # Seconds a sub-agent waits for a user interaction response before timing out (default: 5 minutes) +escalation_timeout: 0 # 0 = wait indefinitely for the reply (default); N = give up after N seconds mcp_servers: # Optional list of MCP servers that the agent utilizes - github # Corresponds to the name of an MCP server in the `/mcp.json` file mcp_tools: # Optional per-server tool allowlist for the agent's MCP servers diff --git a/src/config/agent.rs b/src/config/agent.rs index 8951d1a..d0a4a32 100644 --- a/src/config/agent.rs +++ b/src/config/agent.rs @@ -889,7 +889,7 @@ fn default_summarization_threshold() -> usize { } fn default_escalation_timeout() -> u64 { - 300 + 0 } impl AgentConfig { diff --git a/src/config/prompts.rs b/src/config/prompts.rs index c37c13b..f70e231 100644 --- a/src/config/prompts.rs +++ b/src/config/prompts.rs @@ -200,7 +200,7 @@ pub(in crate::config) const DEFAULT_SPAWN_INSTRUCTIONS: &str = indoc! {" 1. Read the child's question and options. 2. If you can answer from context, call `agent__reply_escalation` with your answer. 3. If you need the user's input, call the appropriate `user__*` tool yourself, then relay the answer via `agent__reply_escalation`. - 4. **Respond promptly**; the child agent is blocked and waiting (5-minute timeout). + 4. **Respond promptly**; the child blocks until answered — no timeout by default (`escalation_timeout` per agent). "}; pub(in crate::config) const DEFAULT_JOB_INSTRUCTIONS: &str = indoc! {" diff --git a/src/function/user_interaction.rs b/src/function/user_interaction.rs index 1d5988b..6900091 100644 --- a/src/function/user_interaction.rs +++ b/src/function/user_interaction.rs @@ -13,7 +13,7 @@ use tokio::sync::oneshot; pub const USER_FUNCTION_PREFIX: &str = "user__"; -const DEFAULT_ESCALATION_TIMEOUT_SECS: u64 = 300; +const DEFAULT_ESCALATION_TIMEOUT_SECS: u64 = 0; const CUSTOM_MULTI_CHOICE_ANSWER_OPTION: &str = "Other (custom)"; pub fn user_interaction_function_declarations() -> Vec { @@ -289,17 +289,32 @@ async fn handle_escalated(ctx: &RequestContext, action: &str, args: &Value) -> R root_queue.submit(request); - let timeout = Duration::from_secs(timeout_secs); - match tokio::time::timeout(timeout, rx).await { - Ok(Ok(reply)) => Ok(json!({ "answer": reply })), - Ok(Err(_)) => Ok(json!({ - "error": "Escalation was cancelled. The parent agent dropped the request", - "fallback": "Make your best judgment and proceed", - })), + await_escalation_reply(rx, timeout_secs).await +} + +/// Waits for the parent's reply. `timeout_secs == 0` waits indefinitely; a +/// dropped sender (the parent cancelled the request) resolves either way. +async fn await_escalation_reply(rx: oneshot::Receiver, timeout_secs: u64) -> Result { + let reply = if timeout_secs == 0 { + rx.await + } else { + match tokio::time::timeout(Duration::from_secs(timeout_secs), rx).await { + Ok(reply) => reply, + Err(_) => { + return Ok(json!({ + "error": format!( + "Escalation timed out after {timeout_secs} seconds waiting for user response" + ), + "fallback": "Make your best judgment and proceed", + })); + } + } + }; + + match reply { + Ok(reply) => Ok(json!({ "answer": reply })), Err(_) => Ok(json!({ - "error": format!( - "Escalation timed out after {timeout_secs} seconds waiting for user response" - ), + "error": "Escalation was cancelled. The parent agent dropped the request", "fallback": "Make your best judgment and proceed", })), } @@ -351,4 +366,96 @@ mod tests { assert_eq!(v["action"], "confirm"); assert_eq!(v["options"], json!([])); } + + #[test] + fn escalation_timeout_defaults_to_unlimited() { + assert_eq!(DEFAULT_ESCALATION_TIMEOUT_SECS, 0); + } + + #[tokio::test(start_paused = true)] + async fn zero_timeout_waits_indefinitely_for_the_reply() { + let (tx, rx) = oneshot::channel(); + let wait = tokio::spawn(await_escalation_reply(rx, 0)); + + tokio::time::advance(Duration::from_secs(3600)).await; + assert!(!wait.is_finished()); + + tx.send("approved".to_string()).unwrap(); + let v = wait.await.unwrap().unwrap(); + assert_eq!(v, json!({ "answer": "approved" })); + } + + #[tokio::test(start_paused = true)] + async fn nonzero_timeout_still_expires_with_the_existing_message() { + let (tx, rx) = oneshot::channel::(); + let wait = tokio::spawn(await_escalation_reply(rx, 5)); + + tokio::time::advance(Duration::from_secs(6)).await; + let v = wait.await.unwrap().unwrap(); + assert_eq!( + v["error"], + "Escalation timed out after 5 seconds waiting for user response" + ); + assert_eq!(v["fallback"], "Make your best judgment and proceed"); + assert!(tx.send("too late".to_string()).is_err()); + } + + #[tokio::test(start_paused = true)] + async fn zero_timeout_resolves_when_the_sender_is_dropped() { + let (tx, rx) = oneshot::channel::(); + drop(tx); + + let v = tokio::time::timeout(Duration::from_secs(1), await_escalation_reply(rx, 0)) + .await + .expect("the wait must complete once the sender is gone") + .unwrap(); + assert!( + v["error"] + .as_str() + .unwrap() + .starts_with("Escalation was cancelled"), + "{v}" + ); + assert_eq!(v["fallback"], "Make your best judgment and proceed"); + } + + #[test] + fn stale_timeout_wording_is_gone_from_shipped_text() { + let needles = [ + concat!("(5-minute", " timeout)"), + concat!("escalation_timeout:", " 300"), + concat!("default:", " 5 minutes"), + ]; + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + let mut files = vec![root.join("config.agent.example.yaml")]; + collect_files(&root.join("src"), &mut files); + collect_files(&root.join("assets"), &mut files); + assert!(files.len() > 3, "walked only {} files", files.len()); + + let mut hits = Vec::new(); + for path in &files { + let text = String::from_utf8_lossy(&std::fs::read(path).unwrap()).into_owned(); + for needle in needles { + if text.contains(needle) { + hits.push(format!("{}: {needle:?}", path.display())); + } + } + } + assert!( + hits.is_empty(), + "stale timeout wording found:\n{}", + hits.join("\n") + ); + } + + fn collect_files(dir: &std::path::Path, out: &mut Vec) { + for entry in std::fs::read_dir(dir).unwrap().flatten() { + let path = entry.path(); + if path.is_dir() { + collect_files(&path, out); + } else if path.is_file() { + out.push(path); + } + } + } } From 33399203778d1897ac7db418541dca8d09aaf77f Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 14 Sep 2026 00:31:34 -0600 Subject: [PATCH 02/15] fix(TASK-028): pin global-template exclusion, tighten timeout boundary test, reword stale hint --- src/config/agent.rs | 1 + src/config/mod.rs | 6 ++++++ src/function/agents.rs | 2 +- src/function/user_interaction.rs | 7 ++++++- 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/config/agent.rs b/src/config/agent.rs index d0a4a32..18bf661 100644 --- a/src/config/agent.rs +++ b/src/config/agent.rs @@ -1341,6 +1341,7 @@ variables: assert_eq!(config.max_concurrent_agents, 4); assert_eq!(config.max_agent_depth, 3); assert_eq!(config.max_auto_continues, 10); + assert_eq!(config.escalation_timeout, 0); assert!(config.mcp_servers.is_empty()); assert!(config.global_tools.is_empty()); assert!(config.conversation_starters.is_empty()); diff --git a/src/config/mod.rs b/src/config/mod.rs index efacc93..1016014 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -1383,6 +1383,12 @@ clients: assert_eq!(cfg.clients.len(), 1); } + #[test] + fn config_template_does_not_carry_the_per_agent_escalation_timeout_key() { + // `escalation_timeout` is a per-agent setting; the global template must never grow it. + assert!(!CONFIG_TEMPLATE.contains("escalation_timeout")); + } + #[test] fn config_enabled_macros_empty_string_is_some_empty() { let cfg: Config = serde_yaml::from_str("enabled_macros: \"\"").unwrap(); diff --git a/src/function/agents.rs b/src/function/agents.rs index 77ca043..5637a86 100644 --- a/src/function/agents.rs +++ b/src/function/agents.rs @@ -1711,7 +1711,7 @@ fn handle_reply_escalation(ctx: &mut RequestContext, args: &Value) -> Result Ok(json!({ "status": "error", - "message": format!("No pending escalation found with id '{escalation_id}'. It may have already been replied to or timed out."), + "message": format!("No pending escalation found with id '{escalation_id}'. It may have already been replied to, or the child is no longer waiting (aborted, or its configured `escalation_timeout` expired)."), })), } } diff --git a/src/function/user_interaction.rs b/src/function/user_interaction.rs index 6900091..f9a2923 100644 --- a/src/function/user_interaction.rs +++ b/src/function/user_interaction.rs @@ -389,8 +389,12 @@ mod tests { async fn nonzero_timeout_still_expires_with_the_existing_message() { let (tx, rx) = oneshot::channel::(); let wait = tokio::spawn(await_escalation_reply(rx, 5)); + tokio::task::yield_now().await; - tokio::time::advance(Duration::from_secs(6)).await; + tokio::time::advance(Duration::from_secs(4)).await; + assert!(!wait.is_finished()); + + tokio::time::advance(Duration::from_secs(2)).await; let v = wait.await.unwrap().unwrap(); assert_eq!( v["error"], @@ -421,6 +425,7 @@ mod tests { #[test] fn stale_timeout_wording_is_gone_from_shipped_text() { + // Split so this file does not match its own needles. let needles = [ concat!("(5-minute", " timeout)"), concat!("escalation_timeout:", " 300"), From b90e04f227e0257074095b836752f61c783515f4 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 14 Sep 2026 00:39:20 -0600 Subject: [PATCH 03/15] fix(TASK-028): pin example-doc wording; make not-found escalation hint match queue semantics --- src/function/agents.rs | 6 +++++- src/function/user_interaction.rs | 19 +++++++++++++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/function/agents.rs b/src/function/agents.rs index 5637a86..3baafaa 100644 --- a/src/function/agents.rs +++ b/src/function/agents.rs @@ -1711,7 +1711,7 @@ fn handle_reply_escalation(ctx: &mut RequestContext, args: &Value) -> Result Ok(json!({ "status": "error", - "message": format!("No pending escalation found with id '{escalation_id}'. It may have already been replied to, or the child is no longer waiting (aborted, or its configured `escalation_timeout` expired)."), + "message": format!("No pending escalation found with id '{escalation_id}'. It may have already been replied to."), })), } } @@ -3093,6 +3093,10 @@ mod tests { ) .unwrap(); assert_eq!(result["status"], "error"); + assert_eq!( + result["message"], + "No pending escalation found with id 'missing'. It may have already been replied to." + ); } #[test] diff --git a/src/function/user_interaction.rs b/src/function/user_interaction.rs index f9a2923..d698fa7 100644 --- a/src/function/user_interaction.rs +++ b/src/function/user_interaction.rs @@ -432,10 +432,25 @@ mod tests { concat!("default:", " 5 minutes"), ]; let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); - let mut files = vec![root.join("config.agent.example.yaml")]; + let example = root.join("config.agent.example.yaml"); + let example_text = std::fs::read_to_string(&example).unwrap(); + for needle in ["escalation_timeout: 0", "0 = wait indefinitely"] { + assert!( + example_text.contains(needle), + "{} lost the shipped wording {needle:?}", + example.display() + ); + } + + let mut files = vec![example]; collect_files(&root.join("src"), &mut files); collect_files(&root.join("assets"), &mut files); - assert!(files.len() > 3, "walked only {} files", files.len()); + let this_file = root.join(file!()); + assert!( + files.contains(&this_file), + "walk did not reach {}", + this_file.display() + ); let mut hits = Vec::new(); for path in &files { From 18f5ada58585b263923077ee2ddd9e8dbb8e7393 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 14 Sep 2026 01:41:52 -0600 Subject: [PATCH 04/15] feat(TASK-029): review-gauntlet fault-only lane re-run loop Give the review gauntlet a deterministic retry loop for lanes that FAULT (PIPELINE-FAULT marker, missing verdict sentinel, or a nested graph's own degraded-run wording) without ever re-running a lane that reached a real verdict. - graph.yaml: the four lane maps route to a new `retry_gate` script node (static `next: verdict_gate`; emits `_next: build_items` to retry). initial_state gains lane_attempts / kept_results / retry_lanes / review_incomplete / gauntlet_started_at. settings.timeout 24000 -> 69600 (3 passes x 2 attempts x 11400 + 1200 margin). - scripts/retry_gate.py (new): stashes every lane report that ran (completed and faulted alike) and restores reports a re-run pass overwrote with []; classifies with verdict_gate.py's sentinel regexes verbatim; counts attempts; retries while attempts < 3 and inside a 45600 s wall-clock budget; otherwise records the lane in review_incomplete and falls through. Crash guard appends to signals_error and never retries. - scripts/signals.py: stamps gauntlet_started_at before the git work so a failed diff still starts the clock. - scripts/build_items.py: retry pass emits items only for retry_lanes (with attempt numbers), adds a `retried:` note to lanes_summary, resets retry_lanes; falsy-guarded stamp fallback; crash guard appends to signals_error and preserves lanes_summary. - Tests: hermetic python3-guarded script tests for retry_gate / build_items / signals, graph wiring + initial_state pins, sentinel and LANE_KEYS cross-pins, retry-budget envelope vs settings.timeout, a first-pass golden payload, and an engine-level executor test proving a script `_next` back-edge re-fires a static fan-out and an empty map re-run clears its collect_into. Plan: PLAN-review-retry section A items 1-3 and 5. Verdict/report surface (section A items 4 and 6) is TASK-032. --- assets/agents/review-gauntlet/graph.yaml | 38 +- .../review-gauntlet/scripts/build_items.py | 52 +- .../review-gauntlet/scripts/retry_gate.py | 159 +++++ .../agents/review-gauntlet/scripts/signals.py | 5 + src/config/request_context.rs | 625 ++++++++++++++++++ src/graph/executor.rs | 111 ++++ 6 files changed, 976 insertions(+), 14 deletions(-) create mode 100644 assets/agents/review-gauntlet/scripts/retry_gate.py diff --git a/assets/agents/review-gauntlet/graph.yaml b/assets/agents/review-gauntlet/graph.yaml index 0c08bb8..e2aeaf5 100644 --- a/assets/agents/review-gauntlet/graph.yaml +++ b/assets/agents/review-gauntlet/graph.yaml @@ -41,13 +41,14 @@ variables: settings: max_loop_iterations: 10 log_state_snapshots: true - # max over agent lanes of (max_attempts × lane timeout) = 2 × 11400 = 22800 - # (agent-node retries each get a fresh per-attempt budget, - # src/graph/agent.rs retry_transient/bounded_attempt) + 1200 margin for - # the script stages (signals/select/build/gate) = 24000; the parse and - # select_lanes llm nodes carry no `timeout:` and are bounded by - # max_iterations. - timeout: 24000 + # A faulted lane can be re-run by retry_gate for up to 3 passes, and each + # pass's agent node retries up to max_attempts = 2 with a fresh per-attempt + # budget (src/graph/agent.rs retry_transient/bounded_attempt): + # 3 passes × 2 attempts × 11400 (max lane timeout) = 68400, + 1200 margin + # for the script stages (signals/select/build/retry gate/verdict gate) + # = 69600; the parse and select_lanes llm nodes carry no `timeout:` and + # are bounded by max_iterations. + timeout: 69600 initial_state: diff_spec: worktree @@ -84,6 +85,11 @@ initial_state: adversary_results: [] security_results: [] probe_results: [] + lane_attempts: {} + kept_results: {} + retry_lanes: [] + review_incomplete: [] + gauntlet_started_at: 0 lanes_summary: '' gauntlet_verdict: '' gauntlet_report: '' @@ -251,7 +257,7 @@ nodes: build_items: id: build_items type: script - description: Deterministic lane builder — hard rules are the floor, LLM flags may only add, forced lanes override exactly. Emits one 0-or-1-item list per lane; an empty list skips that lane's map. + description: Deterministic lane builder — hard rules are the floor, LLM flags may only add, forced lanes override exactly. Emits one 0-or-1-item list per lane; an empty list skips that lane's map. On a re-run pass (retry_lanes non-empty) it skips selection and emits items only for those lanes, each carrying its attempt number. script: scripts/build_items.py timeout: 30 next: [map_code_review, map_adversary, map_security, map_probe] @@ -265,7 +271,7 @@ nodes: output_key: lane_out collect_into: code_review_results max_concurrency: 1 - next: verdict_gate + next: retry_gate map_adversary: id: map_adversary @@ -276,7 +282,7 @@ nodes: output_key: lane_out collect_into: adversary_results max_concurrency: 1 - next: verdict_gate + next: retry_gate map_security: id: map_security @@ -287,7 +293,7 @@ nodes: output_key: lane_out collect_into: security_results max_concurrency: 1 - next: verdict_gate + next: retry_gate map_probe: id: map_probe @@ -298,7 +304,7 @@ nodes: output_key: lane_out collect_into: probe_results max_concurrency: 1 - next: verdict_gate + next: retry_gate run_code_review: id: run_code_review @@ -412,6 +418,14 @@ nodes: script: scripts/lane_fault.py timeout: 30 + retry_gate: + id: retry_gate + type: script + description: Deterministic fault-only re-run gate. Stashes every lane report that ran this pass, restores reports a re-run pass overwrote with [] (a lane not being re-run maps over nothing), classifies each lane that ran as completed / faulted / skipped, counts attempts, and routes back into build_items for faulted lanes that still have attempts left inside the wall-clock budget; otherwise it records them in review_incomplete and falls through to verdict_gate. A real verdict is never re-run, and a gate crash never retries — it records a PIPELINE-FAULT and falls through. + script: scripts/retry_gate.py + timeout: 30 + next: verdict_gate + verdict_gate: id: verdict_gate type: script diff --git a/assets/agents/review-gauntlet/scripts/build_items.py b/assets/agents/review-gauntlet/scripts/build_items.py index f8198eb..0183c91 100644 --- a/assets/agents/review-gauntlet/scripts/build_items.py +++ b/assets/agents/review-gauntlet/scripts/build_items.py @@ -9,6 +9,10 @@ skip mechanism — no dynamic routing anywhere. The LLM's add_* flags can only ADD lanes. A flag that is false never removes a lane the hard rules selected. +A re-run pass (retry_gate emitted `retry_lanes`) skips selection: only the +named lanes get an item, stamped with their attempt number, and every other +lane maps over nothing — retry_gate restores their earlier reports. + Fail-closed: a builder crash emits all four lists empty plus a PIPELINE-FAULT in signals_error — verdict_gate blocks on it and the crash never kills the graph. @@ -16,6 +20,7 @@ never kills the graph. import json import os +import time # Keep in sync with default_lanes.ALIASES — a caller-forced lane must # canonicalize the same way on the degraded path as it does here. @@ -35,6 +40,18 @@ ALIASES = { "usage-pattern-testing": "probe", } +# Keep in sync with retry_gate.LANES +LANE_KEYS = { + "code-review": "code_review_items", + "adversary": "adversary_items", + "security": "security_items", + "probe": "probe_items", +} + +# Filled by main() so the crash guard can append to an upstream fault +# instead of overwriting it, and keep the first pass's lanes_summary. +crash_context = {"signals_error": "", "lanes_summary": ""} + def load_state(): if path := os.environ.get("GRAPH_STATE_FILE"): @@ -45,11 +62,38 @@ def load_state(): def main(): state = load_state() + prior = state.get("signals_error") + crash_context["signals_error"] = prior if isinstance(prior, str) else "" + # retry_gate's wall-clock budget is measured from this stamp; signals.py + # is the normal stamper, this is the defensive fallback. + stamp = {} if state.get("gauntlet_started_at") else {"gauntlet_started_at": time.time()} def text(key): v = state.get(key) return v.strip() if isinstance(v, str) else "" + crash_context["lanes_summary"] = text("lanes_summary") + retry_lanes = [lane for lane in (state.get("retry_lanes") or []) if lane in LANE_KEYS] + if retry_lanes: + attempts = state.get("lane_attempts") or {} + retried = [(lane, int(attempts.get(lane) or 0) + 1) for lane in retry_lanes] + note = "- retried: [" + ", ".join(f"{lane} (attempt {n})" for lane, n in retried) + "]" + summary = text("lanes_summary") + print( + json.dumps( + { + **{key: [] for key in LANE_KEYS.values()}, + **{ + LANE_KEYS[lane]: [{"lane": lane, "attempt": n}] for lane, n in retried + }, + "lanes_summary": f"{summary}\n{note}" if summary else note, + "retry_lanes": [], + **stamp, + } + ) + ) + return + forced_raw = state.get("forced_lanes") or [] forced, unknown = [], [] for lane in forced_raw: @@ -126,6 +170,7 @@ def main(): "security_items": [{"lane": "security"}] if "security" in lanes else [], "probe_items": [{"lane": "probe"}] if "probe" in lanes else [], "lanes_summary": "\n".join(f"- {r}" for r in reasons), + **stamp, } ) ) @@ -134,6 +179,9 @@ def main(): try: main() except Exception as e: # noqa: BLE001 — a builder crash must never kill the graph + fault = f"PIPELINE-FAULT: lane builder crashed — {e}; no lanes were run" + if crash_context["signals_error"]: + fault = f"{crash_context['signals_error']}; {fault}" print( json.dumps( { @@ -141,8 +189,8 @@ except Exception as e: # noqa: BLE001 — a builder crash must never kill the g "adversary_items": [], "security_items": [], "probe_items": [], - "lanes_summary": "", - "signals_error": f"PIPELINE-FAULT: lane builder crashed — {e}; no lanes were run", + "lanes_summary": crash_context["lanes_summary"], + "signals_error": fault, } ) ) diff --git a/assets/agents/review-gauntlet/scripts/retry_gate.py b/assets/agents/review-gauntlet/scripts/retry_gate.py new file mode 100644 index 0000000..2831e47 --- /dev/null +++ b/assets/agents/review-gauntlet/scripts/retry_gate.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Deterministic fault-only re-run gate for the review gauntlet. + +Sits between the four lane maps and the verdict gate. Every lane map runs +on every pass — a lane that is not being re-run maps over [] — so a re-run +pass overwrites the untouched lanes' collected results with []. The gate's +hard rules: + + - every lane that ran this pass is stashed in kept_results, completed + and faulted alike, and a lane whose map ran nothing this pass gets + its stashed report restored — the verdict gate always sees each + lane's LAST report; + - a lane is FAULTED only when its report is missing, non-string, the + lane_fault.py marker, sentinel-less, or carries the nested graph's own + degraded-run wording; a real verdict (DIVERGES, FAIL, INCONCLUSIVE, + NEEDS-HUMAN, …) is never re-run — the review's judgment stands; + - a faulted lane is re-run only while it has attempts left AND the + gauntlet is inside its wall-clock budget; otherwise it is recorded in + review_incomplete and its fault flows to the verdict gate as-is; + - the gate itself must never retry by accident: any internal error + records a PIPELINE-FAULT in signals_error and falls through to the + verdict gate, which blocks on it. +""" + +import json +import os +import re +import time + +MAX_RETRY_ELAPSED_SECS = 45600 +MAX_ATTEMPTS = 3 + +# lane name -> (items key, results key) +LANES = { + "code-review": ("code_review_items", "code_review_results"), + "adversary": ("adversary_items", "adversary_results"), + "security": ("security_items", "security_results"), + "probe": ("probe_items", "probe_results"), +} + +# Must stay identical to verdict_gate.py's sentinel regexes: a lane this +# gate calls completed must be one the verdict gate can parse, or a drift +# would re-run lanes the verdict gate accepts (or accept lanes it cannot +# read). A cross-pin test compares the literals in both files. +SENTINELS = { + "code-review": r"Verdict:\**\s*\**\s*(MERGE-READY|NEEDS-HUMAN)", + "adversary": r"ADVERSARIAL_REVIEW:\s*(CONFORMS|DIVERGES)", + "security": r"SECURITY_REVIEW:\s*(PASS|FAIL)", + "probe": r"USAGE_PROBE:\s*(PASS|FAIL|INCONCLUSIVE)", +} + +# Filled by main() so the crash guard can append to an upstream fault +# instead of overwriting it, and report the lanes it had already declined. +crash_context = {"signals_error": "", "review_incomplete": []} + + +def load_state(): + if path := os.environ.get("GRAPH_STATE_FILE"): + with open(path) as f: + return json.load(f) + return json.loads(os.environ.get("GRAPH_STATE", "{}")) + + +def nested_degraded(lane, report): + # The nested graphs complete with a sentinel even when a stage inside + # them died; their own degraded-run wording marks a partial review, + # which is not a completed one. Same anchoring as verdict_gate.py. + if lane == "code-review": + verdict_line = re.search(r"^\*\*Verdict: .*$", report, re.MULTILINE) + return bool( + verdict_line + and re.search( + r"^\*\*Verdict: NEEDS-HUMAN\*\* — .*" + r"(pipeline fault\(s\) recorded|PIPELINE-FAULT:|report rendering error)", + verdict_line.group(0), + ) + ) + if lane == "adversary": + return bool( + re.search( + r"^Criteria: .*degraded run: pipeline fault recorded", + report, + re.MULTILINE, + ) + ) + return False + + +def is_faulted(lane, report): + if not isinstance(report, str): + return True + if report.strip().startswith("PIPELINE-FAULT:"): + return True + if not re.search(SENTINELS[lane], report): + return True + return nested_degraded(lane, report) + + +def main(): + state = load_state() + prior = state.get("signals_error") + crash_context["signals_error"] = prior if isinstance(prior, str) else "" + incomplete = list(state.get("review_incomplete") or []) + crash_context["review_incomplete"] = incomplete + attempts = dict(state.get("lane_attempts") or {}) + kept = dict(state.get("kept_results") or {}) + out = {} + faulted = [] + + for lane, (items_key, results_key) in LANES.items(): + results = state.get(results_key) or [] + if not state.get(items_key): + if not results and lane in kept: + out[results_key] = [kept[lane]] + continue + report = results[0] if results else None + if report is not None: + kept[lane] = report + if is_faulted(lane, report): + faulted.append(lane) + attempts[lane] = int(attempts.get(lane) or 0) + 1 + + # A missing/zero stamp reads as an ancient start and declines every + # retry — the safe direction when the clock's origin is unknown. + started = state.get("gauntlet_started_at") or 0 + within_budget = (time.time() - float(started)) <= MAX_RETRY_ELAPSED_SECS + retry = [lane for lane in faulted if within_budget and attempts[lane] < MAX_ATTEMPTS] + declined = [lane for lane in faulted if lane not in retry] + incomplete = sorted(set(incomplete) | set(declined)) + crash_context["review_incomplete"] = incomplete + + out.update( + { + "lane_attempts": attempts, + "kept_results": kept, + "retry_lanes": retry, + "review_incomplete": incomplete, + } + ) + if retry: + out["_next"] = "build_items" + print(json.dumps(out)) + + +try: + main() +except Exception as e: # noqa: BLE001 — a gate crash must fall through, never retry + fault = f"PIPELINE-FAULT: retry gate crashed — {e}" + if crash_context["signals_error"]: + fault = f"{crash_context['signals_error']}; {fault}" + print( + json.dumps( + { + "retry_lanes": [], + "review_incomplete": crash_context["review_incomplete"], + "signals_error": fault, + } + ) + ) diff --git a/assets/agents/review-gauntlet/scripts/signals.py b/assets/agents/review-gauntlet/scripts/signals.py index 6ccecb4..2e5e365 100644 --- a/assets/agents/review-gauntlet/scripts/signals.py +++ b/assets/agents/review-gauntlet/scripts/signals.py @@ -12,6 +12,7 @@ import json import os import re import subprocess +import time def load_state(): if path := os.environ.get("GRAPH_STATE_FILE"): @@ -37,6 +38,10 @@ def git(*args): out = {"project_dir": proj} +# retry_gate measures its wall-clock budget from this stamp; it is set +# before the git work so a failed diff still starts the clock. +if not state.get("gauntlet_started_at"): + out["gauntlet_started_at"] = time.time() try: if spec in ("", "worktree"): names = git("diff", "--name-only", "HEAD") diff --git a/src/config/request_context.rs b/src/config/request_context.rs index ef594d5..7710ef5 100644 --- a/src/config/request_context.rs +++ b/src/config/request_context.rs @@ -11663,6 +11663,631 @@ mod tests { ); } + fn now_secs() -> f64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs_f64() + } + + /// State for a single gauntlet lane whose map ran one item and collected + /// `report`, stamped with a fresh start so the retry budget is open. + fn gauntlet_lane_ran(lane: &str, report: &str) -> serde_json::Value { + let key = lane.replace('-', "_"); + let mut state = json!({ "gauntlet_started_at": now_secs() }); + state[format!("{key}_items").as_str()] = json!([{ "lane": lane }]); + state[format!("{key}_results").as_str()] = json!([report]); + state + } + + const GAUNTLET_MERGE_READY: &str = "# Review\n\n**Verdict: MERGE-READY**\n"; + const GAUNTLET_ADVERSARY_FAULT: &str = + "PIPELINE-FAULT: adversary lane failed after retries — Agent node failed: boom"; + + #[test] + fn gauntlet_retry_gate_stashes_and_restores_reports() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + let first = run_gauntlet_script( + "retry_gate.py", + &json!({ + "code_review_items": [{"lane": "code-review"}], + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_items": [{"lane": "adversary"}], + "adversary_results": [GAUNTLET_ADVERSARY_FAULT], + "gauntlet_started_at": now_secs(), + }), + ); + assert_eq!( + first["kept_results"], + json!({"code-review": GAUNTLET_MERGE_READY, "adversary": GAUNTLET_ADVERSARY_FAULT}), + "completed and faulted lanes must both be stashed verbatim: {first}" + ); + assert_eq!(first["retry_lanes"], json!(["adversary"]), "{first}"); + assert_eq!(first["_next"], "build_items", "{first}"); + + // The re-run pass: code-review's map ran nothing and overwrote its + // collected results with []; adversary ran again and completed. + let second = run_gauntlet_script( + "retry_gate.py", + &json!({ + "code_review_items": [], + "code_review_results": [], + "adversary_items": [{"lane": "adversary", "attempt": 2}], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS\nCriteria: all verified"], + "kept_results": first["kept_results"], + "lane_attempts": first["lane_attempts"], + "gauntlet_started_at": now_secs(), + }), + ); + assert_eq!( + second["code_review_results"], + json!([GAUNTLET_MERGE_READY]), + "the overwritten code-review report must be restored from the stash: {second}" + ); + assert_eq!(second["retry_lanes"], json!([]), "{second}"); + assert!( + second.get("_next").is_none(), + "a pass with nothing to retry must fall through statically: {second}" + ); + assert_eq!( + second["kept_results"]["adversary"], + "ADVERSARIAL_REVIEW: CONFORMS\nCriteria: all verified", + "the stash must carry the lane's latest report: {second}" + ); + assert_eq!(second["lane_attempts"]["adversary"], 2, "{second}"); + + // A pass where nothing ran: both lanes come back from the stash, + // the faulted adversary report included — it is restored, not + // re-classified for another retry. + let restored = run_gauntlet_script( + "retry_gate.py", + &json!({ + "code_review_items": [], + "code_review_results": [], + "adversary_items": [], + "adversary_results": [], + "security_items": [], + "security_results": [], + "probe_items": [], + "probe_results": [], + "kept_results": first["kept_results"], + "lane_attempts": first["lane_attempts"], + "gauntlet_started_at": now_secs(), + }), + ); + assert_eq!( + restored["code_review_results"], + json!([GAUNTLET_MERGE_READY]), + "{restored}" + ); + assert_eq!( + restored["adversary_results"], + json!([GAUNTLET_ADVERSARY_FAULT]), + "a faulted stash is restored as-is for the verdict gate: {restored}" + ); + assert_eq!(restored["retry_lanes"], json!([]), "{restored}"); + assert!( + restored.get("_next").is_none(), + "restoring a stash must never trigger a retry: {restored}" + ); + assert_eq!( + restored["lane_attempts"], first["lane_attempts"], + "a lane that did not run must not be charged an attempt: {restored}" + ); + } + + #[test] + fn gauntlet_retry_gate_classifies_lane_reports() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + let completed = run_gauntlet_script( + "retry_gate.py", + &gauntlet_lane_ran("code-review", GAUNTLET_MERGE_READY), + ); + assert_eq!( + completed["retry_lanes"], + json!([]), + "a report carrying its sentinel is completed, never re-run: {completed}" + ); + assert!(completed.get("_next").is_none(), "{completed}"); + + let faulted = [ + ("adversary", GAUNTLET_ADVERSARY_FAULT), + ("security", "some text"), + ( + "code-review", + "# Review\n\n**Verdict: NEEDS-HUMAN** — 1 pipeline fault(s) recorded; see Human attention required\n\n## Findings\n", + ), + ( + "adversary", + "ADVERSARIAL_REVIEW: DIVERGES\nCriteria: none verified — degraded run: pipeline fault recorded (fail-closed).\nComplaints:\n- none", + ), + ]; + for (lane, report) in faulted { + let out = run_gauntlet_script("retry_gate.py", &gauntlet_lane_ran(lane, report)); + assert_eq!( + out["retry_lanes"], + json!([lane]), + "{lane} report {report:?} must classify as faulted: {out}" + ); + assert_eq!(out["_next"], "build_items", "{lane}: {out}"); + } + + // A lane whose map ran nothing (items []) is skipped: no attempt, no + // retry, no incompleteness — and nothing to restore without a stash. + let mut state = gauntlet_lane_ran("adversary", GAUNTLET_ADVERSARY_FAULT); + state["probe_items"] = json!([]); + state["probe_results"] = json!([]); + let out = run_gauntlet_script("retry_gate.py", &state); + assert_eq!(out["retry_lanes"], json!(["adversary"]), "{out}"); + assert_eq!(out["review_incomplete"], json!([]), "{out}"); + assert!( + out["lane_attempts"].get("probe").is_none(), + "a skipped lane must not count an attempt: {out}" + ); + assert!(out.get("probe_results").is_none(), "{out}"); + } + + #[test] + fn gauntlet_retry_gate_counts_attempts_per_lane_that_ran() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + let out = run_gauntlet_script( + "retry_gate.py", + &json!({ + "code_review_items": [{"lane": "code-review"}], + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_items": [{"lane": "adversary"}], + "adversary_results": [GAUNTLET_ADVERSARY_FAULT], + "security_items": [], + "probe_items": [], + "gauntlet_started_at": now_secs(), + }), + ); + assert_eq!( + out["lane_attempts"], + json!({"code-review": 1, "adversary": 1}), + "only lanes that ran count an attempt: {out}" + ); + + let mut state = gauntlet_lane_ran("adversary", GAUNTLET_ADVERSARY_FAULT); + state["lane_attempts"] = json!({"adversary": 1}); + let out = run_gauntlet_script("retry_gate.py", &state); + assert_eq!(out["lane_attempts"]["adversary"], 2, "{out}"); + } + + #[test] + fn gauntlet_retry_gate_retries_until_attempts_are_exhausted() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + for prior in [0, 1] { + let mut state = gauntlet_lane_ran("adversary", GAUNTLET_ADVERSARY_FAULT); + state["lane_attempts"] = json!({"adversary": prior}); + let out = run_gauntlet_script("retry_gate.py", &state); + assert_eq!( + out["_next"], + "build_items", + "attempt {} must route back into the builder: {out}", + prior + 1 + ); + assert_eq!(out["retry_lanes"], json!(["adversary"]), "{out}"); + assert_eq!( + out["kept_results"]["adversary"], GAUNTLET_ADVERSARY_FAULT, + "{out}" + ); + assert_eq!(out["review_incomplete"], json!([]), "{out}"); + } + + let mut state = gauntlet_lane_ran("adversary", GAUNTLET_ADVERSARY_FAULT); + state["lane_attempts"] = json!({"adversary": 2}); + let out = run_gauntlet_script("retry_gate.py", &state); + assert!( + out.get("_next").is_none(), + "the third attempt must fall through to the verdict gate: {out}" + ); + assert_eq!(out["lane_attempts"]["adversary"], 3, "{out}"); + assert_eq!(out["retry_lanes"], json!([]), "{out}"); + assert_eq!(out["review_incomplete"], json!(["adversary"]), "{out}"); + } + + #[test] + fn gauntlet_retry_gate_declines_outside_wall_clock_budget() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + let mut state = gauntlet_lane_ran("adversary", GAUNTLET_ADVERSARY_FAULT); + state["gauntlet_started_at"] = json!(now_secs() - 100000.0); + let out = run_gauntlet_script("retry_gate.py", &state); + assert!( + out.get("_next").is_none(), + "a first-attempt fault past the budget must not retry: {out}" + ); + assert_eq!(out["retry_lanes"], json!([]), "{out}"); + assert_eq!(out["review_incomplete"], json!(["adversary"]), "{out}"); + } + + #[test] + fn gauntlet_retry_gate_crash_falls_through_without_retry() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // dict(42) raises inside main after the upstream fault was read. + let out = run_gauntlet_script( + "retry_gate.py", + &json!({ + "lane_attempts": 42, + "adversary_items": [{"lane": "adversary"}], + "adversary_results": ["x"], + "signals_error": "PIPELINE-FAULT: earlier fault", + }), + ); + assert!( + out["signals_error"] + .as_str() + .unwrap() + .starts_with("PIPELINE-FAULT: earlier fault; PIPELINE-FAULT: retry gate crashed"), + "the crash must append to the upstream fault, not overwrite it: {out}" + ); + assert_eq!(out["retry_lanes"], json!([]), "{out}"); + assert!( + out["review_incomplete"].is_array(), + "review_incomplete must stay a list on the crash path: {out}" + ); + assert!( + out.get("_next").is_none(), + "a crashed gate must never retry: {out}" + ); + } + + /// retry_gate.py decides "completed" with the same sentinel regexes + /// verdict_gate.py parses; a drift would re-run lanes the verdict gate + /// accepts, or accept lanes it cannot read. + #[test] + fn gauntlet_retry_gate_sentinels_match_verdict_gate() { + let scripts = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("assets/agents/review-gauntlet/scripts"); + let retry = read_to_string(scripts.join("retry_gate.py")).unwrap(); + let verdict = read_to_string(scripts.join("verdict_gate.py")).unwrap(); + for sentinel in [ + r"Verdict:\**\s*\**\s*(MERGE-READY|NEEDS-HUMAN)", + r"ADVERSARIAL_REVIEW:\s*(CONFORMS|DIVERGES)", + r"SECURITY_REVIEW:\s*(PASS|FAIL)", + r"USAGE_PROBE:\s*(PASS|FAIL|INCONCLUSIVE)", + ] { + assert!( + retry.contains(sentinel), + "retry_gate.py must carry the sentinel regex {sentinel}" + ); + assert!( + verdict.contains(sentinel), + "verdict_gate.py must carry the sentinel regex {sentinel}" + ); + } + for fragment in [ + r"^\*\*Verdict: NEEDS-HUMAN\*\* — .*", + r"(pipeline fault\(s\) recorded|PIPELINE-FAULT:|report rendering error)", + ] { + assert!( + retry.contains(fragment), + "retry_gate.py must carry the nested-degraded fragment {fragment}" + ); + assert!( + verdict.contains(fragment), + "verdict_gate.py must carry the nested-degraded fragment {fragment}" + ); + } + let adversary_verdict = read_to_string( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("assets/agents/adversary/scripts/verdict.py"), + ) + .unwrap(); + let degraded = "degraded run: pipeline fault recorded"; + assert!( + retry.contains(degraded), + "retry_gate.py must match the adversary's degraded-run wording {degraded:?}" + ); + assert!( + adversary_verdict.contains(degraded), + "assets/agents/adversary/scripts/verdict.py must still emit {degraded:?}" + ); + } + + /// build_items.py's LANE_KEYS and retry_gate.py's LANES name the same + /// four lanes and item keys; a lane present in one but not the other + /// would be built but never stashed, or stashed but never rebuilt. + #[test] + fn gauntlet_build_items_lane_keys_match_retry_gate_lanes() { + let scripts = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("assets/agents/review-gauntlet/scripts"); + let build = read_to_string(scripts.join("build_items.py")).unwrap(); + let retry = read_to_string(scripts.join("retry_gate.py")).unwrap(); + assert!( + build.contains("Keep in sync with retry_gate.LANES"), + "build_items.py LANE_KEYS must carry its sync note" + ); + for lane in ["code-review", "adversary", "security", "probe"] { + let name = format!("\"{lane}\""); + let items = format!("{}_items", lane.replace('-', "_")); + for (file, source) in [("build_items.py", &build), ("retry_gate.py", &retry)] { + assert!(source.contains(&name), "{file} must name the lane {name}"); + assert!( + source.contains(&items), + "{file} must carry the items key {items}" + ); + } + } + } + + #[test] + fn gauntlet_build_items_retry_pass_rebuilds_only_retried_lanes() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + let out = run_gauntlet_script( + "build_items.py", + &json!({ + "retry_lanes": ["adversary"], + "lane_attempts": {"adversary": 1, "code-review": 1}, + "lanes_summary": "- code-review: always on", + "forced_lanes": ["code-review", "adversary"], + "gauntlet_started_at": 123.0, + }), + ); + assert_eq!( + out["adversary_items"], + json!([{"lane": "adversary", "attempt": 2}]), + "{out}" + ); + for key in ["code_review_items", "security_items", "probe_items"] { + assert_eq!( + out[key], + json!([]), + "{key} must map over nothing on a re-run pass, even when forced: {out}" + ); + } + assert_eq!(out["retry_lanes"], json!([]), "{out}"); + let summary = out["lanes_summary"].as_str().unwrap(); + assert!( + summary.contains("retried: [adversary (attempt 2)]") + && summary.contains("code-review: always on"), + "the retry note must append to the first pass's summary: {summary}" + ); + assert!( + out.get("gauntlet_started_at").is_none(), + "an existing start stamp must be left alone: {out}" + ); + } + + #[test] + fn gauntlet_build_items_stamps_start_time_once() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + let out = run_gauntlet_script("build_items.py", &json!({"forced_lanes": ["code-review"]})); + assert!( + out["gauntlet_started_at"].as_f64().is_some_and(|t| t > 0.0), + "a first pass without a stamp must start the clock: {out}" + ); + let out = run_gauntlet_script( + "build_items.py", + &json!({"forced_lanes": ["code-review"], "gauntlet_started_at": 5.0}), + ); + assert!( + out.get("gauntlet_started_at").is_none(), + "an existing start stamp must be left alone: {out}" + ); + } + + /// With the clock already stamped (the normal graph path — signals.py + /// runs first), a first pass must serialize exactly the five-key payload + /// it always has: same keys, same order, same values, nothing extra. + #[test] + fn gauntlet_build_items_first_pass_payload_is_unchanged_when_stamped() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("assets/agents/review-gauntlet/scripts/build_items.py"); + let state = + json!({"forced_lanes": ["code-review", "adversary"], "gauntlet_started_at": 5.0}); + let out = std::process::Command::new("python3") + .arg(&path) + .env("GRAPH_STATE", state.to_string()) + .env_remove("GRAPH_STATE_FILE") + .output() + .expect("failed to invoke python3 build_items.py"); + assert!( + out.status.success(), + "build_items.py exited nonzero: {}", + String::from_utf8_lossy(&out.stderr) + ); + let stdout = String::from_utf8_lossy(&out.stdout); + assert_eq!( + stdout.trim_end(), + r#"{"code_review_items": [{"lane": "code-review"}], "adversary_items": [{"lane": "adversary"}], "security_items": [], "probe_items": [], "lanes_summary": "- forced lanes honored exactly: ['adversary', 'code-review']"}"#, + "the stamped first-pass payload must be byte-identical to the pre-loop form" + ); + } + + #[test] + fn gauntlet_build_items_crash_appends_to_prior_fault() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + let out = run_gauntlet_script( + "build_items.py", + &json!({ + "forced_lanes": 42, + "signals_error": "PIPELINE-FAULT: earlier", + "lanes_summary": "- code-review: always on", + }), + ); + assert!( + out["signals_error"] + .as_str() + .unwrap() + .starts_with("PIPELINE-FAULT: earlier; PIPELINE-FAULT: lane builder crashed"), + "the crash must append to the upstream fault, not overwrite it: {out}" + ); + assert_eq!( + out["lanes_summary"], "- code-review: always on", + "a crash must not wipe the first pass's lanes_summary: {out}" + ); + } + + #[test] + fn gauntlet_signals_stamps_start_time_before_git_runs() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + let dir = utils::temp_file("-gauntlet-signals-", ""); + create_dir_all(&dir).unwrap(); + let project_dir = dir.to_string_lossy().to_string(); + let out = run_gauntlet_script( + "signals.py", + &json!({"project_dir": project_dir, "diff_spec": "worktree"}), + ); + let stamped = run_gauntlet_script( + "signals.py", + &json!({"project_dir": project_dir, "diff_spec": "worktree", "gauntlet_started_at": 77.0}), + ); + let _ = remove_dir_all(&dir); + assert!( + out["signals_error"] + .as_str() + .is_some_and(|e| e.contains("diff signals could not be computed")), + "a non-repo project_dir must degrade the signals: {out}" + ); + assert!( + out["gauntlet_started_at"].as_f64().is_some_and(|t| t > 0.0), + "the start stamp must be set even when git fails: {out}" + ); + assert!( + stamped.get("gauntlet_started_at").is_none(), + "an existing start stamp must be left alone: {stamped}" + ); + } + + #[test] + fn gauntlet_retry_loop_wiring() { + use crate::graph::NodeType; + let graph = load_bundled_graph("review-gauntlet"); + let maps = [ + "map_code_review", + "map_adversary", + "map_security", + "map_probe", + ]; + for id in maps { + let node = graph + .get_node(id) + .unwrap_or_else(|| panic!("review-gauntlet must have a {id} node")); + assert_eq!( + node.next.as_ref().map(|n| n.as_slice()), + Some(&["retry_gate".to_string()][..]), + "{id} must route into retry_gate: {:?}", + node.next + ); + } + let gate = graph + .get_node("retry_gate") + .expect("review-gauntlet must have a retry_gate node"); + let NodeType::Script(s) = &gate.node_type else { + panic!("retry_gate must be a script node"); + }; + assert_eq!(s.script, "scripts/retry_gate.py"); + assert_eq!( + gate.next.as_ref().map(|n| n.as_slice()), + Some(&["verdict_gate".to_string()][..]), + "retry_gate must fall through statically to verdict_gate: {:?}", + gate.next + ); + let build = graph + .get_node("build_items") + .expect("review-gauntlet must have a build_items node"); + let fan_out: Vec = maps.iter().map(|m| m.to_string()).collect(); + assert_eq!( + build.next.as_ref().map(|n| n.as_slice()), + Some(fan_out.as_slice()), + "build_items must still fan out to the four maps: {:?}", + build.next + ); + for (key, expected) in [ + ("lane_attempts", json!({})), + ("kept_results", json!({})), + ("retry_lanes", json!([])), + ("review_incomplete", json!([])), + ("gauntlet_started_at", json!(0)), + ] { + assert_eq!( + graph.initial_state.get(key), + Some(&expected), + "initial_state.{key} must seed the retry loop" + ); + } + assert_eq!(graph.settings.timeout, Some(69600)); + let script = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("assets/agents/review-gauntlet/scripts/retry_gate.py"); + assert!(script.exists(), "{} must exist", script.display()); + } + + /// MAX_RETRY_ELAPSED_SECS in retry_gate.py and settings.timeout in + /// graph.yaml live in different files. A re-run admitted at the very end + /// of the retry budget can still burn a full lane envelope (max_attempts + /// × the slowest lane) plus the script stages, and a graph timeout kills + /// from outside — no verdict would be rendered. + #[test] + fn gauntlet_retry_budget_fits_inside_graph_timeout() { + use crate::graph::NodeType; + const OTHER_STAGES_MARGIN_SECS: u64 = 1200; + let script = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("assets/agents/review-gauntlet/scripts/retry_gate.py"); + let source = read_to_string(&script).unwrap(); + let line = source + .lines() + .find(|l| l.starts_with("MAX_RETRY_ELAPSED_SECS = ")) + .expect("retry_gate.py must define MAX_RETRY_ELAPSED_SECS"); + let max_retry: u64 = line + .split_once(" = ") + .and_then(|(_, v)| v.trim().parse().ok()) + .unwrap_or_else(|| panic!("MAX_RETRY_ELAPSED_SECS no longer parses: {line}")); + + let graph = load_bundled_graph("review-gauntlet"); + let max_lane_envelope = graph + .nodes + .values() + .filter_map(|node| match &node.node_type { + NodeType::Agent(a) => a.timeout.map(|t| u64::from(a.max_attempts) * t), + _ => None, + }) + .max() + .expect("review-gauntlet must have agent lanes with timeouts"); + let graph_timeout = graph + .settings + .timeout + .expect("review-gauntlet must set settings.timeout"); + assert!( + max_retry + max_lane_envelope + OTHER_STAGES_MARGIN_SECS <= graph_timeout, + "assets/agents/review-gauntlet/scripts/retry_gate.py MAX_RETRY_ELAPSED_SECS ({max_retry}) + the largest lane envelope, max_attempts × timeout ({max_lane_envelope}) + {OTHER_STAGES_MARGIN_SECS}s margin must fit inside assets/agents/review-gauntlet/graph.yaml settings.timeout ({graph_timeout})" + ); + } + // The value run_checks executes with a shell must never be something an // LLM lifted out of the prompt — the prompt also carries pasted plan/diff // text from the repo under review. diff --git a/src/graph/executor.rs b/src/graph/executor.rs index 2e1c4dc..71a87b5 100644 --- a/src/graph/executor.rs +++ b/src/graph/executor.rs @@ -1325,6 +1325,117 @@ nodes: ); } + /// A script `_next` back-edge into a node with a static fan-out must + /// re-fire every fan-out target on the next pass, and a map that + /// re-runs over zero items must overwrite its `collect_into` with `[]` + /// rather than leave the previous pass's results in place. + #[tokio::test] + async fn script_next_back_edge_refires_static_fan_out_and_empty_map_rerun_clears_collect_into() + { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + let ws = TestWorkspace::new(); + ws.write_script( + "build.py", + r#"#!/usr/bin/env python3 +import os, json +state = json.loads(os.environ.get("GRAPH_STATE", "{}")) +if state["pass"] == 0: + print(json.dumps({"a_items": [1], "b_items": [1]})) +else: + print(json.dumps({"a_items": [], "b_items": [2]})) +"#, + ); + ws.write_script( + "worker.py", + r#"#!/usr/bin/env python3 +import os, json +state = json.loads(os.environ.get("GRAPH_STATE", "{}")) +print(json.dumps({"output": f"report-{state['item']}"})) +"#, + ); + ws.write_script( + "gate.py", + r#"#!/usr/bin/env python3 +import os, json +state = json.loads(os.environ.get("GRAPH_STATE", "{}")) +if state["pass"] == 0: + print(json.dumps({"pass": 1, "_next": "build"})) +else: + print(json.dumps({})) +"#, + ); + + let yaml = r#" +name: fan_out_rerun_test +start: build +initial_state: + pass: 0 + a_items: [] + b_items: [] + a_results: [] + b_results: [] +nodes: + build: + type: script + script: build.py + next: [map_a, map_b] + map_a: + type: map + over: "{{a_items}}" + as: item + branch: worker + collect_into: a_results + next: gate + map_b: + type: map + over: "{{b_items}}" + as: item + branch: worker + collect_into: b_results + next: gate + worker: + type: script + script: worker.py + state_updates: {} + timeout: 60 + gate: + type: script + script: gate.py + next: done + done: + type: end + output: "{{a_results}}|{{b_results}}" +"#; + let graph: Graph = serde_yaml::from_str(yaml).unwrap(); + let mut ctx = make_ctx(); + let abort = create_abort_signal(); + let result = GraphExecutor::new(graph, &ws.dir) + .execute(&mut ctx, abort) + .await + .unwrap_or_else(|e| panic!("executor failed: {e:#}")); + + let (a, b) = result + .split_once('|') + .unwrap_or_else(|| panic!("expected `a|b`, got: {result}")); + let a: Value = + serde_json::from_str(a).unwrap_or_else(|_| panic!("a_results not JSON: {result}")); + let b: Value = + serde_json::from_str(b).unwrap_or_else(|_| panic!("b_results not JSON: {result}")); + assert_eq!( + b, + serde_json::json!(["report-2"]), + "the back-edge into build must re-fire both fan-out targets: {result}" + ); + assert_eq!( + a, + serde_json::json!([]), + "map_a must re-fire on the back-edge pass and clear its collect_into over zero items: {result}" + ); + } + #[tokio::test] async fn parallel_branch_error_aborts_super_step() { if !cmd_available("bash") { From 786e5542d41a9330784192e92af14577a6e0e54b Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 14 Sep 2026 02:59:58 -0600 Subject: [PATCH 05/15] =?UTF-8?q?feat(TASK-032):=20review-gauntlet=20incom?= =?UTF-8?q?plete-review=20machine=20line=20+=20=C3=97N=20attempt=20details?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit verdict_gate.py now names every incomplete review on a machine line the done template renders immediately after the verdict: `GAUNTLET_REVIEW_INCOMPLETE: [, ]` (sorted). Names are the lanes retry_gate gave up on (`review_incomplete`) plus the `pipeline` pseudo-lane whenever `signals_error` carries a `PIPELINE-FAULT:` — parse death, builder / retry-gate / default-lanes crash guards — and the gate's own crash guard emits the `pipeline` line too. The report gains a `## Review incomplete` section (after the signals_error blockquote, immediately before `## Full lane reports`) and the three PIPELINE-FAULT lane rows carry ` ×N` when the lane was re-run (N = 1 texts unchanged). retry_gate.py stands in a synthetic `PIPELINE-FAULT: lane produced no report` marker for a lane that ran but collected nothing, so it is stashed, classified FAULTED and never renders SKIPPED downstream. MAX_RETRY_ELAPSED_SECS trimmed 45600 → 45000 for margin slack. graph.yaml: `gauntlet_incomplete_line` seeded in initial_state and spliced into the done output. README + mermaid document retry_gate, the dotted _next back-edge, the 3-attempt / wall-clock budget, that retry never narrows selection, the crash-guard append semantics, and the machine-line contract (present whenever any lane or the pipeline was incomplete; it can accompany real findings). Tests: ×N on all three branches (N>1, N=1 byte-identical, no leak onto non-fault rows), exact incomplete line / section placement, pipeline pseudo-lane alone and alongside real lanes (via default_lanes.py's real crash guard), mixed finding+fault, crash-guard chaining from build_items / retry_gate / parse_fault / verdict_gate, synthetic no-report marker, done-template wiring, README pins (positive + stale negative). --- assets/agents/review-gauntlet/README.md | 73 ++- assets/agents/review-gauntlet/graph.yaml | 5 +- .../review-gauntlet/scripts/retry_gate.py | 14 +- .../review-gauntlet/scripts/verdict_gate.py | 69 ++- src/config/request_context.rs | 438 ++++++++++++++++++ 5 files changed, 568 insertions(+), 31 deletions(-) diff --git a/assets/agents/review-gauntlet/README.md b/assets/agents/review-gauntlet/README.md index 39785f6..d4aae09 100644 --- a/assets/agents/review-gauntlet/README.md +++ b/assets/agents/review-gauntlet/README.md @@ -26,8 +26,10 @@ flowchart TD radv -. "fallback" .-> lfault rsec -. "fallback" .-> lfault rpb -. "fallback" .-> lfault - mcr & madv & msec & mpb --> gate["verdict_gate
(script: sentinel regex — deterministic)"] - gate --> done(["GAUNTLET: PASS | BLOCKED"]) + mcr & madv & msec & mpb --> rgate["retry_gate
(script: fault-only re-run gate;
stash + restore, ≤3 attempts,
wall-clock budget)"] + rgate -. "_next: retry faulted lanes" .-> build + rgate --> gate["verdict_gate
(script: sentinel regex — deterministic)"] + gate --> done(["GAUNTLET: PASS | BLOCKED
(+ GAUNTLET_REVIEW_INCOMPLETE: … when any lane or the pipeline was incomplete)"]) ``` ## Why a graph @@ -49,7 +51,8 @@ those rules are *structure*: - **Conditional lanes without dynamic routing**: each lane is a `map` node over a 0-or-1-item list — an empty list means the map runs zero branches (map-over-nothing), which *is* the skip mechanism. All four maps join at - the verdict gate on equal-length paths. + the retry gate on equal-length paths; it falls through to the verdict + gate, or routes back to `build_items` for faulted lanes only. - **The verdict is a regex, not an opinion** (`verdict_gate.py`): a missing sentinel is a lane FAILURE, never a pass; the critical count from the code-review report's summary line (raw 🔴 count if absent) blocks @@ -77,6 +80,45 @@ those rules are *structure*: `GAUNTLET:` sentinel is always emitted: even a crashed gate prints `GAUNTLET: BLOCKED`, never silence. +## Fault-only re-runs + +`retry_gate.py` sits between the lane maps and the verdict gate and re-runs +only lanes that **faulted** — a missing report (stood in for by a synthetic +`PIPELINE-FAULT: lane produced no report` marker so it never renders as +SKIPPED), a non-string report, the `lane_fault` marker, a sentinel-less +report, or a nested graph's own degraded-run wording. A real verdict +(DIVERGES, FAIL, INCONCLUSIVE, NEEDS-HUMAN, …) is never re-run: the review's +judgment stands. Each lane gets up to 3 attempts, all inside a wall-clock +budget (`MAX_RETRY_ELAPSED_SECS`) sized to fit the graph timeout. + +- **Stash + restore**: every lane report that ran this pass is stashed in + `kept_results`; a re-run pass overwrites the untouched lanes' collected + results with `[]`, so their stashed reports are restored before the gate. + The verdict gate always sees each lane's *last* report. +- **Retry never narrows selection**: lane selection is not re-run. A re-run + pass rebuilds items only for the retried lanes, and the stashed results + stand in for the rest. +- **Exhaustion is recorded, not hidden**: a lane still faulted after its + attempts or the budget lands in `review_incomplete`, and its fault flows to + the verdict gate as-is. +- **Crash guards append**: the `build_items.py` and `retry_gate.py` crash + guards append to an existing `signals_error` rather than overwrite it, and + `build_items.py` preserves the prior `lanes_summary`. + +Whenever at least one lane (or the pipeline) could not complete, the first +line reads `GAUNTLET: BLOCKED` followed immediately by +`GAUNTLET_REVIEW_INCOMPLETE: [, ]`. The names are the exhausted +lanes from `review_incomplete`, plus the `pipeline` pseudo-lane whenever a +pipeline-level `PIPELINE-FAULT:` fired — parse death, or a crash in the +builder, the retry gate, the default-lanes fallback, or the verdict gate +itself (the default-lanes fallback's crash still runs the widened lanes, so +`pipeline` appears alongside real lane rows). The report gains a +`## Review incomplete` section naming each exhausted lane with its +attempt count; `PIPELINE-FAULT` lane rows carry ` ×N` when the lane was +re-run (N > 1). The *named* entries are faults to re-run, never code +findings; the line can accompany real findings from the other lanes, which +still appear under `## Blockers` and must be fixed as usual. + ## Sentinels parsed | Lane | Sentinel | Blocks on | @@ -115,22 +157,29 @@ coyote -a review-gauntlet --agent-variable project_dir /abs/path/to/repo \ --agent-variable verification_commands '["cargo test --all"]' "" ``` -The reply ends with `GAUNTLET: PASS` or `GAUNTLET: BLOCKED`, a lane status +The reply opens with `GAUNTLET: PASS` or `GAUNTLET: BLOCKED` (plus the +`GAUNTLET_REVIEW_INCOMPLETE:` line when applicable), followed by a lane status table, the lane-selection reasons, blockers, attention items, and each lane's -full report in a collapsible section. On BLOCKED: fix the findings per your -own findings-handling rules, then spawn a **fresh** gauntlet run. On PASS with -*Human attention required*: complete the work but carry those items into your -final report verbatim. +full report in a collapsible section. On BLOCKED with a +`GAUNTLET_REVIEW_INCOMPLETE:` line: the named lanes never completed — fix any +non-fault blockers listed under `## Blockers`, then spawn a **fresh** gauntlet +run (a fault is never a code finding). On BLOCKED without it: fix the findings +per your own findings-handling rules, then spawn a **fresh** gauntlet run. On +PASS with *Human attention required*: complete the work but carry those items +into your final report verbatim. ## Notes - The caller decides *whether* review is warranted at all (the gauntlet is for non-trivial work — that's why code-review is unconditionally on); the gauntlet decides *which* of the other lanes apply and what the verdicts mean. -- Lane sub-agents get generous finite timeouts (2h each; 3h whole-gauntlet) - and one retry (`max_attempts: 2`). A lane that still dies or times out falls - back to the `lane_fault` marker — BLOCKED naming the lane, never a silent - pass. +- Lane sub-agents get generous finite per-attempt timeouts (see the + `timeout:` on each `run_*` node) and one retry (`max_attempts: 2`) inside + the lane, then up to 3 attempts of the whole lane from `retry_gate.py` while + the wall-clock budget (`MAX_RETRY_ELAPSED_SECS`) allows; `settings.timeout` + is derived from that envelope in graph.yaml's comment. A lane that still + dies or times out falls back to the `lane_fault` marker — BLOCKED naming the + lane and `GAUNTLET_REVIEW_INCOMPLETE`, never a silent pass. - The `verification_commands` variable is forwarded to the adversary lane as a structured `inputs:` passthrough on `run_adversary` (a lone `{{…}}` template lands the raw value in the child's state) — never through the lane prompt diff --git a/assets/agents/review-gauntlet/graph.yaml b/assets/agents/review-gauntlet/graph.yaml index e2aeaf5..cc58f67 100644 --- a/assets/agents/review-gauntlet/graph.yaml +++ b/assets/agents/review-gauntlet/graph.yaml @@ -93,6 +93,7 @@ initial_state: lanes_summary: '' gauntlet_verdict: '' gauntlet_report: '' + gauntlet_incomplete_line: '' start: parse @@ -429,7 +430,7 @@ nodes: verdict_gate: id: verdict_gate type: script - description: Deterministic sentinel parser and gate. A missing sentinel is a lane failure, never a pass; probe INCONCLUSIVE blocks with an environment note; the code-review critical count is read from render.py's summary line (raw 🔴 count if absent) and blocks regardless of the lane's own verdict line; a code-review verdict line that records the nested graph's own pipeline fault blocks as a degraded lane. + description: Deterministic sentinel parser and gate. A missing sentinel is a lane failure, never a pass; probe INCONCLUSIVE blocks with an environment note; the code-review critical count is read from render.py's summary line (raw 🔴 count if absent) and blocks regardless of the lane's own verdict line; a code-review verdict line that records the nested graph's own pipeline fault blocks as a degraded lane. Lanes retry_gate gave up on, and any pipeline-level fault, are named on a GAUNTLET_REVIEW_INCOMPLETE machine line so a caller can tell an incomplete review from a failed one. script: scripts/verdict_gate.py timeout: 60 next: done @@ -438,6 +439,6 @@ nodes: id: done type: end output: | - GAUNTLET: {{gauntlet_verdict}} + GAUNTLET: {{gauntlet_verdict}}{{gauntlet_incomplete_line}} {{gauntlet_report}} diff --git a/assets/agents/review-gauntlet/scripts/retry_gate.py b/assets/agents/review-gauntlet/scripts/retry_gate.py index 2831e47..45f1ecd 100644 --- a/assets/agents/review-gauntlet/scripts/retry_gate.py +++ b/assets/agents/review-gauntlet/scripts/retry_gate.py @@ -10,7 +10,9 @@ hard rules: and faulted alike, and a lane whose map ran nothing this pass gets its stashed report restored — the verdict gate always sees each lane's LAST report; - - a lane is FAULTED only when its report is missing, non-string, the + - a lane is FAULTED only when its report is missing (the map ran but + collected nothing — replaced by a synthetic PIPELINE-FAULT marker so + the verdict gate never reads it as SKIPPED), non-string, the lane_fault.py marker, sentinel-less, or carries the nested graph's own degraded-run wording; a real verdict (DIVERGES, FAIL, INCONCLUSIVE, NEEDS-HUMAN, …) is never re-run — the review's judgment stands; @@ -27,7 +29,9 @@ import os import re import time -MAX_RETRY_ELAPSED_SECS = 45600 +# 69600 (graph timeout) − 2 × 11400 (largest lane envelope) − 1200 (script stages) +# = 45600; 600 s of slack below that. +MAX_RETRY_ELAPSED_SECS = 45000 MAX_ATTEMPTS = 3 # lane name -> (items key, results key) @@ -114,8 +118,10 @@ def main(): out[results_key] = [kept[lane]] continue report = results[0] if results else None - if report is not None: - kept[lane] = report + if report is None: + report = f"PIPELINE-FAULT: {lane} lane produced no report" + out[results_key] = [report] + kept[lane] = report if is_faulted(lane, report): faulted.append(lane) attempts[lane] = int(attempts.get(lane) or 0) + 1 diff --git a/assets/agents/review-gauntlet/scripts/verdict_gate.py b/assets/agents/review-gauntlet/scripts/verdict_gate.py index f995a0d..cbeab90 100644 --- a/assets/agents/review-gauntlet/scripts/verdict_gate.py +++ b/assets/agents/review-gauntlet/scripts/verdict_gate.py @@ -8,10 +8,11 @@ re-reads, no judgment. The gate's hard rules: - a non-string lane result (the map collected an object instead of the lane's rendered text) blocks — it cannot carry a sentinel; - a PIPELINE-FAULT lane result (the lane's agent died after retries; - detected only when the report IS the lane_fault.py marker, i.e. it - STARTS with it — a real review merely quoting the marker flows to the - normal rules) or a PIPELINE-FAULT in signals_error (builder/parse - fault) blocks — a degraded pipeline is never a pass; + detected only when the report IS the lane_fault.py marker or + retry_gate.py's no-report marker, i.e. it STARTS with it — a real + review merely quoting the marker flows to the normal rules) or a + PIPELINE-FAULT in signals_error (builder/parse fault) blocks — a + degraded pipeline is never a pass; - a code-review lane whose OWN verdict line records a pipeline fault (code-reviewer's verdict.py / render.py degraded-run wording) blocks — the nested graph completed, but with a dead domain lane, verifier, or @@ -24,8 +25,16 @@ re-reads, no judgment. The gate's hard rules: - probe INCONCLUSIVE blocks with an ENVIRONMENT note — it is never treated as PASS or FAIL (fix the environment per the plan's local-run recipe, then re-run); + - a PIPELINE-FAULT lane row carries its attempt count (` ×N`) when retry_gate + re-ran it, so the reader can tell a first-pass death from an exhausted + retry budget; + - an incomplete review — lanes retry_gate gave up on (review_incomplete) + or a pipeline-level fault (reported under the `pipeline` pseudo-lane) — + emits the GAUNTLET_REVIEW_INCOMPLETE: machine line and a "Review + incomplete" section: that is a fault to re-run, never a code finding; - the gate itself must never crash into a silent pass: any internal - error emits GAUNTLET: BLOCKED naming the error. + error emits GAUNTLET: BLOCKED naming the error, with the pipeline + marked incomplete. """ import json @@ -43,6 +52,7 @@ def main(): state = load_state() blockers, attention, lane_lines, reports = [], [], [], [] malformed = set() + lane_attempts = state.get("lane_attempts") or {} def lane_report(key): arr = state.get(key) or [] @@ -57,6 +67,10 @@ def main(): def record(lane, status, detail=""): lane_lines.append(f"| {lane} | {status} | {detail} |") + def attempts_suffix(lane): + n = int(lane_attempts.get(lane) or 0) + return f" ×{n}" if n > 1 else "" + def fault_excerpt(report): line = next( (ln for ln in report.splitlines() if "PIPELINE-FAULT:" in ln), report @@ -64,10 +78,11 @@ def main(): return line.strip()[:200] def is_lane_fault(report): - # A genuine lane fault is the ENTIRE report emitted by lane_fault.py, - # which always begins with the marker. Matching the marker anywhere - # would let a real review that merely QUOTES "PIPELINE-FAULT:" bypass - # 🔴 counting and sentinel parsing. + # A genuine lane fault is the ENTIRE report — the lane_fault.py marker + # or retry_gate.py's no-report marker — which always begins with + # "PIPELINE-FAULT:". Matching the marker anywhere would let a real + # review that merely QUOTES it bypass 🔴 counting and sentinel + # parsing. return report.strip().startswith("PIPELINE-FAULT:") def lane_fault_blocker(lane, report): @@ -75,7 +90,7 @@ def main(): f"{lane}: PIPELINE-FAULT — the lane failed after retries and produced " f"no verdict; a degraded lane is never a pass ({fault_excerpt(report)})" ) - record(lane, "BLOCKED", "PIPELINE-FAULT (lane failed)") + record(lane, "BLOCKED", f"PIPELINE-FAULT (lane failed{attempts_suffix(lane)})") def malformed_blocker(lane, report): blockers.append(f"{lane}: malformed lane result (non-string item) — never a pass") @@ -129,10 +144,11 @@ def main(): blockers.append( f"code-review: {reds} 🔴 CRITICAL finding(s) — fix before claiming done" ) + suffix = attempts_suffix("code-review") if m_fault and reds: - record("code-review", "BLOCKED", f"PIPELINE-FAULT (degraded lane); {reds} 🔴") + record("code-review", "BLOCKED", f"PIPELINE-FAULT (degraded lane{suffix}); {reds} 🔴") elif m_fault: - record("code-review", "BLOCKED", "PIPELINE-FAULT (degraded lane)") + record("code-review", "BLOCKED", f"PIPELINE-FAULT (degraded lane{suffix})") elif reds: record("code-review", "BLOCKED", f"{reds} 🔴 finding(s)") elif m and m.group(1) == "MERGE-READY": @@ -231,11 +247,27 @@ def main(): # A gauntlet-level fault (build_items crash, dead parse stage) means the # lanes above were never built — SKIPPED rows alone must not read as PASS. signals_error = state.get("signals_error") + incomplete = set(state.get("review_incomplete") or []) if isinstance(signals_error, str) and "PIPELINE-FAULT:" in signals_error: blockers.append(f"pipeline: {signals_error}") + incomplete.add("pipeline") verdict = "BLOCKED" if blockers else "PASS" + incomplete_lines = [] + for name in sorted(incomplete): + if name == "pipeline": + incomplete_lines.append(f"- pipeline: {signals_error}") + elif n := int(lane_attempts.get(name) or 0): + incomplete_lines.append(f"- {name}: no completed review after {n} attempt(s)") + else: + incomplete_lines.append(f"- {name}: no completed review") + incomplete_line = ( + "\nGAUNTLET_REVIEW_INCOMPLETE: " + ", ".join(sorted(incomplete)) + if incomplete + else "" + ) + md = "\n".join( ["| lane | status | detail |", "|------|--------|--------|"] + lane_lines ) @@ -246,11 +278,21 @@ def main(): md += "\n\n## Human attention required\n" + "\n".join(f"- {a}" for a in attention) if state.get("signals_error"): md += f"\n\n> {state['signals_error']}" + if incomplete_lines: + md += "\n\n## Review incomplete\n" + "\n".join(incomplete_lines) md += "\n\n## Full lane reports\n" for lane, rep in reports: md += f"\n
{lane}\n\n{rep}\n\n
\n" - print(json.dumps({"gauntlet_verdict": verdict, "gauntlet_report": md})) + print( + json.dumps( + { + "gauntlet_verdict": verdict, + "gauntlet_report": md, + "gauntlet_incomplete_line": incomplete_line, + } + ) + ) try: @@ -261,6 +303,7 @@ except Exception as e: # noqa: BLE001 — the gate must never crash into a sile { "gauntlet_verdict": "BLOCKED", "gauntlet_report": f"verdict gate error: {e} — treat as a failed gate, not a pass.", + "gauntlet_incomplete_line": "\nGAUNTLET_REVIEW_INCOMPLETE: pipeline", } ) ) diff --git a/src/config/request_context.rs b/src/config/request_context.rs index 7710ef5..7261f8e 100644 --- a/src/config/request_context.rs +++ b/src/config/request_context.rs @@ -8335,6 +8335,50 @@ mod tests { ); } + #[test] + fn review_gauntlet_readme_states_retry_and_incomplete_contract() { + let readmes = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("assets/agents"); + let gauntlet = read_to_string(readmes.join("review-gauntlet/README.md")).unwrap(); + for edge in [ + "mcr & madv & msec & mpb --> rgate[\"retry_gate", + "rgate --> gate[\"verdict_gate", + "rgate -. \"_next: retry faulted lanes\" .-> build", + ] { + assert!( + gauntlet.contains(edge), + "review-gauntlet README mermaid must route the lane maps through the retry gate: {edge}" + ); + } + assert!(gauntlet.contains("the retry gate on equal-length paths")); + for claim in [ + "when any lane or the pipeline was incomplete", + "Whenever at least one lane (or the pipeline) could not complete", + "GAUNTLET_REVIEW_INCOMPLETE: [, ]", + "`pipeline` pseudo-lane", + "the named lanes never completed", + "never narrows selection", + "up to 3 attempts", + "MAX_RETRY_ELAPSED_SECS", + "preserves the prior `lanes_summary`", + "append to an existing `signals_error`", + ] { + assert!( + gauntlet.contains(claim), + "review-gauntlet README must state the retry/incomplete contract: {claim}" + ); + } + for stale in [ + "on fault-only BLOCKED", + "there is nothing to fix in the code yet", + "3h whole-gauntlet", + ] { + assert!( + !gauntlet.contains(stale), + "review-gauntlet README must not carry the stale claim: {stale}" + ); + } + } + // The spawner's own `max_agent_depth` is checked against the child's // absolute depth (root = 0), so every hop along a chain needs its own // limit >= child depth; +1 leaves one level of headroom. @@ -10916,6 +10960,10 @@ mod tests { .contains("verdict gate error"), "the internal error must be named in the report: {out}" ); + assert_eq!( + out["gauntlet_incomplete_line"], "\nGAUNTLET_REVIEW_INCOMPLETE: pipeline", + "a crashed gate is an incomplete review, never a code finding: {out}" + ); } #[test] @@ -11105,6 +11153,269 @@ mod tests { ); } + #[test] + fn gauntlet_verdict_gate_attempt_counts_on_faulted_rows() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + let degraded = "# Code Review Summary\n\n**Verdict: NEEDS-HUMAN** — pipeline fault(s) recorded — degraded run; always-human trigger(s) fired\n\n## Walkthrough\n(no walkthrough provided)\n\n---\n*Reviewed 3 files, found 0 critical, 0 warnings, 0 suggestions, 0 nitpicks (0 deferred by quality bar)*"; + let state = json!({ + "lane_attempts": {"adversary": 3, "code-review": 2}, + "code_review_results": [degraded], + "adversary_results": [GAUNTLET_ADVERSARY_FAULT], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| adversary | BLOCKED | PIPELINE-FAULT (lane failed ×3) |"), + "a re-run faulted lane must show its attempt count: {report}" + ); + assert!( + report.contains("| code-review | BLOCKED | PIPELINE-FAULT (degraded lane ×2) |"), + "a re-run degraded lane must show its attempt count: {report}" + ); + + let state = json!({ + "lane_attempts": {"code-review": 2}, + "code_review_results": ["# Code Review Summary\n\n**Verdict: NEEDS-HUMAN** — 2 🔴 CRITICAL finding(s); pipeline fault(s) recorded — degraded run\n\n---\n*Reviewed 3 files, found 2 critical, 0 warnings, 0 suggestions, 0 nitpicks (0 deferred by quality bar)*"], + "adversary_results": [], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| code-review | BLOCKED | PIPELINE-FAULT (degraded lane ×2); 2 🔴 |"), + "the attempt count must sit inside the degraded detail, before the reds: {report}" + ); + + let state = json!({ + "lane_attempts": {"adversary": 1}, + "code_review_results": [], + "adversary_results": [GAUNTLET_ADVERSARY_FAULT], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| adversary | BLOCKED | PIPELINE-FAULT (lane failed) |"), + "a single attempt must render without a suffix: {report}" + ); + + let state = json!({ + "lane_attempts": {"code-review": 1}, + "code_review_results": [degraded], + "adversary_results": [], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| code-review | BLOCKED | PIPELINE-FAULT (degraded lane) |"), + "a single-attempt degraded lane must render without a suffix: {report}" + ); + + let state = json!({ + "lane_attempts": {"code-review": 1}, + "code_review_results": ["# Code Review Summary\n\n**Verdict: NEEDS-HUMAN** — 2 🔴 CRITICAL finding(s); pipeline fault(s) recorded — degraded run\n\n---\n*Reviewed 3 files, found 2 critical, 0 warnings, 0 suggestions, 0 nitpicks (0 deferred by quality bar)*"], + "adversary_results": [], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| code-review | BLOCKED | PIPELINE-FAULT (degraded lane); 2 🔴 |"), + "a single-attempt degraded lane with reds must render without a suffix: {report}" + ); + + let state = json!({ + "lane_attempts": {"adversary": 2}, + "code_review_results": [], + "adversary_results": ["ADVERSARIAL_REVIEW: DIVERGES"], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| adversary | BLOCKED | DIVERGES |"), + "a real verdict must render its own detail: {report}" + ); + assert!( + !report.contains("×2"), + "the attempt suffix must stay on PIPELINE-FAULT rows only: {report}" + ); + } + + #[test] + fn gauntlet_verdict_gate_incomplete_line_names_exhausted_lanes() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + let state = json!({ + "review_incomplete": ["probe", "adversary"], + "lane_attempts": {"adversary": 3, "probe": 3}, + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": [GAUNTLET_ADVERSARY_FAULT], + "security_results": [], + "probe_results": ["PIPELINE-FAULT: probe lane failed after retries — Agent node failed: boom"], + "signals_error": "git diff unavailable" + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + assert_eq!( + out["gauntlet_incomplete_line"], "\nGAUNTLET_REVIEW_INCOMPLETE: adversary, probe", + "the machine line must name every exhausted lane, sorted: {out}" + ); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains( + "## Review incomplete\n- adversary: no completed review after 3 attempt(s)\n- probe: no completed review after 3 attempt(s)\n\n## Full lane reports" + ), + "the section must list the exhausted lanes sorted, with attempt counts, directly above the full lane reports: {report}" + ); + assert!( + report.find("> git diff unavailable").unwrap() + < report.find("## Review incomplete").unwrap(), + "the section must follow the signals_error blockquote: {report}" + ); + + // No lane_attempts at all (retry_gate never ran, or crashed before + // recording): the section still names the lane, without a count. + let state = json!({ + "review_incomplete": ["probe"], + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": ["PIPELINE-FAULT: probe lane failed after retries — Agent node failed: boom"] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!( + out["gauntlet_incomplete_line"], "\nGAUNTLET_REVIEW_INCOMPLETE: probe", + "{out}" + ); + assert!( + out["gauntlet_report"].as_str().unwrap().contains( + "## Review incomplete\n- probe: no completed review\n\n## Full lane reports" + ), + "a lane with no recorded attempts must render without a count: {out}" + ); + + // The incomplete line must coexist with real findings, never + // suppress them. + let state = json!({ + "review_incomplete": ["adversary"], + "lane_attempts": {"adversary": 3}, + "code_review_results": ["# Code Review Summary\n\n**Verdict: NEEDS-HUMAN** — 2 🔴 CRITICAL finding(s)\n\n---\n*Reviewed 3 files, found 2 critical, 0 warnings, 0 suggestions, 0 nitpicks (0 deferred by quality bar)*"], + "adversary_results": [GAUNTLET_ADVERSARY_FAULT], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + assert_eq!( + out["gauntlet_incomplete_line"], "\nGAUNTLET_REVIEW_INCOMPLETE: adversary", + "{out}" + ); + assert!( + out["gauntlet_report"] + .as_str() + .unwrap() + .contains("code-review: 2 🔴 CRITICAL finding(s) — fix before claiming done"), + "a real finding must still be named as a blocker alongside the incomplete line: {out}" + ); + + let state = json!({ + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "PASS", "{out}"); + assert_eq!( + out["gauntlet_incomplete_line"], "", + "a complete review must emit an empty line, not a missing key: {out}" + ); + assert!( + !out["gauntlet_report"] + .as_str() + .unwrap() + .contains("## Review incomplete"), + "a complete review must not render the section: {out}" + ); + } + + #[test] + fn gauntlet_verdict_gate_pipeline_incomplete_alongside_real_lanes() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // default_lanes' own crash guard fired (the ordinary degraded fallback + // emits no PIPELINE-FAULT and is not incomplete): real lanes ran to a + // verdict, but the selection stage faulted, so the review is incomplete + // at the pipeline level even though no lane row is faulted. + let fault = run_gauntlet_script( + "default_lanes.py", + &json!({"forced_lanes": 42, "touches_auth": true}), + )["signals_error"] + .clone(); + let fault = fault + .as_str() + .expect("default_lanes crash guard must fire on a non-list forced_lanes"); + assert!( + fault.starts_with("PIPELINE-FAULT: lane-selection fallback crashed"), + "{fault}" + ); + let state = json!({ + "signals_error": fault, + "code_review_results": ["**Verdict: MERGE-READY**"], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": ["SECURITY_REVIEW: PASS"], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + assert_eq!( + out["gauntlet_incomplete_line"], "\nGAUNTLET_REVIEW_INCOMPLETE: pipeline", + "a pipeline-level fault must name the pipeline pseudo-lane: {out}" + ); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| code-review | GREEN | MERGE-READY |"), + "real lane verdicts must still render: {report}" + ); + assert!( + report.contains( + "## Review incomplete\n- pipeline: PIPELINE-FAULT: lane-selection fallback crashed" + ), + "the section must carry the pipeline fault text: {report}" + ); + + let state = json!({ + "signals_error": fault, + "review_incomplete": ["adversary"], + "lane_attempts": {"adversary": 3}, + "code_review_results": ["**Verdict: MERGE-READY**"], + "adversary_results": [GAUNTLET_ADVERSARY_FAULT], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!( + out["gauntlet_incomplete_line"], "\nGAUNTLET_REVIEW_INCOMPLETE: adversary, pipeline", + "exhausted lanes and the pipeline pseudo-lane must sort together: {out}" + ); + } + #[test] fn gauntlet_verdict_gate_non_string_lane_result_blocks() { if !cmd_available("python3") { @@ -11464,6 +11775,10 @@ mod tests { report.contains("provider exploded"), "the failure detail must survive into the report: {report}" ); + assert_eq!( + out["gauntlet_incomplete_line"], "\nGAUNTLET_REVIEW_INCOMPLETE: pipeline", + "a dead parse stage is an incomplete review under the pipeline pseudo-lane: {out}" + ); } #[test] @@ -11916,6 +12231,46 @@ mod tests { assert_eq!(out["review_incomplete"], json!(["adversary"]), "{out}"); } + #[test] + fn gauntlet_retry_gate_missing_report_becomes_synthetic_fault() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // The map ran an item but collected nothing: without a synthetic + // marker the lane would have no report to stash or restore and the + // verdict gate would render it SKIPPED. + let marker = "PIPELINE-FAULT: adversary lane produced no report"; + let out = run_gauntlet_script( + "retry_gate.py", + &json!({ + "adversary_items": [{"lane": "adversary"}], + "adversary_results": [], + "gauntlet_started_at": now_secs(), + }), + ); + assert_eq!(out["adversary_results"], json!([marker]), "{out}"); + assert_eq!(out["kept_results"]["adversary"], marker, "{out}"); + assert_eq!(out["retry_lanes"], json!(["adversary"]), "{out}"); + assert_eq!(out["_next"], "build_items", "{out}"); + assert_eq!(out["lane_attempts"]["adversary"], 1, "{out}"); + + let verdict = run_gauntlet_script( + "verdict_gate.py", + &json!({ + "code_review_results": [], + "adversary_results": out["adversary_results"], + "security_results": [], + "probe_results": [] + }), + ); + let report = verdict["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| adversary | BLOCKED | PIPELINE-FAULT (lane failed) |"), + "a lane that ran but produced nothing must block, never SKIPPED: {report}" + ); + } + #[test] fn gauntlet_retry_gate_crash_falls_through_without_retry() { if !cmd_available("python3") { @@ -11950,6 +12305,63 @@ mod tests { ); } + #[test] + fn gauntlet_pipeline_crash_guards_name_pipeline_incomplete() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + let out = run_gauntlet_script("build_items.py", &json!({"forced_lanes": 42})); + let gate = run_gauntlet_script( + "verdict_gate.py", + &json!({ + "code_review_results": [], + "adversary_results": [], + "security_results": [], + "probe_results": [], + "signals_error": out["signals_error"], + }), + ); + assert_eq!(gate["gauntlet_verdict"], "BLOCKED", "{gate}"); + assert_eq!( + gate["gauntlet_incomplete_line"], "\nGAUNTLET_REVIEW_INCOMPLETE: pipeline", + "an all-SKIPPED gate downstream of a builder crash must name the pipeline: {gate}" + ); + + let out = run_gauntlet_script( + "retry_gate.py", + &json!({ + "lane_attempts": 42, + "adversary_items": [{"lane": "adversary"}], + "adversary_results": ["x"], + "signals_error": "git diff unavailable", + }), + ); + let crashed = out["signals_error"] + .as_str() + .expect("retry_gate crash guard must record signals_error"); + assert!( + crashed.contains("PIPELINE-FAULT: retry gate crashed"), + "the pipeline name below must come from the crash guard, not the seed: {out}" + ); + let gate = run_gauntlet_script( + "verdict_gate.py", + &json!({ + "code_review_results": [], + "adversary_results": [], + "security_results": [], + "probe_results": [], + "signals_error": out["signals_error"], + "review_incomplete": out["review_incomplete"], + }), + ); + assert_eq!(gate["gauntlet_verdict"], "BLOCKED", "{gate}"); + assert_eq!( + gate["gauntlet_incomplete_line"], "\nGAUNTLET_REVIEW_INCOMPLETE: pipeline", + "the verdict gate must name the pipeline after a retry-gate crash: {gate}" + ); + } + /// retry_gate.py decides "completed" with the same sentinel regexes /// verdict_gate.py parses; a drift would re-run lanes the verdict gate /// accepts, or accept lanes it cannot read. @@ -12247,6 +12659,32 @@ mod tests { assert!(script.exists(), "{} must exist", script.display()); } + #[test] + fn gauntlet_done_template_carries_incomplete_line() { + use crate::graph::NodeType; + let graph = load_bundled_graph("review-gauntlet"); + let NodeType::End(done) = &graph.get_node("done").unwrap().node_type else { + panic!("done must be an end node") + }; + assert!( + done.output + .contains("GAUNTLET: {{gauntlet_verdict}}{{gauntlet_incomplete_line}}"), + "the incomplete line must follow the verdict immediately: {}", + done.output + ); + assert!( + done.output + .contains("{{gauntlet_incomplete_line}}\n\n{{gauntlet_report}}"), + "the report must follow the incomplete line after one blank line: {}", + done.output + ); + assert_eq!( + graph.initial_state.get("gauntlet_incomplete_line"), + Some(&json!("")), + "initial_state must declare the incomplete line so the template key resolves" + ); + } + /// MAX_RETRY_ELAPSED_SECS in retry_gate.py and settings.timeout in /// graph.yaml live in different files. A re-run admitted at the very end /// of the retry budget can still burn a full lane envelope (max_attempts From f18c436e2c7ebc3786d8d1cce8afa2688f5fb14d Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 14 Sep 2026 03:58:56 -0600 Subject: [PATCH 06/15] =?UTF-8?q?fix(TASK-032):=20enforce=20incomplete?= =?UTF-8?q?=E2=87=92BLOCKED=20structurally;=20anchor=20adversary=20degrade?= =?UTF-8?q?d=20detection?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- assets/agents/review-gauntlet/README.md | 18 +- .../review-gauntlet/scripts/retry_gate.py | 67 +- .../review-gauntlet/scripts/verdict_gate.py | 112 ++- src/config/request_context.rs | 715 +++++++++++++++++- 4 files changed, 882 insertions(+), 30 deletions(-) diff --git a/assets/agents/review-gauntlet/README.md b/assets/agents/review-gauntlet/README.md index d4aae09..86afa34 100644 --- a/assets/agents/review-gauntlet/README.md +++ b/assets/agents/review-gauntlet/README.md @@ -57,7 +57,11 @@ those rules are *structure*: sentinel is a lane FAILURE, never a pass; the critical count from the code-review report's summary line (raw 🔴 count if absent) blocks regardless of its verdict line; a code-review lane whose own verdict line - records a pipeline fault blocks (a degraded review is never a pass); + records a pipeline fault blocks (a degraded review is never a pass); the + adversary's own degraded-run header (the `Criteria: …` line directly under + its sentinel recording a pipeline fault, died criterion checks, or its + verdict script's crash stub) blocks as a pipeline fault, + not as a DIVERGES finding; `NEEDS-HUMAN` without 🔴 passes but is surfaced under *Human attention required*; probe `INCONCLUSIVE` blocks with an environment note — it is never treated as PASS or FAIL. @@ -86,7 +90,8 @@ those rules are *structure*: only lanes that **faulted** — a missing report (stood in for by a synthetic `PIPELINE-FAULT: lane produced no report` marker so it never renders as SKIPPED), a non-string report, the `lane_fault` marker, a sentinel-less -report, or a nested graph's own degraded-run wording. A real verdict +report, or a nested graph's own degraded-run wording (the code-review verdict +line / the adversary's header line directly under its sentinel). A real verdict (DIVERGES, FAIL, INCONCLUSIVE, NEEDS-HUMAN, …) is never re-run: the review's judgment stands. Each lane gets up to 3 attempts, all inside a wall-clock budget (`MAX_RETRY_ELAPSED_SECS`) sized to fit the graph timeout. @@ -105,6 +110,13 @@ budget (`MAX_RETRY_ELAPSED_SECS`) sized to fit the graph timeout. guards append to an existing `signals_error` rather than overwrite it, and `build_items.py` preserves the prior `lanes_summary`. +A non-empty `GAUNTLET_REVIEW_INCOMPLETE:` line structurally implies +`GAUNTLET: BLOCKED`: the verdict gate forces every lane named in +`review_incomplete` to a BLOCKED row (`review incomplete after N attempt(s)`) +and a matching blocker, so a restored earlier report can never turn an +incomplete review into a pass — and malformed `review_incomplete` bookkeeping +is itself recorded as a `pipeline` fault, never dropped. + Whenever at least one lane (or the pipeline) could not complete, the first line reads `GAUNTLET: BLOCKED` followed immediately by `GAUNTLET_REVIEW_INCOMPLETE: [, ]`. The names are the exhausted @@ -124,7 +136,7 @@ still appear under `## Blockers` and must be fixed as usual. | Lane | Sentinel | Blocks on | |------|----------|-----------| | code-reviewer | `**Verdict: MERGE-READY \| NEEDS-HUMAN**` | critical count from the report's summary line (raw 🔴 count if absent); missing sentinel; own verdict line records a pipeline fault | -| adversary | `ADVERSARIAL_REVIEW: CONFORMS \| DIVERGES` | DIVERGES; missing sentinel | +| adversary | `ADVERSARIAL_REVIEW: CONFORMS \| DIVERGES` | DIVERGES; own degraded-run header (pipeline fault, died criterion checks, or its verdict script's crash stub); missing sentinel | | security-reviewer | `SECURITY_REVIEW: PASS \| FAIL` | FAIL; missing sentinel | | probe | `USAGE_PROBE: PASS \| FAIL \| INCONCLUSIVE` | FAIL; INCONCLUSIVE (environment note); missing sentinel | diff --git a/assets/agents/review-gauntlet/scripts/retry_gate.py b/assets/agents/review-gauntlet/scripts/retry_gate.py index 45f1ecd..8346a56 100644 --- a/assets/agents/review-gauntlet/scripts/retry_gate.py +++ b/assets/agents/review-gauntlet/scripts/retry_gate.py @@ -19,6 +19,9 @@ hard rules: - a faulted lane is re-run only while it has attempts left AND the gauntlet is inside its wall-clock budget; otherwise it is recorded in review_incomplete and its fault flows to the verdict gate as-is; + - malformed review_incomplete bookkeeping (not a list, or non-str + entries) keeps its str entries and records a PIPELINE-FAULT in + signals_error — never silently dropped, never split into lane names; - the gate itself must never retry by accident: any internal error records a PIPELINE-FAULT in signals_error and falls through to the verdict gate, which blocks on it. @@ -57,6 +60,15 @@ SENTINELS = { # instead of overwriting it, and report the lanes it had already declined. crash_context = {"signals_error": "", "review_incomplete": []} +# Duplicated verbatim in the sibling gate script — the two copies must stay identical. +ADVERSARY_DEGRADED_HEADER = re.compile( + r"Criteria: .*(" + r"degraded run: pipeline fault recorded" + r"|degraded run: \d+ criterion check\(s\) died" + r"|verdict computation error:" + r")" +) + def load_state(): if path := os.environ.get("GRAPH_STATE_FILE"): @@ -65,6 +77,18 @@ def load_state(): return json.loads(os.environ.get("GRAPH_STATE", "{}")) +def adversary_degraded(report): + # The adversary's verdict.py emits its degraded-run header on the line + # directly under the FIRST sentinel; anchoring there keeps a report that + # merely quotes the wording later (observations, complaints) a real verdict. + lines = report.splitlines() + for i, ln in enumerate(lines): + if re.search(r"ADVERSARIAL_REVIEW:\s*(CONFORMS|DIVERGES)", ln): + nxt = lines[i + 1] if i + 1 < len(lines) else "" + return bool(ADVERSARY_DEGRADED_HEADER.match(nxt)) + return False + + def nested_degraded(lane, report): # The nested graphs complete with a sentinel even when a stage inside # them died; their own degraded-run wording marks a partial review, @@ -80,13 +104,7 @@ def nested_degraded(lane, report): ) ) if lane == "adversary": - return bool( - re.search( - r"^Criteria: .*degraded run: pipeline fault recorded", - report, - re.MULTILINE, - ) - ) + return adversary_degraded(report) return False @@ -100,15 +118,38 @@ def is_faulted(lane, report): return nested_degraded(lane, report) +def attempt_count(attempts, lane): + try: + return int(attempts.get(lane) or 0) + except (TypeError, ValueError): + return 0 + + def main(): state = load_state() prior = state.get("signals_error") - crash_context["signals_error"] = prior if isinstance(prior, str) else "" - incomplete = list(state.get("review_incomplete") or []) - crash_context["review_incomplete"] = incomplete - attempts = dict(state.get("lane_attempts") or {}) - kept = dict(state.get("kept_results") or {}) + signals_error = prior if isinstance(prior, str) else "" + crash_context["signals_error"] = signals_error out = {} + declared = state.get("review_incomplete") + if isinstance(declared, list): + incomplete = [name for name in declared if isinstance(name, str)] + malformed = len(incomplete) != len(declared) + else: + incomplete = [] + malformed = declared is not None + if malformed: + fault = ( + "PIPELINE-FAULT: malformed review_incomplete bookkeeping " + f"({repr(declared)[:80]})" + ) + signals_error = f"{signals_error}; {fault}" if signals_error else fault + crash_context["signals_error"] = signals_error + out["signals_error"] = signals_error + crash_context["review_incomplete"] = incomplete + raw_attempts = state.get("lane_attempts") + attempts = dict(raw_attempts) if isinstance(raw_attempts, dict) else {} + kept = dict(state.get("kept_results") or {}) faulted = [] for lane, (items_key, results_key) in LANES.items(): @@ -124,7 +165,7 @@ def main(): kept[lane] = report if is_faulted(lane, report): faulted.append(lane) - attempts[lane] = int(attempts.get(lane) or 0) + 1 + attempts[lane] = attempt_count(attempts, lane) + 1 # A missing/zero stamp reads as an ancient start and declines every # retry — the safe direction when the clock's origin is unknown. diff --git a/assets/agents/review-gauntlet/scripts/verdict_gate.py b/assets/agents/review-gauntlet/scripts/verdict_gate.py index cbeab90..5f496bc 100644 --- a/assets/agents/review-gauntlet/scripts/verdict_gate.py +++ b/assets/agents/review-gauntlet/scripts/verdict_gate.py @@ -17,6 +17,11 @@ re-reads, no judgment. The gate's hard rules: (code-reviewer's verdict.py / render.py degraded-run wording) blocks — the nested graph completed, but with a dead domain lane, verifier, or synthesis behind it, so its critical count is not trustworthy; + - an adversary lane whose OWN header line (the line directly under its + sentinel) records a degraded run — a pipeline fault, died criterion + checks, or its verdict script's crash stub — blocks as a PIPELINE-FAULT, + not as a DIVERGES finding: the nested graph completed with a dead stage + behind it, so its verdict is not a judgment on the code; - a non-zero critical count in the code-review report blocks regardless of the lane's own verdict line — derived from render.py's deterministic summary line (`*Reviewed N files, found X critical, …*`); the raw 🔴 @@ -31,7 +36,12 @@ re-reads, no judgment. The gate's hard rules: - an incomplete review — lanes retry_gate gave up on (review_incomplete) or a pipeline-level fault (reported under the `pipeline` pseudo-lane) — emits the GAUNTLET_REVIEW_INCOMPLETE: machine line and a "Review - incomplete" section: that is a fault to re-run, never a code finding; + incomplete" section: that is a fault to re-run, never a code finding. + Every lane named in review_incomplete is forced to a BLOCKED row, so a + restored earlier report can never turn an incomplete review into a + pass: a non-empty machine line structurally implies GAUNTLET: BLOCKED. + Malformed review_incomplete bookkeeping (not a list, or non-str + entries) is itself recorded as a `pipeline` fault, never dropped; - the gate itself must never crash into a silent pass: any internal error emits GAUNTLET: BLOCKED naming the error, with the pipeline marked incomplete. @@ -41,6 +51,16 @@ import json import os import re +# Duplicated verbatim in the sibling gate script — the two copies must stay identical. +ADVERSARY_DEGRADED_HEADER = re.compile( + r"Criteria: .*(" + r"degraded run: pipeline fault recorded" + r"|degraded run: \d+ criterion check\(s\) died" + r"|verdict computation error:" + r")" +) + + def load_state(): if path := os.environ.get("GRAPH_STATE_FILE"): with open(path) as f: @@ -48,11 +68,26 @@ def load_state(): return json.loads(os.environ.get("GRAPH_STATE", "{}")) +def adversary_degraded(report): + # The adversary's verdict.py emits its degraded-run header on the line + # directly under the FIRST sentinel; anchoring there keeps a report that + # merely quotes the wording later (observations, complaints) a real verdict. + lines = report.splitlines() + for i, ln in enumerate(lines): + if re.search(r"ADVERSARIAL_REVIEW:\s*(CONFORMS|DIVERGES)", ln): + nxt = lines[i + 1] if i + 1 < len(lines) else "" + return bool(ADVERSARY_DEGRADED_HEADER.match(nxt)) + return False + + def main(): state = load_state() - blockers, attention, lane_lines, reports = [], [], [], [] + blockers, attention, reports = [], [], [] + rows = {} malformed = set() - lane_attempts = state.get("lane_attempts") or {} + lane_attempts = state.get("lane_attempts") + if not isinstance(lane_attempts, dict): + lane_attempts = {} def lane_report(key): arr = state.get(key) or [] @@ -65,10 +100,16 @@ def main(): return json.dumps(item) def record(lane, status, detail=""): - lane_lines.append(f"| {lane} | {status} | {detail} |") + rows[lane] = (status, detail) + + def attempt_count(lane): + try: + return int(lane_attempts.get(lane) or 0) + except (TypeError, ValueError): + return 0 def attempts_suffix(lane): - n = int(lane_attempts.get(lane) or 0) + n = attempt_count(lane) return f" ×{n}" if n > 1 else "" def fault_excerpt(report): @@ -178,7 +219,17 @@ def main(): else: reports.append(("adversary", adv)) m = re.search(r"ADVERSARIAL_REVIEW:\s*(CONFORMS|DIVERGES)", adv) - if m and m.group(1) == "CONFORMS": + if adversary_degraded(adv): + blockers.append( + "adversary: PIPELINE-FAULT — degraded run, pipeline fault recorded " + "inside the adversary; a degraded lane is never a pass" + ) + record( + "adversary", + "BLOCKED", + f"PIPELINE-FAULT (degraded lane{attempts_suffix('adversary')})", + ) + elif m and m.group(1) == "CONFORMS": record("adversary", "GREEN", "CONFORMS") elif m: blockers.append( @@ -244,21 +295,57 @@ def main(): ) record("probe", "BLOCKED", "missing verdict sentinel") + # retry_gate restores an exhausted lane's LAST report, which may be an + # earlier clean verdict — an incomplete lane must block regardless of + # what its row parsed to. + declared = state.get("review_incomplete") + incomplete = set() + malformed_bookkeeping = None + if isinstance(declared, list): + incomplete = {name for name in declared if isinstance(name, str)} + if any(not isinstance(name, str) for name in declared): + malformed_bookkeeping = repr(declared)[:80] + elif declared is not None: + malformed_bookkeeping = repr(declared)[:80] + for lane in sorted(incomplete - {"pipeline"}): + if rows.get(lane, ("", ""))[0] == "BLOCKED": + continue + after = f" after {n} attempt(s)" if (n := attempt_count(lane)) else "" + blockers.append( + f"{lane}: review incomplete{after} — the lane never produced a usable verdict" + ) + record(lane, "BLOCKED", f"review incomplete{after}") + # A gauntlet-level fault (build_items crash, dead parse stage) means the # lanes above were never built — SKIPPED rows alone must not read as PASS. signals_error = state.get("signals_error") - incomplete = set(state.get("review_incomplete") or []) - if isinstance(signals_error, str) and "PIPELINE-FAULT:" in signals_error: + pipeline_fault = isinstance(signals_error, str) and "PIPELINE-FAULT:" in signals_error + if pipeline_fault: blockers.append(f"pipeline: {signals_error}") incomplete.add("pipeline") + if malformed_bookkeeping is not None: + blockers.append( + "pipeline: PIPELINE-FAULT — malformed review_incomplete bookkeeping " + f"({malformed_bookkeeping}); never a pass" + ) + incomplete.add("pipeline") + if "pipeline" in incomplete and not any(b.startswith("pipeline:") for b in blockers): + blockers.append( + "pipeline: review incomplete — a pipeline stage never completed " + "(no fault text recorded)" + ) - verdict = "BLOCKED" if blockers else "PASS" + verdict = "BLOCKED" if blockers or incomplete else "PASS" incomplete_lines = [] for name in sorted(incomplete): if name == "pipeline": - incomplete_lines.append(f"- pipeline: {signals_error}") - elif n := int(lane_attempts.get(name) or 0): + incomplete_lines.append( + f"- pipeline: {signals_error}" + if pipeline_fault + else "- pipeline: review incomplete (no fault text recorded)" + ) + elif n := attempt_count(name): incomplete_lines.append(f"- {name}: no completed review after {n} attempt(s)") else: incomplete_lines.append(f"- {name}: no completed review") @@ -269,7 +356,8 @@ def main(): ) md = "\n".join( - ["| lane | status | detail |", "|------|--------|--------|"] + lane_lines + ["| lane | status | detail |", "|------|--------|--------|"] + + [f"| {lane} | {status} | {detail} |" for lane, (status, detail) in rows.items()] ) md += "\n\n## Lane selection\n" + (state.get("lanes_summary") or "(none recorded)") if blockers: diff --git a/src/config/request_context.rs b/src/config/request_context.rs index 7261f8e..4262da2 100644 --- a/src/config/request_context.rs +++ b/src/config/request_context.rs @@ -8361,16 +8361,25 @@ mod tests { "MAX_RETRY_ELAPSED_SECS", "preserves the prior `lanes_summary`", "append to an existing `signals_error`", + "structurally implies", + "not as a DIVERGES finding", + "never dropped", + "| adversary | `ADVERSARIAL_REVIEW: CONFORMS \\| DIVERGES` | DIVERGES; own degraded-run header (pipeline fault, died criterion checks, or its verdict script's crash stub); missing sentinel |", ] { assert!( gauntlet.contains(claim), "review-gauntlet README must state the retry/incomplete contract: {claim}" ); } + assert!( + gauntlet.contains("never dropped.\n\nWhenever at least one lane"), + "review-gauntlet README must keep the incomplete-line contract in its own paragraph" + ); for stale in [ "on fault-only BLOCKED", "there is nothing to fix in the code yet", "3h whole-gauntlet", + "| DIVERGES; missing sentinel |", ] { assert!( !gauntlet.contains(stale), @@ -11416,6 +11425,352 @@ mod tests { ); } + #[test] + fn gauntlet_verdict_gate_incomplete_lane_forces_blocked() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // retry_gate restores an exhausted lane's LAST report, which can be + // an earlier clean verdict; the incomplete lane must block anyway. + let state = json!({ + "review_incomplete": ["adversary"], + "lane_attempts": {"adversary": 3}, + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + assert_eq!( + out["gauntlet_incomplete_line"], "\nGAUNTLET_REVIEW_INCOMPLETE: adversary", + "{out}" + ); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| adversary | BLOCKED | review incomplete after 3 attempt(s) |"), + "an incomplete lane must render as BLOCKED with its attempt count: {report}" + ); + assert!( + report.contains( + "adversary: review incomplete after 3 attempt(s) — the lane never produced a usable verdict" + ), + "an incomplete lane must be named as a blocker: {report}" + ); + assert!( + !report.contains("| adversary | GREEN | CONFORMS |"), + "a restored clean verdict must not render as GREEN for an incomplete lane: {report}" + ); + assert!( + !(out["gauntlet_verdict"] == "PASS" && out["gauntlet_incomplete_line"] != ""), + "PASS must never co-occur with an incomplete line: {out}" + ); + + let mut state = state; + state.as_object_mut().unwrap().remove("lane_attempts"); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| adversary | BLOCKED | review incomplete |"), + "no recorded attempts must render without a count: {report}" + ); + assert!( + report.contains( + "adversary: review incomplete — the lane never produced a usable verdict" + ), + "{report}" + ); + assert!( + !(out["gauntlet_verdict"] == "PASS" && out["gauntlet_incomplete_line"] != ""), + "PASS must never co-occur with an incomplete line: {out}" + ); + + state.as_object_mut().unwrap().remove("review_incomplete"); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "PASS", "{out}"); + assert_eq!(out["gauntlet_incomplete_line"], "", "{out}"); + assert!( + out["gauntlet_report"] + .as_str() + .unwrap() + .contains("| adversary | GREEN | CONFORMS |"), + "a complete clean lane must still render GREEN: {out}" + ); + assert!( + !(out["gauntlet_verdict"] == "PASS" && out["gauntlet_incomplete_line"] != ""), + "PASS must never co-occur with an incomplete line: {out}" + ); + + // A bare `pipeline` entry with no fault text must still block and + // must never render Python's None into the report. + let state = json!({ + "review_incomplete": ["pipeline"], + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + assert_eq!( + out["gauntlet_incomplete_line"], "\nGAUNTLET_REVIEW_INCOMPLETE: pipeline", + "{out}" + ); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("pipeline: review incomplete"), + "a pipeline entry without fault text must still be named as a blocker: {report}" + ); + assert!( + !report.contains("pipeline: None"), + "missing fault text must not render as None: {report}" + ); + + // An incomplete lane whose map collected nothing must not fall back + // to the SKIPPED row. + let state = json!({ + "review_incomplete": ["security"], + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| security | BLOCKED | review incomplete |"), + "an incomplete lane with no report must render BLOCKED: {report}" + ); + assert!( + !report.contains("| security | SKIPPED | not selected |"), + "an incomplete lane must never read as SKIPPED: {report}" + ); + } + + #[test] + fn gauntlet_verdict_gate_adversary_degraded_header_is_fault_not_finding() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + let state = json!({ + "lane_attempts": {"adversary": 2}, + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": [GAUNTLET_ADVERSARY_DEGRADED], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| adversary | BLOCKED | PIPELINE-FAULT (degraded lane ×2) |"), + "the adversary's own degraded-run header is a pipeline fault with its attempt count: {report}" + ); + assert!( + report.contains( + "adversary: PIPELINE-FAULT — degraded run, pipeline fault recorded inside the adversary" + ), + "{report}" + ); + assert!( + !report.contains("adversary: DIVERGES — the implementation does not conform"), + "a degraded run must not be reported as a DIVERGES finding: {report}" + ); + + let mut state = state; + state.as_object_mut().unwrap().remove("lane_attempts"); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert!( + out["gauntlet_report"] + .as_str() + .unwrap() + .contains("| adversary | BLOCKED | PIPELINE-FAULT (degraded lane) |"), + "a first-pass degraded run renders without a count: {out}" + ); + + // The predicate is anchored to the line under the sentinel: a clean + // report that quotes the wording later stays a real verdict. + state["adversary_results"] = json!([GAUNTLET_ADVERSARY_QUOTED_DEGRADED]); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "PASS", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| adversary | GREEN | CONFORMS |"), + "a quoted degraded-run phrase must not trip the fault predicate: {report}" + ); + assert!(!report.contains("adversary: PIPELINE-FAULT"), "{report}"); + + // Every degraded header the adversary's verdict.py can emit is a + // pipeline fault, not a DIVERGES finding. + for degraded in [ + GAUNTLET_ADVERSARY_DEGRADED_COUNTED, + GAUNTLET_ADVERSARY_DEGRADED_DIED, + GAUNTLET_ADVERSARY_CRASH_STUB, + ] { + state["adversary_results"] = json!([degraded]); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{degraded:?}: {out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| adversary | BLOCKED | PIPELINE-FAULT (degraded lane) |"), + "{degraded:?} must render as a degraded lane: {report}" + ); + assert!( + !report.contains("adversary: DIVERGES — the implementation does not conform"), + "{degraded:?} must not be reported as a DIVERGES finding: {report}" + ); + } + + // A plain DIVERGES header is a real finding. + state["adversary_results"] = json!([GAUNTLET_ADVERSARY_DIVERGES]); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| adversary | BLOCKED | DIVERGES |"), + "a real DIVERGES must stay a finding: {report}" + ); + assert!(!report.contains("adversary: PIPELINE-FAULT"), "{report}"); + } + + #[test] + fn gauntlet_verdict_gate_malformed_attempts_keeps_lane_rows() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // Malformed bookkeeping must not crash the gate into the error stub + // that loses every lane row: a non-dict lane_attempts only affects + // counts, while a non-list review_incomplete is itself a pipeline + // fault — a bare string is never adopted as a lane name. + let state = json!({ + "lane_attempts": 42, + "review_incomplete": "adversary", + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": [GAUNTLET_ADVERSARY_FAULT], + "security_results": [], + "probe_results": [], + "signals_error": "PIPELINE-FAULT: lane-selection fallback crashed — boom" + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + !report.contains("verdict gate error"), + "malformed bookkeeping must not reach the crash stub: {report}" + ); + assert!( + report.contains("| code-review | GREEN | MERGE-READY |"), + "{report}" + ); + assert!( + report.contains("| adversary | BLOCKED | PIPELINE-FAULT (lane failed) |"), + "unreadable attempts must render without a count: {report}" + ); + assert_eq!( + out["gauntlet_incomplete_line"], "\nGAUNTLET_REVIEW_INCOMPLETE: pipeline", + "a string-valued review_incomplete is a pipeline fault, not a lane name; the pipeline is named: {out}" + ); + + // Without any upstream fault text, the malformed bookkeeping alone + // must block as a pipeline fault. + let state = json!({ + "lane_attempts": 42, + "review_incomplete": "adversary", + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + assert_eq!( + out["gauntlet_incomplete_line"], "\nGAUNTLET_REVIEW_INCOMPLETE: pipeline", + "{out}" + ); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("pipeline: PIPELINE-FAULT — malformed review_incomplete bookkeeping"), + "malformed bookkeeping must be named as a pipeline fault: {report}" + ); + assert!( + report.contains("| code-review | GREEN | MERGE-READY |"), + "{report}" + ); + assert!(!report.contains("verdict gate error"), "{report}"); + + // A list with a non-str entry keeps its str lanes and still records + // the fault. + let state = json!({ + "review_incomplete": ["adversary", 7], + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + assert_eq!( + out["gauntlet_incomplete_line"], "\nGAUNTLET_REVIEW_INCOMPLETE: adversary, pipeline", + "{out}" + ); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| adversary | BLOCKED | review incomplete |"), + "the str lane entries must still force their rows: {report}" + ); + assert!( + report.contains("pipeline: PIPELINE-FAULT — malformed review_incomplete bookkeeping"), + "{report}" + ); + + // Per-lane values that are not ints count as zero: the row renders + // without a ×N suffix and the incomplete line without "after N". + let state = json!({ + "lane_attempts": {"adversary": "three", "code-review": [1]}, + "review_incomplete": ["adversary"], + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": [GAUNTLET_ADVERSARY_FAULT], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!(!report.contains("verdict gate error"), "{report}"); + assert!( + report.contains("| adversary | BLOCKED | PIPELINE-FAULT (lane failed) |"), + "a non-int per-lane count must render without a count: {report}" + ); + assert!(!report.contains('×'), "{report}"); + assert!( + report.contains("## Review incomplete\n- adversary: no completed review\n"), + "a non-int per-lane count must not render an attempt count: {report}" + ); + + let state = json!({ + "lane_attempts": {"security": "x"}, + "review_incomplete": ["security"], + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!(!report.contains("verdict gate error"), "{report}"); + assert!( + report.contains("| security | BLOCKED | review incomplete |"), + "{report}" + ); + assert!(!report.contains("review incomplete after"), "{report}"); + } + #[test] fn gauntlet_verdict_gate_non_string_lane_result_blocks() { if !cmd_available("python3") { @@ -11998,6 +12353,12 @@ mod tests { const GAUNTLET_MERGE_READY: &str = "# Review\n\n**Verdict: MERGE-READY**\n"; const GAUNTLET_ADVERSARY_FAULT: &str = "PIPELINE-FAULT: adversary lane failed after retries — Agent node failed: boom"; + const GAUNTLET_ADVERSARY_DEGRADED: &str = "ADVERSARIAL_REVIEW: DIVERGES\nCriteria: none verified — degraded run: pipeline fault recorded (fail-closed).\nComplaints:\n1. PIPELINE-FAULT: run_checks died"; + const GAUNTLET_ADVERSARY_DEGRADED_COUNTED: &str = "ADVERSARIAL_REVIEW: DIVERGES\nCriteria: 2/3 met, 0 partial, 1 unmet/diverged — degraded run: pipeline fault recorded (fail-closed).\nComplaints:\n1. PIPELINE-FAULT: run_checks died"; + const GAUNTLET_ADVERSARY_DEGRADED_DIED: &str = "ADVERSARIAL_REVIEW: DIVERGES\nCriteria: 2/3 met, 0 partial, 1 unmet/diverged — degraded run: 1 criterion check(s) died (fail-closed).\nComplaints:\n1. c3: check died — boom"; + const GAUNTLET_ADVERSARY_CRASH_STUB: &str = "ADVERSARIAL_REVIEW: DIVERGES\nCriteria: verdict computation error: boom — treat as failed, not as conforming."; + const GAUNTLET_ADVERSARY_DIVERGES: &str = "ADVERSARIAL_REVIEW: DIVERGES\nCriteria: 2/3 met, 0 partial, 1 unmet/diverged.\nComplaints:\n1. c3: nothing in the diff does Z"; + const GAUNTLET_ADVERSARY_QUOTED_DEGRADED: &str = "ADVERSARIAL_REVIEW: CONFORMS\nCriteria: 3/3 met, 0 partial, 0 unmet/diverged.\nComplaints:\n- none\n\nObservations:\n- an earlier run printed 'Criteria: none verified — degraded run: pipeline fault recorded' but this one is clean"; #[test] fn gauntlet_retry_gate_stashes_and_restores_reports() { @@ -12148,6 +12509,311 @@ mod tests { assert!(out.get("probe_results").is_none(), "{out}"); } + #[test] + fn gauntlet_retry_gate_adversary_degraded_predicate_is_anchored() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + for degraded in [ + GAUNTLET_ADVERSARY_DEGRADED, + GAUNTLET_ADVERSARY_DEGRADED_COUNTED, + GAUNTLET_ADVERSARY_DEGRADED_DIED, + GAUNTLET_ADVERSARY_CRASH_STUB, + ] { + let out = + run_gauntlet_script("retry_gate.py", &gauntlet_lane_ran("adversary", degraded)); + assert_eq!( + out["retry_lanes"], + json!(["adversary"]), + "the degraded-run header {degraded:?} under the sentinel is a fault: {out}" + ); + assert_eq!(out["_next"], "build_items", "{out}"); + } + + // Anchored to the line under the sentinel: a clean report quoting + // the wording later is a completed review. + let out = run_gauntlet_script( + "retry_gate.py", + &gauntlet_lane_ran("adversary", GAUNTLET_ADVERSARY_QUOTED_DEGRADED), + ); + assert_eq!( + out["retry_lanes"], + json!([]), + "a quoted degraded-run phrase must not trigger a re-run: {out}" + ); + assert!(out.get("_next").is_none(), "{out}"); + assert_eq!(out["review_incomplete"], json!([]), "{out}"); + + // A plain DIVERGES is a real finding, never re-run. + let out = run_gauntlet_script( + "retry_gate.py", + &gauntlet_lane_ran("adversary", GAUNTLET_ADVERSARY_DIVERGES), + ); + assert_eq!( + out["retry_lanes"], + json!([]), + "a real DIVERGES must not trigger a re-run: {out}" + ); + assert!(out.get("_next").is_none(), "{out}"); + } + + #[test] + fn gauntlet_retry_gate_malformed_bookkeeping_degrades() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // Malformed bookkeeping must not crash the gate: attempts are still + // counted, the faulted lane is still retried, a stashed report is + // still restored, and the malformation is recorded as a pipeline + // fault rather than adopted as lane names. + let mut state = gauntlet_lane_ran("adversary", GAUNTLET_ADVERSARY_FAULT); + state["lane_attempts"] = json!(42); + state["review_incomplete"] = json!("probe"); + state["kept_results"] = json!({"code-review": GAUNTLET_MERGE_READY}); + state["code_review_items"] = json!([]); + state["code_review_results"] = json!([]); + let out = run_gauntlet_script("retry_gate.py", &state); + let fault = out["signals_error"].as_str().unwrap_or_default(); + assert!( + !fault.starts_with("PIPELINE-FAULT: retry gate crashed"), + "malformed bookkeeping must not reach the crash stub: {out}" + ); + assert_eq!(out["retry_lanes"], json!(["adversary"]), "{out}"); + assert_eq!(out["lane_attempts"], json!({"adversary": 1}), "{out}"); + assert_eq!( + out["review_incomplete"], + json!([]), + "a bare string is never split or adopted as a lane name: {out}" + ); + assert_eq!( + out["code_review_results"], + json!([GAUNTLET_MERGE_READY]), + "the stashed report must still be restored: {out}" + ); + assert!( + fault.contains("malformed review_incomplete bookkeeping"), + "the malformation must be recorded as a fault: {out}" + ); + assert!( + fault.starts_with("PIPELINE-FAULT: malformed review_incomplete bookkeeping ("), + "{out}" + ); + + // Merged over the input state as the graph would, the verdict gate + // must block and name the pipeline. + let mut merged = state; + for (k, v) in out.as_object().unwrap() { + merged[k.as_str()] = v.clone(); + } + let verdict = run_gauntlet_script("verdict_gate.py", &merged); + assert_eq!(verdict["gauntlet_verdict"], "BLOCKED", "{verdict}"); + assert!( + verdict["gauntlet_incomplete_line"] + .as_str() + .unwrap() + .contains("pipeline"), + "{verdict}" + ); + + // Per-lane values that are not ints count as zero, so the lane that + // ran lands on its first attempt. + let mut state = gauntlet_lane_ran("adversary", GAUNTLET_ADVERSARY_FAULT); + state["lane_attempts"] = json!({"adversary": [1], "probe": "x"}); + let out = run_gauntlet_script("retry_gate.py", &state); + assert!( + !out["signals_error"] + .as_str() + .unwrap_or_default() + .contains("retry gate crashed"), + "{out}" + ); + assert_eq!(out["retry_lanes"], json!(["adversary"]), "{out}"); + assert_eq!(out["lane_attempts"]["adversary"], json!(1), "{out}"); + + // A list with a non-str entry keeps its str lanes and records the + // fault; the retried lane is not declined. + let mut state = gauntlet_lane_ran("adversary", GAUNTLET_ADVERSARY_FAULT); + state["review_incomplete"] = json!(["probe", 7]); + let out = run_gauntlet_script("retry_gate.py", &state); + assert_eq!(out["retry_lanes"], json!(["adversary"]), "{out}"); + assert_eq!(out["review_incomplete"], json!(["probe"]), "{out}"); + assert!( + out["signals_error"] + .as_str() + .unwrap_or_default() + .starts_with("PIPELINE-FAULT: malformed review_incomplete bookkeeping ("), + "{out}" + ); + + // The bookkeeping fault appends to a prior signals_error. + let mut state = gauntlet_lane_ran("adversary", GAUNTLET_ADVERSARY_FAULT); + state["review_incomplete"] = json!("probe"); + state["signals_error"] = json!("PIPELINE-FAULT: earlier"); + let out = run_gauntlet_script("retry_gate.py", &state); + assert!( + out["signals_error"].as_str().unwrap_or_default().starts_with( + "PIPELINE-FAULT: earlier; PIPELINE-FAULT: malformed review_incomplete bookkeeping (" + ), + "{out}" + ); + } + + #[test] + fn gauntlet_gates_block_real_adversary_degraded_verdict() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // Pins the sentinel→header adjacency both gates rely on against the + // adversary's real verdict.py output. + let met = json!({"id": "c1", "text": "does X", "status": "MET", + "evidence": "src/x.rs:1 + test src/x.rs:99", "complaint": ""}); + let degraded = run_adversary_script( + "verdict.py", + &json!({ + "pipeline_faults": ["PIPELINE-FAULT: run_checks died — boom"], + "crit_verdicts": [met], + "extra_complaints": [], + "observations": "", + "exec_results": "" + }), + ); + let degraded = degraded["adv_report"].as_str().unwrap(); + assert!( + degraded.contains("degraded run: pipeline fault recorded"), + "{degraded}" + ); + let out = run_gauntlet_script( + "verdict_gate.py", + &json!({ + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": [degraded], + "security_results": [], + "probe_results": [] + }), + ); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| adversary | BLOCKED | PIPELINE-FAULT (degraded lane) |"), + "the real degraded header must render as a pipeline fault: {report}" + ); + assert!( + !report.contains("adversary: DIVERGES — the implementation does not conform"), + "{report}" + ); + let out = run_gauntlet_script("retry_gate.py", &gauntlet_lane_ran("adversary", degraded)); + assert_eq!( + out["retry_lanes"], + json!(["adversary"]), + "the real degraded header must be re-run: {out}" + ); + + let diverges = run_adversary_script( + "verdict.py", + &json!({ + "pipeline_faults": [], + "crit_verdicts": [{"id": "c1", "text": "does Y", "status": "UNMET", + "evidence": "", "complaint": "nothing in the diff does Y"}], + "extra_complaints": [], + "observations": "", + "exec_results": "" + }), + ); + let diverges = diverges["adv_report"].as_str().unwrap(); + let out = run_gauntlet_script( + "verdict_gate.py", + &json!({ + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": [diverges], + "security_results": [], + "probe_results": [] + }), + ); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + assert!( + out["gauntlet_report"] + .as_str() + .unwrap() + .contains("| adversary | BLOCKED | DIVERGES |"), + "a real DIVERGES must stay a finding: {out}" + ); + let out = run_gauntlet_script("retry_gate.py", &gauntlet_lane_ran("adversary", diverges)); + assert_eq!( + out["retry_lanes"], + json!([]), + "a real DIVERGES must never be re-run: {out}" + ); + + // The other two degraded-header forms verdict.py emits: a criterion + // check that died, and the top-level crash stub. + let died = run_adversary_script( + "verdict.py", + &json!({ + "pipeline_faults": [], + "crit_verdicts": [ + met, + {"id": "c2", "text": "does Y", "status": "UNMET", + "evidence": "PIPELINE-FAULT: criterion check died — boom", + "complaint": "boom"} + ], + "extra_complaints": [], + "observations": "", + "exec_results": "" + }), + ); + let died = died["adv_report"].as_str().unwrap(); + assert!( + died.contains("degraded run: 1 criterion check(s) died"), + "{died}" + ); + let crashed = run_adversary_script( + "verdict.py", + &json!({ + "pipeline_faults": [], + "crit_verdicts": [{"status": "UNMET", "text": 123}], + "extra_complaints": [], + "observations": "", + "exec_results": "" + }), + ); + let crashed = crashed["adv_report"].as_str().unwrap(); + assert!( + crashed.contains("Criteria: verdict computation error:"), + "{crashed}" + ); + for degraded in [died, crashed] { + let out = run_gauntlet_script( + "verdict_gate.py", + &json!({ + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": [degraded], + "security_results": [], + "probe_results": [] + }), + ); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| adversary | BLOCKED | PIPELINE-FAULT (degraded lane) |"), + "the real degraded header must render as a pipeline fault: {report}" + ); + assert!( + !report.contains("adversary: DIVERGES — the implementation does not conform"), + "{report}" + ); + let out = + run_gauntlet_script("retry_gate.py", &gauntlet_lane_ran("adversary", degraded)); + assert_eq!( + out["retry_lanes"], + json!(["adversary"]), + "the real degraded header must be re-run: {out}" + ); + } + } + #[test] fn gauntlet_retry_gate_counts_attempts_per_lane_that_ran() { if !cmd_available("python3") { @@ -12281,7 +12947,7 @@ mod tests { let out = run_gauntlet_script( "retry_gate.py", &json!({ - "lane_attempts": 42, + "kept_results": 42, "adversary_items": [{"lane": "adversary"}], "adversary_results": ["x"], "signals_error": "PIPELINE-FAULT: earlier fault", @@ -12331,7 +12997,7 @@ mod tests { let out = run_gauntlet_script( "retry_gate.py", &json!({ - "lane_attempts": 42, + "kept_results": 42, "adversary_items": [{"lane": "adversary"}], "adversary_results": ["x"], "signals_error": "git diff unavailable", @@ -12409,10 +13075,55 @@ mod tests { retry.contains(degraded), "retry_gate.py must match the adversary's degraded-run wording {degraded:?}" ); + assert!( + verdict.contains(degraded), + "verdict_gate.py must match the adversary's degraded-run wording {degraded:?}" + ); assert!( adversary_verdict.contains(degraded), "assets/agents/adversary/scripts/verdict.py must still emit {degraded:?}" ); + for fragment in [ + "degraded run: pipeline fault recorded", + r"criterion check\(s\) died", + "verdict computation error:", + ] { + assert!( + retry.contains(fragment), + "retry_gate.py must match the adversary degraded header fragment {fragment:?}" + ); + assert!( + verdict.contains(fragment), + "verdict_gate.py must match the adversary degraded header fragment {fragment:?}" + ); + } + for emitted in ["criterion check(s) died", "verdict computation error:"] { + assert!( + adversary_verdict.contains(emitted), + "assets/agents/adversary/scripts/verdict.py must still emit {emitted:?}" + ); + } + + fn slice_between<'a>(src: &'a str, start: &str, end: &str) -> &'a str { + let from = src + .find(start) + .unwrap_or_else(|| panic!("missing {start:?}")); + let rest = &src[from..]; + let to = rest + .find(end) + .unwrap_or_else(|| panic!("missing {end:?} after {start:?}")); + &rest[..to + end.len()] + } + for (start, end) in [ + ("ADVERSARY_DEGRADED_HEADER = re.compile(", "\n)\n"), + ("def adversary_degraded(report):", " return False\n"), + ] { + assert_eq!( + slice_between(&retry, start, end), + slice_between(&verdict, start, end), + "the adversary degraded predicate must stay byte-identical in both gate scripts" + ); + } } /// build_items.py's LANE_KEYS and retry_gate.py's LANES name the same From 78e326039d145e593f8a1135961f5d4628ced5c6 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 14 Sep 2026 04:12:56 -0600 Subject: [PATCH 07/15] feat(TASK-030): callers escalate on GAUNTLET_REVIEW_INCOMPLETE instead of findings-routing --- assets/agents/architect/README.md | 2 +- assets/agents/architect/config.yaml | 25 +++++++++++++-- assets/agents/sisyphus/README.md | 2 +- assets/agents/sisyphus/config.yaml | 26 +++++++++++++--- src/config/request_context.rs | 48 +++++++++++++++++++++++++++++ 5 files changed, 93 insertions(+), 10 deletions(-) diff --git a/assets/agents/architect/README.md b/assets/agents/architect/README.md index 96df479..b36f360 100644 --- a/assets/agents/architect/README.md +++ b/assets/agents/architect/README.md @@ -177,7 +177,7 @@ By default (`parallel_tasks: 0`) tasks run **sequentially** on the single run br | [`explore`](../explore/README.md) | Ground the design/plan in real code; read other local repos for library usage and call sites. | | [`librarian`](../librarian/README.md) | External docs / OSS examples for unfamiliar libraries. | | [`whetstone`](../whetstone/README.md) | Continuous improvement — spawned once before the final report when the run addressed PR review feedback from outside its own lanes; converts review misses into user-approved suite edits / ledger entries. Never blocks the run. | -| [`review-gauntlet`](../review-gauntlet/README.md) | Preferred per-task conformance gate — one spawn running adversary (+ probe for consumer-facing tasks) with a machine-parsed GAUNTLET: PASS/BLOCKED verdict; direct adversary/probe spawns remain the fallback. | +| [`review-gauntlet`](../review-gauntlet/README.md) | Preferred per-task conformance gate — one spawn running adversary (+ probe for consumer-facing tasks) with a machine-parsed GAUNTLET: PASS/BLOCKED verdict; a `GAUNTLET_REVIEW_INCOMPLETE:` line is an infrastructure fault escalated to the user (retry / accept NEEDS-HUMAN / abort), never a divergence; direct adversary/probe spawns remain the fallback. | ## Related skills diff --git a/assets/agents/architect/config.yaml b/assets/agents/architect/config.yaml index 4058e04..ea64257 100644 --- a/assets/agents/architect/config.yaml +++ b/assets/agents/architect/config.yaml @@ -341,9 +341,28 @@ instructions: | prompt also carries pasted plan text from the repo under review, and whatever is declared here is executed with a shell — a trust boundary the prose must not cross. - It returns `GAUNTLET: PASS/BLOCKED`. On BLOCKED apply the same fix-and-rerun rules as below - (resume the SAME Sisyphus session with the findings verbatim, then a FRESH gauntlet run). - The direct `adversary`/probe spawns below are the FALLBACK when the gauntlet is not installed. + It returns `GAUNTLET: PASS/BLOCKED`. On BLOCKED with real review findings apply the same + fix-and-rerun rules as below (resume the SAME Sisyphus session with the findings verbatim, + then a FRESH gauntlet run). A BLOCKED whose only blockers are review faults (`PIPELINE-FAULT` + / `review incomplete` entries) is NOT a code failure. A `GAUNTLET_REVIEW_INCOMPLETE: [, + ]` line directly under `GAUNTLET: BLOCKED` means the review could NOT be completed: the + named lanes still faulted after 3 attempts, or the gauntlet pipeline itself faulted + (`pipeline`). This is an infrastructure fault, not a code finding. + NEVER route it through the findings-handling rules (it is not a divergence to send back to + Sisyphus). NEVER claim done while it stands. + Escalate to the user via `user__select` with exactly these three options: + - "Retry the review again" + - "Accept NEEDS-HUMAN: proceed without the review (recorded in the report / PR body)" + - "Abort this task" + Retry → a FRESH gauntlet run. Accept → close the task and record the skipped review (lane + names or `pipeline`) in the task's `log.md`, the Phase F digest, and the PR body. Abort → + leave the task `in-progress`, log the abort in its `log.md`, and stop to report. Findings AND + incomplete on the same output → fix the findings first, then re-run the gauntlet (the re-run + retries the lane) — escalate only if the line persists. Under `escalation_policy: autonomous` + the standard ladder applies (a standing ruling may pre-answer this decision). + The direct `adversary`/probe spawns below are the FALLBACK when the gauntlet is not installed; + on that path an adversary/probe result that is ONLY a `PIPELINE-FAULT` (no verdict on the + code) is likewise an infrastructure fault to retry, never a divergence. - **Spawn `adversary`** with the diff base and the criteria pasted in: ``` diff --git a/assets/agents/sisyphus/README.md b/assets/agents/sisyphus/README.md index 8c91715..6aeb602 100644 --- a/assets/agents/sisyphus/README.md +++ b/assets/agents/sisyphus/README.md @@ -47,7 +47,7 @@ Spawnable sub-agents (from `config.yaml`): - **[code-reviewer](../code-reviewer/README.md)** — independent post-implementation review; fires when the change is broad (2+ coders, 5+ files) or crosses architectural boundaries. - **[adversary](../adversary/README.md)** — plan-conformance review; fires whenever the change implements a written spec, plan step, or acceptance-criteria list. Orthogonal to `code-reviewer` — both can run. - **[security-reviewer](../security-reviewer/README.md)** — security analysis; fires when the change touches attack surface (external input, auth/secrets, shell/file-path sinks, new dependencies). Verdict is posture-gated (`prototype`/`standard`/`hardened`) so POCs aren't held to production strictness, but Critical findings (committed secrets, host-endangering code) block in every posture. Orthogonal to both other reviewers — all three can run. -- **[review-gauntlet](../review-gauntlet/README.md)** — preferred single entry point for the post-implementation review passes: a graph that computes diff signals deterministically, runs the applicable lanes (code-reviewer / adversary / security-reviewer / probe) in parallel, and machine-parses their verdict sentinels into GAUNTLET: PASS/BLOCKED — skipping a lane or misreading a verdict becomes structurally impossible. The reviewer bullets above describe the lanes it runs and remain the spawn fallback when it is not installed. +- **[review-gauntlet](../review-gauntlet/README.md)** — preferred single entry point for the post-implementation review passes: a graph that computes diff signals deterministically, runs the applicable lanes (code-reviewer / adversary / security-reviewer / probe) in parallel, and machine-parses their verdict sentinels into GAUNTLET: PASS/BLOCKED — skipping a lane or misreading a verdict becomes structurally impossible. A `GAUNTLET_REVIEW_INCOMPLETE:` line under BLOCKED is an infrastructure fault (a lane exhausted its retries, or the pipeline faulted) that Sisyphus escalates to the user — retry / accept NEEDS-HUMAN / abort — never a finding to fix. The reviewer bullets above describe the lanes it runs and remain the spawn fallback when it is not installed. - **[step-runner](../step-runner/README.md)** — graph agent that executes one step of a phased plan repo. Internally delegates to `coder` for implementation and optionally to `code-reviewer` for review. - **[whetstone](../whetstone/README.md)** — continuous improvement; spawned ONCE at the end of a run that addressed review feedback from outside Sisyphus's own lanes (humans, or a teammate's independent review agent — never its own reviewers' output). Classifies each piece of feedback CATCH/PARTIAL/MISS against what the run's own reviewers flagged and converts the misses into user-approved suite improvements (skill edits, ledger entries). Never blocks run completion. diff --git a/assets/agents/sisyphus/config.yaml b/assets/agents/sisyphus/config.yaml index d5c5f6f..c81c5f5 100644 --- a/assets/agents/sisyphus/config.yaml +++ b/assets/agents/sisyphus/config.yaml @@ -329,11 +329,27 @@ instructions: | parallel verdict-independent, and returns a machine-parsed `GAUNTLET: PASS` or `GAUNTLET: BLOCKED` with a lane table, blockers, attention items, and every lane's full report. - - `GAUNTLET: BLOCKED` blocks completion. Route each blocker through the findings-handling rules in - the reviewer sections below (🔴 → coder fix in the SAME session; DIVERGES → the adversary rules; - security FAIL → the security rules; probe INCONCLUSIVE → fix the ENVIRONMENT per the plan's - local-run recipe first — it is never a code verdict). Then spawn a FRESH gauntlet run; lane - selection re-runs from the new diff. + - `GAUNTLET: BLOCKED` blocks completion. Route each blocker that is a real review finding through + the findings-handling rules in the reviewer sections below (🔴 → coder fix in the SAME session; + DIVERGES → the adversary rules; security FAIL → the security rules; probe INCONCLUSIVE → fix the + ENVIRONMENT per the plan's local-run recipe first — it is never a code verdict). Then spawn a + FRESH gauntlet run; lane selection re-runs from the new diff. A BLOCKED whose only blockers are + review faults (`PIPELINE-FAULT` / `review incomplete` entries) is NOT a code failure — the next + rule owns it. + - `GAUNTLET_REVIEW_INCOMPLETE: [, ]` (the line directly under `GAUNTLET: BLOCKED`) + means the review could NOT be completed: the named lanes still faulted after 3 attempts, or the + gauntlet pipeline itself faulted (`pipeline`). This is an infrastructure fault, not a code + finding. NEVER route it through the findings-handling rules. NEVER claim done while it stands. + Escalate to the user via `user__select` with exactly these three options: + - "Retry the review again" + - "Accept NEEDS-HUMAN: proceed without the review (recorded in the report / PR body)" + - "Abort this task" + Retry → spawn a FRESH gauntlet run. Accept → finish the work and record the skipped review + (lane names or `pipeline`) in your final report and in the PR body when you author one. Abort → + stop; leave the code in a working state and report. Findings AND incomplete on the same output → + fix the findings first, then re-run the gauntlet (the re-run retries the lane) — escalate only if + the line persists. Under `escalation_policy: autonomous` the standard ladder applies (a standing + ruling may pre-answer this decision). - `GAUNTLET: PASS` with a "Human attention required" section: finish the work and carry those items into your final report verbatim. - The reviewer sections below remain authoritative for HANDLING each lane's findings, and are the diff --git a/src/config/request_context.rs b/src/config/request_context.rs index 4262da2..b184cbe 100644 --- a/src/config/request_context.rs +++ b/src/config/request_context.rs @@ -8223,6 +8223,54 @@ mod tests { ); } + // A gauntlet output carrying the review-incomplete line is an infrastructure + // fault: both callers must escalate it with the exact same three options + // instead of routing it through their findings-handling rules. + #[test] + #[serial] + fn bundled_assets_pin_review_incomplete_escalation() { + let _guard = TestConfigDirGuard::new(); + Agent::install_builtin_agents(false).unwrap(); + + const SHARED_ANCHORS: [&str; 7] = [ + "GAUNTLET_REVIEW_INCOMPLETE:", + "\"Retry the review again\"", + "\"Accept NEEDS-HUMAN: proceed without the review (recorded in the report / PR body)\"", + "\"Abort this task\"", + "NEVER route it through the findings-handling rules", + "NEVER claim done while it stands", + "is NOT a code failure", + ]; + + let mut configs = std::collections::HashMap::new(); + for name in ["architect", "sisyphus"] { + let config = + read_to_string(paths::agents_data_dir().join(name).join("config.yaml")).unwrap(); + for anchor in SHARED_ANCHORS { + assert!( + config.contains(anchor), + "{name} config lost review-incomplete escalation anchor: {anchor:?}" + ); + } + assert!( + config.contains("user__select"), + "{name} config must escalate review-incomplete via user__select" + ); + configs.insert(name, config); + } + + // The architect also owns the no-gauntlet fallback path: a + // fault-only adversary/probe result is retried, never treated as + // a divergence. + let architect = &configs["architect"]; + assert!( + architect.contains( + "an adversary/probe result that is ONLY a `PIPELINE-FAULT` (no verdict on the\n code) is likewise an infrastructure fault to retry, never a divergence." + ), + "architect config lost the fallback adversary/probe pipeline-fault sentence" + ); + } + // Shell commands the adversary's run_checks stage executes must be // declared as a variable, never as prompt prose an LLM would re-extract. #[test] From fcf67b07a7f317c6bf4eb853af0c29e0afd527b1 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 14 Sep 2026 04:21:42 -0600 Subject: [PATCH 08/15] test(TASK-030): pin placement, ladder clause, findings-first rule, README sync; reconcile gauntlet README caller guidance --- assets/agents/review-gauntlet/README.md | 16 +-- src/config/request_context.rs | 123 ++++++++++++++++++++++-- 2 files changed, 123 insertions(+), 16 deletions(-) diff --git a/assets/agents/review-gauntlet/README.md b/assets/agents/review-gauntlet/README.md index 86afa34..f58f865 100644 --- a/assets/agents/review-gauntlet/README.md +++ b/assets/agents/review-gauntlet/README.md @@ -173,12 +173,16 @@ The reply opens with `GAUNTLET: PASS` or `GAUNTLET: BLOCKED` (plus the `GAUNTLET_REVIEW_INCOMPLETE:` line when applicable), followed by a lane status table, the lane-selection reasons, blockers, attention items, and each lane's full report in a collapsible section. On BLOCKED with a -`GAUNTLET_REVIEW_INCOMPLETE:` line: the named lanes never completed — fix any -non-fault blockers listed under `## Blockers`, then spawn a **fresh** gauntlet -run (a fault is never a code finding). On BLOCKED without it: fix the findings -per your own findings-handling rules, then spawn a **fresh** gauntlet run. On -PASS with *Human attention required*: complete the work but carry those items -into your final report verbatim. +`GAUNTLET_REVIEW_INCOMPLETE:` line: the named lanes never completed — this is an +infrastructure fault, never a code finding. Fix any non-fault blockers listed +under `## Blockers` first (the re-run retries the lane); if the line persists, +escalate to the user with three options — retry the review again (a **fresh** +gauntlet run), accept NEEDS-HUMAN and proceed without that lane (recorded in the +report / PR body), or abort — never route it through your findings-handling +rules. On BLOCKED without it: fix the findings per your own findings-handling +rules, then spawn a **fresh** gauntlet run. On PASS with *Human attention +required*: complete the work but carry those items into your final report +verbatim. ## Notes diff --git a/src/config/request_context.rs b/src/config/request_context.rs index b184cbe..f48540b 100644 --- a/src/config/request_context.rs +++ b/src/config/request_context.rs @@ -8232,7 +8232,10 @@ mod tests { let _guard = TestConfigDirGuard::new(); Agent::install_builtin_agents(false).unwrap(); - const SHARED_ANCHORS: [&str; 7] = [ + // Every anchor is a single-line prefix so YAML re-wrapping cannot + // break the pin; clauses that wrap differently per config are split + // at the wrap point. + const SHARED_ANCHORS: [&str; 10] = [ "GAUNTLET_REVIEW_INCOMPLETE:", "\"Retry the review again\"", "\"Accept NEEDS-HUMAN: proceed without the review (recorded in the report / PR body)\"", @@ -8240,12 +8243,24 @@ mod tests { "NEVER route it through the findings-handling rules", "NEVER claim done while it stands", "is NOT a code failure", + // Findings AND incomplete on the same output: findings first. + "fix the findings first, then re-run the gauntlet (the re-run", + // The autonomous ladder still governs this escalation. + "Under `escalation_policy: autonomous`", + "the standard ladder applies (a standing", ]; + // The pin must never leak run-local provenance into the shipped + // assets: task IDs, plan branch names, or plan-repo paths. + const PROVENANCE_LEAKS: [&str; 3] = ["TASK-030", "PLAN-review-retry", "plans/tasks/"]; + let mut configs = std::collections::HashMap::new(); for name in ["architect", "sisyphus"] { - let config = - read_to_string(paths::agents_data_dir().join(name).join("config.yaml")).unwrap(); + // Normalize line endings: the ordering slices below span line + // breaks and Windows checkouts carry CRLF. + let config = read_to_string(paths::agents_data_dir().join(name).join("config.yaml")) + .unwrap() + .replace("\r\n", "\n"); for anchor in SHARED_ANCHORS { assert!( config.contains(anchor), @@ -8256,19 +8271,103 @@ mod tests { config.contains("user__select"), "{name} config must escalate review-incomplete via user__select" ); + for leak in PROVENANCE_LEAKS { + assert!( + !config.contains(leak), + "{name} config must not carry run-local provenance: {leak:?}" + ); + } configs.insert(name, config); } + // Sisyphus PLACEMENT: the incomplete rule is its own bullet directly + // between the BLOCKED and PASS handling bullets, so a follower reading + // the BLOCKED rule meets "the next rule owns it" immediately. + let sisyphus = &configs["sisyphus"]; + let find = |needle: &str| { + sisyphus + .find(needle) + .unwrap_or_else(|| panic!("sisyphus config lost placement anchor: {needle:?}")) + }; + let blocked = find("- `GAUNTLET: BLOCKED` blocks completion"); + let incomplete = + find("- `GAUNTLET_REVIEW_INCOMPLETE: [, ]` (the line directly under"); + let pass = find("- `GAUNTLET: PASS` with a"); + assert!( + blocked < incomplete && incomplete < pass, + "sisyphus config must order the gauntlet rules BLOCKED < REVIEW_INCOMPLETE < PASS \ + (got {blocked} / {incomplete} / {pass})" + ); + assert!( + !sisyphus[blocked + 1..incomplete].contains("\n - "), + "sisyphus config must keep the REVIEW_INCOMPLETE bullet directly after the BLOCKED \ + bullet (no other same-level bullet in between)" + ); + + // Architect PLACEMENT: the incomplete contract lives inside Phase E's + // divergence check, under the preferred gauntlet path and before the + // fallback adversary spawn. + let architect = &configs["architect"]; + let find = |needle: &str| { + architect + .find(needle) + .unwrap_or_else(|| panic!("architect config lost placement anchor: {needle:?}")) + }; + let phase_e = find("### Phase E"); + let verify = find("4. **Verify against the plan"); + let preferred = find("**Preferred: run the conformance checks through `review-gauntlet`**"); + let incomplete = find("GAUNTLET_REVIEW_INCOMPLETE:"); + let adversary = find("- **Spawn `adversary`**"); + assert!( + phase_e < verify + && verify < preferred + && preferred < incomplete + && incomplete < adversary, + "architect config must place the REVIEW_INCOMPLETE contract inside Phase E's verify \ + step, under the preferred gauntlet path and before the adversary fallback \ + (got {phase_e} / {verify} / {preferred} / {incomplete} / {adversary})" + ); + // The architect also owns the no-gauntlet fallback path: a // fault-only adversary/probe result is retried, never treated as // a divergence. - let architect = &configs["architect"]; - assert!( - architect.contains( - "an adversary/probe result that is ONLY a `PIPELINE-FAULT` (no verdict on the\n code) is likewise an infrastructure fault to retry, never a divergence." - ), - "architect config lost the fallback adversary/probe pipeline-fault sentence" - ); + for anchor in [ + "an adversary/probe result that is ONLY a `PIPELINE-FAULT`", + "is likewise an infrastructure fault to retry, never a divergence", + ] { + assert!( + architect.contains(anchor), + "architect config lost the fallback adversary/probe pipeline-fault sentence: {anchor:?}" + ); + } + + // The agent READMEs describe the same contract to humans: the line is + // an infrastructure fault escalated with the three caller options, + // never a finding (Sisyphus) / never a divergence (Architect). + let readmes = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("assets/agents"); + for (name, own_claim) in [ + ("sisyphus", "never a finding to fix"), + ("architect", "never a divergence"), + ] { + let readme = read_to_string(readmes.join(name).join("README.md")).unwrap(); + for anchor in [ + "`GAUNTLET_REVIEW_INCOMPLETE:` line", + "infrastructure fault", + "retry / accept NEEDS-HUMAN / abort", + own_claim, + ] { + assert!( + readme.contains(anchor), + "{name} README lost review-incomplete contract anchor: {anchor:?}" + ); + } + for leak in PROVENANCE_LEAKS { + assert!( + !readme.contains(leak), + "{name} README must not carry run-local provenance: {leak:?}" + ); + } + } } // Shell commands the adversary's run_checks stage executes must be @@ -8404,6 +8503,10 @@ mod tests { "GAUNTLET_REVIEW_INCOMPLETE: [, ]", "`pipeline` pseudo-lane", "the named lanes never completed", + // Caller guidance mirrors the sisyphus/architect contract: the + // caller escalates; a fresh run is only the "retry" choice. + "escalate to the user with three options — retry the review again (a **fresh**", + "accept NEEDS-HUMAN and proceed without that lane", "never narrows selection", "up to 3 attempts", "MAX_RETRY_ELAPSED_SECS", From 4f0c7e44315dc9f941990e57c8d4192710d381cc Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 14 Sep 2026 04:29:26 -0600 Subject: [PATCH 09/15] test(TASK-030): pin fallback-note placement; detect concrete task ids (with illustrative-example allowlist) instead of run-local literals --- src/config/request_context.rs | 52 +++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/src/config/request_context.rs b/src/config/request_context.rs index f48540b..8a19462 100644 --- a/src/config/request_context.rs +++ b/src/config/request_context.rs @@ -8251,8 +8251,25 @@ mod tests { ]; // The pin must never leak run-local provenance into the shipped - // assets: task IDs, plan branch names, or plan-repo paths. - const PROVENANCE_LEAKS: [&str; 3] = ["TASK-030", "PLAN-review-retry", "plans/tasks/"]; + // assets: concrete task ids (`TASK-` followed by digits) or plan-repo + // paths. `TASK-NNN` / `PLAN-` template vocabulary is legitimate + // orchestrator prose, so only concrete ids are rejected — except the + // ids the configs use as illustrative anti-pattern examples (the + // "never cite TASK numbers in code comments" rule quotes `// TASK-002`). + const ILLUSTRATIVE_IDS: [&str; 1] = ["TASK-002"]; + fn provenance_leak(text: &str) -> Option { + if text.contains("plans/tasks/") { + return Some("plans/tasks/".to_string()); + } + text.match_indices("TASK-").find_map(|(start, _)| { + let digits = text[start + "TASK-".len()..] + .chars() + .take_while(|c| c.is_ascii_digit()) + .count(); + let id = text[start..start + "TASK-".len() + digits].to_string(); + (digits >= 3 && !ILLUSTRATIVE_IDS.contains(&id.as_str())).then_some(id) + }) + } let mut configs = std::collections::HashMap::new(); for name in ["architect", "sisyphus"] { @@ -8271,12 +8288,11 @@ mod tests { config.contains("user__select"), "{name} config must escalate review-incomplete via user__select" ); - for leak in PROVENANCE_LEAKS { - assert!( - !config.contains(leak), - "{name} config must not carry run-local provenance: {leak:?}" - ); - } + assert!( + provenance_leak(&config).is_none(), + "{name} config must not carry run-local provenance: {:?}", + provenance_leak(&config) + ); configs.insert(name, config); } @@ -8317,15 +8333,18 @@ mod tests { let verify = find("4. **Verify against the plan"); let preferred = find("**Preferred: run the conformance checks through `review-gauntlet`**"); let incomplete = find("GAUNTLET_REVIEW_INCOMPLETE:"); + let fallback = find("an adversary/probe result that is ONLY a `PIPELINE-FAULT`"); let adversary = find("- **Spawn `adversary`**"); assert!( phase_e < verify && verify < preferred && preferred < incomplete - && incomplete < adversary, + && incomplete < fallback + && fallback < adversary, "architect config must place the REVIEW_INCOMPLETE contract inside Phase E's verify \ - step, under the preferred gauntlet path and before the adversary fallback \ - (got {phase_e} / {verify} / {preferred} / {incomplete} / {adversary})" + step, under the preferred gauntlet path, followed by the fallback-path note, all \ + before the adversary fallback bullet \ + (got {phase_e} / {verify} / {preferred} / {incomplete} / {fallback} / {adversary})" ); // The architect also owns the no-gauntlet fallback path: a @@ -8361,12 +8380,11 @@ mod tests { "{name} README lost review-incomplete contract anchor: {anchor:?}" ); } - for leak in PROVENANCE_LEAKS { - assert!( - !readme.contains(leak), - "{name} README must not carry run-local provenance: {leak:?}" - ); - } + assert!( + provenance_leak(&readme).is_none(), + "{name} README must not carry run-local provenance: {:?}", + provenance_leak(&readme) + ); } } From 342f39f30e681745e8c28f96aa1e69a4a37ce0c1 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 14 Sep 2026 04:56:42 -0600 Subject: [PATCH 10/15] ci: run pull_request checks for PRs based on any branch '*' does not match '/' in GitHub branch filters, so a PR whose base is a feature branch (e.g. a stacked PR) never triggered CI. --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fc98eba..4aa891d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -3,7 +3,7 @@ name: CI on: pull_request: branches: - - '*' + - '**' push: branches: - main From d89592fee6e2d1656775cb39f4ccca5977953d7e Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 14 Sep 2026 05:18:52 -0600 Subject: [PATCH 11/15] test: normalize CRLF in the gauntlet script/README source pins (Windows CI) --- src/config/request_context.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/config/request_context.rs b/src/config/request_context.rs index 8a19462..326d11f 100644 --- a/src/config/request_context.rs +++ b/src/config/request_context.rs @@ -8454,7 +8454,10 @@ mod tests { && adversary.contains("never degrades to an ENVIRONMENT marker"), "adversary README must describe a malformed declaration as fail-closed, not a soft marker" ); - let gauntlet = read_to_string(readmes.join("review-gauntlet/README.md")).unwrap(); + // Windows checkouts carry CRLF; the paragraph assertions below span line breaks. + let gauntlet = read_to_string(readmes.join("review-gauntlet/README.md")) + .unwrap() + .replace("\r\n", "\n"); assert!( !gauntlet.contains("Verification commands:"), "review-gauntlet README must not show the prose Verification commands: line" @@ -13204,8 +13207,13 @@ mod tests { fn gauntlet_retry_gate_sentinels_match_verdict_gate() { let scripts = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) .join("assets/agents/review-gauntlet/scripts"); - let retry = read_to_string(scripts.join("retry_gate.py")).unwrap(); - let verdict = read_to_string(scripts.join("verdict_gate.py")).unwrap(); + // Windows checkouts carry CRLF; `slice_between` anchors on "\n)\n". + let retry = read_to_string(scripts.join("retry_gate.py")) + .unwrap() + .replace("\r\n", "\n"); + let verdict = read_to_string(scripts.join("verdict_gate.py")) + .unwrap() + .replace("\r\n", "\n"); for sentinel in [ r"Verdict:\**\s*\**\s*(MERGE-READY|NEEDS-HUMAN)", r"ADVERSARIAL_REVIEW:\s*(CONFORMS|DIVERGES)", From 85cb7b0fbf87c9da2bec947cff76d5bf430d7cab Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 14 Sep 2026 05:37:37 -0600 Subject: [PATCH 12/15] test: normalize CRLF in the gauntlet README contract pin (Windows CI) --- src/config/request_context.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/config/request_context.rs b/src/config/request_context.rs index 326d11f..2421809 100644 --- a/src/config/request_context.rs +++ b/src/config/request_context.rs @@ -8506,7 +8506,10 @@ mod tests { #[test] fn review_gauntlet_readme_states_retry_and_incomplete_contract() { let readmes = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("assets/agents"); - let gauntlet = read_to_string(readmes.join("review-gauntlet/README.md")).unwrap(); + // Windows checkouts carry CRLF; the paragraph assertion below spans line breaks. + let gauntlet = read_to_string(readmes.join("review-gauntlet/README.md")) + .unwrap() + .replace("\r\n", "\n"); for edge in [ "mcr & madv & msec & mpb --> rgate[\"retry_gate", "rgate --> gate[\"verdict_gate", From cee5cf0f36db9536beeda1163a963e957f0fdaec Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 14 Sep 2026 05:52:54 -0600 Subject: [PATCH 13/15] test(rag): make TempDir names unique across parallel tests SystemTime nanos alone collide on coarse clocks, so concurrent tests shared a scratch dir and read each other's generated sbx mixins (flaky generated_sbx_mixin_* failures in CI). --- src/rag/mod.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/rag/mod.rs b/src/rag/mod.rs index 758d0ee..a88af63 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -2289,10 +2289,17 @@ mod tests { impl TempDir { fn new(tag: &str) -> Self { - let unique = SystemTime::now() + // Wall-clock nanos alone are not unique across parallel tests + // (coarse clocks hand the same tick to concurrent callers, so two + // tests would share a directory and overwrite each other's files); + // a process-wide counter makes every name distinct. + static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let nanos = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() .as_nanos(); + let seq = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let unique = format!("{nanos}-{seq}"); let path = env::temp_dir().join(format!("coyote-rag-{tag}-{unique}")); fs::create_dir_all(&path).unwrap(); Self { path } From c7ec4ed1243aff698dd778656d5c841ee14674d5 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 14 Sep 2026 11:03:24 -0600 Subject: [PATCH 14/15] docs: restored formatting to agent example config file --- config.agent.example.yaml | 122 ++++++++++++++++++-------------------- 1 file changed, 59 insertions(+), 63 deletions(-) diff --git a/config.agent.example.yaml b/config.agent.example.yaml index bcdd79a..1ed57d8 100644 --- a/config.agent.example.yaml +++ b/config.agent.example.yaml @@ -11,74 +11,70 @@ # - _AGENT_SESSION # - _VARIABLES (as JSON array of key-value pairs; e.g. '[{"name": "username", "value": "alex"}]') -model: openai:gpt-4o # Specify the LLM to use -temperature: null # Set default temperature parameter, range (0, 1) -top_p: null # Set default top-p parameter, with a range of (0, 1) or (0, 2) depending on the model -reasoning_effort: - null # Reasoning effort level for models that support it (e.g. low, medium, high). - # Only valid when the agent's model declares reasoning_levels. -agent_session: null # Set a session to use when starting the agent. (e.g. temp, default); defaults to globally set agent_session -name: # Name of the agent, used in the UI and logs -description: # Description of the agent, used in the UI -version: 1 # Version of the agent +model: openai:gpt-4o # Specify the LLM to use +temperature: null # Set default temperature parameter, range (0, 1) +top_p: null # Set default top-p parameter, with a range of (0, 1) or (0, 2) depending on the model +reasoning_effort: null # Reasoning effort level for models that support it (e.g. low, medium, high). + # Only valid when the agent's model declares reasoning_levels. +agent_session: null # Set a session to use when starting the agent. (e.g. temp, default); defaults to globally set agent_session +name: # Name of the agent, used in the UI and logs +description: # Description of the agent, used in the UI +version: 1 # Version of the agent + # Auto-Continue (Todo System) # The auto-continue system provides built-in task tracking for improved reliability. # When enabled, the model can create todo lists and the system will automatically # prompt it to continue when incomplete tasks remain. # See the [Todo System documentation](https://github.com/Dark-Alex-17/coyote/wiki/TODO-System) for more information -auto_continue: false # Enable automatic continuation when incomplete todos remain -max_auto_continues: 10 # Maximum number of automatic continuations before stopping -inject_todo_instructions: true # Inject the default todo tool usage instructions into the agent's system prompt -continuation_prompt: null # Custom prompt used when auto-continuing (optional; uses default if null) +auto_continue: false # Enable automatic continuation when incomplete todos remain +max_auto_continues: 10 # Maximum number of automatic continuations before stopping +inject_todo_instructions: true # Inject the default todo tool usage instructions into the agent's system prompt +continuation_prompt: null # Custom prompt used when auto-continuing (optional; uses default if null) + # Sub-Agent Spawning System # Enable this agent to spawn and manage child agents in parallel. # See https://github.com/Dark-Alex-17/coyote/wiki/Agents for detailed documentation. -can_spawn_agents: false # Enable the agent to spawn child agents +can_spawn_agents: false # Enable the agent to spawn child agents # spawnable_agents: # Optional whitelist restricting which agents can be spawned via `agent__spawn`. # - explore # If omitted (the default), ALL installed agents are spawnable. This is the unrestricted default. # - coder # Provide a list to restrict. Match is exact and case-sensitive (use directory names). # - oracle # An empty list ([]) means literally nothing spawnable. -# Also filters `agent__list_available` output so the LLM only sees what it can spawn. -# Graph agents (graph.yaml) ignore this; they declare spawn targets in agent nodes. -max_concurrent_agents: 4 # Maximum number of agents that can run simultaneously -max_agent_depth: 3 # Maximum nesting depth for sub-agents (prevents runaway spawning) -max_concurrent_jobs: - 5 # Max background jobs (`job__*` tools) running at once for this agent - # (overrides the global setting; 0 disables background jobs for this agent) -inject_spawn_instructions: true # Inject the default agent spawning instructions into the agent's system prompt -summarization_model: null # Model to use for summarizing sub-agent output (e.g. 'openai:gpt-4o-mini'); defaults to current model -summarization_threshold: 4000 # Character threshold above which sub-agent output is summarized before returning to parent -escalation_timeout: 0 # 0 = wait indefinitely for the reply (default); N = give up after N seconds -mcp_servers: # Optional list of MCP servers that the agent utilizes - - github # Corresponds to the name of an MCP server in the `/mcp.json` file -mcp_tools: # Optional per-server tool allowlist for the agent's MCP servers - github: # (glob patterns: * and ?). Intersects with the global config, - - get_* # mcp.json `allowedTools`, and every other configured layer. It - - search_* # can only narrow access, never widen it. -global_tools: # Optional list of additional global tools to enable for the agent; i.e. not tools specific to the agent + # Also filters `agent__list_available` output so the LLM only sees what it can spawn. + # Graph agents (graph.yaml) ignore this; they declare spawn targets in agent nodes. +max_concurrent_agents: 4 # Maximum number of agents that can run simultaneously +max_agent_depth: 3 # Maximum nesting depth for sub-agents (prevents runaway spawning) +max_concurrent_jobs: 5 # Max background jobs (`job__*` tools) running at once for this agent + # (overrides the global setting; 0 disables background jobs for this agent) +inject_spawn_instructions: true # Inject the default agent spawning instructions into the agent's system prompt +summarization_model: null # Model to use for summarizing sub-agent output (e.g. 'openai:gpt-4o-mini'); defaults to current model +summarization_threshold: 4000 # Character threshold above which sub-agent output is summarized before returning to parent +escalation_timeout: 0 # 0 = wait indefinitely for the reply (default); N = give up after N seconds +mcp_servers: # Optional list of MCP servers that the agent utilizes + - github # Corresponds to the name of an MCP server in the `/mcp.json` file +mcp_tools: # Optional per-server tool allowlist for the agent's MCP servers + github: # (glob patterns: * and ?). Intersects with the global config, + - get_* # mcp.json `allowedTools`, and every other configured layer. It + - search_* # can only narrow access, never widen it. +global_tools: # Optional list of additional global tools to enable for the agent; i.e. not tools specific to the agent - web_search - fs - python -skills_enabled: - true # Master switch for skills in this agent (default: inherit from global). - # Skills also require `function_calling_support: true` in the global config. -enabled_skills: # Optional list of skills available when this agent runs. - # Must be a subset of global `visible_skills`. Omit to inherit the global default. - - git-master +skills_enabled: true # Master switch for skills in this agent (default: inherit from global). + # Skills also require `function_calling_support: true` in the global config. +enabled_skills: # Optional list of skills available when this agent runs. + - git-master # Must be a subset of global `visible_skills`. Omit to inherit the global default. - ai-slop-remover -inject_skill_instructions: - true # Inject a short hint pointing the model at `skill__list` when skills are enabled - # (default: true). Suppressed automatically when no skills are available. -skill_instructions: null # Custom text for the skill hint (optional; uses built-in default if null) -enabled_macros: # Optional list of macros invocable when this agent is active in the REPL. - - generate-commit-message # An empty list disables all macros. Omit to inherit the role/global default. -memory: - null # Per-agent memory override (default: inherit). Set to `false` to disable memory - # for this agent regardless of workspace/global presence. See the Memory wiki page. +inject_skill_instructions: true # Inject a short hint pointing the model at `skill__list` when skills are enabled + # (default: true). Suppressed automatically when no skills are available. +skill_instructions: null # Custom text for the skill hint (optional; uses built-in default if null) +enabled_macros: # Optional list of macros invocable when this agent is active in the REPL. + - generate-commit-message # An empty list disables all macros. Omit to inherit the role/global default. +memory: null # Per-agent memory override (default: inherit). Set to `false` to disable memory + # for this agent regardless of workspace/global presence. See the Memory wiki page. dynamic_instructions: false # Whether to use dynamic instructions for the agent; if false, static instructions are used -instructions: - | # Static instructions for the agent; ignored if dynamic instructions are used +instructions: # Static instructions for the agent; ignored if dynamic instructions are used + | You are a AI agent designed to demonstrate agent capabilities. @@ -99,10 +95,10 @@ instructions: username: {{username}} variables: # Optional variables for the agent - # The variables defined above like {{__variable_name__}} are automatically available + # The variables defined above like {{__variable_name__}} are automatically available - name: username description: Your user name - default: null # A default value for this variable; if null, the variable must be provided when starting the agent + default: null # A default value for this variable; if null, the variable must be provided when starting the agent conversation_starters: # Optional conversation starters for the agent - What is the meaning of life? - Tell me a joke. @@ -114,15 +110,15 @@ conversation_starters: # Optional conversation starters for the agent - How do I stay motivated? - What is the best way to exercise? - How do I manage my time effectively? -documents: # Optional documents to load for the agent - # To enable graph-based RAG (entity/relationship extraction + knowledge graph retrieval), - # set `rag_extractor_model` in your global config.yaml. - # See https://github.com/Dark-Alex-17/coyote/wiki/RAG#graph-based-rag - - git:/some/repo # Explicitly tell Coyote to use the 'git' document loader using an absolute path - - pdf:some-pdf-file.pdf # Explicitly tell Coyote to use the 'pdf' document loader using a relative path +documents: # Optional documents to load for the agent + # To enable graph-based RAG (entity/relationship extraction + knowledge graph retrieval), + # set `rag_extractor_model` in your global config.yaml. + # See https://github.com/Dark-Alex-17/coyote/wiki/RAG#graph-based-rag + - git:/some/repo # Explicitly tell Coyote to use the 'git' document loader using an absolute path + - pdf:some-pdf-file.pdf # Explicitly tell Coyote to use the 'pdf' document loader using a relative path - https://some-website.com/some-page - - some-file.pdf # File with relative path to the /agents/ directory; i.e. file in the same directory as this config file - - ~/some-file.txt # File in the user's home directory - - /absolute/path/to/some-file.md # File with absolute path - - /absolute/path/**/NAME.txt # Find all NAME.txt files in the specified directory and all its subdirectories - - /absolute/path/to/*/README.md # Find all README.md files in all immediate subdirectories of the specified directory (depth=1) + - some-file.pdf # File with relative path to the /agents/ directory; i.e. file in the same directory as this config file + - ~/some-file.txt # File in the user's home directory + - /absolute/path/to/some-file.md # File with absolute path + - /absolute/path/**/NAME.txt # Find all NAME.txt files in the specified directory and all its subdirectories + - /absolute/path/to/*/README.md # Find all README.md files in all immediate subdirectories of the specified directory (depth=1) From beca13308287e3e354477f372b578e1b5820b0ab Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 14 Sep 2026 11:03:43 -0600 Subject: [PATCH 15/15] style: removed unneccessary path prefixes and organized imports --- src/config/request_context.rs | 29 +++++++++++++++-------------- src/function/user_interaction.rs | 12 +++++++----- src/rag/mod.rs | 5 +++-- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/config/request_context.rs b/src/config/request_context.rs index 2421809..b669ed6 100644 --- a/src/config/request_context.rs +++ b/src/config/request_context.rs @@ -5268,6 +5268,7 @@ mod tests { use serial_test::serial; use std::fs::{create_dir_all, remove_dir_all, write}; use std::path::PathBuf; + use std::process::Command; use std::time::{Instant, SystemTime, UNIX_EPOCH}; use std::{env, mem}; @@ -8505,7 +8506,7 @@ mod tests { #[test] fn review_gauntlet_readme_states_retry_and_incomplete_contract() { - let readmes = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("assets/agents"); + let readmes = Path::new(env!("CARGO_MANIFEST_DIR")).join("assets/agents"); // Windows checkouts carry CRLF; the paragraph assertion below spans line breaks. let gauntlet = read_to_string(readmes.join("review-gauntlet/README.md")) .unwrap() @@ -9050,11 +9051,11 @@ mod tests { // Scripts' load_state() prefers GRAPH_STATE_FILE over GRAPH_STATE, so an // inherited live state file (adversary verifying this repo) must not win. - fn adversary_script_command(script: &str, state: &serde_json::Value) -> std::process::Command { + fn adversary_script_command(script: &str, state: &serde_json::Value) -> Command { let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) .join("assets/agents/adversary/scripts") .join(script); - let mut cmd = std::process::Command::new("python3"); + let mut cmd = Command::new("python3"); cmd.arg(&path) .env("GRAPH_STATE", state.to_string()) .env_remove("GRAPH_STATE_FILE"); @@ -9591,7 +9592,7 @@ mod tests { ); let alive = |pid: &str| { - std::process::Command::new("kill") + Command::new("kill") .args(["-0", pid]) .output() .unwrap() @@ -10950,7 +10951,7 @@ mod tests { let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) .join("assets/agents/review-gauntlet/scripts") .join(script); - let out = std::process::Command::new("python3") + let out = Command::new("python3") .arg(&path) .env("GRAPH_STATE", state.to_string()) .env_remove("GRAPH_STATE_FILE") @@ -13246,7 +13247,7 @@ mod tests { ); } let adversary_verdict = read_to_string( - std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + Path::new(env!("CARGO_MANIFEST_DIR")) .join("assets/agents/adversary/scripts/verdict.py"), ) .unwrap(); @@ -13403,11 +13404,11 @@ mod tests { eprintln!("skipping: python3 not available"); return; } - let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + let path = Path::new(env!("CARGO_MANIFEST_DIR")) .join("assets/agents/review-gauntlet/scripts/build_items.py"); let state = json!({"forced_lanes": ["code-review", "adversary"], "gauntlet_started_at": 5.0}); - let out = std::process::Command::new("python3") + let out = Command::new("python3") .arg(&path) .env("GRAPH_STATE", state.to_string()) .env_remove("GRAPH_STATE_FILE") @@ -13459,7 +13460,7 @@ mod tests { eprintln!("skipping: python3 not available"); return; } - let dir = utils::temp_file("-gauntlet-signals-", ""); + let dir = temp_file("-gauntlet-signals-", ""); create_dir_all(&dir).unwrap(); let project_dir = dir.to_string_lossy().to_string(); let out = run_gauntlet_script( @@ -13585,7 +13586,7 @@ mod tests { fn gauntlet_retry_budget_fits_inside_graph_timeout() { use crate::graph::NodeType; const OTHER_STAGES_MARGIN_SECS: u64 = 1200; - let script = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + let script = Path::new(env!("CARGO_MANIFEST_DIR")) .join("assets/agents/review-gauntlet/scripts/retry_gate.py"); let source = read_to_string(&script).unwrap(); let line = source @@ -13707,7 +13708,7 @@ mod tests { let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) .join("assets/agents/code-reviewer/scripts") .join(script); - let out = std::process::Command::new("python3") + let out = Command::new("python3") .arg(&path) .env("GRAPH_STATE", raw_state) .env_remove("GRAPH_STATE_FILE") @@ -14517,7 +14518,7 @@ mod tests { let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) .join("assets/agents/finding-verifier/scripts") .join(script); - let out = std::process::Command::new("python3") + let out = Command::new("python3") .arg(&path) .env("GRAPH_STATE", raw_state) .env_remove("GRAPH_STATE_FILE") @@ -14819,7 +14820,7 @@ mod tests { let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) .join("assets/agents/step-runner/scripts") .join(script); - let out = std::process::Command::new("bash") + let out = Command::new("bash") .arg(&path) .env("GRAPH_STATE", state.to_string()) .env_remove("GRAPH_STATE_FILE") @@ -15123,7 +15124,7 @@ mod tests { let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) .join("assets/agents/deep-research/scripts") .join(script); - let out = std::process::Command::new("python3") + let out = Command::new("python3") .arg(&path) .env("GRAPH_STATE", raw_state) .env_remove("GRAPH_STATE_FILE") diff --git a/src/function/user_interaction.rs b/src/function/user_interaction.rs index d698fa7..1e62d19 100644 --- a/src/function/user_interaction.rs +++ b/src/function/user_interaction.rs @@ -346,6 +346,8 @@ fn parse_options(args: &Value) -> Result> { #[cfg(test)] mod tests { use super::*; + use std::fs; + use std::path::{Path, PathBuf}; #[test] fn headless_select_returns_structured_json() { @@ -431,9 +433,9 @@ mod tests { concat!("escalation_timeout:", " 300"), concat!("default:", " 5 minutes"), ]; - let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + let root = Path::new(env!("CARGO_MANIFEST_DIR")); let example = root.join("config.agent.example.yaml"); - let example_text = std::fs::read_to_string(&example).unwrap(); + let example_text = fs::read_to_string(&example).unwrap(); for needle in ["escalation_timeout: 0", "0 = wait indefinitely"] { assert!( example_text.contains(needle), @@ -454,7 +456,7 @@ mod tests { let mut hits = Vec::new(); for path in &files { - let text = String::from_utf8_lossy(&std::fs::read(path).unwrap()).into_owned(); + let text = String::from_utf8_lossy(&fs::read(path).unwrap()).into_owned(); for needle in needles { if text.contains(needle) { hits.push(format!("{}: {needle:?}", path.display())); @@ -468,8 +470,8 @@ mod tests { ); } - fn collect_files(dir: &std::path::Path, out: &mut Vec) { - for entry in std::fs::read_dir(dir).unwrap().flatten() { + fn collect_files(dir: &Path, out: &mut Vec) { + for entry in fs::read_dir(dir).unwrap().flatten() { let path = entry.path(); if path.is_dir() { collect_files(&path, out); diff --git a/src/rag/mod.rs b/src/rag/mod.rs index a88af63..6f5c627 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -2281,6 +2281,7 @@ where mod tests { use super::*; use std::path::PathBuf; + use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; struct TempDir { @@ -2293,12 +2294,12 @@ mod tests { // (coarse clocks hand the same tick to concurrent callers, so two // tests would share a directory and overwrite each other's files); // a process-wide counter makes every name distinct. - static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + static SEQ: AtomicU64 = AtomicU64::new(0); let nanos = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() .as_nanos(); - let seq = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let seq = SEQ.fetch_add(1, Ordering::Relaxed); let unique = format!("{nanos}-{seq}"); let path = env::temp_dir().join(format!("coyote-rag-{tag}-{unique}")); fs::create_dir_all(&path).unwrap();