From 3f49db86b2501ca1894e5a37d5fac159bbb0b3e4 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Sun, 13 Sep 2026 00:13:54 -0600 Subject: [PATCH] fix(adversary): fail closed per criterion when check_criterion dies A live run died at map_criteria: one heavy criterion exhausted max_iterations (10) with no fallback and sank the whole 17-criterion review. check_criterion now has a 30-iteration budget, a tool-budget instruction, and falls back to criterion_fault, which records a schema-shaped UNMET verdict with a PIPELINE-FAULT evidence marker for that one criterion; verdict.py renders it as DIED rather than judged. --- assets/agents/adversary/README.md | 3 + assets/agents/adversary/graph.yaml | 14 +- .../adversary/scripts/criterion_fault.py | 91 ++++++ assets/agents/adversary/scripts/verdict.py | 22 +- src/config/request_context.rs | 269 ++++++++++++++++++ 5 files changed, 391 insertions(+), 8 deletions(-) create mode 100644 assets/agents/adversary/scripts/criterion_fault.py diff --git a/assets/agents/adversary/README.md b/assets/agents/adversary/README.md index 9ea5a7a..d1b918a 100644 --- a/assets/agents/adversary/README.md +++ b/assets/agents/adversary/README.md @@ -78,6 +78,8 @@ flowchart TD A -. "fallback" .-> P["pipeline_fault (script): record PIPELINE-FAULT marker"] B -. "fallback" .-> P R -. "fallback" .-> P + D -. "fallback" .-> CF["criterion_fault (script): UNMET + PIPELINE-FAULT evidence for that ONE criterion"] + CF --> F F -. "fallback" .-> G P --> G ``` @@ -89,6 +91,7 @@ The conformance doctrine is baked into the graph's own nodes: 3. Execution evidence is recorded **once, deterministically**: `run_checks` runs ONLY the caller-declared verification commands (never invented or auto-detected), sequentially under a 3300s total deadline — commands the deadline leaves no budget for are recorded as SKIPPED. Criterion branches have no execution tools by design: "tests pass"-style criteria are judged against the record — a recorded green run is MET citing it; no declared command ⇒ at best PARTIAL. 4. Hunt adversarially for the **absent**: skipped criteria, scope creep, interface/approach substitution, out-of-scope touches, downstream contract breakage. 5. **Fail closed on pipeline faults**: if parse, diff resolution, or the verification runner dies, its fallback routes to `pipeline_fault`, which records a `PIPELINE-FAULT:` marker and continues to the verdict (a failed holistic pass falls back straight to the verdict, which synthesizes the same marker). Any recorded fault forces DIVERGES with the fault as complaint #1 — degraded is never passing, and the sentinel is always emitted: the graph never dies without one. +6. **Fail closed per criterion**: a criterion check that dies (max_iterations exhausted, API failure) falls back to `criterion_fault`, which records a schema-shaped UNMET verdict carrying a `PIPELINE-FAULT:` evidence marker for that one criterion. The other criteria still get verdicts, the report says the check DIED rather than judged, and the run is DIVERGES. It is **read-only** — it produces a verdict, never a fix. diff --git a/assets/agents/adversary/graph.yaml b/assets/agents/adversary/graph.yaml index 8b962d9..cdc2d36 100644 --- a/assets/agents/adversary/graph.yaml +++ b/assets/agents/adversary/graph.yaml @@ -215,6 +215,8 @@ nodes: Your response is machine-validated: a malformed response is rejected back to you exactly once with the reason, then recorded as PARTIAL (unproven). The id is stamped from the criterion itself — your echo is informative only. + You have a bounded tool budget — prefer fs_grep and line-ranged fs_read over + whole-file reads, and conclude as soon as the evidence is decisive. prompt: | Repository: {{project_dir}} Changed files: {{changed_files}} @@ -236,10 +238,11 @@ nodes: - fs_cat - fs_grep - ast_grep - max_iterations: 10 + max_iterations: 30 max_attempts: 2 state_updates: crit_verdict: '{{output}}' + fallback: criterion_fault next: crit_gate crit_gate: @@ -249,6 +252,13 @@ nodes: script: scripts/crit_gate.py timeout: 30 + criterion_fault: + id: criterion_fault + type: script + description: Fail-closed per-criterion fault marker (reached only via check_criterion's fallback) — converts the captured "LLM node failed" text into a schema-shaped UNMET verdict carrying a PIPELINE-FAULT evidence marker, so one dead criterion check can never sink the map or read as met. Never sinks the map. + script: scripts/criterion_fault.py + timeout: 30 + holistic: id: holistic type: llm @@ -320,7 +330,7 @@ nodes: observations: type: string required: [extra_complaints, observations] - max_iterations: 10 + max_iterations: 20 max_attempts: 2 state_updates: holistic_failure: '{{output}}' diff --git a/assets/agents/adversary/scripts/criterion_fault.py b/assets/agents/adversary/scripts/criterion_fault.py new file mode 100644 index 0000000..eb7b601 --- /dev/null +++ b/assets/agents/adversary/scripts/criterion_fault.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Fail-closed per-criterion fault marker for the adversary graph. + +Reached ONLY via check_criterion's `fallback:` route, after the engine has +written its "LLM node …failed: " text into `crit_verdict` through +state_updates. Converts that text into a schema-shaped UNMET verdict whose +evidence carries a "PIPELINE-FAULT:" marker, so one criterion check that dies +(max_iterations exhausted, API failure) is recorded as DIED rather than +judged, the other criteria still get verdicts, and the run is DIVERGES. The +map collects it like any other verdict — the branch never sinks the map. +""" + +import json +import os + +MAX_DETAIL_CHARS = 500 + + +def load_state(): + if path := os.environ.get("GRAPH_STATE_FILE"): + with open(path) as f: + return json.load(f) + return json.loads(os.environ.get("GRAPH_STATE", "{}")) + + +def criterion_fields(state): + c = state.get("criterion") + if isinstance(c, dict): + cid = c.get("id") + text = c.get("text") + return ( + cid.strip() if isinstance(cid, str) and cid.strip() else "unknown", + text.strip() if isinstance(text, str) else "", + ) + return "unknown", "" + + +def failure_detail(state): + captured = state.get("crit_verdict") + detail = captured.strip() if isinstance(captured, str) else "" + if not detail.startswith("LLM node"): + return "no failure text captured" + detail = detail.replace("\n", " ") + if len(detail) > MAX_DETAIL_CHARS: + detail = detail[: MAX_DETAIL_CHARS - 1] + "…" + return detail + + +def main(): + state = load_state() + cid, ctext = criterion_fields(state) + detail = failure_detail(state) + print( + json.dumps( + { + "crit_verdict": { + "id": cid, + "text": ctext, + "status": "UNMET", + "evidence": f"PIPELINE-FAULT: criterion check failed — {detail}", + "complaint": ( + f"criterion check DIED (pipeline fault) — {detail}; the " + "criterion was NOT verified and is treated as unmet " + "(fail-closed)" + ), + } + } + ) + ) + + +try: + main() +except Exception as e: # noqa: BLE001 — the fault marker itself must never crash + print( + json.dumps( + { + "crit_verdict": { + "id": "unknown", + "text": "", + "status": "UNMET", + "evidence": f"PIPELINE-FAULT: criterion fault-marker script error: {e}", + "complaint": ( + f"criterion check DIED (pipeline fault) — fault-marker script " + f"error: {e}; the criterion was NOT verified and is treated as " + "unmet (fail-closed)" + ), + } + } + ) + ) diff --git a/assets/agents/adversary/scripts/verdict.py b/assets/agents/adversary/scripts/verdict.py index 127ae28..377ac36 100644 --- a/assets/agents/adversary/scripts/verdict.py +++ b/assets/agents/adversary/scripts/verdict.py @@ -6,14 +6,17 @@ holistic complaints, and zero pipeline faults. Everything else — including a missing plan or a degraded (faulted) pipeline — is DIVERGES (fail-closed). Any "PIPELINE-FAULT:" marker recorded in `pipeline_faults` (or a holistic llm-node failure captured in `holistic_failure`) becomes complaint #1; -per-criterion results are still reported. Assembles the exact sentinel -format the callers route on. Never crashes into a silent verdict. +per-criterion results are still reported. A criterion whose check DIED +(criterion_fault's UNMET verdict, evidence prefixed "PIPELINE-FAULT:") is +rendered as died rather than judged. Assembles the exact sentinel format the +callers route on. Never crashes into a silent verdict. """ import json import os MAX_FAULT_DETAIL_CHARS = 300 +DIED_MARKER = "criterion check DIED (pipeline fault)" def load_state(): @@ -57,10 +60,17 @@ def append_criterion_complaints(lines, verdicts, extra, start): text = (v.get("text") or "").strip().replace("\n", " ") if len(text) > 140: text = text[:137] + "…" - lines.append( - f'{i}. Acceptance criterion "{text}" — {v.get("status", "?").title()} — ' - f'{(v.get("complaint") or "").strip()}' - ) + complaint = (v.get("complaint") or "").strip() + evidence = v.get("evidence") + if isinstance(evidence, str) and evidence.startswith("PIPELINE-FAULT:"): + if not complaint.startswith(DIED_MARKER): + complaint = f"{DIED_MARKER} — {complaint}" + lines.append(f'{i}. Acceptance criterion "{text}" — {complaint}') + else: + lines.append( + f'{i}. Acceptance criterion "{text}" — {v.get("status", "?").title()} — ' + f"{complaint}" + ) for c in extra: i += 1 lines.append( diff --git a/src/config/request_context.rs b/src/config/request_context.rs index cd7eda8..f759fe7 100644 --- a/src/config/request_context.rs +++ b/src/config/request_context.rs @@ -8810,6 +8810,275 @@ mod tests { ); } + #[test] + fn adversary_criterion_fault_happy_path() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + let state = json!({ + "criterion": {"id": "c7", "text": "the spec"}, + "crit_verdict": "LLM node failed: LLM call failed: llm node hit max_iterations (10) before LLM concluded" + }); + let out = run_adversary_script("criterion_fault.py", &state); + let v = &out["crit_verdict"]; + assert_eq!(v["status"], "UNMET", "a dead check must fail closed: {out}"); + assert_eq!( + v["id"], "c7", + "id must be stamped from the criterion: {out}" + ); + assert_eq!(v["text"], "the spec"); + assert!( + v["evidence"] + .as_str() + .unwrap() + .starts_with("PIPELINE-FAULT: criterion check failed — LLM node failed:"), + "evidence must carry the fault marker and the captured chain: {out}" + ); + assert!( + v["complaint"].as_str().unwrap().contains("DIED"), + "the complaint must say the check died rather than judged: {out}" + ); + } + + #[test] + fn adversary_criterion_fault_degraded_inputs() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // No captured failure text at all. + let out = run_adversary_script( + "criterion_fault.py", + &json!({"criterion": {"id": "c1", "text": "x"}}), + ); + assert_eq!(out["crit_verdict"]["status"], "UNMET"); + assert!( + out["crit_verdict"]["evidence"] + .as_str() + .unwrap() + .ends_with("no failure text captured"), + "{out}" + ); + + // crit_verdict present but not an engine failure string. + let out = run_adversary_script( + "criterion_fault.py", + &json!({"criterion": {"id": "c1", "text": "x"}, "crit_verdict": "{\"status\": \"MET\"}"}), + ); + assert_eq!(out["crit_verdict"]["status"], "UNMET"); + assert!( + out["crit_verdict"]["evidence"] + .as_str() + .unwrap() + .ends_with("no failure text captured"), + "non-failure text must not be echoed as a failure: {out}" + ); + + // Criterion missing → id falls back to "unknown". + let out = run_adversary_script( + "criterion_fault.py", + &json!({"crit_verdict": "LLM node failed: boom"}), + ); + assert_eq!(out["crit_verdict"]["id"], "unknown", "{out}"); + assert_eq!(out["crit_verdict"]["status"], "UNMET"); + + // Overlong, multi-line failure text is flattened and truncated. + let long = format!("LLM node failed: {}\nline two", "x".repeat(600)); + let out = run_adversary_script( + "criterion_fault.py", + &json!({"criterion": {"id": "c1", "text": "x"}, "crit_verdict": long}), + ); + let evidence = out["crit_verdict"]["evidence"].as_str().unwrap(); + assert!( + evidence.ends_with('…'), + "must truncate with an ellipsis: {evidence}" + ); + assert!( + !evidence.contains('\n'), + "must flatten newlines: {evidence}" + ); + assert!( + evidence.chars().count() < 600, + "{}", + evidence.chars().count() + ); + } + + #[test] + fn adversary_criterion_fault_crash_fails_closed() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // Unparseable state makes load_state raise before main runs; the + // fault marker itself must still emit a schema-shaped UNMET verdict. + let out = run_adversary_script_env( + "criterion_fault.py", + &json!({}), + &[("GRAPH_STATE", "not json")], + ); + let v = &out["crit_verdict"]; + assert_eq!( + v["status"], "UNMET", + "a crashed fault marker must fail closed: {out}" + ); + assert_eq!(v["id"], "unknown"); + assert!( + v["evidence"] + .as_str() + .unwrap() + .starts_with("PIPELINE-FAULT:"), + "the fault prefix must survive a crash: {out}" + ); + assert!( + v["complaint"] + .as_str() + .unwrap() + .contains("fault-marker script error"), + "the crash must be named: {out}" + ); + } + + #[test] + fn adversary_verdict_renders_died_criterion_and_never_conforms() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + let state = json!({ + "pipeline_faults": [], + "crit_verdicts": [ + {"id": "c1", "text": "does X", "status": "MET", + "evidence": "src/x.rs:1 + test src/x.rs:99", "complaint": ""}, + {"id": "c2", "text": "does Y", "status": "UNMET", + "evidence": "PIPELINE-FAULT: criterion check failed — LLM node failed: boom", + "complaint": "criterion check DIED (pipeline fault) — LLM node failed: boom; the criterion was NOT verified and is treated as unmet (fail-closed)"} + ], + "extra_complaints": [], + "observations": "", + "exec_results": [] + }); + let out = run_adversary_script("verdict.py", &state); + let report = out["adv_report"].as_str().unwrap(); + assert!( + report.starts_with("ADVERSARIAL_REVIEW: DIVERGES"), + "a died criterion must never conform: {report}" + ); + assert!( + report.contains("Criteria: 1/2 met, 0 partial, 1 unmet/diverged."), + "the died criterion counts as unmet: {report}" + ); + assert!( + report.contains( + "Acceptance criterion \"does Y\" — criterion check DIED (pipeline fault) — " + ), + "a died criterion must render as died rather than judged: {report}" + ); + assert!( + !report.contains("— Unmet —"), + "a died criterion must not render as a judged Unmet: {report}" + ); + assert_eq!( + report + .matches("criterion check DIED (pipeline fault)") + .count(), + 1, + "the died marker must not be doubled: {report}" + ); + } + + #[test] + fn adversary_crit_gate_passes_fault_shaped_verdict_through() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + let evidence = "PIPELINE-FAULT: criterion check failed — LLM node failed: boom"; + let fault = json!({ + "id": "c1", "status": "UNMET", "evidence": evidence, + "complaint": "criterion check DIED (pipeline fault) — LLM node failed: boom" + }); + let out = run_adversary_script( + "crit_gate.py", + &json!({ + "criterion": {"id": "c1", "text": "x"}, + "gate_attempts": 0, + "crit_verdict": fault.to_string() + }), + ); + assert!( + out.get("_next").is_none(), + "a fault-shaped verdict must not retry: {out}" + ); + assert_eq!(out["crit_verdict"]["status"], "UNMET"); + assert_eq!(out["crit_verdict"]["evidence"], evidence, "{out}"); + } + + #[test] + fn adversary_check_criterion_fails_closed_per_criterion() { + use crate::graph::{GraphParser, NodeType}; + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("assets/agents/adversary"); + let graph = GraphParser::new(&dir) + .load_from_file(dir.join("graph.yaml")) + .expect("adversary graph.yaml must parse"); + + let node = graph + .nodes + .get("check_criterion") + .expect("adversary graph must have a check_criterion node"); + let NodeType::Llm(llm) = &node.node_type else { + panic!("check_criterion must be an llm node"); + }; + assert_eq!( + llm.fallback.as_deref(), + Some("criterion_fault"), + "a dead criterion check must fall back instead of sinking the map" + ); + assert_eq!(llm.max_iterations, 30); + assert_eq!(llm.max_attempts, 2); + assert!( + llm.state_updates + .as_ref() + .is_some_and(|u| u.contains_key("crit_verdict")), + "the failure text must land in crit_verdict for criterion_fault to read" + ); + assert_eq!(node.next_target(), Some("crit_gate")); + assert!( + llm.instructions + .as_deref() + .is_some_and(|s| s.contains("bounded tool budget")), + "check_criterion must be told its tool budget is bounded" + ); + + let fault = graph + .nodes + .get("criterion_fault") + .expect("adversary graph must have a criterion_fault node"); + let NodeType::Script(script) = &fault.node_type else { + panic!("criterion_fault must be a script node"); + }; + assert!( + script.script.ends_with("criterion_fault.py"), + "{}", + script.script + ); + assert_eq!( + fault.next_target(), + None, + "criterion_fault must terminate the branch so the map collects it" + ); + + let holistic = graph + .nodes + .get("holistic") + .expect("adversary graph must have a holistic node"); + let NodeType::Llm(holistic) = &holistic.node_type else { + panic!("holistic must be an llm node"); + }; + assert_eq!(holistic.max_iterations, 20); + } + // ---- review-gauntlet suite-script regression tests (R5) ---- // // Degradation paths of the gauntlet's lane/builder/gate scripts,