From d50059a56f90dcfb977a224f2751b1c749131dc7 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Sat, 12 Sep 2026 15:06:28 -0600 Subject: [PATCH] feat(TASK-017): adversary execution evidence + fail-closed routing - parse extracts caller-declared verification_commands (schema + initial_state); never invents commands. parse gains fallback + state_updates capturing the failure string (TASK-016 fallback-capture warning stays silent). - new run_checks script node (facts -> run_checks -> map_criteria, timeout 3600): runs each declared command in the project dir (900s per-command timeout), records exec_results {cmd, exit, duration_s, tail~50}; 'none declared' / ENVIRONMENT markers instead of guessing or failing the node. - check_criterion judges execution criteria against {{exec_results}} only: recorded green run = MET citing it, recorded failure = UNMET, markers = PARTIAL with an ENVIRONMENT note. - fail-closed routing (R1/R2): parse/facts fallback -> pipeline_fault marker script -> verdict; holistic fallback -> verdict with holistic_failure captured. verdict.py: any PIPELINE-FAULT forces DIVERGES with the fault as complaint #1, per-criterion results still reported; report gains a 'Verification runs:' section. Non-degraded semantics unchanged. - R5 rust tests (python3-guarded, local cmd_available): verdict fault path, holistic-failure fault, CONFORMS regression, run_checks markers + recorded runs, crit_gate contract regression. --- assets/agents/adversary/graph.yaml | 61 ++++- .../adversary/scripts/pipeline_fault.py | 58 +++++ assets/agents/adversary/scripts/run_checks.py | 113 +++++++++ assets/agents/adversary/scripts/verdict.py | 136 +++++++++-- src/config/request_context.rs | 222 ++++++++++++++++++ 5 files changed, 564 insertions(+), 26 deletions(-) create mode 100644 assets/agents/adversary/scripts/pipeline_fault.py create mode 100644 assets/agents/adversary/scripts/run_checks.py diff --git a/assets/agents/adversary/graph.yaml b/assets/agents/adversary/graph.yaml index 9ad6001..e267aee 100644 --- a/assets/agents/adversary/graph.yaml +++ b/assets/agents/adversary/graph.yaml @@ -12,7 +12,11 @@ description: | tests — ADDITIVE-only: it can add complaints, never flip a criterion verdict. The final CONFORMS/DIVERGES sentinel is computed by a script: CONFORMS requires every criterion MET and zero holistic complaints. Missing plan = - DIVERGES (fail-closed) naming the missing spec. + DIVERGES (fail-closed) naming the missing spec. Caller-declared verification + commands are executed ONCE by a deterministic stage and recorded — criterion + branches cite the record and never run anything themselves. Any pipeline + fault (parse, diff resolution, or holistic failure) lands a PIPELINE-FAULT + marker and fails closed into DIVERGES: degraded is never passing. version: '1.0' @@ -42,12 +46,17 @@ initial_state: changed_files: [] diff_text: '' diff_note: '' + verification_commands: [] + exec_results: '' crit_verdicts: [] extra_complaints: [] observations: '' adv_report: '' gate_feedback: '' gate_attempts: 0 + parse_failure: '' + holistic_failure: '' + pipeline_faults: [] start: parse @@ -75,6 +84,10 @@ nodes: verbatim — the holistic pass checks the diff didn't silently override them. - `scope_notes`: what the plan says NOT to touch / explicit boundaries. - `extra_context`: anything else conformance-relevant, condensed. + - `verification_commands`: the exact build/test/lint shell commands the + caller explicitly declares for verification (verbatim strings, in the + order given). Empty array when the caller declares none — NEVER invent, + guess, or auto-detect commands; undeclared stays undeclared. prompt: | Extract the conformance-review facts from this request: @@ -103,9 +116,16 @@ nodes: type: string project_dir_in: type: string - required: [criteria, diff_spec, plan_notes, scope_notes, extra_context, project_dir_in] + verification_commands: + type: array + items: + type: string + required: [criteria, diff_spec, plan_notes, scope_notes, extra_context, project_dir_in, verification_commands] max_iterations: 3 max_attempts: 2 + state_updates: + parse_failure: '{{output}}' + fallback: pipeline_fault next: facts facts: @@ -114,6 +134,15 @@ nodes: description: Deterministic diff resolution — changed files and the (bounded) diff text every criterion branch verifies against. script: scripts/diff_facts.py timeout: 120 + fallback: pipeline_fault + next: run_checks + + run_checks: + id: run_checks + type: script + description: Deterministic execution evidence — runs ONLY the caller-declared verification commands in the project dir and records exec_results; criterion branches cite the record and never execute anything. Undeclared commands degrade to a "none declared" marker, runner errors to an ENVIRONMENT marker; never fails the node. + script: scripts/run_checks.py + timeout: 3600 next: map_criteria map_criteria: @@ -158,6 +187,18 @@ nodes: values shaped to the criterion's example, TODO/stub presented as done — those are PARTIAL or DIVERGED with the gaming named in the complaint. + Execution criteria ("tests pass", "lint clean", "build succeeds") are + judged ONLY against the recorded verification runs in your prompt — you + have no execution tools BY DESIGN and must never claim to have run + anything yourself: + - a recorded green run (exit 0) of a command covering the criterion is + the proving evidence — MET, citing that run (the command + exit 0). + - a recorded failing run covering the criterion ⇒ UNMET, citing the + recorded exit code and output tail. + - "none declared" or an ENVIRONMENT marker ⇒ at best PARTIAL, with an + ENVIRONMENT note in the complaint naming the exact command the caller + should declare via verification_commands (e.g. "cargo test --all"). + Output format — respond with ONLY this JSON object, no prose around it: {"id": "", "status": "MET" | "PARTIAL" | "UNMET" | "DIVERGED", @@ -181,6 +222,9 @@ nodes: The diff{{diff_note}}: {{diff_text}} + + Verification runs (recorded by the deterministic run_checks stage): + {{exec_results}} {{gate_feedback}} tools: - fs_read @@ -273,12 +317,23 @@ nodes: required: [extra_complaints, observations] max_iterations: 10 max_attempts: 2 + state_updates: + holistic_failure: '{{output}}' + fallback: verdict + next: verdict + + pipeline_fault: + id: pipeline_fault + type: script + description: Fail-closed fault marker (reached only via fallback routes from parse/facts) — records a PIPELINE-FAULT in state and continues to verdict, which turns any fault into DIVERGES. The graph never dies without its sentinel. + script: scripts/pipeline_fault.py + timeout: 30 next: verdict verdict: id: verdict type: script - description: Deterministic verdict — CONFORMS requires every criterion MET and zero holistic complaints; no criteria at all fails closed. Assembles the exact sentinel format. + description: Deterministic verdict — CONFORMS requires every criterion MET, zero holistic complaints, and zero pipeline faults; no criteria at all fails closed and any recorded PIPELINE-FAULT forces DIVERGES (degraded is never passing). Assembles the exact sentinel format. script: scripts/verdict.py timeout: 30 next: done diff --git a/assets/agents/adversary/scripts/pipeline_fault.py b/assets/agents/adversary/scripts/pipeline_fault.py new file mode 100644 index 0000000..a16bfc7 --- /dev/null +++ b/assets/agents/adversary/scripts/pipeline_fault.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Fail-closed fault marker for the adversary graph. + +Reached ONLY via `fallback:` routes (parse, facts). Appends a machine-readable +"PIPELINE-FAULT:" marker to `pipeline_faults` and continues to `verdict`, +which turns any recorded fault into DIVERGES — the graph never dies without +its sentinel, and a degraded run can never read as CONFORMS. + +Stage attribution: an llm-node failure writes an "LLM node …failed: " +string into `parse_failure` via parse's `state_updates`; anything else means +the deterministic facts (diff resolution) script failed, which leaves no +failure text in state by design. +""" + +import json +import os + +MAX_DETAIL_CHARS = 500 + + +def load_state(): + if path := os.environ.get("GRAPH_STATE_FILE"): + with open(path) as f: + return json.load(f) + return json.loads(os.environ.get("GRAPH_STATE", "{}")) + + +def main(): + state = load_state() + faults = [f for f in (state.get("pipeline_faults") or []) if isinstance(f, str)] + parse_failure = state.get("parse_failure") + detail = parse_failure.strip() if isinstance(parse_failure, str) else "" + if detail.startswith("LLM node"): + detail = detail.replace("\n", " ") + if len(detail) > MAX_DETAIL_CHARS: + detail = detail[: MAX_DETAIL_CHARS - 1] + "…" + faults.append(f"PIPELINE-FAULT: parse failed — cannot review: {detail}") + else: + faults.append( + "PIPELINE-FAULT: diff resolution failed — changed files and diff text " + "could not be resolved; criterion verification never ran" + ) + print(json.dumps({"pipeline_faults": faults})) + + +try: + main() +except Exception as e: # noqa: BLE001 — the fault marker itself must never crash + print( + json.dumps( + { + "pipeline_faults": [ + f"PIPELINE-FAULT: fault-marker script error: {e} — treat the " + "run as degraded" + ] + } + ) + ) diff --git a/assets/agents/adversary/scripts/run_checks.py b/assets/agents/adversary/scripts/run_checks.py new file mode 100644 index 0000000..c82e225 --- /dev/null +++ b/assets/agents/adversary/scripts/run_checks.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Deterministic execution-evidence recorder for the adversary graph. + +Runs ONLY the caller-declared verification commands (verification_commands) +in the project dir and records the outcomes as `exec_results` — a list of +{cmd, exit, duration_s, tail} records. Criterion branches cite this record; +they never execute anything themselves. + +NO auto-detection: undeclared commands mean a "none declared" marker +(fail-visible, not fail-guessed). Any runner error degrades into an +ENVIRONMENT marker — this script never fails the node. +""" + +import json +import os +import subprocess +import time + +PER_COMMAND_TIMEOUT_SECS = 900 +TAIL_LINES = 50 +MAX_TAIL_CHARS = 8000 + + +def load_state(): + if path := os.environ.get("GRAPH_STATE_FILE"): + with open(path) as f: + return json.load(f) + return json.loads(os.environ.get("GRAPH_STATE", "{}")) + + +def tail_of(text): + tail = "\n".join(text.splitlines()[-TAIL_LINES:]) + if len(tail) > MAX_TAIL_CHARS: + tail = "…" + tail[-MAX_TAIL_CHARS:] + return tail + + +def main(): + state = load_state() + declared = state.get("verification_commands") + cmds = ( + [c.strip() for c in declared if isinstance(c, str) and c.strip()] + if isinstance(declared, list) + else [] + ) + if not cmds: + print( + json.dumps( + { + "exec_results": ( + "none declared — the caller supplied no verification_commands; " + "execution criteria (build/test/lint) cannot be independently " + "confirmed. Declare the exact commands in the spawn prompt to " + "get recorded runs." + ) + } + ) + ) + return + + proj = os.path.expanduser( + (state.get("project_dir") or "").strip() or "." + ) + results = [] + for cmd in cmds: + start = time.monotonic() + try: + r = subprocess.run( + cmd, + shell=True, + cwd=proj, + capture_output=True, + text=True, + timeout=PER_COMMAND_TIMEOUT_SECS, + ) + combined = (r.stdout or "") + (("\n" + r.stderr) if r.stderr else "") + results.append( + { + "cmd": cmd, + "exit": r.returncode, + "duration_s": round(time.monotonic() - start, 1), + "tail": tail_of(combined), + } + ) + except subprocess.TimeoutExpired: + results.append( + { + "cmd": cmd, + "exit": -1, + "duration_s": round(time.monotonic() - start, 1), + "tail": ( + f"TIMEOUT: no exit within {PER_COMMAND_TIMEOUT_SECS}s — " + "recorded as a failing run" + ), + } + ) + print(json.dumps({"exec_results": results})) + + +try: + main() +except Exception as e: # noqa: BLE001 — never fail the node; degrade visibly + print( + json.dumps( + { + "exec_results": ( + f"ENVIRONMENT: verification runner error: {e} — the declared " + "commands could not be executed; treat execution criteria as " + "PARTIAL (unproven), not as failed" + ) + } + ) + ) diff --git a/assets/agents/adversary/scripts/verdict.py b/assets/agents/adversary/scripts/verdict.py index f604e0b..524baca 100644 --- a/assets/agents/adversary/scripts/verdict.py +++ b/assets/agents/adversary/scripts/verdict.py @@ -1,15 +1,21 @@ #!/usr/bin/env python3 """Deterministic conformance verdict for the adversary graph. -CONFORMS requires: at least one criterion, every criterion MET, and zero -holistic complaints. Everything else — including a missing plan — is -DIVERGES (fail-closed). Assembles the exact sentinel format the callers -route on. Never crashes into a silent verdict. +CONFORMS requires: at least one criterion, every criterion MET, zero +holistic complaints, and zero pipeline faults. Everything else — including +a missing plan or a degraded (faulted) pipeline — is DIVERGES (fail-closed). +Any "PIPELINE-FAULT:" marker recorded in `pipeline_faults` (or a holistic +llm-node failure captured in `holistic_failure`) becomes complaint #1; +per-criterion results are still reported. Assembles the exact sentinel +format the callers route on. Never crashes into a silent verdict. """ import json import os +MAX_FAULT_DETAIL_CHARS = 300 + + def load_state(): if path := os.environ.get("GRAPH_STATE_FILE"): with open(path) as f: @@ -17,11 +23,85 @@ def load_state(): return json.loads(os.environ.get("GRAPH_STATE", "{}")) +def collect_pipeline_faults(state): + """Recorded fault markers, plus a holistic llm-node failure if one occurred. + + `holistic_failure` holds the "LLM node …failed: " string when the + holistic pass fell back to verdict; on a successful pass it holds the + rendered schema output (never that prefix). + """ + faults = [ + f.strip() + for f in (state.get("pipeline_faults") or []) + if isinstance(f, str) and f.strip() + ] + hf = state.get("holistic_failure") + if isinstance(hf, str) and hf.strip().startswith("LLM node"): + detail = hf.strip().replace("\n", " ") + if len(detail) > MAX_FAULT_DETAIL_CHARS: + detail = detail[: MAX_FAULT_DETAIL_CHARS - 1] + "…" + faults.append( + "PIPELINE-FAULT: holistic pass failed — criterion verdicts stand " + f"but cross-cutting hunt did not run ({detail})" + ) + return faults + + +def append_criterion_complaints(lines, verdicts, extra, start): + """Numbered complaints for non-MET verdicts and holistic extras.""" + i = start + for v in verdicts: + if v.get("status") == "MET": + continue + i += 1 + text = (v.get("text") or "").strip().replace("\n", " ") + if len(text) > 140: + text = text[:137] + "…" + lines.append( + f'{i}. Acceptance criterion "{text}" — {v.get("status", "?").title()} — ' + f'{(v.get("complaint") or "").strip()}' + ) + for c in extra: + i += 1 + lines.append( + f"{i}. {c.get('kind', 'violation')} — {c.get('location', '?')} — " + f"{c.get('violation', '')} — {c.get('fix', '')}" + ) + return i + + +def render_exec_results(exec_results): + """Lines for the "Verification runs:" report section. + + exec_results is a list of {cmd, exit, duration_s, tail} records from + run_checks.py, or a string marker ("none declared —…" / "ENVIRONMENT: …"), + or empty when the run never reached the verification stage. + """ + if isinstance(exec_results, list): + out = [] + for r in exec_results: + if not isinstance(r, dict): + continue + status = "PASS" if r.get("exit") == 0 else "FAIL" + out.append( + f"- [{status}] `{r.get('cmd', '?')}` — exit {r.get('exit', '?')}, " + f"{r.get('duration_s', '?')}s" + ) + if status == "FAIL": + tail = (r.get("tail") or "").strip() + out.extend(" " + t for t in tail.splitlines()[-15:]) + return out or ["- (no verification commands were run)"] + if isinstance(exec_results, str) and exec_results.strip(): + return [f"- {exec_results.strip()}"] + return ["- none recorded (the run did not reach the verification stage)"] + + def main(): state = load_state() verdicts = [v for v in (state.get("crit_verdicts") or []) if isinstance(v, dict)] extra = [c for c in (state.get("extra_complaints") or []) if isinstance(c, dict)] observations = (state.get("observations") or "").strip() + faults = collect_pipeline_faults(state) n = len(verdicts) met = [v for v in verdicts if v.get("status") == "MET"] @@ -29,7 +109,30 @@ def main(): bad = [v for v in verdicts if v.get("status") in ("UNMET", "DIVERGED")] lines = [] - if n == 0: + if faults: + lines.append("ADVERSARIAL_REVIEW: DIVERGES") + if n: + lines.append( + f"Criteria: {len(met)}/{n} met, {len(partial)} partial, " + f"{len(bad)} unmet/diverged — degraded run: pipeline fault recorded " + "(fail-closed)." + ) + else: + lines.append( + "Criteria: none verified — degraded run: pipeline fault recorded " + "(fail-closed)." + ) + lines.append("Complaints:") + i = 0 + for f in faults: + i += 1 + lines.append(f"{i}. {f}") + append_criterion_complaints(lines, verdicts, extra, i) + if observations: + lines.append("") + lines.append("Non-blocking observations:") + lines.append(observations) + elif n == 0: lines.append("ADVERSARIAL_REVIEW: DIVERGES") lines.append("Criteria: none provided.") lines.append("Complaints:") @@ -51,24 +154,7 @@ def main(): f"Criteria: {len(met)}/{n} met, {len(partial)} partial, {len(bad)} unmet/diverged." ) lines.append("Complaints:") - i = 0 - for v in verdicts: - if v.get("status") == "MET": - continue - i += 1 - text = (v.get("text") or "").strip().replace("\n", " ") - if len(text) > 140: - text = text[:137] + "…" - lines.append( - f'{i}. Acceptance criterion "{text}" — {v.get("status", "?").title()} — ' - f'{(v.get("complaint") or "").strip()}' - ) - for c in extra: - i += 1 - lines.append( - f"{i}. {c.get('kind', 'violation')} — {c.get('location', '?')} — " - f"{c.get('violation', '')} — {c.get('fix', '')}" - ) + append_criterion_complaints(lines, verdicts, extra, 0) if observations: lines.append("") lines.append("Non-blocking observations:") @@ -84,6 +170,10 @@ def main(): text = text[:97] + "…" lines.append(f'- "{text}" — {(v.get("evidence") or "").strip()}') + lines.append("") + lines.append("Verification runs:") + lines.extend(render_exec_results(state.get("exec_results"))) + print(json.dumps({"adv_report": "\n".join(lines)})) diff --git a/src/config/request_context.rs b/src/config/request_context.rs index 27ce19b..be469cd 100644 --- a/src/config/request_context.rs +++ b/src/config/request_context.rs @@ -8348,6 +8348,228 @@ mod tests { ); } + // ---- adversary suite-script regression tests (R5) ---- + // + // Fault paths of the adversary's verdict/gate scripts, exercised by + // invoking `python3