From 8407729d5a3d73224b10970420e1ef04fc15e119 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Sat, 12 Sep 2026 15:44:23 -0600 Subject: [PATCH] feat(TASK-018): review-gauntlet lane degradation + verification passthrough - lane_fault.py + fallback on all four run_* lane nodes: an agent-node failure is normalized into a PIPELINE-FAULT lane result instead of killing the graph - verdict_gate.py: PIPELINE-FAULT lane results block distinctly from SKIPPED and missing-sentinel; a PIPELINE-FAULT in signals_error blocks at the gauntlet level - build_items.py: restructured into main() + fail-closed guard (crash emits empty lanes + PIPELINE-FAULT) and folds lanes_degraded into lanes_summary - parse: state_updates/fallback -> parse_fault.py, which records the fault in signals_error and jumps to the verdict gate - select_lanes: state_updates/fallback -> default_lanes.py, a deterministic wider-never-narrower lane fallback - verification_commands threaded through parse into run_adversary's prompt (declared-only, never invented) - run_probe prompt gains the aborted-attempt reconcile line - 8 gauntlet_* R5 tests in src/config/request_context.rs (script harness + graph interpolation) --- assets/agents/review-gauntlet/graph.yaml | 52 +++- .../review-gauntlet/scripts/build_items.py | 186 +++++++------ .../review-gauntlet/scripts/default_lanes.py | 51 ++++ .../review-gauntlet/scripts/lane_fault.py | 59 +++++ .../review-gauntlet/scripts/parse_fault.py | 52 ++++ .../review-gauntlet/scripts/verdict_gate.py | 34 +++ src/config/request_context.rs | 249 ++++++++++++++++++ 7 files changed, 603 insertions(+), 80 deletions(-) create mode 100644 assets/agents/review-gauntlet/scripts/default_lanes.py create mode 100644 assets/agents/review-gauntlet/scripts/lane_fault.py create mode 100644 assets/agents/review-gauntlet/scripts/parse_fault.py diff --git a/assets/agents/review-gauntlet/graph.yaml b/assets/agents/review-gauntlet/graph.yaml index e975f17..56a42a0 100644 --- a/assets/agents/review-gauntlet/graph.yaml +++ b/assets/agents/review-gauntlet/graph.yaml @@ -39,6 +39,7 @@ initial_state: security_posture: standard probe_context: '' extra_context: '' + verification_commands: [] forced_lanes: [] changed_files: [] file_count: 0 @@ -56,6 +57,9 @@ initial_state: add_security: false add_probe: false lane_reasons: '' + parse_failure: '' + select_lanes_failure: '' + lanes_degraded: '' code_review_items: [] adversary_items: [] security_items: [] @@ -100,6 +104,10 @@ nodes: names). Empty array = automatic selection. - `extra_context`: any remaining caller context reviewers should see, condensed. + - `verification_commands`: the exact build/test/lint shell commands the + caller explicitly declares for verification (verbatim strings, in the + order given). Empty array when the caller declares none — NEVER invent, + guess, or auto-detect commands; undeclared stays undeclared. prompt: | Extract the review-request facts from this request: @@ -126,11 +134,26 @@ nodes: type: string project_dir_in: type: string - required: [diff_spec, plan_context, rigor, security_posture, probe_context, forced_lanes, extra_context, project_dir_in] + verification_commands: + type: array + items: + type: string + required: [diff_spec, plan_context, rigor, security_posture, probe_context, forced_lanes, extra_context, project_dir_in, verification_commands] max_iterations: 3 max_attempts: 2 + state_updates: + parse_failure: '{{output}}' + fallback: parse_fault next: signals + parse_fault: + id: parse_fault + type: script + description: Fail-closed marker for a dead parse stage (reached only via parse's fallback) — records a PIPELINE-FAULT in signals_error and jumps straight to the verdict gate, which blocks on it. Lane item lists stay their initial [] so every lane records SKIPPED. + script: scripts/parse_fault.py + timeout: 30 + next: verdict_gate + signals: id: signals type: script @@ -204,6 +227,17 @@ nodes: required: [add_code_review, add_adversary, add_security, add_probe, lane_reasons] max_iterations: 3 max_attempts: 2 + state_updates: + select_lanes_failure: '{{output}}' + fallback: default_lanes + next: build_items + + default_lanes: + id: default_lanes + type: script + description: Deterministic lane-selection fallback (reached only via select_lanes' fallback) — forces code-review + adversary, plus probe when consumer_surface, when the additive LLM judgment dies. Selection degrades WIDER, never narrower; the degradation note flows into lanes_summary via build_items. + script: scripts/default_lanes.py + timeout: 30 next: build_items build_items: @@ -279,6 +313,7 @@ nodes: max_attempts: 2 state_updates: lane_out: '{{output}}' + fallback: lane_fault run_adversary: id: run_adversary @@ -294,12 +329,15 @@ nodes: The spec / plan / acceptance criteria it must satisfy: {{plan_context}} + Verification commands (declared by the caller; empty = none declared): {{verification_commands}} + {{extra_context}} Return your ADVERSARIAL_REVIEW verdict. max_attempts: 2 state_updates: lane_out: '{{output}}' + fallback: lane_fault run_security: id: run_security @@ -318,6 +356,7 @@ nodes: max_attempts: 2 state_updates: lane_out: '{{output}}' + fallback: lane_fault run_probe: id: run_probe @@ -335,12 +374,23 @@ nodes: Plan / acceptance-criteria context: {{plan_context}} + The workspace may contain test files from an aborted earlier attempt of + this lane; reconcile rather than duplicate. + Derive expected behaviors from the plan/contract BEFORE reading implementation code, run existing suites first, then author spec-first tests for the uncovered delta. Return your USAGE_PROBE verdict. max_attempts: 2 state_updates: lane_out: '{{output}}' + fallback: lane_fault + + lane_fault: + id: lane_fault + type: script + description: Fail-closed lane fault marker (reached only via the run_* lanes' fallback routes; shared across the four map branches). Normalizes the engine's "Agent node failed" text in lane_out into a PIPELINE-FAULT lane result the map collects; verdict_gate turns it into BLOCKED — a degraded lane is never a pass. + script: scripts/lane_fault.py + timeout: 30 verdict_gate: id: verdict_gate diff --git a/assets/agents/review-gauntlet/scripts/build_items.py b/assets/agents/review-gauntlet/scripts/build_items.py index 4a31e58..b104b94 100644 --- a/assets/agents/review-gauntlet/scripts/build_items.py +++ b/assets/agents/review-gauntlet/scripts/build_items.py @@ -8,20 +8,15 @@ skip mechanism — no dynamic routing anywhere. The LLM's add_* flags can only ADD lanes. A flag that is false never removes a lane the hard rules selected. + +Fail-closed: a builder crash emits all four lists empty plus a +PIPELINE-FAULT in signals_error — verdict_gate blocks on it and the crash +never kills the graph. """ import json import os -def load_state(): - if path := os.environ.get("GRAPH_STATE_FILE"): - with open(path) as f: - return json.load(f) - return json.loads(os.environ.get("GRAPH_STATE", "{}")) - - -state = load_state() - ALIASES = { "code-review": "code-review", "code_review": "code-review", @@ -39,80 +34,113 @@ ALIASES = { } -def text(key): - v = state.get(key) - return v.strip() if isinstance(v, str) else "" +def load_state(): + if path := os.environ.get("GRAPH_STATE_FILE"): + with open(path) as f: + return json.load(f) + return json.loads(os.environ.get("GRAPH_STATE", "{}")) -forced_raw = state.get("forced_lanes") or [] -forced, unknown = [], [] -for lane in forced_raw: - canon = ALIASES.get(str(lane).strip().lower()) - (forced.append(canon) if canon else unknown.append(str(lane))) +def main(): + state = load_state() -reasons = [] -if forced: - lanes = set(forced) - reasons.append(f"forced lanes honored exactly: {sorted(lanes)}") -else: - lanes = {"code-review"} - reasons.append( - "code-review: always on — the gauntlet is only spawned for non-trivial work" - ) - if text("plan_context"): - lanes.add("adversary") - reasons.append("adversary: a spec/plan/acceptance-criteria context is present") - if ( - state.get("touches_auth") - or state.get("touches_deps") - or state.get("touches_exec") - or state.get("security_posture") == "hardened" - ): - lanes.add("security") + def text(key): + v = state.get(key) + return v.strip() if isinstance(v, str) else "" + + forced_raw = state.get("forced_lanes") or [] + forced, unknown = [], [] + for lane in forced_raw: + canon = ALIASES.get(str(lane).strip().lower()) + (forced.append(canon) if canon else unknown.append(str(lane))) + + reasons = [] + if forced: + lanes = set(forced) + reasons.append(f"forced lanes honored exactly: {sorted(lanes)}") + else: + lanes = {"code-review"} reasons.append( - "security: attack-surface signals (auth/deps/exec) or hardened posture" + "code-review: always on — the gauntlet is only spawned for non-trivial work" ) - if state.get("consumer_surface") and text("probe_context"): - lanes.add("probe") - reasons.append( - "probe: consumer-facing surface changed and a local-run recipe/suite pointer exists" + if text("plan_context"): + lanes.add("adversary") + reasons.append("adversary: a spec/plan/acceptance-criteria context is present") + if ( + state.get("touches_auth") + or state.get("touches_deps") + or state.get("touches_exec") + or state.get("security_posture") == "hardened" + ): + lanes.add("security") + reasons.append( + "security: attack-surface signals (auth/deps/exec) or hardened posture" + ) + if state.get("consumer_surface") and text("probe_context"): + lanes.add("probe") + reasons.append( + "probe: consumer-facing surface changed and a local-run recipe/suite pointer exists" + ) + + # LLM judgment is additive-only. + llm_why = text("lane_reasons")[:300] + for flag, lane in ( + ("add_code_review", "code-review"), + ("add_adversary", "adversary"), + ("add_security", "security"), + ("add_probe", "probe"), + ): + if state.get(flag) and lane not in lanes: + if lane == "probe" and not text("probe_context"): + reasons.append( + "probe: requested by lane judgment but skipped — no local-run " + "recipe/suite pointer (a probe without one is guaranteed INCONCLUSIVE)" + ) + continue + if lane == "adversary" and not text("plan_context"): + reasons.append( + "adversary: requested by lane judgment but skipped — no spec/plan " + "to check conformance against" + ) + continue + lanes.add(lane) + reasons.append(f"{lane}: added by lane-selection judgment — {llm_why}") + + if unknown: + reasons.append(f"ignored unknown forced lane name(s): {unknown}") + + # A degraded lane selection (default_lanes fallback) must survive into the + # final report; lanes_summary is the only channel this script doesn't + # overwrite silently. + degraded = text("lanes_degraded") + if degraded: + reasons.insert(0, degraded) + + print( + json.dumps( + { + "code_review_items": [{"lane": "code-review"}] if "code-review" in lanes else [], + "adversary_items": [{"lane": "adversary"}] if "adversary" in lanes else [], + "security_items": [{"lane": "security"}] if "security" in lanes else [], + "probe_items": [{"lane": "probe"}] if "probe" in lanes else [], + "lanes_summary": "\n".join(f"- {r}" for r in reasons), + } + ) + ) + + +try: + main() +except Exception as e: # noqa: BLE001 — a builder crash must never kill the graph + print( + json.dumps( + { + "code_review_items": [], + "adversary_items": [], + "security_items": [], + "probe_items": [], + "lanes_summary": "", + "signals_error": f"PIPELINE-FAULT: lane builder crashed — {e}; no lanes were run", + } ) - - # LLM judgment is additive-only. - llm_why = text("lane_reasons")[:300] - for flag, lane in ( - ("add_code_review", "code-review"), - ("add_adversary", "adversary"), - ("add_security", "security"), - ("add_probe", "probe"), - ): - if state.get(flag) and lane not in lanes: - if lane == "probe" and not text("probe_context"): - reasons.append( - "probe: requested by lane judgment but skipped — no local-run " - "recipe/suite pointer (a probe without one is guaranteed INCONCLUSIVE)" - ) - continue - if lane == "adversary" and not text("plan_context"): - reasons.append( - "adversary: requested by lane judgment but skipped — no spec/plan " - "to check conformance against" - ) - continue - lanes.add(lane) - reasons.append(f"{lane}: added by lane-selection judgment — {llm_why}") - -if unknown: - reasons.append(f"ignored unknown forced lane name(s): {unknown}") - -print( - json.dumps( - { - "code_review_items": [{"lane": "code-review"}] if "code-review" in lanes else [], - "adversary_items": [{"lane": "adversary"}] if "adversary" in lanes else [], - "security_items": [{"lane": "security"}] if "security" in lanes else [], - "probe_items": [{"lane": "probe"}] if "probe" in lanes else [], - "lanes_summary": "\n".join(f"- {r}" for r in reasons), - } ) -) diff --git a/assets/agents/review-gauntlet/scripts/default_lanes.py b/assets/agents/review-gauntlet/scripts/default_lanes.py new file mode 100644 index 0000000..d33bca6 --- /dev/null +++ b/assets/agents/review-gauntlet/scripts/default_lanes.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Deterministic lane-selection fallback for the review gauntlet. + +Reached ONLY via select_lanes' `fallback:` route, when the additive LLM +judgment dies after retries. Emits `forced_lanes` — code-review + adversary, +plus probe when the deterministic signals flagged a consumer-facing +surface — which build_items honors exactly, so selection degrades WIDER, +never narrower. The degradation note goes into `lanes_degraded`; build_items +folds it into lanes_summary so the final report names the degradation. +""" + +import json +import os + +MAX_DETAIL_CHARS = 500 + + +def load_state(): + if path := os.environ.get("GRAPH_STATE_FILE"): + with open(path) as f: + return json.load(f) + return json.loads(os.environ.get("GRAPH_STATE", "{}")) + + +def main(): + state = load_state() + forced = ["code-review", "adversary"] + if state.get("consumer_surface"): + forced.append("probe") + raw = state.get("select_lanes_failure") + detail = raw.strip().replace("\n", " ") if isinstance(raw, str) else "" + if len(detail) > MAX_DETAIL_CHARS: + detail = detail[: MAX_DETAIL_CHARS - 1] + "…" + note = "lane selection degraded to deterministic defaults" + if detail: + note += f" — {detail}" + print(json.dumps({"forced_lanes": forced, "lanes_degraded": note})) + + +try: + main() +except Exception as e: # noqa: BLE001 — the fallback itself must never crash + print( + json.dumps( + { + "forced_lanes": ["code-review", "adversary"], + "lanes_degraded": "lane selection degraded to deterministic " + f"defaults (fallback script error: {e})", + } + ) + ) diff --git a/assets/agents/review-gauntlet/scripts/lane_fault.py b/assets/agents/review-gauntlet/scripts/lane_fault.py new file mode 100644 index 0000000..56a53ff --- /dev/null +++ b/assets/agents/review-gauntlet/scripts/lane_fault.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Fail-closed lane fault marker for the review gauntlet. + +Reached ONLY via the run_* lane nodes' `fallback:` routes inside their map +branches. Grounded in engine behavior: on agent-node failure the engine +writes "Agent node failed: {e:#}" into `lane_out` via the node's +state_updates BEFORE routing to the fallback, so the raw failure text is +already in state. This node normalizes it into a +"PIPELINE-FAULT: lane failed after retries — …" marker that the map +collects as the lane result; verdict_gate turns any PIPELINE-FAULT lane +result into BLOCKED — a degraded lane can never read as a pass. + +Lane identity comes from `lane_ctx` (the map item build_items emitted, which +carries a `lane` field); a missing/malformed item degrades to "unknown". +""" + +import json +import os + +MAX_DETAIL_CHARS = 500 + + +def load_state(): + if path := os.environ.get("GRAPH_STATE_FILE"): + with open(path) as f: + return json.load(f) + return json.loads(os.environ.get("GRAPH_STATE", "{}")) + + +def main(): + state = load_state() + lane = "unknown" + lane_ctx = state.get("lane_ctx") + if isinstance(lane_ctx, dict): + name = lane_ctx.get("lane") + if isinstance(name, str) and name.strip(): + lane = name.strip() + raw = state.get("lane_out") + detail = raw.strip().replace("\n", " ") if isinstance(raw, str) else "" + if len(detail) > MAX_DETAIL_CHARS: + detail = detail[: MAX_DETAIL_CHARS - 1] + "…" + print( + json.dumps( + {"lane_out": f"PIPELINE-FAULT: {lane} lane failed after retries — {detail}"} + ) + ) + + +try: + main() +except Exception as e: # noqa: BLE001 — the fault marker itself must never crash + print( + json.dumps( + { + "lane_out": "PIPELINE-FAULT: lane failed after retries — " + f"fault-marker script error: {e}" + } + ) + ) diff --git a/assets/agents/review-gauntlet/scripts/parse_fault.py b/assets/agents/review-gauntlet/scripts/parse_fault.py new file mode 100644 index 0000000..7ace95d --- /dev/null +++ b/assets/agents/review-gauntlet/scripts/parse_fault.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Fail-closed marker for a dead parse stage of the review gauntlet. + +Reached ONLY via parse's `fallback:` route. parse's state_updates wrote an +"LLM node …failed: " string into `parse_failure` before routing here; +this node records the failure as a PIPELINE-FAULT in `signals_error` and the +graph jumps straight to the verdict gate. Every lane item list is still its +initial [] so every lane records SKIPPED, and verdict_gate's signals_error +rule makes the verdict BLOCKED — a gauntlet that could not even parse its +request never reads as a pass. +""" + +import json +import os + +MAX_DETAIL_CHARS = 500 + + +def load_state(): + if path := os.environ.get("GRAPH_STATE_FILE"): + with open(path) as f: + return json.load(f) + return json.loads(os.environ.get("GRAPH_STATE", "{}")) + + +def main(): + state = load_state() + raw = state.get("parse_failure") + detail = raw.strip().replace("\n", " ") if isinstance(raw, str) else "" + if len(detail) > MAX_DETAIL_CHARS: + detail = detail[: MAX_DETAIL_CHARS - 1] + "…" + print( + json.dumps( + { + "signals_error": "PIPELINE-FAULT: parse failed — cannot run the " + f"gauntlet: {detail}" + } + ) + ) + + +try: + main() +except Exception as e: # noqa: BLE001 — the fault marker itself must never crash + print( + json.dumps( + { + "signals_error": "PIPELINE-FAULT: parse failed — cannot run the " + f"gauntlet: fault-marker script error: {e}" + } + ) + ) diff --git a/assets/agents/review-gauntlet/scripts/verdict_gate.py b/assets/agents/review-gauntlet/scripts/verdict_gate.py index d1eea2d..6664624 100644 --- a/assets/agents/review-gauntlet/scripts/verdict_gate.py +++ b/assets/agents/review-gauntlet/scripts/verdict_gate.py @@ -5,6 +5,9 @@ Parses each configured lane's verdict SENTINEL with a regex — no LLM re-reads, no judgment. The gate's hard rules: - a missing sentinel is a LANE FAILURE and blocks — never a pass; + - a PIPELINE-FAULT lane result (the lane's agent died after retries) or a + PIPELINE-FAULT in signals_error (builder/parse fault) blocks — a + degraded pipeline is never a pass; - any 🔴 finding in the code-review report blocks regardless of the lane's own verdict line; - NEEDS-HUMAN without 🔴 passes but is surfaced as attention required; @@ -40,10 +43,26 @@ def main(): def record(lane, status, detail=""): lane_lines.append(f"| {lane} | {status} | {detail} |") + def fault_excerpt(report): + line = next( + (ln for ln in report.splitlines() if "PIPELINE-FAULT:" in ln), report + ) + return line.strip()[:200] + + def lane_fault_blocker(lane, report): + blockers.append( + f"{lane}: PIPELINE-FAULT — the lane failed after retries and produced " + f"no verdict; a degraded lane is never a pass ({fault_excerpt(report)})" + ) + record(lane, "BLOCKED", "PIPELINE-FAULT (lane failed)") + # --- code-review ------------------------------------------------------ cr = lane_report("code_review_results") if cr is None: record("code-review", "SKIPPED", "not selected") + elif "PIPELINE-FAULT:" in cr: + lane_fault_blocker("code-review", cr) + reports.append(("code-review", cr)) else: reports.append(("code-review", cr)) m = re.search(r"Verdict:\**\s*\**\s*(MERGE-READY|NEEDS-HUMAN)", cr) @@ -72,6 +91,9 @@ def main(): adv = lane_report("adversary_results") if adv is None: record("adversary", "SKIPPED", "not selected") + elif "PIPELINE-FAULT:" in adv: + lane_fault_blocker("adversary", adv) + reports.append(("adversary", adv)) else: reports.append(("adversary", adv)) m = re.search(r"ADVERSARIAL_REVIEW:\s*(CONFORMS|DIVERGES)", adv) @@ -92,6 +114,9 @@ def main(): sec = lane_report("security_results") if sec is None: record("security", "SKIPPED", "not selected") + elif "PIPELINE-FAULT:" in sec: + lane_fault_blocker("security", sec) + reports.append(("security", sec)) else: reports.append(("security", sec)) m = re.search(r"SECURITY_REVIEW:\s*(PASS|FAIL)", sec) @@ -110,6 +135,9 @@ def main(): pb = lane_report("probe_results") if pb is None: record("probe", "SKIPPED", "not selected") + elif "PIPELINE-FAULT:" in pb: + lane_fault_blocker("probe", pb) + reports.append(("probe", pb)) else: reports.append(("probe", pb)) m = re.search(r"USAGE_PROBE:\s*(PASS|FAIL|INCONCLUSIVE)", pb) @@ -131,6 +159,12 @@ def main(): ) record("probe", "BLOCKED", "missing verdict sentinel") + # A gauntlet-level fault (build_items crash, dead parse stage) means the + # lanes above were never built — SKIPPED rows alone must not read as PASS. + signals_error = state.get("signals_error") + if isinstance(signals_error, str) and "PIPELINE-FAULT:" in signals_error: + blockers.append(f"pipeline: {signals_error}") + verdict = "BLOCKED" if blockers else "PASS" md = "\n".join( diff --git a/src/config/request_context.rs b/src/config/request_context.rs index ee87eea..a91cce7 100644 --- a/src/config/request_context.rs +++ b/src/config/request_context.rs @@ -8762,6 +8762,255 @@ mod tests { ); } + // ---- review-gauntlet suite-script regression tests (R5) ---- + // + // Degradation paths of the gauntlet's lane/builder/gate scripts, + // exercised the same way as the adversary suite above: `python3