diff --git a/assets/agents/review-gauntlet/README.md b/assets/agents/review-gauntlet/README.md index 82c698c..39785f6 100644 --- a/assets/agents/review-gauntlet/README.md +++ b/assets/agents/review-gauntlet/README.md @@ -51,8 +51,11 @@ those rules are *structure*: (map-over-nothing), which *is* the skip mechanism. All four maps join at the verdict gate on equal-length paths. - **The verdict is a regex, not an opinion** (`verdict_gate.py`): a missing - sentinel is a lane FAILURE, never a pass; any 🔴 in the code-review report - blocks regardless of its verdict line; `NEEDS-HUMAN` without 🔴 passes but + sentinel is a lane FAILURE, never a pass; the critical count from the + code-review report's summary line (raw 🔴 count if absent) blocks + regardless of its verdict line; a code-review lane whose own verdict line + records a pipeline fault blocks (a degraded review is never a pass); + `NEEDS-HUMAN` without 🔴 passes but is surfaced under *Human attention required*; probe `INCONCLUSIVE` blocks with an environment note — it is never treated as PASS or FAIL. - **Verdict independence is structural**: lanes run as isolated sub-agents @@ -65,7 +68,8 @@ those rules are *structure*: lane* — distinct from SKIPPED (not selected). A dead `parse` records a fault and jumps straight to the gate (`parse_fault.py`). A dead `select_lanes` degrades to a deterministic selection (`default_lanes.py`): code-review + - adversary, plus probe on consumer surface, plus security on auth/deps/exec + adversary, plus probe on consumer surface *with a local-run recipe*, plus + security on auth/deps/exec signals or hardened posture, unioned with caller-forced lanes — wider, never narrower. A crashed `build_items` records a fault so an all-SKIPPED gate still blocks. Fault detection is prefix-anchored — a real review that merely @@ -77,7 +81,7 @@ those rules are *structure*: | Lane | Sentinel | Blocks on | |------|----------|-----------| -| code-reviewer | `**Verdict: MERGE-READY \| NEEDS-HUMAN**` | any 🔴 finding; missing sentinel | +| code-reviewer | `**Verdict: MERGE-READY \| NEEDS-HUMAN**` | critical count from the report's summary line (raw 🔴 count if absent); missing sentinel; own verdict line records a pipeline fault | | adversary | `ADVERSARIAL_REVIEW: CONFORMS \| DIVERGES` | DIVERGES; missing sentinel | | security-reviewer | `SECURITY_REVIEW: PASS \| FAIL` | FAIL; missing sentinel | | probe | `USAGE_PROBE: PASS \| FAIL \| INCONCLUSIVE` | FAIL; INCONCLUSIVE (environment note); missing sentinel | diff --git a/assets/agents/review-gauntlet/graph.yaml b/assets/agents/review-gauntlet/graph.yaml index 7ad2482..0c08bb8 100644 --- a/assets/agents/review-gauntlet/graph.yaml +++ b/assets/agents/review-gauntlet/graph.yaml @@ -41,7 +41,13 @@ variables: settings: max_loop_iterations: 10 log_state_snapshots: true - timeout: 10800 + # max over agent lanes of (max_attempts × lane timeout) = 2 × 11400 = 22800 + # (agent-node retries each get a fresh per-attempt budget, + # src/graph/agent.rs retry_transient/bounded_attempt) + 1200 margin for + # the script stages (signals/select/build/gate) = 24000; the parse and + # select_lanes llm nodes carry no `timeout:` and are bounded by + # max_iterations. + timeout: 24000 initial_state: diff_spec: worktree @@ -237,7 +243,7 @@ nodes: default_lanes: id: default_lanes type: script - description: Deterministic lane-selection fallback (reached only via select_lanes' fallback) — when the additive LLM judgment dies, forces code-review + adversary, plus probe when consumer_surface, plus security on attack-surface signals (auth/deps/exec) or hardened posture, unioned with any caller-forced lanes. Selection degrades WIDER, never narrower; the degradation note flows into lanes_summary via build_items. + description: Deterministic lane-selection fallback (reached only via select_lanes' fallback) — when the additive LLM judgment dies, forces code-review + adversary, plus probe when consumer_surface AND a probe_context (local-run recipe) is present, plus security on attack-surface signals (auth/deps/exec) or hardened posture, unioned with any caller-forced lanes. Selection degrades WIDER, never narrower; the degradation note flows into lanes_summary via build_items. script: scripts/default_lanes.py timeout: 30 next: build_items @@ -299,7 +305,12 @@ nodes: type: agent description: Independent code-quality review lane. agent: code-reviewer - timeout: 7200 + # Exceeds the code-reviewer graph's own settings.timeout (10800) by 600 so + # the child's graph timeout, not this lane timeout, is the binding bound. + # NOTE: the child's own internal retry envelope (its nodes' max_attempts × + # timeouts) is not derived here — deliberately out of scope for this + # change; tracked as a follow-up. + timeout: 11400 prompt: | Review the code changes in {{project_dir}} (diff: {{diff_spec}}) as an independent post-implementation review. @@ -322,7 +333,12 @@ nodes: type: agent description: Plan-conformance review lane. agent: adversary - timeout: 7200 + # Exceeds the adversary graph's own settings.timeout (9000) by 600 so the + # child's graph timeout, not this lane timeout, is the binding bound. + # NOTE: the child's own internal retry envelope (its nodes' max_attempts × + # timeouts) is not derived here — deliberately out of scope for this + # change; tracked as a follow-up. + timeout: 9600 inputs: verification_commands: '{{verification_commands}}' prompt: | @@ -399,7 +415,7 @@ nodes: verdict_gate: id: verdict_gate type: script - description: Deterministic sentinel parser and gate. A missing sentinel is a lane failure, never a pass; probe INCONCLUSIVE blocks with an environment note; 🔴 findings block regardless of the lane's own verdict line. + description: Deterministic sentinel parser and gate. A missing sentinel is a lane failure, never a pass; probe INCONCLUSIVE blocks with an environment note; the code-review critical count is read from render.py's summary line (raw 🔴 count if absent) and blocks regardless of the lane's own verdict line; a code-review verdict line that records the nested graph's own pipeline fault blocks as a degraded lane. script: scripts/verdict_gate.py timeout: 60 next: done diff --git a/assets/agents/review-gauntlet/scripts/build_items.py b/assets/agents/review-gauntlet/scripts/build_items.py index b104b94..f8198eb 100644 --- a/assets/agents/review-gauntlet/scripts/build_items.py +++ b/assets/agents/review-gauntlet/scripts/build_items.py @@ -17,6 +17,8 @@ never kills the graph. import json import os +# Keep in sync with default_lanes.ALIASES — a caller-forced lane must +# canonicalize the same way on the degraded path as it does here. ALIASES = { "code-review": "code-review", "code_review": "code-review", diff --git a/assets/agents/review-gauntlet/scripts/default_lanes.py b/assets/agents/review-gauntlet/scripts/default_lanes.py index 4387b3a..ffaeabf 100644 --- a/assets/agents/review-gauntlet/scripts/default_lanes.py +++ b/assets/agents/review-gauntlet/scripts/default_lanes.py @@ -3,13 +3,19 @@ Reached ONLY via select_lanes' `fallback:` route, when the additive LLM judgment dies after retries. Emits `forced_lanes` — code-review + adversary, -plus probe when the deterministic signals flagged a consumer-facing surface, +plus probe when the deterministic signals flagged a consumer-facing surface +AND a local-run recipe exists (build_items gates probe the same way — a +probe without a recipe is INCONCLUSIVE by construction), plus security when the attack-surface signals (auth/deps/exec) or a hardened posture demand it, unioned with any lanes the caller forced (canonicalized with the same aliases build_items uses) — which build_items honors exactly, so selection degrades WIDER, never narrower. The degradation note goes into `lanes_degraded`; build_items folds it into lanes_summary so the final report names the degradation. + +Fail-closed: if the fallback itself crashes it emits the widest safe +selection (defaults ∪ security) plus a PIPELINE-FAULT in signals_error so +verdict_gate blocks — a crashed fallback never narrows selection silently. """ import json @@ -49,7 +55,12 @@ def load_state(): def main(): state = load_state() lanes = {"code-review", "adversary"} - if state.get("consumer_surface"): + probe_context = state.get("probe_context") + if ( + state.get("consumer_surface") + and isinstance(probe_context, str) + and probe_context.strip() + ): lanes.add("probe") # The deterministic security signals are already in state when this # fallback runs; a degraded selection must never drop the security lane @@ -82,9 +93,13 @@ except Exception as e: # noqa: BLE001 — the fallback itself must never crash print( json.dumps( { - "forced_lanes": ["code-review", "adversary"], + "forced_lanes": ["code-review", "adversary", "security"], "lanes_degraded": "lane selection degraded to deterministic " f"defaults (fallback script error: {e})", + "signals_error": ( + f"PIPELINE-FAULT: lane-selection fallback crashed — {e}; " + "lanes widened to defaults ∪ security" + ), } ) ) diff --git a/assets/agents/review-gauntlet/scripts/verdict_gate.py b/assets/agents/review-gauntlet/scripts/verdict_gate.py index 4d6fa39..f995a0d 100644 --- a/assets/agents/review-gauntlet/scripts/verdict_gate.py +++ b/assets/agents/review-gauntlet/scripts/verdict_gate.py @@ -5,13 +5,21 @@ Parses each configured lane's verdict SENTINEL with a regex — no LLM re-reads, no judgment. The gate's hard rules: - a missing sentinel is a LANE FAILURE and blocks — never a pass; + - a non-string lane result (the map collected an object instead of the + lane's rendered text) blocks — it cannot carry a sentinel; - a PIPELINE-FAULT lane result (the lane's agent died after retries; detected only when the report IS the lane_fault.py marker, i.e. it STARTS with it — a real review merely quoting the marker flows to the normal rules) or a PIPELINE-FAULT in signals_error (builder/parse fault) blocks — a degraded pipeline is never a pass; - - any 🔴 finding in the code-review report blocks regardless of the - lane's own verdict line; + - a code-review lane whose OWN verdict line records a pipeline fault + (code-reviewer's verdict.py / render.py degraded-run wording) blocks — + the nested graph completed, but with a dead domain lane, verifier, or + synthesis behind it, so its critical count is not trustworthy; + - a non-zero critical count in the code-review report blocks regardless + of the lane's own verdict line — derived from render.py's deterministic + summary line (`*Reviewed N files, found X critical, …*`); the raw 🔴 + count is used only when that summary line is missing; - NEEDS-HUMAN without 🔴 passes but is surfaced as attention required; - probe INCONCLUSIVE blocks with an ENVIRONMENT note — it is never treated as PASS or FAIL (fix the environment per the plan's local-run @@ -34,13 +42,17 @@ def load_state(): def main(): state = load_state() blockers, attention, lane_lines, reports = [], [], [], [] + malformed = set() def lane_report(key): arr = state.get(key) or [] if not arr: return None item = arr[0] - return item if isinstance(item, str) else json.dumps(item) + if isinstance(item, str): + return item + malformed.add(key) + return json.dumps(item) def record(lane, status, detail=""): lane_lines.append(f"| {lane} | {status} | {detail} |") @@ -65,21 +77,63 @@ def main(): ) record(lane, "BLOCKED", "PIPELINE-FAULT (lane failed)") + def malformed_blocker(lane, report): + blockers.append(f"{lane}: malformed lane result (non-string item) — never a pass") + record(lane, "BLOCKED", "malformed lane result (non-string)") + reports.append((lane, report)) + # --- code-review ------------------------------------------------------ cr = lane_report("code_review_results") if cr is None: record("code-review", "SKIPPED", "not selected") + elif "code_review_results" in malformed: + malformed_blocker("code-review", cr) elif is_lane_fault(cr): lane_fault_blocker("code-review", cr) reports.append(("code-review", cr)) else: reports.append(("code-review", cr)) m = re.search(r"Verdict:\**\s*\**\s*(MERGE-READY|NEEDS-HUMAN)", cr) - reds = cr.count("🔴") + # code-reviewer's verdict.py forces NEEDS-HUMAN on its own internal + # faults; render.py emits that as `**Verdict: NEEDS-HUMAN** — ` + # (CONTRACT comments at both emitters). Scoped to the FIRST rendered + # verdict line — render.py emits it before any finding body — so a + # finding that quotes the wording cannot trip it. + verdict_line = re.search(r"^\*\*Verdict: .*$", cr, re.MULTILINE) + m_fault = verdict_line and re.search( + r"^\*\*Verdict: NEEDS-HUMAN\*\* — .*" + r"(pipeline fault\(s\) recorded|PIPELINE-FAULT:|report rendering error)", + verdict_line.group(0), + ) + # Same quoted-marker class as the PIPELINE-FAULT prefix anchoring: a + # 🔴 quoted inside finding text or a Changes-table row is not a + # finding. render.py's summary line is the authoritative count; the + # raw count is the fail-closed fallback when that line is absent. The + # LAST match wins: render.py emits the line after every finding body, + # so a quoted look-alike earlier in the report cannot shadow it. The + # pattern is the FULL shape of render.py's line (CONTRACT comment + # there) so a partial look-alike in LLM-authored text never matches. + sums = re.findall( + r"^\*Reviewed \d+ files, found (\d+) critical, \d+ warnings, \d+ suggestions, " + r"\d+ nitpicks \(\d+ deferred by quality bar\)\*$", + cr, + re.MULTILINE, + ) + reds = int(sums[-1]) if sums else cr.count("🔴") + if m_fault: + blockers.append( + "code-review: the lane reported an internal PIPELINE-FAULT " + "(degraded review) — a degraded lane is never a pass" + ) if reds: blockers.append( f"code-review: {reds} 🔴 CRITICAL finding(s) — fix before claiming done" ) + if m_fault and reds: + record("code-review", "BLOCKED", f"PIPELINE-FAULT (degraded lane); {reds} 🔴") + elif m_fault: + record("code-review", "BLOCKED", "PIPELINE-FAULT (degraded lane)") + elif reds: record("code-review", "BLOCKED", f"{reds} 🔴 finding(s)") elif m and m.group(1) == "MERGE-READY": record("code-review", "GREEN", "MERGE-READY") @@ -100,6 +154,8 @@ def main(): adv = lane_report("adversary_results") if adv is None: record("adversary", "SKIPPED", "not selected") + elif "adversary_results" in malformed: + malformed_blocker("adversary", adv) elif is_lane_fault(adv): lane_fault_blocker("adversary", adv) reports.append(("adversary", adv)) @@ -123,6 +179,8 @@ def main(): sec = lane_report("security_results") if sec is None: record("security", "SKIPPED", "not selected") + elif "security_results" in malformed: + malformed_blocker("security", sec) elif is_lane_fault(sec): lane_fault_blocker("security", sec) reports.append(("security", sec)) @@ -144,6 +202,8 @@ def main(): pb = lane_report("probe_results") if pb is None: record("probe", "SKIPPED", "not selected") + elif "probe_results" in malformed: + malformed_blocker("probe", pb) elif is_lane_fault(pb): lane_fault_blocker("probe", pb) reports.append(("probe", pb)) diff --git a/src/config/request_context.rs b/src/config/request_context.rs index b4c10f0..946340c 100644 --- a/src/config/request_context.rs +++ b/src/config/request_context.rs @@ -10386,6 +10386,392 @@ mod tests { ); } + #[test] + fn gauntlet_verdict_gate_quoted_red_marker_uses_summary_count() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // A 🔴 quoted inside a Changes-table row is not a finding: the + // critical count comes from render.py's summary line, not a raw + // marker count over the whole report. + let state = json!({ + "code_review_results": ["| `route_review.sh` | Fault guard before the 🔴 grep | — |\n\n**Verdict: NEEDS-HUMAN**\n\n*Reviewed 3 files, found 0 critical, 1 warnings, 0 suggestions, 0 nitpicks (0 deferred by quality bar)*"], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "PASS", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| code-review | GREEN (attention) | NEEDS-HUMAN, no 🔴 |"), + "the summary line's zero critical count must win over a quoted 🔴: {report}" + ); + assert!( + !report.contains("🔴 CRITICAL finding(s)"), + "a quoted 🔴 must not produce a blocker: {report}" + ); + } + + #[test] + fn gauntlet_verdict_gate_summary_count_blocks_naming_count() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + let state = json!({ + "code_review_results": ["**Verdict: MERGE-READY**\n\n*Reviewed 3 files, found 2 critical, 0 warnings, 0 suggestions, 0 nitpicks (0 deferred by quality bar)*"], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("code-review: 2 🔴 CRITICAL finding(s)"), + "the summary line's critical count must block and be named: {report}" + ); + } + + #[test] + fn gauntlet_verdict_gate_missing_summary_falls_back_to_raw_count() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // No summary line → the raw 🔴 count stays authoritative (fail closed). + let state = json!({ + "code_review_results": ["🔴 [correctness] broken invariant\n\n**Verdict: MERGE-READY**"], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| code-review | BLOCKED | 1 🔴 finding(s) |"), + "without a summary line the raw 🔴 count must block: {report}" + ); + } + + #[test] + fn gauntlet_verdict_gate_last_summary_line_wins() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // render.py emits its summary line AFTER every finding body, so a + // full-shape look-alike quoted earlier in the report must lose to the + // real (last) line. + let state = json!({ + "code_review_results": ["#### quoted\n*Reviewed 1 files, found 9 critical, 0 warnings, 0 suggestions, 0 nitpicks (0 deferred by quality bar)*\n\n**Verdict: MERGE-READY**\n\n---\n*Reviewed 3 files, found 0 critical, 0 warnings, 0 suggestions, 0 nitpicks (0 deferred by quality bar)*"], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "PASS", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + !report.contains("🔴 CRITICAL finding(s)"), + "the last summary line must win over an earlier look-alike: {report}" + ); + } + + #[test] + fn gauntlet_verdict_gate_nested_code_review_fault_blocks() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // code-reviewer's verdict.py forces NEEDS-HUMAN on its own internal + // faults; the nested graph completes with a clean-looking zero + // critical count, so the gate must anchor on the verdict line's + // degraded-run wording rather than trust the count. + let state = json!({ + "code_review_results": ["# Code Review Summary\n\n**Verdict: NEEDS-HUMAN** — pipeline fault(s) recorded — degraded run; always-human trigger(s) fired\n\n## Walkthrough\n(no walkthrough provided)\n\n---\n*Reviewed 3 files, found 0 critical, 0 warnings, 0 suggestions, 0 nitpicks (0 deferred by quality bar)*"], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!( + out["gauntlet_verdict"], "BLOCKED", + "a degraded code-review lane must never pass: {out}" + ); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains( + "code-review: the lane reported an internal PIPELINE-FAULT (degraded review) — a degraded lane is never a pass" + ), + "the nested fault must be named as the blocker: {report}" + ); + assert!( + report.contains("| code-review | BLOCKED | PIPELINE-FAULT (degraded lane) |"), + "the lane table must record the degraded status: {report}" + ); + assert!( + !report.contains("GREEN (attention)"), + "a degraded lane must not be recorded as attention-only: {report}" + ); + + // A degraded lane that ALSO carries 🔴 findings must name both + // blockers — neither may shadow the other. + let state = json!({ + "code_review_results": ["# Code Review Summary\n\n**Verdict: NEEDS-HUMAN** — 2 🔴 CRITICAL finding(s); pipeline fault(s) recorded — degraded run\n\n---\n*Reviewed 3 files, found 2 critical, 0 warnings, 0 suggestions, 0 nitpicks (0 deferred by quality bar)*"], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains( + "code-review: the lane reported an internal PIPELINE-FAULT (degraded review) — a degraded lane is never a pass" + ), + "the fault blocker must survive alongside the reds: {report}" + ); + assert!( + report.contains("code-review: 2 🔴 CRITICAL finding(s) — fix before claiming done"), + "the reds blocker must survive alongside the fault: {report}" + ); + assert!( + report.contains("| code-review | BLOCKED | PIPELINE-FAULT (degraded lane); 2 🔴 |"), + "the lane table must record both: {report}" + ); + } + + #[test] + fn gauntlet_verdict_gate_non_string_lane_result_blocks() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // The map collected an object instead of the lane's rendered text; + // it cannot carry a sentinel and must never read as a pass. + let state = json!({ + "code_review_results": [{"verdict": "MERGE-READY"}], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("code-review: malformed lane result (non-string item) — never a pass"), + "{report}" + ); + assert!( + report.contains("| code-review | BLOCKED | malformed lane result (non-string) |"), + "{report}" + ); + } + + #[test] + fn gauntlet_verdict_gate_non_fault_needs_human_reason_passes() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // The fault anchor must not over-match: a rendered NEEDS-HUMAN reason + // made of ordinary findings still passes with attention. + let state = json!({ + "code_review_results": ["# Code Review Summary\n\n**Verdict: NEEDS-HUMAN** — 1 🟡 [correctness] finding(s) outside the deferred section\n\n---\n*Reviewed 2 files, found 0 critical, 1 warnings, 0 suggestions, 0 nitpicks (0 deferred by quality bar)*"], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "PASS", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| code-review | GREEN (attention) | NEEDS-HUMAN, no 🔴 |"), + "a non-fault NEEDS-HUMAN must surface as attention: {report}" + ); + assert!( + !report.contains("PIPELINE-FAULT"), + "no fault may be inferred from an ordinary reason: {report}" + ); + } + + #[test] + fn gauntlet_verdict_gate_parses_real_render_output() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // Pins the render.py ↔ verdict_gate.py contract end to end: the + // gate's summary-line regex must match what render.py actually emits, + // and a 🔴 quoted inside a finding body must not count. + let finding = json!({ + "id": "F1", + "file": "src/lib.rs", + "icon": "🟡", + "section": "blocking", + "marker": "correctness", + "title": "Fault guard removed", + "block": "#### 🟡 [correctness] Fault guard removed\nThe guard before the 🔴 grep was dropped; a quoted marker now leaks through." + }); + let rendered = run_code_reviewer_script( + "render.py", + &json!({ + "changed_files": ["src/lib.rs"], + "resolved_rigor": "production", + "walkthrough": "Removes a guard.", + "changes_rows": [{"file": "src/lib.rs", "desc": "guard removal"}], + "verdict_out": { + "verdict": "NEEDS-HUMAN", + "reason": "1 🟡 [correctness] finding(s) outside the deferred section; always-human trigger(s) fired", + "counts": {"🔴": 0, "🟡": 1, "🟢": 0, "💡": 0}, + "deferred_count": 0, + "dropped_count": 0, + "dropped_titles": [], + "attention": ["x"], + "findings_final": [finding] + } + }), + ); + let report = rendered["final_report"].as_str().unwrap(); + assert!( + report.contains("found 0 critical, 1 warnings"), + "render.py must emit the summary line: {report}" + ); + let out = run_gauntlet_script( + "verdict_gate.py", + &json!({ + "code_review_results": [report], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }), + ); + assert_eq!(out["gauntlet_verdict"], "PASS", "{out}"); + let gauntlet = out["gauntlet_report"].as_str().unwrap(); + assert!( + !gauntlet.contains("🔴 CRITICAL finding(s)"), + "the rendered summary line must win over the quoted 🔴: {gauntlet}" + ); + assert!( + gauntlet.contains("| code-review | GREEN (attention) | NEEDS-HUMAN, no 🔴 |"), + "the gate must parse render.py's real output: {gauntlet}" + ); + } + + #[test] + fn gauntlet_verdict_gate_blocks_real_render_of_degraded_code_review() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // Pins the verdict.py → render.py → verdict_gate.py fault contract end + // to end: the degraded-run reason wording render.py emits on the + // verdict line must be what the gate's fault regex anchors on. + let rendered = run_code_reviewer_script( + "render.py", + &json!({ + "changed_files": ["a.rs"], + "verdict_out": { + "verdict": "NEEDS-HUMAN", + "reason": "pipeline fault(s) recorded — degraded run", + "counts": {"🔴": 0, "🟡": 0, "🟢": 0, "💡": 0}, + "attention": ["PIPELINE-FAULT: x"], + "findings_final": [] + } + }), + ); + let report = rendered["final_report"].as_str().unwrap(); + let out = run_gauntlet_script( + "verdict_gate.py", + &json!({ + "code_review_results": [report], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }), + ); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + assert!( + out["gauntlet_report"] + .as_str() + .unwrap() + .contains("internal PIPELINE-FAULT (degraded review)"), + "a rendered degraded run must block as a fault, not pass on 🔴 = 0: {out}" + ); + } + + #[test] + fn gauntlet_verdict_gate_blocks_real_render_crash_report() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // A non-dict verdict_out makes render.py's main() raise; its crash + // guard emits a bare verdict line that the gate must still block on. + let rendered = run_code_reviewer_script( + "render.py", + &json!({"changed_files": ["a.rs"], "verdict_out": "not a dict"}), + ); + let report = rendered["final_report"].as_str().unwrap(); + assert!( + report.starts_with( + "# Code Review Summary\n\n**Verdict: NEEDS-HUMAN** — report rendering error:" + ), + "the crash guard must emit the anchored verdict line: {report}" + ); + let out = run_gauntlet_script( + "verdict_gate.py", + &json!({ + "code_review_results": [report], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }), + ); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + assert!( + out["gauntlet_report"] + .as_str() + .unwrap() + .contains("internal PIPELINE-FAULT (degraded review)"), + "a crashed render must block via the fault hook: {out}" + ); + } + + #[test] + fn gauntlet_alias_tables_stay_in_sync() { + // build_items.py and default_lanes.py each carry a copy of the lane + // alias table; a caller-forced lane must canonicalize identically on + // the normal and degraded paths. + fn aliases_block(src: &str) -> String { + let start = src + .find("ALIASES = {") + .unwrap_or_else(|| panic!("no ALIASES table in: {src}")); + let end = start + src[start..].find("\n}").unwrap(); + src[start..end] + .split_whitespace() + .collect::>() + .join(" ") + } + let scripts = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("assets/agents/review-gauntlet/scripts"); + let build_items = std::fs::read_to_string(scripts.join("build_items.py")).unwrap(); + let default_lanes = std::fs::read_to_string(scripts.join("default_lanes.py")).unwrap(); + assert_eq!( + aliases_block(&build_items), + aliases_block(&default_lanes), + "build_items.ALIASES and default_lanes.ALIASES must be identical" + ); + assert!( + build_items.contains("Keep in sync with default_lanes.ALIASES"), + "build_items.py must point at its twin table" + ); + } + #[test] fn gauntlet_build_items_crash_fails_closed() { if !cmd_available("python3") { @@ -10536,11 +10922,14 @@ mod tests { "the degradation note must name the deterministic defaults: {out}" ); - let probe = run_gauntlet_script("default_lanes.py", &json!({"consumer_surface": true})); + let probe = run_gauntlet_script( + "default_lanes.py", + &json!({"consumer_surface": true, "probe_context": "make run && hurl tests/"}), + ); assert_eq!( probe["forced_lanes"], json!(["code-review", "adversary", "probe"]), - "a consumer surface must widen selection to include probe: {probe}" + "a consumer surface with a local-run recipe must widen selection to include probe: {probe}" ); let again = run_gauntlet_script("default_lanes.py", &json!({})); @@ -10605,6 +10994,66 @@ mod tests { } } + #[test] + fn gauntlet_default_lanes_probe_requires_probe_context() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // build_items gates probe on a local-run recipe; the fallback must + // not force a probe that is INCONCLUSIVE by construction. + let out = run_gauntlet_script("default_lanes.py", &json!({"consumer_surface": true})); + assert_eq!( + out["forced_lanes"], + json!(["code-review", "adversary"]), + "a consumer surface without a probe_context must not add probe: {out}" + ); + + let out = run_gauntlet_script( + "default_lanes.py", + &json!({"consumer_surface": true, "probe_context": "make run && hurl tests/"}), + ); + assert_eq!( + out["forced_lanes"], + json!(["code-review", "adversary", "probe"]), + "a consumer surface with a probe_context must add probe: {out}" + ); + } + + #[test] + fn gauntlet_default_lanes_crash_widens_and_records_fault() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // A truthy non-iterable forced_lanes raises inside main; the guard + // must emit the WIDEST safe selection (defaults ∪ security) plus a + // PIPELINE-FAULT so verdict_gate blocks — never a narrower pass. + let out = run_gauntlet_script( + "default_lanes.py", + &json!({"forced_lanes": 42, "touches_auth": true}), + ); + assert_eq!( + out["forced_lanes"], + json!(["code-review", "adversary", "security"]), + "a crashed fallback must widen to defaults ∪ security: {out}" + ); + assert!( + out["signals_error"] + .as_str() + .unwrap() + .starts_with("PIPELINE-FAULT: lane-selection fallback crashed"), + "the crash must surface as a PIPELINE-FAULT: {out}" + ); + assert!( + out["lanes_degraded"] + .as_str() + .unwrap() + .contains("fallback script error"), + "the degradation note must name the crash: {out}" + ); + } + #[test] fn gauntlet_lane_prompts_carry_passthroughs() { use crate::graph::{GraphParser, NodeType};