diff --git a/assets/agents/review-gauntlet/README.md b/assets/agents/review-gauntlet/README.md index d4aae09..86afa34 100644 --- a/assets/agents/review-gauntlet/README.md +++ b/assets/agents/review-gauntlet/README.md @@ -57,7 +57,11 @@ those rules are *structure*: sentinel is a lane FAILURE, never a pass; the critical count from the code-review report's summary line (raw πŸ”΄ count if absent) blocks regardless of its verdict line; a code-review lane whose own verdict line - records a pipeline fault blocks (a degraded review is never a pass); + records a pipeline fault blocks (a degraded review is never a pass); the + adversary's own degraded-run header (the `Criteria: …` line directly under + its sentinel recording a pipeline fault, died criterion checks, or its + verdict script's crash stub) blocks as a pipeline fault, + not as a DIVERGES finding; `NEEDS-HUMAN` without πŸ”΄ passes but is surfaced under *Human attention required*; probe `INCONCLUSIVE` blocks with an environment note β€” it is never treated as PASS or FAIL. @@ -86,7 +90,8 @@ those rules are *structure*: only lanes that **faulted** β€” a missing report (stood in for by a synthetic `PIPELINE-FAULT: lane produced no report` marker so it never renders as SKIPPED), a non-string report, the `lane_fault` marker, a sentinel-less -report, or a nested graph's own degraded-run wording. A real verdict +report, or a nested graph's own degraded-run wording (the code-review verdict +line / the adversary's header line directly under its sentinel). A real verdict (DIVERGES, FAIL, INCONCLUSIVE, NEEDS-HUMAN, …) is never re-run: the review's judgment stands. Each lane gets up to 3 attempts, all inside a wall-clock budget (`MAX_RETRY_ELAPSED_SECS`) sized to fit the graph timeout. @@ -105,6 +110,13 @@ budget (`MAX_RETRY_ELAPSED_SECS`) sized to fit the graph timeout. guards append to an existing `signals_error` rather than overwrite it, and `build_items.py` preserves the prior `lanes_summary`. +A non-empty `GAUNTLET_REVIEW_INCOMPLETE:` line structurally implies +`GAUNTLET: BLOCKED`: the verdict gate forces every lane named in +`review_incomplete` to a BLOCKED row (`review incomplete after N attempt(s)`) +and a matching blocker, so a restored earlier report can never turn an +incomplete review into a pass β€” and malformed `review_incomplete` bookkeeping +is itself recorded as a `pipeline` fault, never dropped. + Whenever at least one lane (or the pipeline) could not complete, the first line reads `GAUNTLET: BLOCKED` followed immediately by `GAUNTLET_REVIEW_INCOMPLETE: [, ]`. The names are the exhausted @@ -124,7 +136,7 @@ still appear under `## Blockers` and must be fixed as usual. | Lane | Sentinel | Blocks on | |------|----------|-----------| | code-reviewer | `**Verdict: MERGE-READY \| NEEDS-HUMAN**` | critical count from the report's summary line (raw πŸ”΄ count if absent); missing sentinel; own verdict line records a pipeline fault | -| adversary | `ADVERSARIAL_REVIEW: CONFORMS \| DIVERGES` | DIVERGES; missing sentinel | +| adversary | `ADVERSARIAL_REVIEW: CONFORMS \| DIVERGES` | DIVERGES; own degraded-run header (pipeline fault, died criterion checks, or its verdict script's crash stub); missing sentinel | | security-reviewer | `SECURITY_REVIEW: PASS \| FAIL` | FAIL; missing sentinel | | probe | `USAGE_PROBE: PASS \| FAIL \| INCONCLUSIVE` | FAIL; INCONCLUSIVE (environment note); missing sentinel | diff --git a/assets/agents/review-gauntlet/scripts/retry_gate.py b/assets/agents/review-gauntlet/scripts/retry_gate.py index 45f1ecd..8346a56 100644 --- a/assets/agents/review-gauntlet/scripts/retry_gate.py +++ b/assets/agents/review-gauntlet/scripts/retry_gate.py @@ -19,6 +19,9 @@ hard rules: - a faulted lane is re-run only while it has attempts left AND the gauntlet is inside its wall-clock budget; otherwise it is recorded in review_incomplete and its fault flows to the verdict gate as-is; + - malformed review_incomplete bookkeeping (not a list, or non-str + entries) keeps its str entries and records a PIPELINE-FAULT in + signals_error β€” never silently dropped, never split into lane names; - the gate itself must never retry by accident: any internal error records a PIPELINE-FAULT in signals_error and falls through to the verdict gate, which blocks on it. @@ -57,6 +60,15 @@ SENTINELS = { # instead of overwriting it, and report the lanes it had already declined. crash_context = {"signals_error": "", "review_incomplete": []} +# Duplicated verbatim in the sibling gate script β€” the two copies must stay identical. +ADVERSARY_DEGRADED_HEADER = re.compile( + r"Criteria: .*(" + r"degraded run: pipeline fault recorded" + r"|degraded run: \d+ criterion check\(s\) died" + r"|verdict computation error:" + r")" +) + def load_state(): if path := os.environ.get("GRAPH_STATE_FILE"): @@ -65,6 +77,18 @@ def load_state(): return json.loads(os.environ.get("GRAPH_STATE", "{}")) +def adversary_degraded(report): + # The adversary's verdict.py emits its degraded-run header on the line + # directly under the FIRST sentinel; anchoring there keeps a report that + # merely quotes the wording later (observations, complaints) a real verdict. + lines = report.splitlines() + for i, ln in enumerate(lines): + if re.search(r"ADVERSARIAL_REVIEW:\s*(CONFORMS|DIVERGES)", ln): + nxt = lines[i + 1] if i + 1 < len(lines) else "" + return bool(ADVERSARY_DEGRADED_HEADER.match(nxt)) + return False + + def nested_degraded(lane, report): # The nested graphs complete with a sentinel even when a stage inside # them died; their own degraded-run wording marks a partial review, @@ -80,13 +104,7 @@ def nested_degraded(lane, report): ) ) if lane == "adversary": - return bool( - re.search( - r"^Criteria: .*degraded run: pipeline fault recorded", - report, - re.MULTILINE, - ) - ) + return adversary_degraded(report) return False @@ -100,15 +118,38 @@ def is_faulted(lane, report): return nested_degraded(lane, report) +def attempt_count(attempts, lane): + try: + return int(attempts.get(lane) or 0) + except (TypeError, ValueError): + return 0 + + def main(): state = load_state() prior = state.get("signals_error") - crash_context["signals_error"] = prior if isinstance(prior, str) else "" - incomplete = list(state.get("review_incomplete") or []) - crash_context["review_incomplete"] = incomplete - attempts = dict(state.get("lane_attempts") or {}) - kept = dict(state.get("kept_results") or {}) + signals_error = prior if isinstance(prior, str) else "" + crash_context["signals_error"] = signals_error out = {} + declared = state.get("review_incomplete") + if isinstance(declared, list): + incomplete = [name for name in declared if isinstance(name, str)] + malformed = len(incomplete) != len(declared) + else: + incomplete = [] + malformed = declared is not None + if malformed: + fault = ( + "PIPELINE-FAULT: malformed review_incomplete bookkeeping " + f"({repr(declared)[:80]})" + ) + signals_error = f"{signals_error}; {fault}" if signals_error else fault + crash_context["signals_error"] = signals_error + out["signals_error"] = signals_error + crash_context["review_incomplete"] = incomplete + raw_attempts = state.get("lane_attempts") + attempts = dict(raw_attempts) if isinstance(raw_attempts, dict) else {} + kept = dict(state.get("kept_results") or {}) faulted = [] for lane, (items_key, results_key) in LANES.items(): @@ -124,7 +165,7 @@ def main(): kept[lane] = report if is_faulted(lane, report): faulted.append(lane) - attempts[lane] = int(attempts.get(lane) or 0) + 1 + attempts[lane] = attempt_count(attempts, lane) + 1 # A missing/zero stamp reads as an ancient start and declines every # retry β€” the safe direction when the clock's origin is unknown. diff --git a/assets/agents/review-gauntlet/scripts/verdict_gate.py b/assets/agents/review-gauntlet/scripts/verdict_gate.py index cbeab90..5f496bc 100644 --- a/assets/agents/review-gauntlet/scripts/verdict_gate.py +++ b/assets/agents/review-gauntlet/scripts/verdict_gate.py @@ -17,6 +17,11 @@ re-reads, no judgment. The gate's hard rules: (code-reviewer's verdict.py / render.py degraded-run wording) blocks β€” the nested graph completed, but with a dead domain lane, verifier, or synthesis behind it, so its critical count is not trustworthy; + - an adversary lane whose OWN header line (the line directly under its + sentinel) records a degraded run β€” a pipeline fault, died criterion + checks, or its verdict script's crash stub β€” blocks as a PIPELINE-FAULT, + not as a DIVERGES finding: the nested graph completed with a dead stage + behind it, so its verdict is not a judgment on the code; - a non-zero critical count in the code-review report blocks regardless of the lane's own verdict line β€” derived from render.py's deterministic summary line (`*Reviewed N files, found X critical, …*`); the raw πŸ”΄ @@ -31,7 +36,12 @@ re-reads, no judgment. The gate's hard rules: - an incomplete review β€” lanes retry_gate gave up on (review_incomplete) or a pipeline-level fault (reported under the `pipeline` pseudo-lane) β€” emits the GAUNTLET_REVIEW_INCOMPLETE: machine line and a "Review - incomplete" section: that is a fault to re-run, never a code finding; + incomplete" section: that is a fault to re-run, never a code finding. + Every lane named in review_incomplete is forced to a BLOCKED row, so a + restored earlier report can never turn an incomplete review into a + pass: a non-empty machine line structurally implies GAUNTLET: BLOCKED. + Malformed review_incomplete bookkeeping (not a list, or non-str + entries) is itself recorded as a `pipeline` fault, never dropped; - the gate itself must never crash into a silent pass: any internal error emits GAUNTLET: BLOCKED naming the error, with the pipeline marked incomplete. @@ -41,6 +51,16 @@ import json import os import re +# Duplicated verbatim in the sibling gate script β€” the two copies must stay identical. +ADVERSARY_DEGRADED_HEADER = re.compile( + r"Criteria: .*(" + r"degraded run: pipeline fault recorded" + r"|degraded run: \d+ criterion check\(s\) died" + r"|verdict computation error:" + r")" +) + + def load_state(): if path := os.environ.get("GRAPH_STATE_FILE"): with open(path) as f: @@ -48,11 +68,26 @@ def load_state(): return json.loads(os.environ.get("GRAPH_STATE", "{}")) +def adversary_degraded(report): + # The adversary's verdict.py emits its degraded-run header on the line + # directly under the FIRST sentinel; anchoring there keeps a report that + # merely quotes the wording later (observations, complaints) a real verdict. + lines = report.splitlines() + for i, ln in enumerate(lines): + if re.search(r"ADVERSARIAL_REVIEW:\s*(CONFORMS|DIVERGES)", ln): + nxt = lines[i + 1] if i + 1 < len(lines) else "" + return bool(ADVERSARY_DEGRADED_HEADER.match(nxt)) + return False + + def main(): state = load_state() - blockers, attention, lane_lines, reports = [], [], [], [] + blockers, attention, reports = [], [], [] + rows = {} malformed = set() - lane_attempts = state.get("lane_attempts") or {} + lane_attempts = state.get("lane_attempts") + if not isinstance(lane_attempts, dict): + lane_attempts = {} def lane_report(key): arr = state.get(key) or [] @@ -65,10 +100,16 @@ def main(): return json.dumps(item) def record(lane, status, detail=""): - lane_lines.append(f"| {lane} | {status} | {detail} |") + rows[lane] = (status, detail) + + def attempt_count(lane): + try: + return int(lane_attempts.get(lane) or 0) + except (TypeError, ValueError): + return 0 def attempts_suffix(lane): - n = int(lane_attempts.get(lane) or 0) + n = attempt_count(lane) return f" Γ—{n}" if n > 1 else "" def fault_excerpt(report): @@ -178,7 +219,17 @@ def main(): else: reports.append(("adversary", adv)) m = re.search(r"ADVERSARIAL_REVIEW:\s*(CONFORMS|DIVERGES)", adv) - if m and m.group(1) == "CONFORMS": + if adversary_degraded(adv): + blockers.append( + "adversary: PIPELINE-FAULT β€” degraded run, pipeline fault recorded " + "inside the adversary; a degraded lane is never a pass" + ) + record( + "adversary", + "BLOCKED", + f"PIPELINE-FAULT (degraded lane{attempts_suffix('adversary')})", + ) + elif m and m.group(1) == "CONFORMS": record("adversary", "GREEN", "CONFORMS") elif m: blockers.append( @@ -244,21 +295,57 @@ def main(): ) record("probe", "BLOCKED", "missing verdict sentinel") + # retry_gate restores an exhausted lane's LAST report, which may be an + # earlier clean verdict β€” an incomplete lane must block regardless of + # what its row parsed to. + declared = state.get("review_incomplete") + incomplete = set() + malformed_bookkeeping = None + if isinstance(declared, list): + incomplete = {name for name in declared if isinstance(name, str)} + if any(not isinstance(name, str) for name in declared): + malformed_bookkeeping = repr(declared)[:80] + elif declared is not None: + malformed_bookkeeping = repr(declared)[:80] + for lane in sorted(incomplete - {"pipeline"}): + if rows.get(lane, ("", ""))[0] == "BLOCKED": + continue + after = f" after {n} attempt(s)" if (n := attempt_count(lane)) else "" + blockers.append( + f"{lane}: review incomplete{after} β€” the lane never produced a usable verdict" + ) + record(lane, "BLOCKED", f"review incomplete{after}") + # A gauntlet-level fault (build_items crash, dead parse stage) means the # lanes above were never built β€” SKIPPED rows alone must not read as PASS. signals_error = state.get("signals_error") - incomplete = set(state.get("review_incomplete") or []) - if isinstance(signals_error, str) and "PIPELINE-FAULT:" in signals_error: + pipeline_fault = isinstance(signals_error, str) and "PIPELINE-FAULT:" in signals_error + if pipeline_fault: blockers.append(f"pipeline: {signals_error}") incomplete.add("pipeline") + if malformed_bookkeeping is not None: + blockers.append( + "pipeline: PIPELINE-FAULT β€” malformed review_incomplete bookkeeping " + f"({malformed_bookkeeping}); never a pass" + ) + incomplete.add("pipeline") + if "pipeline" in incomplete and not any(b.startswith("pipeline:") for b in blockers): + blockers.append( + "pipeline: review incomplete β€” a pipeline stage never completed " + "(no fault text recorded)" + ) - verdict = "BLOCKED" if blockers else "PASS" + verdict = "BLOCKED" if blockers or incomplete else "PASS" incomplete_lines = [] for name in sorted(incomplete): if name == "pipeline": - incomplete_lines.append(f"- pipeline: {signals_error}") - elif n := int(lane_attempts.get(name) or 0): + incomplete_lines.append( + f"- pipeline: {signals_error}" + if pipeline_fault + else "- pipeline: review incomplete (no fault text recorded)" + ) + elif n := attempt_count(name): incomplete_lines.append(f"- {name}: no completed review after {n} attempt(s)") else: incomplete_lines.append(f"- {name}: no completed review") @@ -269,7 +356,8 @@ def main(): ) md = "\n".join( - ["| lane | status | detail |", "|------|--------|--------|"] + lane_lines + ["| lane | status | detail |", "|------|--------|--------|"] + + [f"| {lane} | {status} | {detail} |" for lane, (status, detail) in rows.items()] ) md += "\n\n## Lane selection\n" + (state.get("lanes_summary") or "(none recorded)") if blockers: diff --git a/src/config/request_context.rs b/src/config/request_context.rs index 7261f8e..4262da2 100644 --- a/src/config/request_context.rs +++ b/src/config/request_context.rs @@ -8361,16 +8361,25 @@ mod tests { "MAX_RETRY_ELAPSED_SECS", "preserves the prior `lanes_summary`", "append to an existing `signals_error`", + "structurally implies", + "not as a DIVERGES finding", + "never dropped", + "| adversary | `ADVERSARIAL_REVIEW: CONFORMS \\| DIVERGES` | DIVERGES; own degraded-run header (pipeline fault, died criterion checks, or its verdict script's crash stub); missing sentinel |", ] { assert!( gauntlet.contains(claim), "review-gauntlet README must state the retry/incomplete contract: {claim}" ); } + assert!( + gauntlet.contains("never dropped.\n\nWhenever at least one lane"), + "review-gauntlet README must keep the incomplete-line contract in its own paragraph" + ); for stale in [ "on fault-only BLOCKED", "there is nothing to fix in the code yet", "3h whole-gauntlet", + "| DIVERGES; missing sentinel |", ] { assert!( !gauntlet.contains(stale), @@ -11416,6 +11425,352 @@ mod tests { ); } + #[test] + fn gauntlet_verdict_gate_incomplete_lane_forces_blocked() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // retry_gate restores an exhausted lane's LAST report, which can be + // an earlier clean verdict; the incomplete lane must block anyway. + let state = json!({ + "review_incomplete": ["adversary"], + "lane_attempts": {"adversary": 3}, + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + assert_eq!( + out["gauntlet_incomplete_line"], "\nGAUNTLET_REVIEW_INCOMPLETE: adversary", + "{out}" + ); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| adversary | BLOCKED | review incomplete after 3 attempt(s) |"), + "an incomplete lane must render as BLOCKED with its attempt count: {report}" + ); + assert!( + report.contains( + "adversary: review incomplete after 3 attempt(s) β€” the lane never produced a usable verdict" + ), + "an incomplete lane must be named as a blocker: {report}" + ); + assert!( + !report.contains("| adversary | GREEN | CONFORMS |"), + "a restored clean verdict must not render as GREEN for an incomplete lane: {report}" + ); + assert!( + !(out["gauntlet_verdict"] == "PASS" && out["gauntlet_incomplete_line"] != ""), + "PASS must never co-occur with an incomplete line: {out}" + ); + + let mut state = state; + state.as_object_mut().unwrap().remove("lane_attempts"); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| adversary | BLOCKED | review incomplete |"), + "no recorded attempts must render without a count: {report}" + ); + assert!( + report.contains( + "adversary: review incomplete β€” the lane never produced a usable verdict" + ), + "{report}" + ); + assert!( + !(out["gauntlet_verdict"] == "PASS" && out["gauntlet_incomplete_line"] != ""), + "PASS must never co-occur with an incomplete line: {out}" + ); + + state.as_object_mut().unwrap().remove("review_incomplete"); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "PASS", "{out}"); + assert_eq!(out["gauntlet_incomplete_line"], "", "{out}"); + assert!( + out["gauntlet_report"] + .as_str() + .unwrap() + .contains("| adversary | GREEN | CONFORMS |"), + "a complete clean lane must still render GREEN: {out}" + ); + assert!( + !(out["gauntlet_verdict"] == "PASS" && out["gauntlet_incomplete_line"] != ""), + "PASS must never co-occur with an incomplete line: {out}" + ); + + // A bare `pipeline` entry with no fault text must still block and + // must never render Python's None into the report. + let state = json!({ + "review_incomplete": ["pipeline"], + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + assert_eq!( + out["gauntlet_incomplete_line"], "\nGAUNTLET_REVIEW_INCOMPLETE: pipeline", + "{out}" + ); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("pipeline: review incomplete"), + "a pipeline entry without fault text must still be named as a blocker: {report}" + ); + assert!( + !report.contains("pipeline: None"), + "missing fault text must not render as None: {report}" + ); + + // An incomplete lane whose map collected nothing must not fall back + // to the SKIPPED row. + let state = json!({ + "review_incomplete": ["security"], + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| security | BLOCKED | review incomplete |"), + "an incomplete lane with no report must render BLOCKED: {report}" + ); + assert!( + !report.contains("| security | SKIPPED | not selected |"), + "an incomplete lane must never read as SKIPPED: {report}" + ); + } + + #[test] + fn gauntlet_verdict_gate_adversary_degraded_header_is_fault_not_finding() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + let state = json!({ + "lane_attempts": {"adversary": 2}, + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": [GAUNTLET_ADVERSARY_DEGRADED], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| adversary | BLOCKED | PIPELINE-FAULT (degraded lane Γ—2) |"), + "the adversary's own degraded-run header is a pipeline fault with its attempt count: {report}" + ); + assert!( + report.contains( + "adversary: PIPELINE-FAULT β€” degraded run, pipeline fault recorded inside the adversary" + ), + "{report}" + ); + assert!( + !report.contains("adversary: DIVERGES β€” the implementation does not conform"), + "a degraded run must not be reported as a DIVERGES finding: {report}" + ); + + let mut state = state; + state.as_object_mut().unwrap().remove("lane_attempts"); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert!( + out["gauntlet_report"] + .as_str() + .unwrap() + .contains("| adversary | BLOCKED | PIPELINE-FAULT (degraded lane) |"), + "a first-pass degraded run renders without a count: {out}" + ); + + // The predicate is anchored to the line under the sentinel: a clean + // report that quotes the wording later stays a real verdict. + state["adversary_results"] = json!([GAUNTLET_ADVERSARY_QUOTED_DEGRADED]); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "PASS", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| adversary | GREEN | CONFORMS |"), + "a quoted degraded-run phrase must not trip the fault predicate: {report}" + ); + assert!(!report.contains("adversary: PIPELINE-FAULT"), "{report}"); + + // Every degraded header the adversary's verdict.py can emit is a + // pipeline fault, not a DIVERGES finding. + for degraded in [ + GAUNTLET_ADVERSARY_DEGRADED_COUNTED, + GAUNTLET_ADVERSARY_DEGRADED_DIED, + GAUNTLET_ADVERSARY_CRASH_STUB, + ] { + state["adversary_results"] = json!([degraded]); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{degraded:?}: {out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| adversary | BLOCKED | PIPELINE-FAULT (degraded lane) |"), + "{degraded:?} must render as a degraded lane: {report}" + ); + assert!( + !report.contains("adversary: DIVERGES β€” the implementation does not conform"), + "{degraded:?} must not be reported as a DIVERGES finding: {report}" + ); + } + + // A plain DIVERGES header is a real finding. + state["adversary_results"] = json!([GAUNTLET_ADVERSARY_DIVERGES]); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| adversary | BLOCKED | DIVERGES |"), + "a real DIVERGES must stay a finding: {report}" + ); + assert!(!report.contains("adversary: PIPELINE-FAULT"), "{report}"); + } + + #[test] + fn gauntlet_verdict_gate_malformed_attempts_keeps_lane_rows() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // Malformed bookkeeping must not crash the gate into the error stub + // that loses every lane row: a non-dict lane_attempts only affects + // counts, while a non-list review_incomplete is itself a pipeline + // fault β€” a bare string is never adopted as a lane name. + let state = json!({ + "lane_attempts": 42, + "review_incomplete": "adversary", + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": [GAUNTLET_ADVERSARY_FAULT], + "security_results": [], + "probe_results": [], + "signals_error": "PIPELINE-FAULT: lane-selection fallback crashed β€” boom" + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + !report.contains("verdict gate error"), + "malformed bookkeeping must not reach the crash stub: {report}" + ); + assert!( + report.contains("| code-review | GREEN | MERGE-READY |"), + "{report}" + ); + assert!( + report.contains("| adversary | BLOCKED | PIPELINE-FAULT (lane failed) |"), + "unreadable attempts must render without a count: {report}" + ); + assert_eq!( + out["gauntlet_incomplete_line"], "\nGAUNTLET_REVIEW_INCOMPLETE: pipeline", + "a string-valued review_incomplete is a pipeline fault, not a lane name; the pipeline is named: {out}" + ); + + // Without any upstream fault text, the malformed bookkeeping alone + // must block as a pipeline fault. + let state = json!({ + "lane_attempts": 42, + "review_incomplete": "adversary", + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + assert_eq!( + out["gauntlet_incomplete_line"], "\nGAUNTLET_REVIEW_INCOMPLETE: pipeline", + "{out}" + ); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("pipeline: PIPELINE-FAULT β€” malformed review_incomplete bookkeeping"), + "malformed bookkeeping must be named as a pipeline fault: {report}" + ); + assert!( + report.contains("| code-review | GREEN | MERGE-READY |"), + "{report}" + ); + assert!(!report.contains("verdict gate error"), "{report}"); + + // A list with a non-str entry keeps its str lanes and still records + // the fault. + let state = json!({ + "review_incomplete": ["adversary", 7], + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + assert_eq!( + out["gauntlet_incomplete_line"], "\nGAUNTLET_REVIEW_INCOMPLETE: adversary, pipeline", + "{out}" + ); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| adversary | BLOCKED | review incomplete |"), + "the str lane entries must still force their rows: {report}" + ); + assert!( + report.contains("pipeline: PIPELINE-FAULT β€” malformed review_incomplete bookkeeping"), + "{report}" + ); + + // Per-lane values that are not ints count as zero: the row renders + // without a Γ—N suffix and the incomplete line without "after N". + let state = json!({ + "lane_attempts": {"adversary": "three", "code-review": [1]}, + "review_incomplete": ["adversary"], + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": [GAUNTLET_ADVERSARY_FAULT], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!(!report.contains("verdict gate error"), "{report}"); + assert!( + report.contains("| adversary | BLOCKED | PIPELINE-FAULT (lane failed) |"), + "a non-int per-lane count must render without a count: {report}" + ); + assert!(!report.contains('Γ—'), "{report}"); + assert!( + report.contains("## Review incomplete\n- adversary: no completed review\n"), + "a non-int per-lane count must not render an attempt count: {report}" + ); + + let state = json!({ + "lane_attempts": {"security": "x"}, + "review_incomplete": ["security"], + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"], + "security_results": [], + "probe_results": [] + }); + let out = run_gauntlet_script("verdict_gate.py", &state); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!(!report.contains("verdict gate error"), "{report}"); + assert!( + report.contains("| security | BLOCKED | review incomplete |"), + "{report}" + ); + assert!(!report.contains("review incomplete after"), "{report}"); + } + #[test] fn gauntlet_verdict_gate_non_string_lane_result_blocks() { if !cmd_available("python3") { @@ -11998,6 +12353,12 @@ mod tests { const GAUNTLET_MERGE_READY: &str = "# Review\n\n**Verdict: MERGE-READY**\n"; const GAUNTLET_ADVERSARY_FAULT: &str = "PIPELINE-FAULT: adversary lane failed after retries β€” Agent node failed: boom"; + const GAUNTLET_ADVERSARY_DEGRADED: &str = "ADVERSARIAL_REVIEW: DIVERGES\nCriteria: none verified β€” degraded run: pipeline fault recorded (fail-closed).\nComplaints:\n1. PIPELINE-FAULT: run_checks died"; + const GAUNTLET_ADVERSARY_DEGRADED_COUNTED: &str = "ADVERSARIAL_REVIEW: DIVERGES\nCriteria: 2/3 met, 0 partial, 1 unmet/diverged β€” degraded run: pipeline fault recorded (fail-closed).\nComplaints:\n1. PIPELINE-FAULT: run_checks died"; + const GAUNTLET_ADVERSARY_DEGRADED_DIED: &str = "ADVERSARIAL_REVIEW: DIVERGES\nCriteria: 2/3 met, 0 partial, 1 unmet/diverged β€” degraded run: 1 criterion check(s) died (fail-closed).\nComplaints:\n1. c3: check died β€” boom"; + const GAUNTLET_ADVERSARY_CRASH_STUB: &str = "ADVERSARIAL_REVIEW: DIVERGES\nCriteria: verdict computation error: boom β€” treat as failed, not as conforming."; + const GAUNTLET_ADVERSARY_DIVERGES: &str = "ADVERSARIAL_REVIEW: DIVERGES\nCriteria: 2/3 met, 0 partial, 1 unmet/diverged.\nComplaints:\n1. c3: nothing in the diff does Z"; + const GAUNTLET_ADVERSARY_QUOTED_DEGRADED: &str = "ADVERSARIAL_REVIEW: CONFORMS\nCriteria: 3/3 met, 0 partial, 0 unmet/diverged.\nComplaints:\n- none\n\nObservations:\n- an earlier run printed 'Criteria: none verified β€” degraded run: pipeline fault recorded' but this one is clean"; #[test] fn gauntlet_retry_gate_stashes_and_restores_reports() { @@ -12148,6 +12509,311 @@ mod tests { assert!(out.get("probe_results").is_none(), "{out}"); } + #[test] + fn gauntlet_retry_gate_adversary_degraded_predicate_is_anchored() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + for degraded in [ + GAUNTLET_ADVERSARY_DEGRADED, + GAUNTLET_ADVERSARY_DEGRADED_COUNTED, + GAUNTLET_ADVERSARY_DEGRADED_DIED, + GAUNTLET_ADVERSARY_CRASH_STUB, + ] { + let out = + run_gauntlet_script("retry_gate.py", &gauntlet_lane_ran("adversary", degraded)); + assert_eq!( + out["retry_lanes"], + json!(["adversary"]), + "the degraded-run header {degraded:?} under the sentinel is a fault: {out}" + ); + assert_eq!(out["_next"], "build_items", "{out}"); + } + + // Anchored to the line under the sentinel: a clean report quoting + // the wording later is a completed review. + let out = run_gauntlet_script( + "retry_gate.py", + &gauntlet_lane_ran("adversary", GAUNTLET_ADVERSARY_QUOTED_DEGRADED), + ); + assert_eq!( + out["retry_lanes"], + json!([]), + "a quoted degraded-run phrase must not trigger a re-run: {out}" + ); + assert!(out.get("_next").is_none(), "{out}"); + assert_eq!(out["review_incomplete"], json!([]), "{out}"); + + // A plain DIVERGES is a real finding, never re-run. + let out = run_gauntlet_script( + "retry_gate.py", + &gauntlet_lane_ran("adversary", GAUNTLET_ADVERSARY_DIVERGES), + ); + assert_eq!( + out["retry_lanes"], + json!([]), + "a real DIVERGES must not trigger a re-run: {out}" + ); + assert!(out.get("_next").is_none(), "{out}"); + } + + #[test] + fn gauntlet_retry_gate_malformed_bookkeeping_degrades() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // Malformed bookkeeping must not crash the gate: attempts are still + // counted, the faulted lane is still retried, a stashed report is + // still restored, and the malformation is recorded as a pipeline + // fault rather than adopted as lane names. + let mut state = gauntlet_lane_ran("adversary", GAUNTLET_ADVERSARY_FAULT); + state["lane_attempts"] = json!(42); + state["review_incomplete"] = json!("probe"); + state["kept_results"] = json!({"code-review": GAUNTLET_MERGE_READY}); + state["code_review_items"] = json!([]); + state["code_review_results"] = json!([]); + let out = run_gauntlet_script("retry_gate.py", &state); + let fault = out["signals_error"].as_str().unwrap_or_default(); + assert!( + !fault.starts_with("PIPELINE-FAULT: retry gate crashed"), + "malformed bookkeeping must not reach the crash stub: {out}" + ); + assert_eq!(out["retry_lanes"], json!(["adversary"]), "{out}"); + assert_eq!(out["lane_attempts"], json!({"adversary": 1}), "{out}"); + assert_eq!( + out["review_incomplete"], + json!([]), + "a bare string is never split or adopted as a lane name: {out}" + ); + assert_eq!( + out["code_review_results"], + json!([GAUNTLET_MERGE_READY]), + "the stashed report must still be restored: {out}" + ); + assert!( + fault.contains("malformed review_incomplete bookkeeping"), + "the malformation must be recorded as a fault: {out}" + ); + assert!( + fault.starts_with("PIPELINE-FAULT: malformed review_incomplete bookkeeping ("), + "{out}" + ); + + // Merged over the input state as the graph would, the verdict gate + // must block and name the pipeline. + let mut merged = state; + for (k, v) in out.as_object().unwrap() { + merged[k.as_str()] = v.clone(); + } + let verdict = run_gauntlet_script("verdict_gate.py", &merged); + assert_eq!(verdict["gauntlet_verdict"], "BLOCKED", "{verdict}"); + assert!( + verdict["gauntlet_incomplete_line"] + .as_str() + .unwrap() + .contains("pipeline"), + "{verdict}" + ); + + // Per-lane values that are not ints count as zero, so the lane that + // ran lands on its first attempt. + let mut state = gauntlet_lane_ran("adversary", GAUNTLET_ADVERSARY_FAULT); + state["lane_attempts"] = json!({"adversary": [1], "probe": "x"}); + let out = run_gauntlet_script("retry_gate.py", &state); + assert!( + !out["signals_error"] + .as_str() + .unwrap_or_default() + .contains("retry gate crashed"), + "{out}" + ); + assert_eq!(out["retry_lanes"], json!(["adversary"]), "{out}"); + assert_eq!(out["lane_attempts"]["adversary"], json!(1), "{out}"); + + // A list with a non-str entry keeps its str lanes and records the + // fault; the retried lane is not declined. + let mut state = gauntlet_lane_ran("adversary", GAUNTLET_ADVERSARY_FAULT); + state["review_incomplete"] = json!(["probe", 7]); + let out = run_gauntlet_script("retry_gate.py", &state); + assert_eq!(out["retry_lanes"], json!(["adversary"]), "{out}"); + assert_eq!(out["review_incomplete"], json!(["probe"]), "{out}"); + assert!( + out["signals_error"] + .as_str() + .unwrap_or_default() + .starts_with("PIPELINE-FAULT: malformed review_incomplete bookkeeping ("), + "{out}" + ); + + // The bookkeeping fault appends to a prior signals_error. + let mut state = gauntlet_lane_ran("adversary", GAUNTLET_ADVERSARY_FAULT); + state["review_incomplete"] = json!("probe"); + state["signals_error"] = json!("PIPELINE-FAULT: earlier"); + let out = run_gauntlet_script("retry_gate.py", &state); + assert!( + out["signals_error"].as_str().unwrap_or_default().starts_with( + "PIPELINE-FAULT: earlier; PIPELINE-FAULT: malformed review_incomplete bookkeeping (" + ), + "{out}" + ); + } + + #[test] + fn gauntlet_gates_block_real_adversary_degraded_verdict() { + if !cmd_available("python3") { + eprintln!("skipping: python3 not available"); + return; + } + // Pins the sentinelβ†’header adjacency both gates rely on against the + // adversary's real verdict.py output. + let met = json!({"id": "c1", "text": "does X", "status": "MET", + "evidence": "src/x.rs:1 + test src/x.rs:99", "complaint": ""}); + let degraded = run_adversary_script( + "verdict.py", + &json!({ + "pipeline_faults": ["PIPELINE-FAULT: run_checks died β€” boom"], + "crit_verdicts": [met], + "extra_complaints": [], + "observations": "", + "exec_results": "" + }), + ); + let degraded = degraded["adv_report"].as_str().unwrap(); + assert!( + degraded.contains("degraded run: pipeline fault recorded"), + "{degraded}" + ); + let out = run_gauntlet_script( + "verdict_gate.py", + &json!({ + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": [degraded], + "security_results": [], + "probe_results": [] + }), + ); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| adversary | BLOCKED | PIPELINE-FAULT (degraded lane) |"), + "the real degraded header must render as a pipeline fault: {report}" + ); + assert!( + !report.contains("adversary: DIVERGES β€” the implementation does not conform"), + "{report}" + ); + let out = run_gauntlet_script("retry_gate.py", &gauntlet_lane_ran("adversary", degraded)); + assert_eq!( + out["retry_lanes"], + json!(["adversary"]), + "the real degraded header must be re-run: {out}" + ); + + let diverges = run_adversary_script( + "verdict.py", + &json!({ + "pipeline_faults": [], + "crit_verdicts": [{"id": "c1", "text": "does Y", "status": "UNMET", + "evidence": "", "complaint": "nothing in the diff does Y"}], + "extra_complaints": [], + "observations": "", + "exec_results": "" + }), + ); + let diverges = diverges["adv_report"].as_str().unwrap(); + let out = run_gauntlet_script( + "verdict_gate.py", + &json!({ + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": [diverges], + "security_results": [], + "probe_results": [] + }), + ); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + assert!( + out["gauntlet_report"] + .as_str() + .unwrap() + .contains("| adversary | BLOCKED | DIVERGES |"), + "a real DIVERGES must stay a finding: {out}" + ); + let out = run_gauntlet_script("retry_gate.py", &gauntlet_lane_ran("adversary", diverges)); + assert_eq!( + out["retry_lanes"], + json!([]), + "a real DIVERGES must never be re-run: {out}" + ); + + // The other two degraded-header forms verdict.py emits: a criterion + // check that died, and the top-level crash stub. + let died = run_adversary_script( + "verdict.py", + &json!({ + "pipeline_faults": [], + "crit_verdicts": [ + met, + {"id": "c2", "text": "does Y", "status": "UNMET", + "evidence": "PIPELINE-FAULT: criterion check died β€” boom", + "complaint": "boom"} + ], + "extra_complaints": [], + "observations": "", + "exec_results": "" + }), + ); + let died = died["adv_report"].as_str().unwrap(); + assert!( + died.contains("degraded run: 1 criterion check(s) died"), + "{died}" + ); + let crashed = run_adversary_script( + "verdict.py", + &json!({ + "pipeline_faults": [], + "crit_verdicts": [{"status": "UNMET", "text": 123}], + "extra_complaints": [], + "observations": "", + "exec_results": "" + }), + ); + let crashed = crashed["adv_report"].as_str().unwrap(); + assert!( + crashed.contains("Criteria: verdict computation error:"), + "{crashed}" + ); + for degraded in [died, crashed] { + let out = run_gauntlet_script( + "verdict_gate.py", + &json!({ + "code_review_results": [GAUNTLET_MERGE_READY], + "adversary_results": [degraded], + "security_results": [], + "probe_results": [] + }), + ); + assert_eq!(out["gauntlet_verdict"], "BLOCKED", "{out}"); + let report = out["gauntlet_report"].as_str().unwrap(); + assert!( + report.contains("| adversary | BLOCKED | PIPELINE-FAULT (degraded lane) |"), + "the real degraded header must render as a pipeline fault: {report}" + ); + assert!( + !report.contains("adversary: DIVERGES β€” the implementation does not conform"), + "{report}" + ); + let out = + run_gauntlet_script("retry_gate.py", &gauntlet_lane_ran("adversary", degraded)); + assert_eq!( + out["retry_lanes"], + json!(["adversary"]), + "the real degraded header must be re-run: {out}" + ); + } + } + #[test] fn gauntlet_retry_gate_counts_attempts_per_lane_that_ran() { if !cmd_available("python3") { @@ -12281,7 +12947,7 @@ mod tests { let out = run_gauntlet_script( "retry_gate.py", &json!({ - "lane_attempts": 42, + "kept_results": 42, "adversary_items": [{"lane": "adversary"}], "adversary_results": ["x"], "signals_error": "PIPELINE-FAULT: earlier fault", @@ -12331,7 +12997,7 @@ mod tests { let out = run_gauntlet_script( "retry_gate.py", &json!({ - "lane_attempts": 42, + "kept_results": 42, "adversary_items": [{"lane": "adversary"}], "adversary_results": ["x"], "signals_error": "git diff unavailable", @@ -12409,10 +13075,55 @@ mod tests { retry.contains(degraded), "retry_gate.py must match the adversary's degraded-run wording {degraded:?}" ); + assert!( + verdict.contains(degraded), + "verdict_gate.py must match the adversary's degraded-run wording {degraded:?}" + ); assert!( adversary_verdict.contains(degraded), "assets/agents/adversary/scripts/verdict.py must still emit {degraded:?}" ); + for fragment in [ + "degraded run: pipeline fault recorded", + r"criterion check\(s\) died", + "verdict computation error:", + ] { + assert!( + retry.contains(fragment), + "retry_gate.py must match the adversary degraded header fragment {fragment:?}" + ); + assert!( + verdict.contains(fragment), + "verdict_gate.py must match the adversary degraded header fragment {fragment:?}" + ); + } + for emitted in ["criterion check(s) died", "verdict computation error:"] { + assert!( + adversary_verdict.contains(emitted), + "assets/agents/adversary/scripts/verdict.py must still emit {emitted:?}" + ); + } + + fn slice_between<'a>(src: &'a str, start: &str, end: &str) -> &'a str { + let from = src + .find(start) + .unwrap_or_else(|| panic!("missing {start:?}")); + let rest = &src[from..]; + let to = rest + .find(end) + .unwrap_or_else(|| panic!("missing {end:?} after {start:?}")); + &rest[..to + end.len()] + } + for (start, end) in [ + ("ADVERSARY_DEGRADED_HEADER = re.compile(", "\n)\n"), + ("def adversary_degraded(report):", " return False\n"), + ] { + assert_eq!( + slice_between(&retry, start, end), + slice_between(&verdict, start, end), + "the adversary degraded predicate must stay byte-identical in both gate scripts" + ); + } } /// build_items.py's LANE_KEYS and retry_gate.py's LANES name the same