feat(TASK-019): code-reviewer fail-closed review pipeline
Every llm/agent stage that can die persistently now falls back to a machine-readable PIPELINE-FAULT sentinel instead of killing the graph: - review_domain -> branch-local domain_fault marker (banner-prefixed domain_report the map collects; completeness_gate precedent) - verify -> verdict (verifier fault renders findings unverified + a PIPELINE-FAULT attention line) - parse -> parse_fault -> render (complete NEEDS-HUMAN verdict_out; render's no-changes stub is bypassed for fault verdicts) - refine_groups -> cover_gate (deterministic-proposal fallback absorbs the failure; degradation noted in groups_note) - aux_lanes -> aux_fault -> synthesize (aux_note fault marker; visible, never blocking - aux is enrichment) - synthesize -> verdict (synth_failure marker; keyed off the failure prefix + non-empty diff, never findings==[], so clean zero-finding reviews stay MERGE-READY) verdict.py gains unified fault accounting (R2): any PIPELINE-FAULT in domain_reports / verifier_output / state fault keys forces NEEDS-HUMAN, prefix-anchored so a report merely quoting the marker mid-text never trips it; old 'PIPELINE FAULT' spelling migrated. Faults are prepended to attention so the [:5] cap can never hide one. Dedicated *_failure state keys capture engine failure text on every fallback+output_schema node, keeping the bundled warning baseline at zero. 15 python3-guarded R5 tests: fault paths, quoted-marker precedence, happy-path byte-compat regression, marker normalization, crash-path fail-closed (verdict/cover_gate/domain_fault), verifier-fault-on-empty- diff render, and the no-changes stub regression.
This commit is contained in:
8 files changed
+687
-4
No files matched your search
@@ -91,6 +91,10 @@ initial_state:
|
|||||||
final_report: ''
|
final_report: ''
|
||||||
gate_feedback: ''
|
gate_feedback: ''
|
||||||
gate_attempts: 0
|
gate_attempts: 0
|
||||||
|
parse_failure: ''
|
||||||
|
refine_failure: ''
|
||||||
|
aux_failure: ''
|
||||||
|
synth_failure: ''
|
||||||
|
|
||||||
start: parse
|
start: parse
|
||||||
|
|
||||||
@@ -139,8 +143,19 @@ nodes:
|
|||||||
required: [diff_spec, intent, rigor_in, surfaces_in, project_dir_in, extra_context]
|
required: [diff_spec, intent, rigor_in, surfaces_in, project_dir_in, extra_context]
|
||||||
max_iterations: 3
|
max_iterations: 3
|
||||||
max_attempts: 2
|
max_attempts: 2
|
||||||
|
state_updates:
|
||||||
|
parse_failure: '{{output}}'
|
||||||
|
fallback: parse_fault
|
||||||
next: facts
|
next: facts
|
||||||
|
|
||||||
|
parse_fault:
|
||||||
|
id: parse_fault
|
||||||
|
type: script
|
||||||
|
description: Fail-closed marker for a dead parse stage (reached only via parse's fallback) — emits a complete PIPELINE-FAULT verdict_out and jumps straight to render; a review that could not parse its request never reads as MERGE-READY.
|
||||||
|
script: scripts/parse_fault.py
|
||||||
|
timeout: 30
|
||||||
|
next: render
|
||||||
|
|
||||||
facts:
|
facts:
|
||||||
id: facts
|
id: facts
|
||||||
type: script
|
type: script
|
||||||
@@ -209,6 +224,9 @@ nodes:
|
|||||||
required: [groups]
|
required: [groups]
|
||||||
max_iterations: 3
|
max_iterations: 3
|
||||||
max_attempts: 2
|
max_attempts: 2
|
||||||
|
state_updates:
|
||||||
|
refine_failure: '{{output}}'
|
||||||
|
fallback: cover_gate
|
||||||
next: cover_gate
|
next: cover_gate
|
||||||
|
|
||||||
cover_gate:
|
cover_gate:
|
||||||
@@ -275,6 +293,7 @@ nodes:
|
|||||||
max_attempts: 2
|
max_attempts: 2
|
||||||
state_updates:
|
state_updates:
|
||||||
domain_report: '{{output}}'
|
domain_report: '{{output}}'
|
||||||
|
fallback: domain_fault
|
||||||
next: completeness_gate
|
next: completeness_gate
|
||||||
|
|
||||||
completeness_gate:
|
completeness_gate:
|
||||||
@@ -284,6 +303,13 @@ nodes:
|
|||||||
script: scripts/completeness_gate.py
|
script: scripts/completeness_gate.py
|
||||||
timeout: 30
|
timeout: 30
|
||||||
|
|
||||||
|
domain_fault:
|
||||||
|
id: domain_fault
|
||||||
|
type: script
|
||||||
|
description: Fail-closed domain lane fault marker (reached only via review_domain's fallback route inside the map branch). Normalizes the engine's "Agent node failed" text in domain_report into a > ⚠️ PIPELINE-FAULT banner the map collects; verdict.py counts any faulted lane and forces NEEDS-HUMAN — a dead domain lane never reads as a clean slice.
|
||||||
|
script: scripts/domain_fault.py
|
||||||
|
timeout: 30
|
||||||
|
|
||||||
aux_lanes:
|
aux_lanes:
|
||||||
id: aux_lanes
|
id: aux_lanes
|
||||||
type: llm
|
type: llm
|
||||||
@@ -339,6 +365,17 @@ nodes:
|
|||||||
max_iterations: 10
|
max_iterations: 10
|
||||||
max_attempts: 2
|
max_attempts: 2
|
||||||
timeout: 3600
|
timeout: 3600
|
||||||
|
state_updates:
|
||||||
|
aux_failure: '{{output}}'
|
||||||
|
fallback: aux_fault
|
||||||
|
next: synthesize
|
||||||
|
|
||||||
|
aux_fault:
|
||||||
|
id: aux_fault
|
||||||
|
type: script
|
||||||
|
description: Fail-closed marker for dead aux context lanes (reached only via aux_lanes' fallback) — records the failure as a PIPELINE-FAULT in aux_note, which synthesize surfaces as a lane skip. Aux lanes are enrichment; the degradation is visible but never blocks.
|
||||||
|
script: scripts/aux_fault.py
|
||||||
|
timeout: 30
|
||||||
next: synthesize
|
next: synthesize
|
||||||
|
|
||||||
downstream_sweep:
|
downstream_sweep:
|
||||||
@@ -457,6 +494,9 @@ nodes:
|
|||||||
required: [walkthrough, changes_rows, findings, cross_file, operational_history, org_context_section, attention_flags, placement_notes]
|
required: [walkthrough, changes_rows, findings, cross_file, operational_history, org_context_section, attention_flags, placement_notes]
|
||||||
max_iterations: 6
|
max_iterations: 6
|
||||||
max_attempts: 2
|
max_attempts: 2
|
||||||
|
state_updates:
|
||||||
|
synth_failure: '{{output}}'
|
||||||
|
fallback: verdict
|
||||||
next: verify
|
next: verify
|
||||||
|
|
||||||
verify:
|
verify:
|
||||||
@@ -476,6 +516,7 @@ nodes:
|
|||||||
max_attempts: 2
|
max_attempts: 2
|
||||||
state_updates:
|
state_updates:
|
||||||
verifier_output: '{{output}}'
|
verifier_output: '{{output}}'
|
||||||
|
fallback: verdict
|
||||||
next: verdict
|
next: verdict
|
||||||
|
|
||||||
verdict:
|
verdict:
|
||||||
|
|||||||
@@ -0,0 +1,48 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Fail-closed marker for dead aux context lanes of the code-reviewer graph.
|
||||||
|
|
||||||
|
Reached ONLY via aux_lanes' `fallback:` route. aux_lanes' state_updates wrote
|
||||||
|
an "LLM node …failed: <chain>" string into `aux_failure` before routing here;
|
||||||
|
this node records the failure as a PIPELINE-FAULT in `aux_note`, which flows
|
||||||
|
into synthesize's "lane skips" prompt line. Aux lanes are ENRICHMENT, not
|
||||||
|
review coverage: the degradation is made VISIBLE in the report's inputs but
|
||||||
|
never blocks — verdict.py deliberately does NOT scan aux_note for faults.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
|
||||||
|
MAX_DETAIL_CHARS = 500
|
||||||
|
|
||||||
|
|
||||||
|
def load_state():
|
||||||
|
if path := os.environ.get("GRAPH_STATE_FILE"):
|
||||||
|
with open(path) as f:
|
||||||
|
return json.load(f)
|
||||||
|
return json.loads(os.environ.get("GRAPH_STATE", "{}"))
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
state = load_state()
|
||||||
|
raw = state.get("aux_failure")
|
||||||
|
detail = raw.strip().replace("\n", " ") if isinstance(raw, str) else ""
|
||||||
|
if len(detail) > MAX_DETAIL_CHARS:
|
||||||
|
detail = detail[: MAX_DETAIL_CHARS - 1] + "…"
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{"aux_note": f"PIPELINE-FAULT: aux context lanes failed after retries — {detail}"}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
main()
|
||||||
|
except Exception as e: # noqa: BLE001 — the fault marker itself must never crash
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"aux_note": "PIPELINE-FAULT: aux context lanes failed after retries — "
|
||||||
|
f"fault-marker script error: {e}"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
@@ -54,6 +54,11 @@ def main():
|
|||||||
else:
|
else:
|
||||||
chosen = [{"domain": g["domain"], "files": g["files"]} for g in refined]
|
chosen = [{"domain": g["domain"], "files": g["files"]} for g in refined]
|
||||||
if chosen is None:
|
if chosen is None:
|
||||||
|
# refine_groups' fallback lands here with `groups` never set; make the
|
||||||
|
# degradation visible in the synthesis prompt via groups_note.
|
||||||
|
refine_failure = state.get("refine_failure")
|
||||||
|
if not refined and isinstance(refine_failure, str) and refine_failure.startswith("LLM node"):
|
||||||
|
note = "refinement lane failed after retries — fell back to the deterministic grouping"
|
||||||
chosen = [{"domain": g["domain"], "files": g["files"]} for g in proposed if isinstance(g, dict)]
|
chosen = [{"domain": g["domain"], "files": g["files"]} for g in proposed if isinstance(g, dict)]
|
||||||
|
|
||||||
print(json.dumps({"group_items": chosen, "groups_note": note}))
|
print(json.dumps({"group_items": chosen, "groups_note": note}))
|
||||||
|
|||||||
@@ -0,0 +1,59 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Fail-closed domain lane fault marker for the code-reviewer graph.
|
||||||
|
|
||||||
|
Reached ONLY via review_domain's `fallback:` route inside the map branch.
|
||||||
|
Grounded in engine behavior: on agent-node failure the engine writes
|
||||||
|
"Agent node failed: {e:#}" into `domain_report` via the node's state_updates
|
||||||
|
BEFORE routing to the fallback, so the raw failure text is already in state.
|
||||||
|
This node normalizes it into a "> ⚠️ PIPELINE-FAULT: domain review lane
|
||||||
|
failed after retries — …" banner the map collects as the lane's report;
|
||||||
|
verdict.py counts any banner-prefixed report as a fault and forces
|
||||||
|
NEEDS-HUMAN — a dead domain lane never reads as a clean slice.
|
||||||
|
|
||||||
|
Slice identity comes from `domain_group` (the map item cover_gate emitted,
|
||||||
|
which carries `domain` and `files`); a missing/malformed item degrades to
|
||||||
|
the failure detail alone.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
|
||||||
|
MAX_DETAIL_CHARS = 500
|
||||||
|
|
||||||
|
FAULT_PREFIX = "> ⚠️ PIPELINE-FAULT: domain review lane failed after retries — "
|
||||||
|
|
||||||
|
|
||||||
|
def load_state():
|
||||||
|
if path := os.environ.get("GRAPH_STATE_FILE"):
|
||||||
|
with open(path) as f:
|
||||||
|
return json.load(f)
|
||||||
|
return json.loads(os.environ.get("GRAPH_STATE", "{}"))
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
state = load_state()
|
||||||
|
slice_desc = ""
|
||||||
|
group = state.get("domain_group")
|
||||||
|
if isinstance(group, dict):
|
||||||
|
domain = group.get("domain")
|
||||||
|
files = [f for f in (group.get("files") or []) if isinstance(f, str)]
|
||||||
|
if isinstance(domain, str) and domain.strip():
|
||||||
|
slice_desc = f"domain '{domain.strip()}' (files: {', '.join(files)}): "
|
||||||
|
raw = state.get("domain_report")
|
||||||
|
detail = raw.strip().replace("\n", " ") if isinstance(raw, str) else ""
|
||||||
|
if len(detail) > MAX_DETAIL_CHARS:
|
||||||
|
detail = detail[: MAX_DETAIL_CHARS - 1] + "…"
|
||||||
|
print(json.dumps({"domain_report": f"{FAULT_PREFIX}{slice_desc}{detail}"}))
|
||||||
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
main()
|
||||||
|
except Exception as e: # noqa: BLE001 — the fault marker itself must never crash
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"domain_report": "> ⚠️ PIPELINE-FAULT: domain review lane failed "
|
||||||
|
f"after retries — fault-marker script error: {e}"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Fail-closed marker for a dead parse stage of the code-reviewer graph.
|
||||||
|
|
||||||
|
Reached ONLY via parse's `fallback:` route. parse's state_updates wrote an
|
||||||
|
"LLM node …failed: <chain>" string into `parse_failure` before routing here;
|
||||||
|
this node emits a complete PIPELINE-FAULT `verdict_out` and the graph jumps
|
||||||
|
straight to render — a review that could not even parse its request never
|
||||||
|
reads as MERGE-READY. render.py renders a PIPELINE-FAULT verdict despite the
|
||||||
|
empty changed-file list (its "No changes to review." early return skips
|
||||||
|
fault-reason verdicts).
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
|
||||||
|
MAX_DETAIL_CHARS = 500
|
||||||
|
|
||||||
|
ATTENTION_LINE = (
|
||||||
|
"PIPELINE-FAULT: parse failed — the review request could not be parsed; "
|
||||||
|
"the review did not run"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def load_state():
|
||||||
|
if path := os.environ.get("GRAPH_STATE_FILE"):
|
||||||
|
with open(path) as f:
|
||||||
|
return json.load(f)
|
||||||
|
return json.loads(os.environ.get("GRAPH_STATE", "{}"))
|
||||||
|
|
||||||
|
|
||||||
|
def fault_verdict(detail):
|
||||||
|
return {
|
||||||
|
"verdict": "NEEDS-HUMAN",
|
||||||
|
"reason": f"PIPELINE-FAULT: parse failed — {detail}",
|
||||||
|
"counts": {},
|
||||||
|
"deferred_count": 0,
|
||||||
|
"dropped_count": 0,
|
||||||
|
"dropped_titles": [],
|
||||||
|
"attention": [ATTENTION_LINE],
|
||||||
|
"findings_final": [],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
state = load_state()
|
||||||
|
raw = state.get("parse_failure")
|
||||||
|
detail = raw.strip().replace("\n", " ") if isinstance(raw, str) else ""
|
||||||
|
if not detail.startswith("LLM node"):
|
||||||
|
detail = "the parse stage died without recording a failure detail"
|
||||||
|
if len(detail) > MAX_DETAIL_CHARS:
|
||||||
|
detail = detail[: MAX_DETAIL_CHARS - 1] + "…"
|
||||||
|
print(json.dumps({"verdict_out": fault_verdict(detail)}))
|
||||||
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
main()
|
||||||
|
except Exception as e: # noqa: BLE001 — the fault marker itself must never crash
|
||||||
|
print(json.dumps({"verdict_out": fault_verdict(f"fault-marker script error: {e}")}))
|
||||||
@@ -24,7 +24,14 @@ def main():
|
|||||||
changed = state.get("changed_files") or []
|
changed = state.get("changed_files") or []
|
||||||
rigor = state.get("resolved_rigor") or "production"
|
rigor = state.get("resolved_rigor") or "production"
|
||||||
|
|
||||||
if not changed:
|
# A fault can leave changed_files empty (a parse fault dies before the
|
||||||
|
# diff is resolved; a verifier fault fires even on an empty diff) — a
|
||||||
|
# fault-degraded verdict must still render as a full report. parse_fault
|
||||||
|
# puts the marker in the reason; verdict.py prepends it to attention.
|
||||||
|
faulted = str(v.get("reason") or "").startswith("PIPELINE-FAULT:") or any(
|
||||||
|
isinstance(a, str) and a.startswith("PIPELINE-FAULT:") for a in v.get("attention") or []
|
||||||
|
)
|
||||||
|
if not changed and not faulted:
|
||||||
print(json.dumps({"final_report": "# Code Review Summary\n\nNo changes to review."}))
|
print(json.dumps({"final_report": "# Code Review Summary\n\nNo changes to review."}))
|
||||||
return
|
return
|
||||||
|
|
||||||
|
|||||||
@@ -8,6 +8,9 @@ The model that wrote the findings never grades its own homework:
|
|||||||
markers, exactly per the published rules;
|
markers, exactly per the published rules;
|
||||||
- MERGE-READY/NEEDS-HUMAN is computed from counts + always-human triggers
|
- MERGE-READY/NEEDS-HUMAN is computed from counts + always-human triggers
|
||||||
(deterministic signals ∪ the synthesis's ADDITIVE-only flags).
|
(deterministic signals ∪ the synthesis's ADDITIVE-only flags).
|
||||||
|
Fail-closed: any PIPELINE-FAULT recorded upstream (dead domain lane, dead
|
||||||
|
verifier, dead synthesis, missing domain reports) forces NEEDS-HUMAN — a
|
||||||
|
degraded run can never read MERGE-READY.
|
||||||
The gate never crashes into a silent verdict: any internal error emits
|
The gate never crashes into a silent verdict: any internal error emits
|
||||||
NEEDS-HUMAN naming the error.
|
NEEDS-HUMAN naming the error.
|
||||||
"""
|
"""
|
||||||
@@ -16,6 +19,9 @@ import json
|
|||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
|
||||||
|
MAX_DETAIL_CHARS = 500
|
||||||
|
|
||||||
|
|
||||||
def load_state():
|
def load_state():
|
||||||
if path := os.environ.get("GRAPH_STATE_FILE"):
|
if path := os.environ.get("GRAPH_STATE_FILE"):
|
||||||
with open(path) as f:
|
with open(path) as f:
|
||||||
@@ -117,16 +123,48 @@ def main():
|
|||||||
seen = set()
|
seen = set()
|
||||||
attention = [a for a in attention if not (a.lower() in seen or seen.add(a.lower()))]
|
attention = [a for a in attention if not (a.lower() in seen or seen.add(a.lower()))]
|
||||||
|
|
||||||
if (state.get("changed_files") or []) and not (state.get("domain_reports") or []):
|
# --- pipeline-fault accounting (fail closed) -----------------------------
|
||||||
attention.append(
|
# PREFIX-anchored: a report that merely QUOTES a marker mid-text must not
|
||||||
"PIPELINE FAULT: the diff is non-empty but no domain reports were produced — "
|
# trip it. Any fault forces NEEDS-HUMAN.
|
||||||
|
faults = []
|
||||||
|
changed = state.get("changed_files") or []
|
||||||
|
reports = state.get("domain_reports") or []
|
||||||
|
faulted = sum(
|
||||||
|
1 for r in reports if isinstance(r, str) and r.lstrip().startswith("> ⚠️ PIPELINE-FAULT:")
|
||||||
|
)
|
||||||
|
if faulted:
|
||||||
|
faults.append(
|
||||||
|
f"PIPELINE-FAULT: {faulted} of {len(reports)} domain review lane(s) "
|
||||||
|
"failed after retries — their slices were not reviewed"
|
||||||
|
)
|
||||||
|
verifier_raw = state.get("verifier_output")
|
||||||
|
if isinstance(verifier_raw, str) and verifier_raw.lstrip().startswith("Agent node failed:"):
|
||||||
|
faults.append(
|
||||||
|
"PIPELINE-FAULT: finding verification failed — findings render as unverified"
|
||||||
|
)
|
||||||
|
# Keyed off synth_failure + a non-empty diff, NEVER off findings == [] —
|
||||||
|
# a clean review legitimately has zero findings and stays MERGE-READY.
|
||||||
|
synth_failure = state.get("synth_failure")
|
||||||
|
if isinstance(synth_failure, str) and synth_failure.startswith("LLM node") and changed:
|
||||||
|
detail = synth_failure.strip().replace("\n", " ")
|
||||||
|
if len(detail) > MAX_DETAIL_CHARS:
|
||||||
|
detail = detail[: MAX_DETAIL_CHARS - 1] + "…"
|
||||||
|
faults.append(f"PIPELINE-FAULT: synthesis failed — findings unavailable; {detail}")
|
||||||
|
if changed and not reports:
|
||||||
|
faults.append(
|
||||||
|
"PIPELINE-FAULT: the diff is non-empty but no domain reports were produced — "
|
||||||
"the review did not actually run; do not trust this verdict."
|
"the review did not actually run; do not trust this verdict."
|
||||||
)
|
)
|
||||||
|
# faults go FIRST so the attention cap can never hide one
|
||||||
|
attention = faults + attention
|
||||||
|
|
||||||
reasons = []
|
reasons = []
|
||||||
if counts["🔴"]:
|
if counts["🔴"]:
|
||||||
reasons.append(f"{counts['🔴']} 🔴 CRITICAL finding(s)")
|
reasons.append(f"{counts['🔴']} 🔴 CRITICAL finding(s)")
|
||||||
if yellow_correctness:
|
if yellow_correctness:
|
||||||
reasons.append(f"{yellow_correctness} 🟡 [correctness] finding(s) outside the deferred section")
|
reasons.append(f"{yellow_correctness} 🟡 [correctness] finding(s) outside the deferred section")
|
||||||
|
if faults:
|
||||||
|
reasons.append("pipeline fault(s) recorded — degraded run")
|
||||||
if attention:
|
if attention:
|
||||||
reasons.append("always-human trigger(s) fired")
|
reasons.append("always-human trigger(s) fired")
|
||||||
verdict = "NEEDS-HUMAN" if reasons else "MERGE-READY"
|
verdict = "NEEDS-HUMAN" if reasons else "MERGE-READY"
|
||||||
|
|||||||
@@ -9248,6 +9248,433 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---- code-reviewer suite-script regression tests (R5) ----
|
||||||
|
//
|
||||||
|
// Fail-closed fault paths of the code-reviewer's verdict/fault/render
|
||||||
|
// scripts, exercised the same way as the suites above: `python3 <script>`
|
||||||
|
// with a synthetic GRAPH_STATE env.
|
||||||
|
|
||||||
|
fn run_code_reviewer_script(script: &str, state: &serde_json::Value) -> serde_json::Value {
|
||||||
|
run_code_reviewer_script_raw(script, &state.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run_code_reviewer_script_raw(script: &str, raw_state: &str) -> serde_json::Value {
|
||||||
|
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||||
|
.join("assets/agents/code-reviewer/scripts")
|
||||||
|
.join(script);
|
||||||
|
let out = std::process::Command::new("python3")
|
||||||
|
.arg(&path)
|
||||||
|
.env("GRAPH_STATE", raw_state)
|
||||||
|
.output()
|
||||||
|
.unwrap_or_else(|e| panic!("failed to invoke python3 {script}: {e}"));
|
||||||
|
assert!(
|
||||||
|
out.status.success(),
|
||||||
|
"{script} exited nonzero: {}",
|
||||||
|
String::from_utf8_lossy(&out.stderr)
|
||||||
|
);
|
||||||
|
serde_json::from_slice(&out.stdout).unwrap_or_else(|e| {
|
||||||
|
panic!(
|
||||||
|
"{script} stdout is not JSON ({e}): {}",
|
||||||
|
String::from_utf8_lossy(&out.stdout)
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn code_reviewer_verdict_domain_fault_forces_needs_human() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let state = json!({
|
||||||
|
"changed_files": ["a.rs"],
|
||||||
|
"domain_reports": ["> ⚠️ PIPELINE-FAULT: domain review lane failed after retries — domain 'engine' (files: a.rs): Agent node failed: dead"],
|
||||||
|
"findings": []
|
||||||
|
});
|
||||||
|
let out = run_code_reviewer_script("verdict.py", &state);
|
||||||
|
let v = &out["verdict_out"];
|
||||||
|
assert_eq!(
|
||||||
|
v["verdict"], "NEEDS-HUMAN",
|
||||||
|
"a faulted domain lane must force NEEDS-HUMAN: {out}"
|
||||||
|
);
|
||||||
|
let first = v["attention"][0].as_str().unwrap();
|
||||||
|
assert!(
|
||||||
|
first.starts_with("PIPELINE-FAULT:"),
|
||||||
|
"the fault must lead the attention list: {out}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn code_reviewer_verdict_synthesis_fault_blocks() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Keyed off synth_failure, never off findings == [] — the empty
|
||||||
|
// findings list here is exactly what a dead synthesis leaves behind.
|
||||||
|
let state = json!({
|
||||||
|
"changed_files": ["a.rs"],
|
||||||
|
"domain_reports": ["clean report. DOMAIN_REVIEW_COMPLETE"],
|
||||||
|
"findings": [],
|
||||||
|
"synth_failure": "LLM node failed: boom"
|
||||||
|
});
|
||||||
|
let out = run_code_reviewer_script("verdict.py", &state);
|
||||||
|
let v = &out["verdict_out"];
|
||||||
|
assert_eq!(
|
||||||
|
v["verdict"], "NEEDS-HUMAN",
|
||||||
|
"a dead synthesis must block: {out}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
v["attention"].as_array().unwrap().iter().any(|a| a
|
||||||
|
.as_str()
|
||||||
|
.unwrap_or("")
|
||||||
|
.contains("PIPELINE-FAULT: synthesis failed")),
|
||||||
|
"the synthesis fault must be in attention: {out}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn code_reviewer_verdict_verifier_fault_attention() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let state = json!({
|
||||||
|
"changed_files": ["a.rs"],
|
||||||
|
"domain_reports": ["clean report. DOMAIN_REVIEW_COMPLETE"],
|
||||||
|
"findings": [{
|
||||||
|
"id": "f1", "severity": "🟡 WARNING", "marker": "",
|
||||||
|
"file": "a.rs", "lines": "10", "title": "possible issue",
|
||||||
|
"block": "#### possible issue"
|
||||||
|
}],
|
||||||
|
"verifier_output": "Agent node failed: dead"
|
||||||
|
});
|
||||||
|
let out = run_code_reviewer_script("verdict.py", &state);
|
||||||
|
let v = &out["verdict_out"];
|
||||||
|
assert_eq!(
|
||||||
|
v["verdict"], "NEEDS-HUMAN",
|
||||||
|
"a dead verifier must block: {out}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
v["attention"].as_array().unwrap().iter().any(|a| a
|
||||||
|
.as_str()
|
||||||
|
.unwrap_or("")
|
||||||
|
.contains("PIPELINE-FAULT: finding verification failed")),
|
||||||
|
"the verifier fault must be in attention: {out}"
|
||||||
|
);
|
||||||
|
let block = v["findings_final"][0]["block"].as_str().unwrap();
|
||||||
|
assert!(
|
||||||
|
block.contains("(unverified: no verifier verdict returned"),
|
||||||
|
"the kept finding must render as unverified: {out}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn code_reviewer_verdict_quoted_marker_precedence() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// PREFIX-anchored fault detection: a clean report that merely QUOTES
|
||||||
|
// the banner mid-text must not read as a faulted lane.
|
||||||
|
let state = json!({
|
||||||
|
"changed_files": ["a.rs"],
|
||||||
|
"domain_reports": ["The gate prepends '> ⚠️ PIPELINE-FAULT:' when a lane dies; this slice reviewed that logic and it is correct. DOMAIN_REVIEW_COMPLETE"],
|
||||||
|
"findings": []
|
||||||
|
});
|
||||||
|
let out = run_code_reviewer_script("verdict.py", &state);
|
||||||
|
assert_eq!(
|
||||||
|
out["verdict_out"]["verdict"], "MERGE-READY",
|
||||||
|
"a quoted marker mid-text must not trip the fault check: {out}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn code_reviewer_verdict_happy_path_regression() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let state = json!({
|
||||||
|
"changed_files": ["a.rs"],
|
||||||
|
"domain_reports": ["clean report. DOMAIN_REVIEW_COMPLETE"],
|
||||||
|
"findings": [],
|
||||||
|
"verifier_output": "",
|
||||||
|
"parse_failure": "",
|
||||||
|
"refine_failure": "",
|
||||||
|
"aux_failure": "",
|
||||||
|
"synth_failure": ""
|
||||||
|
});
|
||||||
|
let out = run_code_reviewer_script("verdict.py", &state);
|
||||||
|
let v = &out["verdict_out"];
|
||||||
|
assert_eq!(
|
||||||
|
v["verdict"], "MERGE-READY",
|
||||||
|
"a clean run must stay MERGE-READY: {out}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
v["reason"], "no blocking findings and no always-human triggers",
|
||||||
|
"the happy-path reason must be byte-compatible: {out}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn code_reviewer_domain_fault_normalizes() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let state = json!({
|
||||||
|
"domain_group": {"domain": "engine", "files": ["a.rs"]},
|
||||||
|
"domain_report": "Agent node failed: spawn dead\nline2"
|
||||||
|
});
|
||||||
|
let out = run_code_reviewer_script("domain_fault.py", &state);
|
||||||
|
let report = out["domain_report"].as_str().unwrap();
|
||||||
|
assert!(
|
||||||
|
report.starts_with("> ⚠️ PIPELINE-FAULT: domain review lane failed after retries — "),
|
||||||
|
"the banner prefix verdict.py anchors on must be exact: {report}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
report.contains("domain 'engine' (files: a.rs)"),
|
||||||
|
"the fault must name the slice: {report}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!report.contains('\n'),
|
||||||
|
"the failure detail must be newline-normalized: {report}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn code_reviewer_parse_fault_emits_needs_human() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let out = run_code_reviewer_script(
|
||||||
|
"parse_fault.py",
|
||||||
|
&json!({"parse_failure": "LLM node failed: x"}),
|
||||||
|
);
|
||||||
|
let v = &out["verdict_out"];
|
||||||
|
assert_eq!(
|
||||||
|
v["verdict"], "NEEDS-HUMAN",
|
||||||
|
"a dead parse must block: {out}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
v["reason"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.starts_with("PIPELINE-FAULT: parse failed"),
|
||||||
|
"the reason must carry the fault marker: {out}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn code_reviewer_parse_fault_renders_well_formed() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// A parse fault dies before facts, so changed_files is empty — the
|
||||||
|
// fault verdict must still render as a full report, not the
|
||||||
|
// "No changes to review." stub.
|
||||||
|
let fault = run_code_reviewer_script(
|
||||||
|
"parse_fault.py",
|
||||||
|
&json!({"parse_failure": "LLM node failed: x"}),
|
||||||
|
);
|
||||||
|
let state = json!({
|
||||||
|
"changed_files": [],
|
||||||
|
"verdict_out": fault["verdict_out"].clone()
|
||||||
|
});
|
||||||
|
let out = run_code_reviewer_script("render.py", &state);
|
||||||
|
let report = out["final_report"].as_str().unwrap();
|
||||||
|
assert!(
|
||||||
|
report.contains("**Verdict: NEEDS-HUMAN**"),
|
||||||
|
"the fault verdict must render: {report}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
report.contains("PIPELINE-FAULT: parse failed"),
|
||||||
|
"the fault reason must render: {report}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn code_reviewer_render_no_changes_regression() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let state = json!({
|
||||||
|
"changed_files": [],
|
||||||
|
"verdict_out": {
|
||||||
|
"verdict": "MERGE-READY",
|
||||||
|
"reason": "no blocking findings and no always-human triggers"
|
||||||
|
}
|
||||||
|
});
|
||||||
|
let out = run_code_reviewer_script("render.py", &state);
|
||||||
|
assert!(
|
||||||
|
out["final_report"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.contains("No changes to review."),
|
||||||
|
"an empty non-fault diff must keep the stub report: {out}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn code_reviewer_aux_fault_normalizes() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let out = run_code_reviewer_script(
|
||||||
|
"aux_fault.py",
|
||||||
|
&json!({"aux_failure": "LLM node failed: y"}),
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
out["aux_note"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.starts_with("PIPELINE-FAULT: aux context lanes failed after retries —"),
|
||||||
|
"the aux fault must land in aux_note: {out}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn code_reviewer_cover_gate_notes_refine_failure() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// refine_groups' fallback lands in cover_gate with `groups` never set;
|
||||||
|
// the deterministic proposal must win and the degradation must be
|
||||||
|
// noted for the synthesis prompt.
|
||||||
|
let state = json!({
|
||||||
|
"changed_files": ["a.rs"],
|
||||||
|
"proposed_groups": [{"domain": "engine", "files": ["a.rs"]}],
|
||||||
|
"refine_failure": "LLM node failed: z"
|
||||||
|
});
|
||||||
|
let out = run_code_reviewer_script("cover_gate.py", &state);
|
||||||
|
assert_eq!(
|
||||||
|
out["group_items"],
|
||||||
|
json!([{"domain": "engine", "files": ["a.rs"]}]),
|
||||||
|
"the deterministic proposal must be used: {out}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
out["groups_note"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.contains("fell back to the deterministic grouping"),
|
||||||
|
"the refine failure must be noted: {out}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn code_reviewer_render_verifier_fault_empty_diff_renders_fault() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// The verifier fault check is deliberately not gated on a non-empty
|
||||||
|
// diff, and verdict.py carries the marker in attention rather than
|
||||||
|
// the reason — render.py must not swallow that verdict into the
|
||||||
|
// "No changes to review." stub.
|
||||||
|
let fault = run_code_reviewer_script(
|
||||||
|
"verdict.py",
|
||||||
|
&json!({
|
||||||
|
"changed_files": [],
|
||||||
|
"domain_reports": [],
|
||||||
|
"findings": [],
|
||||||
|
"verifier_output": "Agent node failed: dead"
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
let state = json!({
|
||||||
|
"changed_files": [],
|
||||||
|
"verdict_out": fault["verdict_out"].clone()
|
||||||
|
});
|
||||||
|
let out = run_code_reviewer_script("render.py", &state);
|
||||||
|
let report = out["final_report"].as_str().unwrap();
|
||||||
|
assert!(
|
||||||
|
report.contains("**Verdict: NEEDS-HUMAN**"),
|
||||||
|
"the fault verdict must render: {report}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
report.contains("PIPELINE-FAULT: finding verification failed"),
|
||||||
|
"the verifier fault must render: {report}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!report.contains("No changes to review."),
|
||||||
|
"a fault-degraded verdict must not collapse into the stub: {report}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn code_reviewer_verdict_crash_fails_closed() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// A truthy non-iterable attention_flags raises TypeError inside main;
|
||||||
|
// the guard must still emit a fail-closed verdict.
|
||||||
|
let state = json!({"changed_files": ["a.rs"], "attention_flags": 42});
|
||||||
|
let out = run_code_reviewer_script("verdict.py", &state);
|
||||||
|
let v = &out["verdict_out"];
|
||||||
|
assert_eq!(
|
||||||
|
v["verdict"], "NEEDS-HUMAN",
|
||||||
|
"a crashed verdict gate must fail closed: {out}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
v["reason"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.contains("verdict computation error"),
|
||||||
|
"the reason must name the error: {out}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn code_reviewer_cover_gate_crash_fails_closed() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// A domain-less proposed group raises KeyError while building the
|
||||||
|
// fallback grouping; the guard must emit one catch-all group, never
|
||||||
|
// an empty review.
|
||||||
|
let state = json!({"changed_files": ["a.rs"], "proposed_groups": [{}]});
|
||||||
|
let out = run_code_reviewer_script("cover_gate.py", &state);
|
||||||
|
assert_eq!(
|
||||||
|
out["group_items"],
|
||||||
|
json!([{"domain": "all-changes", "files": ["a.rs"]}]),
|
||||||
|
"a crashed gate must fall back to a catch-all group: {out}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
out["groups_note"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.contains("cover gate error"),
|
||||||
|
"the crash must be noted: {out}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn code_reviewer_domain_fault_crash_fails_closed() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Unparseable state makes load_state raise before main runs; the
|
||||||
|
// fault marker itself must still emit the banner-prefixed report.
|
||||||
|
let out = run_code_reviewer_script_raw("domain_fault.py", "not json");
|
||||||
|
let report = out["domain_report"].as_str().unwrap();
|
||||||
|
assert!(
|
||||||
|
report.starts_with("> ⚠️ PIPELINE-FAULT: domain review lane failed after retries — "),
|
||||||
|
"the banner prefix verdict.py anchors on must survive a crash: {report}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
report.contains("fault-marker script error"),
|
||||||
|
"the crash must be named: {report}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
#[serial]
|
||||||
fn install_functions_force_preserves_user_mcp_json() {
|
fn install_functions_force_preserves_user_mcp_json() {
|
||||||
|
|||||||
Reference in new issue
Block a user