fix(adversary): fail closed on PIPELINE-FAULT evidence regardless of status; pin bail-chain and validator shape

This commit is contained in:
Dark-Alex-17 committed 2026-09-14 10:50:34 -06:00
1 parent 66b30bb06a
commit 1cf42fffa2
5 files changed
+162 -10

No files matched your search

+1 -1
View File
@@ -91,7 +91,7 @@ The conformance doctrine is baked into the graph's own nodes:
3. Execution evidence is recorded **once, deterministically**: `run_checks` runs ONLY the caller-declared verification commands (never invented or auto-detected), sequentially under a 3300s total deadline — commands the deadline leaves no budget for are recorded as SKIPPED. Criterion branches have no execution tools by design: "tests pass"-style criteria are judged against the record — a recorded green run is MET citing it; no declared command ⇒ at best PARTIAL. 3. Execution evidence is recorded **once, deterministically**: `run_checks` runs ONLY the caller-declared verification commands (never invented or auto-detected), sequentially under a 3300s total deadline — commands the deadline leaves no budget for are recorded as SKIPPED. Criterion branches have no execution tools by design: "tests pass"-style criteria are judged against the record — a recorded green run is MET citing it; no declared command ⇒ at best PARTIAL.
4. Hunt adversarially for the **absent**: skipped criteria, scope creep, interface/approach substitution, out-of-scope touches, downstream contract breakage. 4. Hunt adversarially for the **absent**: skipped criteria, scope creep, interface/approach substitution, out-of-scope touches, downstream contract breakage.
5. **Fail closed on pipeline faults**: if parse, diff resolution, or the verification runner dies, its fallback routes to `pipeline_fault`, which records a `PIPELINE-FAULT:` marker and continues to the verdict (a failed holistic pass falls back straight to the verdict, which synthesizes the same marker). Any recorded fault forces DIVERGES with the fault as complaint #1 — degraded is never passing, and the sentinel is always emitted: the graph never dies without one. 5. **Fail closed on pipeline faults**: if parse, diff resolution, or the verification runner dies, its fallback routes to `pipeline_fault`, which records a `PIPELINE-FAULT:` marker and continues to the verdict (a failed holistic pass falls back straight to the verdict, which synthesizes the same marker). Any recorded fault forces DIVERGES with the fault as complaint #1 — degraded is never passing, and the sentinel is always emitted: the graph never dies without one.
6. **Fail closed per criterion**: a criterion check that dies (max_iterations exhausted, API failure) falls back to `criterion_fault`, which records a schema-shaped UNMET verdict carrying a `PIPELINE-FAULT:` evidence marker for that one criterion. The other criteria still get verdicts, the report says the check DIED rather than judged, and the run is DIVERGES. 6. **Fail closed per criterion**: a criterion check that dies (max_iterations exhausted, API failure) falls back to `criterion_fault`, which records a schema-shaped UNMET verdict carrying a `PIPELINE-FAULT:` evidence marker for that one criterion. The other criteria still get verdicts, the report says the check DIED rather than judged, and the run is DIVERGES. The evidence marker alone forces the criterion into unmet regardless of the status it carries, so a verdict that echoes the marker with `MET` can never count toward CONFORMS.
It is **read-only** — it produces a verdict, never a fix. It is **read-only** — it produces a verdict, never a fix.
+15 -8
View File
@@ -8,8 +8,9 @@ Any "PIPELINE-FAULT:" marker recorded in `pipeline_faults` (or a holistic
llm-node failure captured in `holistic_failure`) becomes complaint #1; llm-node failure captured in `holistic_failure`) becomes complaint #1;
per-criterion results are still reported. A criterion whose check DIED per-criterion results are still reported. A criterion whose check DIED
(criterion_fault's UNMET verdict, evidence prefixed "PIPELINE-FAULT:") is (criterion_fault's UNMET verdict, evidence prefixed "PIPELINE-FAULT:") is
rendered as died rather than judged. Assembles the exact sentinel format the rendered as died rather than judged and counted as unmet regardless of the
callers route on. Never crashes into a silent verdict. status it carries. Assembles the exact sentinel format the callers route on.
Never crashes into a silent verdict.
""" """
import json import json
@@ -19,6 +20,11 @@ MAX_FAULT_DETAIL_CHARS = 300
DIED_MARKER = "criterion check DIED (pipeline fault)" DIED_MARKER = "criterion check DIED (pipeline fault)"
def died(v):
evidence = v.get("evidence")
return isinstance(evidence, str) and evidence.startswith("PIPELINE-FAULT:")
def load_state(): def load_state():
if path := os.environ.get("GRAPH_STATE_FILE"): if path := os.environ.get("GRAPH_STATE_FILE"):
with open(path) as f: with open(path) as f:
@@ -54,15 +60,14 @@ def append_criterion_complaints(lines, verdicts, extra, start):
"""Numbered complaints for non-MET verdicts and holistic extras.""" """Numbered complaints for non-MET verdicts and holistic extras."""
i = start i = start
for v in verdicts: for v in verdicts:
if v.get("status") == "MET": if v.get("status") == "MET" and not died(v):
continue continue
i += 1 i += 1
text = (v.get("text") or "").strip().replace("\n", " ") text = (v.get("text") or "").strip().replace("\n", " ")
if len(text) > 140: if len(text) > 140:
text = text[:137] + "…" text = text[:137] + "…"
complaint = (v.get("complaint") or "").strip() complaint = (v.get("complaint") or "").strip()
evidence = v.get("evidence") if died(v):
if isinstance(evidence, str) and evidence.startswith("PIPELINE-FAULT:"):
if not complaint.startswith(DIED_MARKER): if not complaint.startswith(DIED_MARKER):
complaint = f"{DIED_MARKER} — {complaint}" complaint = f"{DIED_MARKER} — {complaint}"
lines.append(f'{i}. Acceptance criterion "{text}" — {complaint}') lines.append(f'{i}. Acceptance criterion "{text}" — {complaint}')
@@ -121,9 +126,11 @@ def main():
faults = collect_pipeline_faults(state) faults = collect_pipeline_faults(state)
n = len(verdicts) n = len(verdicts)
met = [v for v in verdicts if v.get("status") == "MET"] met = [v for v in verdicts if v.get("status") == "MET" and not died(v)]
partial = [v for v in verdicts if v.get("status") == "PARTIAL"] partial = [v for v in verdicts if v.get("status") == "PARTIAL" and not died(v)]
bad = [v for v in verdicts if v.get("status") in ("UNMET", "DIVERGED")] bad = [
v for v in verdicts if died(v) or v.get("status") in ("UNMET", "DIVERGED")
]
lines = [] lines = []
if faults: if faults:
+84 -1
View File
@@ -8988,6 +8988,83 @@ mod tests {
); );
} }
#[test]
fn adversary_verdict_met_with_fault_evidence_never_conforms() {
if !cmd_available("python3") {
eprintln!("skipping: python3 not available");
return;
}
let state = json!({
"pipeline_faults": [],
"crit_verdicts": [
{"id": "c1", "text": "does X", "status": "MET",
"evidence": "src/x.rs:1 + test src/x.rs:99", "complaint": ""},
{"id": "c2", "text": "does Y", "status": "MET",
"evidence": "PIPELINE-FAULT: criterion check failed — LLM node failed: boom",
"complaint": ""}
],
"extra_complaints": [],
"observations": "",
"exec_results": []
});
let out = run_adversary_script("verdict.py", &state);
let report = out["adv_report"].as_str().unwrap();
assert!(
report.starts_with("ADVERSARIAL_REVIEW: DIVERGES"),
"a fault-marked verdict must never conform, whatever status it carries: {report}"
);
assert!(
report.contains("Criteria: 1/2 met, 0 partial, 1 unmet/diverged."),
"the fault-marked verdict counts as unmet, not met: {report}"
);
assert!(
report.contains("criterion check DIED (pipeline fault)"),
"a fault-marked verdict must render as died: {report}"
);
let met_section = report
.split("Met criteria (evidence):")
.nth(1)
.expect("the genuine MET criterion produces an evidence appendix");
assert!(
!met_section.contains("does Y"),
"the fault-marked verdict must not be listed as met: {report}"
);
}
#[test]
fn adversary_verdict_prepends_died_marker_once_when_complaint_lacks_it() {
if !cmd_available("python3") {
eprintln!("skipping: python3 not available");
return;
}
let state = json!({
"pipeline_faults": [],
"crit_verdicts": [
{"id": "c1", "text": "does Y", "status": "UNMET",
"evidence": "PIPELINE-FAULT: criterion check failed — LLM node failed: boom",
"complaint": "LLM node failed: boom"}
],
"extra_complaints": [],
"observations": "",
"exec_results": []
});
let out = run_adversary_script("verdict.py", &state);
let report = out["adv_report"].as_str().unwrap();
assert!(
report.contains(
"1. Acceptance criterion \"does Y\" — criterion check DIED (pipeline fault) — LLM node failed: boom"
),
"the died marker is prepended to a bare complaint: {report}"
);
assert_eq!(
report
.matches("criterion check DIED (pipeline fault)")
.count(),
1,
"the died marker must appear exactly once: {report}"
);
}
#[test] #[test]
fn adversary_crit_gate_passes_fault_shaped_verdict_through() { fn adversary_crit_gate_passes_fault_shaped_verdict_through() {
if !cmd_available("python3") { if !cmd_available("python3") {
@@ -8996,7 +9073,7 @@ mod tests {
} }
let evidence = "PIPELINE-FAULT: criterion check failed — LLM node failed: boom"; let evidence = "PIPELINE-FAULT: criterion check failed — LLM node failed: boom";
let fault = json!({ let fault = json!({
"id": "c1", "status": "UNMET", "evidence": evidence, "id": "c1", "text": "x", "status": "UNMET", "evidence": evidence,
"complaint": "criterion check DIED (pipeline fault) — LLM node failed: boom" "complaint": "criterion check DIED (pipeline fault) — LLM node failed: boom"
}); });
let out = run_adversary_script( let out = run_adversary_script(
@@ -9011,8 +9088,14 @@ mod tests {
out.get("_next").is_none(), out.get("_next").is_none(),
"a fault-shaped verdict must not retry: {out}" "a fault-shaped verdict must not retry: {out}"
); );
assert_eq!(out["crit_verdict"]["id"], "c1", "{out}");
assert_eq!(out["crit_verdict"]["text"], "x", "{out}");
assert_eq!(out["crit_verdict"]["status"], "UNMET"); assert_eq!(out["crit_verdict"]["status"], "UNMET");
assert_eq!(out["crit_verdict"]["evidence"], evidence, "{out}"); assert_eq!(out["crit_verdict"]["evidence"], evidence, "{out}");
assert_eq!(
out["crit_verdict"]["complaint"], fault["complaint"],
"{out}"
);
} }
#[test] #[test]
+34
View File
@@ -818,4 +818,38 @@ mod tests {
"{captured}" "{captured}"
); );
} }
/// Without a fallback the node bails, and that bail must wrap the same
/// full chain the state_updates text carries.
#[tokio::test]
async fn bail_without_fallback_carries_full_error_chain() {
let mut u = HashMap::new();
u.insert("captured".into(), "{{output}}".into());
let mut node = node_with(Some(u));
node.prompt = "{{nope}}".into();
node.fallback = None;
let mut state = manager_with(&[]);
let mut ctx = RequestContext::new(Arc::new(AppState::test_default()), WorkingMode::Cmd);
ctx.agent = Some(Agent::test_new(AgentConfig::default()));
let abort = create_abort_signal();
let err = LlmNodeExecutor::execute("think", &node, &mut state, &mut ctx, &abort)
.await
.expect_err("no fallback means the failure bails");
let chain = format!("{err:#}");
assert!(
chain.contains(
"LLM node failed and no fallback declared: LLM call failed: Failed to interpolate llm node prompt: Template interpolation failed: 'nope' not found in state"
),
"{chain}"
);
let captured = state
.state()
.get("captured")
.and_then(Value::as_str)
.expect("state_updates still run before the bail")
.to_string();
assert!(captured.starts_with("LLM node failed: "), "{captured}");
}
} }
+28
View File
@@ -3016,6 +3016,34 @@ mod tests {
); );
} }
#[test]
fn map_branch_llm_fallback_to_branch_local_script_passes() {
let map = map_node_basic("m", "br", Some("end"));
let branch = llm_node("br", Some("recover"), None);
let recover = script_node("recover", "Cargo.toml", None);
let graph = graph_with(
vec![
("m", map),
("br", branch),
("recover", recover),
("end", end_node("end")),
],
"m",
);
let result = validator().validate(&graph);
assert!(result.errors.is_empty(), "{:?}", result.errors);
assert!(
!result
.warnings
.iter()
.any(|w| w.node_id.as_deref() == Some("m")),
"a branch-local script fallback raises nothing on the map: {:?}",
result.warnings
);
}
#[test] #[test]
fn map_branch_deep_approval_node_errors_on_the_map() { fn map_branch_deep_approval_node_errors_on_the_map() {
let map = map_node_basic("m", "br", Some("end")); let map = map_node_basic("m", "br", Some("end"));