fix(adversary): red verification runs block CONFORMS; re-derive timeout envelope; robust runner
verdict.py: a recorded verification run with exit≠0 (including the -1 TIMEOUT sentinel) or a skipped record now forces DIVERGES with a numbered complaint naming the command — previously a [FAIL] line sat in the report appendix while the sentinel read CONFORMS, and callers route on the sentinel alone. Fail-closed by the same rule: an ENVIRONMENT runner-error marker when commands WERE declared is one red complaint (the declared commands never ran, so the run is unproven); the "none declared" marker and an empty declaration stay non-red. Headers name the red-run count. run_checks.py: decode command output with encoding=utf-8/errors=replace (a single non-UTF-8 byte used to raise inside communicate() and collapse the whole evidence record to an ENVIRONMENT marker); bound the post-kill pipe drain by DRAIN_TIMEOUT_SECS so a session-escaped grandchild holding the pipe cannot turn a TIMEOUT record into a node kill. graph.yaml: settings.timeout 5400 → 9000 = 3600 (run_checks node timeout) + 5400 (the previous whole-graph budget, kept for every other stage); the old value predated run_checks and a graph-level timeout kills from outside, losing the DIVERGES sentinel. Descriptions and README doctrine updated (CONFORMS = all MET + zero holistic complaints + zero pipeline faults + zero red runs). Tests: envelope pin (settings >= run_checks + 5400), gauntlet lane timeouts > child graph timeouts and gauntlet >= max_attempts × lane + margin over every agent lane, non-UTF-8 output, bounded drain (unix), red/skipped/timeout/none-declared/ENVIRONMENT×declaration matrix, numbering continuity, header punctuation, wording pins on graph.yaml, run_checks.py and both READMEs. The deadline test's hung command is now python3-based instead of GNU sleep (cmd.exe on the Windows CI leg).
This commit is contained in:
5 files changed
+683
-33
No files matched your search
@@ -8280,6 +8280,15 @@ mod tests {
|
||||
) && !adversary.contains("Known residual"),
|
||||
"adversary README must state the schema-declared merge closes the extra-key channel"
|
||||
);
|
||||
assert!(
|
||||
adversary.contains("zero red verification runs"),
|
||||
"adversary README must state the fail-closed CONFORMS doctrine"
|
||||
);
|
||||
assert!(
|
||||
!adversary.contains("soft ENVIRONMENT")
|
||||
&& adversary.contains("never degrades to an ENVIRONMENT marker"),
|
||||
"adversary README must describe a malformed declaration as fail-closed, not a soft marker"
|
||||
);
|
||||
let gauntlet = read_to_string(readmes.join("review-gauntlet/README.md")).unwrap();
|
||||
assert!(
|
||||
!gauntlet.contains("Verification commands:"),
|
||||
@@ -8294,12 +8303,28 @@ mod tests {
|
||||
gauntlet.contains("--agent-variable verification_commands '[\"cargo test --all\"]'"),
|
||||
"review-gauntlet README must show the CLI form"
|
||||
);
|
||||
assert!(gauntlet.contains("plus probe on consumer surface *with a local-run recipe*"));
|
||||
assert!(gauntlet.contains("critical count from the report's summary line"));
|
||||
assert!(gauntlet.contains("raw 🔴 count if absent"));
|
||||
assert!(
|
||||
gauntlet
|
||||
.contains("records a pipeline fault blocks (a degraded review is never a pass)")
|
||||
);
|
||||
assert!(
|
||||
!gauntlet.contains("any 🔴 in the code-review report"),
|
||||
"review-gauntlet README must not describe the retired raw-🔴 gate"
|
||||
);
|
||||
|
||||
let runner = read_to_string(readmes.join("adversary/scripts/run_checks.py")).unwrap();
|
||||
assert!(
|
||||
runner.contains("Trust boundary:") && runner.contains("never a field an LLM"),
|
||||
"run_checks.py docstring must state the declared-variable trust boundary"
|
||||
);
|
||||
assert!(
|
||||
!runner.contains("soft ENVIRONMENT")
|
||||
&& runner.contains("never\ndegrades to an ENVIRONMENT marker"),
|
||||
"run_checks.py docstring must describe a malformed declaration as fail-closed, not a soft marker"
|
||||
);
|
||||
assert!(
|
||||
!runner.contains("Known residual") && !gauntlet.contains("residual:"),
|
||||
"run_checks.py and the review-gauntlet README must not disclose the closed residual"
|
||||
@@ -9223,7 +9248,10 @@ mod tests {
|
||||
// second must be recorded as skipped — the in-script handling stays
|
||||
// authoritative instead of the NODE timeout killing the script from
|
||||
// outside (which would bypass the ENVIRONMENT degradation entirely).
|
||||
let state = json!({"verification_commands": ["sleep 5", "echo never"]});
|
||||
let state = json!({"verification_commands": [
|
||||
r#"python3 -c "import time; time.sleep(5)""#,
|
||||
"echo never"
|
||||
]});
|
||||
let out = run_adversary_script_env(
|
||||
"run_checks.py",
|
||||
&state,
|
||||
@@ -9249,6 +9277,30 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adversary_run_checks_tolerates_non_utf8_output() {
|
||||
if !cmd_available("python3") {
|
||||
eprintln!("skipping: python3 not available");
|
||||
return;
|
||||
}
|
||||
let state = json!({"verification_commands": [
|
||||
r#"python3 -c "import sys; sys.stdout.buffer.write(b'ok \xff\xfe bytes\n')""#
|
||||
]});
|
||||
let out = run_adversary_script("run_checks.py", &state);
|
||||
let results = out["exec_results"].as_array().unwrap_or_else(|| {
|
||||
panic!("invalid UTF-8 output must not collapse the record to a marker: {out}")
|
||||
});
|
||||
assert_eq!(
|
||||
results[0]["exit"], 0,
|
||||
"the command itself succeeded: {results:?}"
|
||||
);
|
||||
let tail = results[0]["tail"].as_str().unwrap();
|
||||
assert!(
|
||||
tail.contains("ok") && tail.contains("bytes"),
|
||||
"decodable parts of the output must survive with replacement: {results:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(unix)]
|
||||
fn adversary_run_checks_timeout_kills_the_whole_process_group() {
|
||||
@@ -9327,6 +9379,39 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(unix)]
|
||||
fn adversary_run_checks_timeout_drain_is_bounded_by_session_escaped_grandchild() {
|
||||
if !cmd_available("python3") || !cmd_available("sh") {
|
||||
eprintln!("skipping: python3 or sh not available");
|
||||
return;
|
||||
}
|
||||
// The grandchild starts its own session, so the SIGKILL of the
|
||||
// command's process group misses it — but it inherited the stdout
|
||||
// pipe, so an unbounded post-kill drain would block until it exits
|
||||
// (~40s). DRAIN_TIMEOUT_SECS (10) must cap that. The orphaned `sleep
|
||||
// 40` exits on its own.
|
||||
let cmd = "python3 -c \"import subprocess,time; \
|
||||
subprocess.Popen(['sleep','40'], start_new_session=True); time.sleep(60)\"";
|
||||
let state = json!({"verification_commands": [cmd]});
|
||||
let started = Instant::now();
|
||||
let out = run_adversary_script_env(
|
||||
"run_checks.py",
|
||||
&state,
|
||||
&[("ADVERSARY_RUN_CHECKS_DEADLINE_SECS", "1")],
|
||||
);
|
||||
let elapsed = started.elapsed();
|
||||
let results = out["exec_results"].as_array().unwrap();
|
||||
assert_eq!(
|
||||
results[0]["exit"], -1,
|
||||
"the hung command must record a timeout: {results:?}"
|
||||
);
|
||||
assert!(
|
||||
elapsed < Duration::from_secs(30),
|
||||
"the post-kill drain must be bounded by DRAIN_TIMEOUT_SECS, took {elapsed:?}: {results:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// TOTAL_DEADLINE_SECS in run_checks.py and the run_checks node's
|
||||
/// `timeout:` in graph.yaml live in different files; if the script's
|
||||
/// deadline ever creeps past the node timeout, the executor kills the
|
||||
@@ -9362,6 +9447,103 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
/// The adversary's graph-level timeout must leave room for run_checks to
|
||||
/// burn its whole node timeout AND for the rest of the pipeline to still
|
||||
/// finish — a graph timeout kills from outside, so no fallback fires and
|
||||
/// the DIVERGES sentinel is lost.
|
||||
#[test]
|
||||
fn adversary_graph_timeout_covers_run_checks_budget() {
|
||||
use crate::graph::NodeType;
|
||||
const OTHER_STAGES_ENVELOPE_SECS: u64 = 5400;
|
||||
let graph = load_bundled_graph("adversary");
|
||||
let node = graph
|
||||
.get_node("run_checks")
|
||||
.expect("adversary graph must have a run_checks node");
|
||||
let NodeType::Script(s) = &node.node_type else {
|
||||
panic!("run_checks must be a script node");
|
||||
};
|
||||
let graph_timeout = graph
|
||||
.settings
|
||||
.timeout
|
||||
.expect("adversary graph must set settings.timeout");
|
||||
assert!(
|
||||
graph_timeout >= s.timeout + OTHER_STAGES_ENVELOPE_SECS,
|
||||
"assets/agents/adversary/graph.yaml settings.timeout ({graph_timeout}) must cover the run_checks node timeout ({}) plus the {OTHER_STAGES_ENVELOPE_SECS}s envelope for the other stages",
|
||||
s.timeout
|
||||
);
|
||||
}
|
||||
|
||||
/// Every gauntlet lane that runs a graph-backed agent must outlive that
|
||||
/// graph's own timeout so the child's own graph timeout, not the lane
|
||||
/// timeout, is the binding bound; the child's internal retry envelope is
|
||||
/// deliberately not derived here (follow-up). And because agent-node
|
||||
/// retries each get a fresh per-attempt budget (src/graph/agent.rs
|
||||
/// retry_transient/bounded_attempt), the gauntlet itself must outlive
|
||||
/// `max_attempts × timeout` for every lane, with headroom for the
|
||||
/// surrounding stages.
|
||||
#[test]
|
||||
fn gauntlet_lane_timeouts_cover_child_graphs_and_retries() {
|
||||
use crate::graph::NodeType;
|
||||
const OTHER_STAGES_MARGIN_SECS: u64 = 1200;
|
||||
let gauntlet = load_bundled_graph("review-gauntlet");
|
||||
let gauntlet_timeout = gauntlet
|
||||
.settings
|
||||
.timeout
|
||||
.expect("review-gauntlet must set settings.timeout");
|
||||
let assets = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("assets/agents");
|
||||
let gauntlet_yaml = read_to_string(assets.join("review-gauntlet/graph.yaml")).unwrap();
|
||||
assert!(
|
||||
gauntlet_yaml.contains("internal retry envelope")
|
||||
&& gauntlet_yaml.contains("not derived here"),
|
||||
"review-gauntlet graph.yaml must keep the NOTE that the child's internal retry envelope is not derived from the lane timeout"
|
||||
);
|
||||
assert!(
|
||||
!gauntlet_yaml.contains("is what arrives"),
|
||||
"review-gauntlet graph.yaml must not claim the lane timeout guarantees the child's verdict arrives"
|
||||
);
|
||||
assert!(
|
||||
gauntlet_yaml.contains("AND a probe_context (local-run recipe) is present"),
|
||||
"review-gauntlet graph.yaml default_lanes description must state the probe_context guard"
|
||||
);
|
||||
let mut seen = 0;
|
||||
let mut graph_backed = 0;
|
||||
for (id, node) in &gauntlet.nodes {
|
||||
let NodeType::Agent(a) = &node.node_type else {
|
||||
continue;
|
||||
};
|
||||
seen += 1;
|
||||
let lane_timeout = a
|
||||
.timeout
|
||||
.unwrap_or_else(|| panic!("review-gauntlet lane {id} must set a timeout"));
|
||||
if assets.join(&a.agent).join("graph.yaml").exists() {
|
||||
graph_backed += 1;
|
||||
let child_timeout = load_bundled_graph(&a.agent)
|
||||
.settings
|
||||
.timeout
|
||||
.unwrap_or_else(|| panic!("{} graph must set settings.timeout", a.agent));
|
||||
assert!(
|
||||
lane_timeout > child_timeout,
|
||||
"assets/agents/review-gauntlet/graph.yaml lane {id} timeout ({lane_timeout}) must exceed assets/agents/{}/graph.yaml settings.timeout ({child_timeout})",
|
||||
a.agent
|
||||
);
|
||||
}
|
||||
let worst_case = u64::from(a.max_attempts) * lane_timeout;
|
||||
assert!(
|
||||
gauntlet_timeout >= worst_case + OTHER_STAGES_MARGIN_SECS,
|
||||
"review-gauntlet settings.timeout ({gauntlet_timeout}) must cover lane {id}'s max_attempts × timeout ({} × {lane_timeout} = {worst_case}) plus a {OTHER_STAGES_MARGIN_SECS}s margin for the other stages",
|
||||
a.max_attempts
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
seen >= 4,
|
||||
"review-gauntlet must have at least 4 agent lanes, saw {seen}"
|
||||
);
|
||||
assert!(
|
||||
graph_backed >= 2,
|
||||
"at least the adversary and code-reviewer lanes must be graph-backed so the child-graph timeout check runs, saw {graph_backed}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adversary_pipeline_fault_parse_stage_attribution() {
|
||||
if !cmd_available("python3") {
|
||||
@@ -9478,6 +9660,380 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adversary_verdict_green_runs_keep_conforms() {
|
||||
if !cmd_available("python3") {
|
||||
eprintln!("skipping: python3 not available");
|
||||
return;
|
||||
}
|
||||
let state = json!({
|
||||
"pipeline_faults": [],
|
||||
"crit_verdicts": [
|
||||
{"id": "c1", "text": "does X", "status": "MET",
|
||||
"evidence": "src/x.rs:1 + test src/x.rs:99", "complaint": ""}
|
||||
],
|
||||
"extra_complaints": [],
|
||||
"observations": "",
|
||||
"exec_results": [{"cmd": "cargo test", "exit": 0, "duration_s": 1.0, "tail": "ok"}]
|
||||
});
|
||||
let out = run_adversary_script("verdict.py", &state);
|
||||
let report = out["adv_report"].as_str().unwrap();
|
||||
assert!(
|
||||
report.starts_with("ADVERSARIAL_REVIEW: CONFORMS"),
|
||||
"an all-green run record must not block CONFORMS: {report}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adversary_verdict_red_run_forces_diverges() {
|
||||
if !cmd_available("python3") {
|
||||
eprintln!("skipping: python3 not available");
|
||||
return;
|
||||
}
|
||||
// A criterion judged MET cannot outrank a recorded failing run of a
|
||||
// declared command — the verdict is fail-closed on exec_results.
|
||||
let mut state = json!({
|
||||
"pipeline_faults": [],
|
||||
"crit_verdicts": [
|
||||
{"id": "c1", "text": "does X", "status": "MET",
|
||||
"evidence": "src/x.rs:1 + test src/x.rs:99", "complaint": ""}
|
||||
],
|
||||
"extra_complaints": [],
|
||||
"observations": "",
|
||||
"exec_results": [{"cmd": "cargo test", "exit": 101, "duration_s": 1.0, "tail": "FAILED"}]
|
||||
});
|
||||
let out = run_adversary_script("verdict.py", &state);
|
||||
let report = out["adv_report"].as_str().unwrap();
|
||||
assert!(
|
||||
report.starts_with("ADVERSARIAL_REVIEW: DIVERGES"),
|
||||
"a recorded red run must force DIVERGES: {report}"
|
||||
);
|
||||
assert!(
|
||||
report.contains("1. Verification run `cargo test` — exit 101"),
|
||||
"the red run must be a numbered complaint naming the command: {report}"
|
||||
);
|
||||
assert!(
|
||||
report.contains("Criteria: 1/1 met, 0 partial, 0 unmet/diverged — 1 red verification run(s) (fail-closed)."),
|
||||
"a reds-only DIVERGES header must count the red runs: {report}"
|
||||
);
|
||||
|
||||
state["exec_results"] = json!([{"cmd": "cargo clippy", "skipped": "deadline"}]);
|
||||
let out = run_adversary_script("verdict.py", &state);
|
||||
let report = out["adv_report"].as_str().unwrap();
|
||||
assert!(
|
||||
report.starts_with("ADVERSARIAL_REVIEW: DIVERGES"),
|
||||
"a skipped (unproven) run must force DIVERGES: {report}"
|
||||
);
|
||||
assert!(
|
||||
report.contains("1. Verification run `cargo clippy` — never ran"),
|
||||
"the skipped run must be a numbered complaint naming the command: {report}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adversary_verdict_none_declared_marker_is_not_red() {
|
||||
if !cmd_available("python3") {
|
||||
eprintln!("skipping: python3 not available");
|
||||
return;
|
||||
}
|
||||
let state = json!({
|
||||
"pipeline_faults": [],
|
||||
"crit_verdicts": [
|
||||
{"id": "c1", "text": "does X", "status": "MET",
|
||||
"evidence": "src/x.rs:1 + test src/x.rs:99", "complaint": ""}
|
||||
],
|
||||
"extra_complaints": [],
|
||||
"observations": "",
|
||||
"exec_results": "none declared — the caller supplied no verification_commands; criteria relying on tests are unproven"
|
||||
});
|
||||
let out = run_adversary_script("verdict.py", &state);
|
||||
let report = out["adv_report"].as_str().unwrap();
|
||||
assert!(
|
||||
report.starts_with("ADVERSARIAL_REVIEW: CONFORMS"),
|
||||
"the none-declared marker must not block CONFORMS: {report}"
|
||||
);
|
||||
assert!(
|
||||
!report.contains("Verification run `"),
|
||||
"the none-declared marker must not yield a run complaint: {report}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adversary_verdict_environment_marker_with_declared_commands_blocks() {
|
||||
if !cmd_available("python3") {
|
||||
eprintln!("skipping: python3 not available");
|
||||
return;
|
||||
}
|
||||
// Commands were declared and the runner died before running them:
|
||||
// the criteria are unproven, and an all-MET set of criterion
|
||||
// judgments must not turn that into CONFORMS.
|
||||
let state = json!({
|
||||
"pipeline_faults": [],
|
||||
"verification_commands": ["cargo test"],
|
||||
"crit_verdicts": [
|
||||
{"id": "c1", "text": "does X", "status": "MET",
|
||||
"evidence": "src/x.rs:1 + test src/x.rs:99", "complaint": ""}
|
||||
],
|
||||
"extra_complaints": [],
|
||||
"observations": "",
|
||||
"exec_results": "ENVIRONMENT: verification runner error: x — the declared commands could not be executed"
|
||||
});
|
||||
let out = run_adversary_script("verdict.py", &state);
|
||||
let report = out["adv_report"].as_str().unwrap();
|
||||
assert!(
|
||||
report.starts_with("ADVERSARIAL_REVIEW: DIVERGES"),
|
||||
"an ENVIRONMENT marker with declared commands must block CONFORMS: {report}"
|
||||
);
|
||||
assert!(
|
||||
report.contains("1. Verification runner error"),
|
||||
"the runner error must be a numbered complaint: {report}"
|
||||
);
|
||||
assert!(
|
||||
report.contains("0 unmet/diverged — 1 red verification run(s) (fail-closed)."),
|
||||
"the header must count the runner error as a red run: {report}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adversary_verdict_environment_marker_without_declaration_is_not_red() {
|
||||
if !cmd_available("python3") {
|
||||
eprintln!("skipping: python3 not available");
|
||||
return;
|
||||
}
|
||||
let state = json!({
|
||||
"pipeline_faults": [],
|
||||
"verification_commands": "[]",
|
||||
"crit_verdicts": [
|
||||
{"id": "c1", "text": "does X", "status": "MET",
|
||||
"evidence": "src/x.rs:1 + test src/x.rs:99", "complaint": ""}
|
||||
],
|
||||
"extra_complaints": [],
|
||||
"observations": "",
|
||||
"exec_results": "ENVIRONMENT: verification runner error: x — the declared commands could not be executed"
|
||||
});
|
||||
let out = run_adversary_script("verdict.py", &state);
|
||||
let report = out["adv_report"].as_str().unwrap();
|
||||
assert!(
|
||||
report.starts_with("ADVERSARIAL_REVIEW: CONFORMS"),
|
||||
"with nothing declared the ENVIRONMENT marker is doctrine-PARTIAL, not a red run: {report}"
|
||||
);
|
||||
assert!(
|
||||
!report.contains("Verification runner error"),
|
||||
"the ENVIRONMENT marker must not yield a run complaint: {report}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adversary_verdict_timeout_run_named_as_timeout() {
|
||||
if !cmd_available("python3") {
|
||||
eprintln!("skipping: python3 not available");
|
||||
return;
|
||||
}
|
||||
let state = json!({
|
||||
"pipeline_faults": [],
|
||||
"crit_verdicts": [
|
||||
{"id": "c1", "text": "does X", "status": "MET",
|
||||
"evidence": "src/x.rs:1 + test src/x.rs:99", "complaint": ""}
|
||||
],
|
||||
"extra_complaints": [],
|
||||
"observations": "",
|
||||
"exec_results": [{"cmd": "cargo test", "exit": -1, "duration_s": 900.0, "tail": "TIMEOUT after 900s"}]
|
||||
});
|
||||
let out = run_adversary_script("verdict.py", &state);
|
||||
let report = out["adv_report"].as_str().unwrap();
|
||||
assert!(
|
||||
report.starts_with("ADVERSARIAL_REVIEW: DIVERGES"),
|
||||
"a timed-out run must force DIVERGES: {report}"
|
||||
);
|
||||
assert!(
|
||||
report.contains("1. Verification run `cargo test` — TIMEOUT (exit -1)"),
|
||||
"the -1 sentinel must be named as a TIMEOUT, not a bare exit code: {report}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adversary_verdict_red_runs_number_after_faults_and_criteria() {
|
||||
if !cmd_available("python3") {
|
||||
eprintln!("skipping: python3 not available");
|
||||
return;
|
||||
}
|
||||
let state = json!({
|
||||
"pipeline_faults": ["PIPELINE-FAULT: x"],
|
||||
"crit_verdicts": [
|
||||
{"id": "c1", "text": "does X", "status": "UNMET",
|
||||
"evidence": "", "complaint": "no impl found"}
|
||||
],
|
||||
"extra_complaints": [],
|
||||
"observations": "",
|
||||
"exec_results": [{"cmd": "cargo test", "exit": 101, "duration_s": 1.0, "tail": "FAILED"}]
|
||||
});
|
||||
let out = run_adversary_script("verdict.py", &state);
|
||||
let report = out["adv_report"].as_str().unwrap();
|
||||
let fault = report
|
||||
.find("1. PIPELINE-FAULT")
|
||||
.expect("the pipeline fault must be complaint 1");
|
||||
let criterion = report
|
||||
.find("2. Acceptance criterion")
|
||||
.expect("the unmet criterion must be complaint 2");
|
||||
let red = report
|
||||
.find("3. Verification run")
|
||||
.expect("the red run must be complaint 3");
|
||||
assert!(
|
||||
fault < criterion && criterion < red,
|
||||
"complaints must be ordered fault, criterion, red run: {report}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adversary_verdict_no_criteria_still_lists_red_runs() {
|
||||
if !cmd_available("python3") {
|
||||
eprintln!("skipping: python3 not available");
|
||||
return;
|
||||
}
|
||||
let state = json!({
|
||||
"pipeline_faults": [],
|
||||
"crit_verdicts": [],
|
||||
"extra_complaints": [],
|
||||
"observations": "",
|
||||
"exec_results": [{"cmd": "cargo test", "exit": 101, "duration_s": 1.0, "tail": "FAILED"}]
|
||||
});
|
||||
let out = run_adversary_script("verdict.py", &state);
|
||||
let report = out["adv_report"].as_str().unwrap();
|
||||
assert!(
|
||||
report.starts_with("ADVERSARIAL_REVIEW: DIVERGES"),
|
||||
"no criteria must fail closed: {report}"
|
||||
);
|
||||
assert!(
|
||||
report.contains("Criteria: none provided."),
|
||||
"the no-criteria header must be kept: {report}"
|
||||
);
|
||||
assert!(
|
||||
report.contains("2. Verification run `cargo test`"),
|
||||
"the red run must be numbered after the no-criteria complaint: {report}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adversary_verdict_environment_marker_with_string_declaration_blocks() {
|
||||
if !cmd_available("python3") {
|
||||
eprintln!("skipping: python3 not available");
|
||||
return;
|
||||
}
|
||||
// Production path: graph variables land in state as strings, so the
|
||||
// declaration is a JSON-encoded list, not a list.
|
||||
let mut state = json!({
|
||||
"pipeline_faults": [],
|
||||
"verification_commands": "[\"cargo test\"]",
|
||||
"crit_verdicts": [
|
||||
{"id": "c1", "text": "does X", "status": "MET",
|
||||
"evidence": "src/x.rs:1 + test src/x.rs:99", "complaint": ""}
|
||||
],
|
||||
"extra_complaints": [],
|
||||
"observations": "",
|
||||
"exec_results": "ENVIRONMENT: verification runner error: x — the declared commands could not be executed"
|
||||
});
|
||||
let out = run_adversary_script("verdict.py", &state);
|
||||
let report = out["adv_report"].as_str().unwrap();
|
||||
assert!(
|
||||
report.starts_with("ADVERSARIAL_REVIEW: DIVERGES"),
|
||||
"a JSON-string declaration must count as declared: {report}"
|
||||
);
|
||||
assert!(
|
||||
report.contains("1. Verification runner error"),
|
||||
"the runner error must be a numbered complaint: {report}"
|
||||
);
|
||||
|
||||
state["verification_commands"] = json!("[\" \"]");
|
||||
let out = run_adversary_script("verdict.py", &state);
|
||||
let report = out["adv_report"].as_str().unwrap();
|
||||
assert!(
|
||||
report.starts_with("ADVERSARIAL_REVIEW: CONFORMS"),
|
||||
"blank items are not a declaration, so the ENVIRONMENT marker is not a red run: {report}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adversary_verdict_died_and_red_header_has_single_period() {
|
||||
if !cmd_available("python3") {
|
||||
eprintln!("skipping: python3 not available");
|
||||
return;
|
||||
}
|
||||
let state = json!({
|
||||
"pipeline_faults": [],
|
||||
"crit_verdicts": [
|
||||
{"id": "c1", "text": "does X", "status": "UNMET",
|
||||
"evidence": "PIPELINE-FAULT: criterion x", "complaint": ""}
|
||||
],
|
||||
"extra_complaints": [],
|
||||
"observations": "",
|
||||
"exec_results": [{"cmd": "cargo test", "exit": 101, "duration_s": 1.0, "tail": "FAILED"}]
|
||||
});
|
||||
let out = run_adversary_script("verdict.py", &state);
|
||||
let report = out["adv_report"].as_str().unwrap();
|
||||
assert!(
|
||||
report.contains("died (fail-closed) — 1 red verification run(s) (fail-closed)."),
|
||||
"both header notes must be joined with a single em dash: {report}"
|
||||
);
|
||||
assert!(
|
||||
!report.contains(".."),
|
||||
"the header must end with exactly one period: {report}"
|
||||
);
|
||||
}
|
||||
|
||||
/// The fail-closed CONFORMS block on unrun declared commands is described
|
||||
/// in three places that must keep agreeing: the runner's ENVIRONMENT
|
||||
/// marker, the node descriptions, and check_criterion's instructions.
|
||||
#[test]
|
||||
fn adversary_fail_closed_wording_pins() {
|
||||
use crate::graph::NodeType;
|
||||
let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("assets/agents/adversary");
|
||||
let runner = read_to_string(dir.join("scripts/run_checks.py")).unwrap();
|
||||
assert!(
|
||||
runner.contains("the verdict blocks CONFORMS when"),
|
||||
"run_checks.py's ENVIRONMENT marker must point at the verdict's fail-closed block"
|
||||
);
|
||||
let graph = load_bundled_graph("adversary");
|
||||
let run_checks = graph
|
||||
.get_node("run_checks")
|
||||
.expect("adversary graph must have a run_checks node");
|
||||
assert!(
|
||||
run_checks.description.contains("blocks CONFORMS"),
|
||||
"run_checks description must state the verdict blocks CONFORMS: {}",
|
||||
run_checks.description
|
||||
);
|
||||
let verdict = graph
|
||||
.get_node("verdict")
|
||||
.expect("adversary graph must have a verdict node");
|
||||
assert!(
|
||||
verdict
|
||||
.description
|
||||
.contains("zero recorded red verification runs"),
|
||||
"verdict description must list red runs among the CONFORMS requirements: {}",
|
||||
verdict.description
|
||||
);
|
||||
let check = graph
|
||||
.get_node("check_criterion")
|
||||
.expect("adversary graph must have a check_criterion node");
|
||||
assert!(
|
||||
check.description.contains(
|
||||
"the deterministic verdict blocks CONFORMS when declared commands did not run"
|
||||
),
|
||||
"check_criterion description must hand the fail-closed block to the verdict: {}",
|
||||
check.description
|
||||
);
|
||||
let NodeType::Llm(llm) = &check.node_type else {
|
||||
panic!("check_criterion must be an llm node");
|
||||
};
|
||||
let instructions = llm
|
||||
.instructions
|
||||
.as_deref()
|
||||
.expect("check_criterion must have instructions");
|
||||
assert!(
|
||||
instructions.contains("deterministic verdict additionally blocks CONFORMS"),
|
||||
"check_criterion instructions must hand the fail-closed block to the verdict"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adversary_crit_gate_contract_regression() {
|
||||
if !cmd_available("python3") {
|
||||
|
||||
Reference in new issue
Block a user