test(adversary): pin run_checks process-group kill and deadline-vs-timeout ordering
run_checks.py SIGKILLs the timed-out command's whole process group so a backgrounded grandchild cannot outlive the record; nothing proved it. Add a unix-only test that backgrounds `sleep 30`, records its pid, and probes `kill -0` after the 1s deadline fires. TOTAL_DEADLINE_SECS (run_checks.py) and the run_checks node's `timeout:` (graph.yaml) live in different files and could drift until the node executor kills the script before its in-script degradation runs. Parse the script's default and assert it stays strictly below the node timeout, without hardcoding either value.
This commit is contained in:
1 file changed
+113
@@ -8793,6 +8793,119 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(unix)]
|
||||
fn adversary_run_checks_timeout_kills_the_whole_process_group() {
|
||||
if !cmd_available("python3") || !cmd_available("sh") {
|
||||
eprintln!("skipping: python3 or sh not available");
|
||||
return;
|
||||
}
|
||||
let unique = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos();
|
||||
let tmp = env::temp_dir().join(format!(
|
||||
"coyote-adversary-run-checks-pgroup-{}-{unique}",
|
||||
std::process::id()
|
||||
));
|
||||
create_dir_all(&tmp).unwrap();
|
||||
let pid_file = tmp.join("grandchild.pid");
|
||||
|
||||
// run_one() starts each command in its own session and SIGKILLs the
|
||||
// process group on timeout. A plain p.kill() would only take the
|
||||
// shell, leaving a backgrounded `sleep 30` holding the stdout pipe —
|
||||
// and, in the real graph, a leaked cargo/pytest tree.
|
||||
let cmd = format!("sleep 30 & echo $! > {}; wait", pid_file.display());
|
||||
let state = json!({"verification_commands": [cmd]});
|
||||
let out = run_adversary_script_env(
|
||||
"run_checks.py",
|
||||
&state,
|
||||
&[("ADVERSARY_RUN_CHECKS_DEADLINE_SECS", "1")],
|
||||
);
|
||||
let results = out["exec_results"].as_array().unwrap();
|
||||
assert_eq!(
|
||||
results[0]["exit"], -1,
|
||||
"the hung shell must record a timeout: {results:?}"
|
||||
);
|
||||
assert!(
|
||||
results[0]["tail"].as_str().unwrap().contains("TIMEOUT"),
|
||||
"the timeout must be named in the tail: {results:?}"
|
||||
);
|
||||
|
||||
let mut pid = String::new();
|
||||
for _ in 0..20 {
|
||||
pid = read_to_string(&pid_file)
|
||||
.unwrap_or_default()
|
||||
.trim()
|
||||
.to_string();
|
||||
if !pid.is_empty() {
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(50));
|
||||
}
|
||||
let _ = remove_dir_all(&tmp);
|
||||
assert!(
|
||||
!pid.is_empty(),
|
||||
"the shell must have recorded its grandchild pid at {}",
|
||||
pid_file.display()
|
||||
);
|
||||
|
||||
let alive = |pid: &str| {
|
||||
std::process::Command::new("kill")
|
||||
.args(["-0", pid])
|
||||
.output()
|
||||
.unwrap()
|
||||
.status
|
||||
.success()
|
||||
};
|
||||
let mut polls = 0;
|
||||
let mut still_alive = alive(&pid);
|
||||
while still_alive && polls < 40 {
|
||||
std::thread::sleep(Duration::from_millis(50));
|
||||
polls += 1;
|
||||
still_alive = alive(&pid);
|
||||
}
|
||||
assert!(
|
||||
!still_alive,
|
||||
"grandchild {pid} must die with the process group, still alive after {polls} polls: {results:?}"
|
||||
);
|
||||
}
|
||||
|
||||
/// TOTAL_DEADLINE_SECS in run_checks.py and the run_checks node's
|
||||
/// `timeout:` in graph.yaml live in different files; if the script's
|
||||
/// deadline ever creeps past the node timeout, the executor kills the
|
||||
/// script from outside and the in-script skipped/ENVIRONMENT degradation
|
||||
/// never gets to run.
|
||||
#[test]
|
||||
fn adversary_run_checks_deadline_stays_inside_the_node_timeout() {
|
||||
use crate::graph::NodeType;
|
||||
let script = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("assets/agents/adversary/scripts/run_checks.py");
|
||||
let source = read_to_string(&script).unwrap();
|
||||
let line = source
|
||||
.lines()
|
||||
.find(|l| l.starts_with("TOTAL_DEADLINE_SECS = "))
|
||||
.expect("run_checks.py must define TOTAL_DEADLINE_SECS");
|
||||
let deadline: u64 = line
|
||||
.rsplit_once(" or ")
|
||||
.and_then(|(_, rest)| rest.strip_suffix(')'))
|
||||
.and_then(|s| s.trim().parse().ok())
|
||||
.unwrap_or_else(|| panic!("TOTAL_DEADLINE_SECS default no longer parses: {line}"));
|
||||
|
||||
let graph = load_bundled_graph("adversary");
|
||||
let node = graph
|
||||
.get_node("run_checks")
|
||||
.expect("adversary graph must have a run_checks node");
|
||||
let NodeType::Script(s) = &node.node_type else {
|
||||
panic!("run_checks must be a script node");
|
||||
};
|
||||
assert!(
|
||||
deadline < s.timeout,
|
||||
"run_checks.py TOTAL_DEADLINE_SECS ({deadline}) must stay below the run_checks node timeout ({}) in assets/agents/adversary/graph.yaml",
|
||||
s.timeout
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adversary_pipeline_fault_parse_stage_attribution() {
|
||||
if !cmd_available("python3") {
|
||||
|
||||
Reference in new issue
Block a user