test(jobs): add feature, hardening, and surface test matrix for background jobs

Covers the plan's T7 matrix: zero-diff invariants when jobs are off
(byte-identical tool lists and prompts, None-vs-Some select_functions),
validation hardening (shell/path-shaped/PATH-resolvable names, undeclared
MCP servers, non-whitelisted and context-filtered tools, mapping-tool
aliases, mid-batch tool-scope freshness), process lifecycle (grandchild
process-group kill, pgid clear after normal completion, panic skips the
completion notification), guardrail behavior (finished-job discard on
force-terminate, bounded inject-then-terminate iteration burn), surface
conformance (concrete_tool_names exclusion, toggle rejection, tools_info
listing, infra preservation under empty filters), supervisor swaps
(use_agent/exit_agent kill running jobs, child contexts cannot reach
parent job ids), and graph-node job lifecycle with deferred notification
drain.
This commit is contained in:
2026-08-25 20:33:10 -06:00
parent 6256b5fcfa
commit 28018f33c9
6 changed files with 862 additions and 0 deletions
+47
View File
@@ -1515,4 +1515,51 @@ nodes: {}
assert_eq!(config.top_k, Some(7)); assert_eq!(config.top_k, Some(7));
assert_eq!(config.embedding_model.as_deref(), Some("some:model")); assert_eq!(config.embedding_model.as_deref(), Some("some:model"));
} }
#[test]
fn interpolated_instructions_without_job_declarations_is_byte_identical_across_job_settings() {
let agent = |max_concurrent_jobs| {
Agent::test_new(AgentConfig {
instructions: "hi".to_string(),
max_concurrent_jobs,
..AgentConfig::default()
})
};
let baseline = agent(None).interpolated_instructions();
assert!(
!baseline.contains(DEFAULT_JOB_INSTRUCTIONS),
"no job guidance may be injected without job__ declarations"
);
assert_eq!(baseline, agent(Some(0)).interpolated_instructions());
assert_eq!(baseline, agent(Some(7)).interpolated_instructions());
let mut with_unrelated = agent(None);
with_unrelated.functions.append_todo_functions();
assert_eq!(
baseline,
with_unrelated.interpolated_instructions(),
"job guidance injection must key strictly on the job__ prefix"
);
}
#[test]
fn interpolated_instructions_with_job_declarations_appends_job_guidance() {
let config = AgentConfig {
instructions: "hi".to_string(),
..AgentConfig::default()
};
let baseline = Agent::test_new(config.clone()).interpolated_instructions();
let mut agent = Agent::test_new(config);
agent.functions.append_job_functions();
let output = agent.interpolated_instructions();
assert!(output.contains(DEFAULT_JOB_INSTRUCTIONS));
let expected = format!(
"hi\n{DEFAULT_JOB_INSTRUCTIONS}{}",
baseline.strip_prefix("hi").unwrap()
);
assert_eq!(output, expected);
}
} }
+250
View File
@@ -7555,4 +7555,254 @@ mod tests {
"global config" "global config"
); );
} }
#[test]
fn select_functions_preserves_job_tools_under_empty_role_filter() {
let mut ctx = create_test_ctx();
ctx.tool_scope.functions.append_job_functions();
let mut role = Role::new("r", "p");
role.set_enabled_tools(Some(vec![]));
let fns = ctx.select_functions(&role).unwrap();
let names: Vec<&str> = fns.iter().map(|f| f.name.as_str()).collect();
assert_eq!(
names,
vec![
"job__start",
"job__check",
"job__collect",
"job__cancel",
"job__list"
],
"job__ tools must survive an empty role tool filter"
);
}
#[test]
#[serial]
fn select_functions_preserves_job_tools_under_empty_agent_filter() {
let _guard = TestConfigDirGuard::new();
let mut ctx = create_test_ctx();
let app = ctx.app.config.clone();
let agent_name = format!(
"test_job_agent_{}",
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
);
let agent_dir = paths::agent_data_dir(&agent_name);
create_dir_all(&agent_dir).unwrap();
write(
agent_dir.join("config.yaml"),
format!("name: {agent_name}\ninstructions: hi\n"),
)
.unwrap();
let abort = utils::create_abort_signal();
run_async(ctx.use_agent(&app, &agent_name, None, abort)).unwrap();
let mut role = Role::new("r", "p");
role.set_enabled_tools(Some(vec![]));
let fns = ctx.select_functions(&role).unwrap();
let names: Vec<&str> = fns.iter().map(|f| f.name.as_str()).collect();
assert!(
names.contains(&"job__start"),
"job__ tools must survive an empty agent tool filter, got: {names:?}"
);
assert!(names.contains(&"job__collect"));
}
#[test]
#[serial]
fn select_functions_when_jobs_disabled_is_byte_identical_to_no_jobs_baseline() {
let _guard = TestConfigDirGuard::new();
let app_state = app_state_with_mcp_config(false, &[]);
let mut ctx = RequestContext::new(app_state, WorkingMode::Repl);
let app = ctx.app.config.clone();
let abort = utils::create_abort_signal();
let mut role = Role::new("r", "p");
role.set_enabled_tools(Some(vec!["all".to_string()]));
let jobs_off = AppConfig {
max_concurrent_jobs: Some(0),
..(*app).clone()
};
run_async(ctx.rebuild_tool_scope(&jobs_off, None, abort.clone())).unwrap();
let without_jobs = serde_json::to_string(&ctx.select_functions(&role)).unwrap();
assert!(
!without_jobs.contains("job__"),
"no job__ declarations may leak when jobs are disabled, got: {without_jobs}"
);
run_async(ctx.rebuild_tool_scope(&app, None, abort)).unwrap();
let with_jobs = ctx.select_functions(&role).unwrap();
assert!(with_jobs.iter().any(|f| f.name.starts_with("job__")));
let stripped: Vec<FunctionDeclaration> = with_jobs
.into_iter()
.filter(|f| !f.name.starts_with("job__"))
.collect();
assert_eq!(
without_jobs,
serde_json::to_string(&Some(stripped)).unwrap(),
"jobs-disabled tool list must be byte-identical to the jobs-enabled list minus job__ declarations"
);
}
#[test]
fn select_functions_returns_none_when_no_tools_enabled_and_jobs_disabled() {
let app_state = {
let config = AppConfig {
max_concurrent_jobs: Some(0),
..AppConfig::default()
};
Arc::new(AppState {
config: Arc::new(config),
vault: Arc::new(Vault::default()),
mcp_factory: Arc::new(McpFactory::default()),
rag_cache: Arc::new(RagCache::default()),
mcp_config: None,
mcp_log_path: None,
mcp_registry: None,
functions: Functions::default(),
})
};
let ctx = RequestContext::new(app_state, WorkingMode::Cmd);
assert!(ctx.select_functions(&Role::default()).is_none());
}
#[test]
fn tools_info_lists_job_tools_when_enabled() {
let mut ctx = create_test_ctx();
ctx.tool_scope.functions.append_job_functions();
let info = ctx.tools_info().unwrap();
for name in [
"job__start",
"job__check",
"job__collect",
"job__cancel",
"job__list",
] {
assert!(
info.contains(name),
"expected {name} in output, got: {info}"
);
}
}
fn make_running_job(abort_signal: utils::AbortSignal) -> crate::supervisor::JobHandle {
// Leak the runtime so the spawned task is never polled and the job
// stays running for the duration of the test.
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let join_handle = rt.spawn(async {
Ok(crate::supervisor::JobResult {
output: serde_json::Value::Null,
exit_code: Some(0),
output_bytes_captured: 0,
})
});
std::mem::forget(rt);
crate::supervisor::JobHandle {
id: "j1".to_string(),
tool: "execute_command".to_string(),
started_at: std::time::Instant::now(),
join_handle,
abort_signal,
state: Arc::new(parking_lot::Mutex::new(crate::supervisor::JobState {
status: crate::supervisor::JobStatus::Running,
pgid: None,
})),
output_buf: Arc::new(parking_lot::Mutex::new(
crate::function::jobs::RingBuf::default(),
)),
no_change_checks: 0,
last_check_state: None,
}
}
#[test]
#[serial]
fn use_agent_cancels_running_jobs_of_previous_supervisor() {
let _guard = TestConfigDirGuard::new();
let mut ctx = create_test_ctx();
let app = ctx.app.config.clone();
let agent_name = format!(
"test_agent_{}",
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_nanos()
);
let agent_dir = paths::agent_data_dir(&agent_name);
create_dir_all(&agent_dir).unwrap();
write(
agent_dir.join("config.yaml"),
format!("name: {agent_name}\ninstructions: hi\n"),
)
.unwrap();
let job_sig = utils::create_abort_signal();
let old_sup = Arc::new(RwLock::new(
Supervisor::new(4, 3).with_max_concurrent_jobs(1),
));
old_sup
.write()
.register(make_running_job(job_sig.clone()))
.unwrap();
ctx.supervisor = Some(old_sup);
run_async(ctx.use_agent(&app, &agent_name, None, utils::create_abort_signal())).unwrap();
assert!(
job_sig.aborted(),
"running jobs of the previous supervisor must be cancelled"
);
assert!(ctx.supervisor.is_some());
}
#[test]
#[serial]
fn exit_agent_cancels_running_jobs() {
let _guard = TestConfigDirGuard::new();
let mut ctx = create_test_ctx();
let app = ctx.app.config.clone();
let job_sig = utils::create_abort_signal();
let sup = Arc::new(RwLock::new(
Supervisor::new(4, 3).with_max_concurrent_jobs(1),
));
sup.write()
.register(make_running_job(job_sig.clone()))
.unwrap();
ctx.agent = Some(Agent::test_new(AgentConfig::default()));
ctx.supervisor = Some(sup);
ctx.exit_agent(&app).unwrap();
assert!(job_sig.aborted(), "exit_agent must cancel running jobs");
assert!(ctx.supervisor.is_none());
}
#[test]
fn toggle_tool_rejects_job_tools_as_unknown() {
let mut ctx = create_test_ctx();
ctx.tool_scope.functions.append_job_functions();
for action in ["enable", "disable"] {
let err = ctx.toggle_tool(action, "job__start").unwrap_err();
assert!(
err.to_string().contains("Unknown tool 'job__start'"),
"expected job__start to be rejected on {action}, got: {err}"
);
}
}
} }
+340
View File
@@ -1928,6 +1928,49 @@ mod tests {
}); });
} }
/// The completion notification is pushed from inside the job task, after
/// the run — a panic unwinds past the push, and neither the supervisor nor
/// collect synthesizes a notification for a panicked job.
#[test]
fn panicked_job_task_skips_the_completion_notification() {
run_async(async {
let ctx = ctx_with_job_supervisor(4);
let join_handle: tokio::task::JoinHandle<Result<JobResult>> =
tokio::spawn(async { panic!("boom") });
let handle = JobHandle {
id: "j1".to_string(),
tool: "execute_command".to_string(),
started_at: Instant::now(),
join_handle,
abort_signal: create_abort_signal(),
state: Arc::new(Mutex::new(JobState {
// A panic never reaches the status update — the cell
// stays Running, which is the real post-panic state.
status: JobStatus::Running,
pgid: None,
})),
output_buf: Arc::new(Mutex::new(RingBuf::default())),
no_change_checks: 0,
last_check_state: None,
};
ctx.supervisor
.as_ref()
.unwrap()
.write()
.register(handle)
.unwrap();
let result = handle_collect(&ctx, &json!({"id": "j1"})).await.unwrap();
assert_eq!(result["status"], "failed");
assert!(result["error"].as_str().unwrap().contains("panicked"));
assert!(
ctx.notification_queue.drain().is_empty(),
"a panicked job must never produce a completion notification"
);
});
}
#[cfg(unix)] #[cfg(unix)]
#[test] #[test]
fn handle_cancel_kills_running_process_job() { fn handle_cancel_kills_running_process_job() {
@@ -2323,4 +2366,301 @@ mod tests {
.unwrap(); .unwrap();
assert_eq!(tasks["tasks"].as_array().unwrap().len(), 1); assert_eq!(tasks["tasks"].as_array().unwrap().len(), 1);
} }
#[cfg(unix)]
#[test]
fn handle_cancel_kills_grandchild_process() {
run_async(async {
let ctx = ctx_with_job_supervisor(4);
let state = Arc::new(Mutex::new(JobState {
status: JobStatus::Running,
pgid: None,
}));
let output_buf = Arc::new(Mutex::new(RingBuf::default()));
let snapshot = test_snapshot("sh", &["-c", "sleep 30 & echo CHILD:$!; wait"], 0);
let task_state = Arc::clone(&state);
let task_buf = Arc::clone(&output_buf);
let join_handle =
tokio::spawn(async move { run_process_job(snapshot, task_state, task_buf).await });
let deadline = time::Instant::now() + Duration::from_secs(5);
let grandchild_pid = loop {
let tail = String::from_utf8_lossy(&output_buf.lock().tail()).to_string();
if let Some(rest) = tail.split("CHILD:").nth(1)
&& let Some(line_end) = rest.find('\n')
{
break rest[..line_end].trim().parse::<i32>().unwrap();
}
assert!(
time::Instant::now() < deadline,
"grandchild pid never appeared in the ring buffer"
);
time::sleep(Duration::from_millis(10)).await;
};
let handle = JobHandle {
id: "j1".to_string(),
tool: "sh".to_string(),
started_at: Instant::now(),
join_handle,
abort_signal: create_abort_signal(),
state: Arc::clone(&state),
output_buf,
no_change_checks: 0,
last_check_state: None,
};
ctx.supervisor
.as_ref()
.unwrap()
.write()
.register(handle)
.unwrap();
let result = handle_cancel(&ctx, &json!({"id": "j1"})).await.unwrap();
assert_eq!(result["status"], "cancelled");
// kill(pid, 0) alone can't observe the death: the orphaned
// grandchild lingers as an unreaped zombie under init/launchd,
// so a Z state also proves the group kill landed.
fn grandchild_is_dead(pid: i32) -> bool {
let esrch = unsafe { libc::kill(pid, 0) } == -1
&& std::io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH);
if esrch {
return true;
}
let stat = std::process::Command::new("ps")
.args(["-o", "stat=", "-p", &pid.to_string()])
.output()
.map(|out| String::from_utf8_lossy(&out.stdout).trim().to_string())
.unwrap_or_default();
stat.is_empty() || stat.starts_with('Z')
}
let deadline = time::Instant::now() + Duration::from_secs(5);
while !grandchild_is_dead(grandchild_pid) {
assert!(
time::Instant::now() < deadline,
"grandchild must die with the process group"
);
time::sleep(Duration::from_millis(10)).await;
}
});
}
#[cfg(unix)]
#[test]
fn run_process_job_clears_pgid_after_normal_completion() {
run_async(async {
let state = Arc::new(Mutex::new(JobState {
status: JobStatus::Running,
pgid: None,
}));
let output_buf = Arc::new(Mutex::new(RingBuf::default()));
let snapshot = test_snapshot("echo", &["done"], 0);
let result = run_process_job(snapshot, Arc::clone(&state), output_buf)
.await
.unwrap();
assert_eq!(result.exit_code, Some(0));
assert!(
state.lock().pgid.is_none(),
"pid-reuse guard must clear pgid"
);
});
}
#[test]
fn handle_start_rejects_shell_and_path_shaped_names_without_spawn() {
let mut ctx = plain_ctx();
for tool in ["bash", "./script.sh", "/usr/bin/env", "ls"] {
let result = run_async(handle_start(
&mut ctx,
&json!({"tool": tool, "arguments": {}}),
))
.unwrap();
assert_eq!(result["status"], "error");
assert!(
result["message"]
.as_str()
.unwrap()
.contains("not enabled in this context")
);
}
assert!(ctx.supervisor.is_none(), "no job may be spawned");
}
#[cfg(unix)]
#[test]
fn handle_start_rejects_context_filtered_tool_and_accepts_in_filter() {
run_async(async {
let mut ctx = plain_ctx();
ctx.declared_function_names.insert("echo".into());
let rejected = handle_start(&mut ctx, &json!({"tool": "git_command", "arguments": {}}))
.await
.unwrap();
assert_eq!(rejected["status"], "error");
assert!(
rejected["message"]
.as_str()
.unwrap()
.contains("not enabled in this context")
);
assert!(ctx.supervisor.is_none(), "no job may be spawned");
let started = handle_start(&mut ctx, &json!({"tool": "echo", "arguments": {}}))
.await
.unwrap();
assert_eq!(started["status"], "ok");
let job_id = started["job_id"].as_str().unwrap().to_string();
let collected = handle_collect(&ctx, &json!({"id": job_id})).await.unwrap();
assert_eq!(collected["status"], "completed");
});
}
#[test]
fn handle_start_rejects_undeclared_mcp_invoke_without_spawn() {
let mut ctx = plain_ctx();
let result = run_async(handle_start(
&mut ctx,
&json!({"tool": "mcp_invoke_someserver", "arguments": {"tool": "search"}}),
))
.unwrap();
assert_eq!(result["status"], "error");
assert!(
result["message"]
.as_str()
.unwrap()
.contains("not enabled in this context")
);
assert!(ctx.supervisor.is_none(), "no job may be spawned");
}
#[test]
fn handle_start_rejects_declared_but_non_whitelisted_tools_without_spawn() {
let mut ctx = plain_ctx();
for tool in ["memory__write", "fs_read", "agent__spawn", "user__select"] {
ctx.declared_function_names.insert(tool.into());
}
for (tool, category) in [
("memory__write", "mutates agent/session state"),
("fs_read", "is fast"),
("agent__spawn", "already asynchronous"),
("user__select", "interactive"),
] {
let result = run_async(handle_start(
&mut ctx,
&json!({"tool": tool, "arguments": {}}),
))
.unwrap();
assert_eq!(result["status"], "error");
assert!(result["message"].as_str().unwrap().contains(category));
}
assert!(ctx.supervisor.is_none(), "no job may be spawned");
}
#[test]
fn handle_start_rejects_mapping_tools_alias() {
let app_state = app_state_with_config(|config| {
config
.mapping_tools
.insert("shell".into(), "execute_command".into());
});
let mut ctx = RequestContext::new(app_state, WorkingMode::Cmd);
ctx.declared_function_names.insert("execute_command".into());
let result = run_async(handle_start(
&mut ctx,
&json!({"tool": "shell", "arguments": {}}),
))
.unwrap();
assert_eq!(result["status"], "error");
assert!(
result["message"]
.as_str()
.unwrap()
.contains("not enabled in this context")
);
assert!(ctx.supervisor.is_none(), "no job may be spawned");
}
#[test]
fn child_context_cannot_reach_parent_job_ids() {
run_async(async {
let parent = ctx_with_job_supervisor(4);
parent
.supervisor
.as_ref()
.unwrap()
.write()
.register(make_running_job("job_p1"))
.unwrap();
let child = RequestContext::new_for_child(
default_app_state(),
&parent,
1,
Arc::new(Inbox::new()),
"c1".into(),
);
assert!(child.supervisor.is_none());
let checked = handle_check(&child, &json!({"id": "job_p1"})).unwrap();
let collected = handle_collect(&child, &json!({"id": "job_p1"}))
.await
.unwrap();
let cancelled = handle_cancel(&child, &json!({"id": "job_p1"}))
.await
.unwrap();
for result in [checked, collected, cancelled] {
assert_eq!(result["status"], "error");
assert!(
result["message"]
.as_str()
.unwrap()
.contains("No job 'job_p1' is registered")
);
}
assert!(parent.supervisor.as_ref().unwrap().read().has_job("job_p1"));
});
}
#[test]
fn handle_start_ignores_mid_batch_tool_scope_additions() {
let mut ctx = plain_ctx();
ctx.declared_function_names.insert("job__start".into());
ctx.tool_scope
.functions
.declarations
.push(FunctionDeclaration {
name: "late_external_tool".into(),
description: String::new(),
parameters: JsonSchema::default(),
agent: false,
});
let result = run_async(handle_start(
&mut ctx,
&json!({"tool": "late_external_tool", "arguments": {}}),
))
.unwrap();
assert_eq!(result["status"], "error");
assert!(
result["message"]
.as_str()
.unwrap()
.contains("not enabled in this context")
);
assert!(ctx.supervisor.is_none(), "no job may be spawned");
}
} }
+61
View File
@@ -2940,6 +2940,51 @@ mod tests {
); );
} }
#[cfg(unix)]
#[test]
fn job_finished_earlier_drains_notification_on_later_batch() {
run_async(async {
let mut ctx = RequestContext::new(Arc::new(AppState::test_default()), WorkingMode::Cmd);
ctx.declared_function_names.insert("echo".into());
let started = jobs::handle_job_tool(
&mut ctx,
"job__start",
&json!({"tool": "echo", "arguments": {}}),
)
.await
.unwrap();
assert_eq!(started["status"], "ok");
let job_id = started["job_id"].as_str().unwrap().to_string();
let supervisor = ctx.supervisor.clone().unwrap();
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5);
while tokio::time::Instant::now() < deadline {
let finished = supervisor
.read()
.job(&job_id)
.is_none_or(|job| job.join_handle.is_finished());
if finished {
break;
}
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
let calls = vec![call("unknown_tool", Some("id-2"))];
let results = eval_tool_calls(&mut ctx, calls).await.unwrap();
let out = &results.last().unwrap().output;
assert_eq!(out["system_notifications"][0]["id"], job_id);
assert_eq!(out["system_notifications"][0]["event"], "job_completed");
assert!(
out["notification_instruction"]
.as_str()
.unwrap()
.contains("next_action")
);
});
}
#[test] #[test]
fn normalize_tool_result_preserves_non_null_values() { fn normalize_tool_result_preserves_non_null_values() {
assert_eq!( assert_eq!(
@@ -3125,6 +3170,22 @@ mod tests {
assert!(msg.contains("repeat_tool")); assert!(msg.contains("repeat_tool"));
} }
#[test]
fn loop_tracker_exempt_list_is_exactly_the_polling_tools() {
let actual: std::collections::HashSet<&str> =
LOOP_TRACKER_EXEMPT_TOOLS.iter().copied().collect();
let expected: std::collections::HashSet<&str> = [
"job__check",
"job__list",
"agent__check",
"agent__list_running",
]
.into_iter()
.collect();
assert_eq!(LOOP_TRACKER_EXEMPT_TOOLS.len(), 4);
assert_eq!(actual, expected);
}
#[test] #[test]
fn tracker_exempt_tools_never_trip() { fn tracker_exempt_tools_never_trip() {
for name in LOOP_TRACKER_EXEMPT_TOOLS { for name in LOOP_TRACKER_EXEMPT_TOOLS {
+82
View File
@@ -2911,4 +2911,86 @@ mod tests {
assert_job_teaching_error(&result, "bg_p"); assert_job_teaching_error(&result, "bg_p");
}); });
} }
#[test]
fn guardrail_burns_bounded_injects_then_force_terminates_running_job() {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
rt.block_on(async {
let mut ctx = ctx_with_job_capable_supervisor();
let abort = create_abort_signal();
let join_handle = tokio::spawn(async {
time::sleep(Duration::from_secs(60)).await;
Ok(JobResult {
output: json!(null),
exit_code: Some(0),
output_bytes_captured: 0,
})
});
let handle = JobHandle {
id: "job_1".to_string(),
tool: "execute_command".to_string(),
started_at: std::time::Instant::now(),
join_handle,
abort_signal: abort.clone(),
state: Arc::new(Mutex::new(JobState {
status: JobStatus::Running,
pgid: None,
})),
output_buf: Arc::new(Mutex::new(RingBuf::default())),
no_change_checks: 0,
last_check_state: None,
};
ctx.supervisor
.as_ref()
.unwrap()
.write()
.register(handle)
.unwrap();
for expected_count in 1..=PENDING_AGENTS_GUARDRAIL_MAX {
match check_pending_agents_guardrail(&mut ctx) {
GuardrailAction::Inject(prompt) => assert!(prompt.contains("job_1")),
_ => panic!("expected Inject below max"),
}
assert_eq!(ctx.pending_agents_guardrail_count, expected_count);
}
match check_pending_agents_guardrail(&mut ctx) {
GuardrailAction::ForceTerminate(ids) => {
assert_eq!(ids, vec!["job_1".to_string()]);
}
_ => panic!("expected ForceTerminate at max"),
}
assert_eq!(ctx.pending_agents_guardrail_count, 0);
assert!(abort.aborted());
});
}
#[test]
fn guardrail_force_terminate_discards_finished_uncollected_job() {
let mut ctx = ctx_with_job_capable_supervisor();
register_fake_job(&mut ctx, "job_1");
let deadline = std::time::Instant::now() + Duration::from_secs(5);
while !pending_tasks(&ctx).iter().any(|t| t.finished) {
assert!(
std::time::Instant::now() < deadline,
"job 'job_1' never finished"
);
std::thread::sleep(Duration::from_millis(10));
}
ctx.pending_agents_guardrail_count = PENDING_AGENTS_GUARDRAIL_MAX;
match check_pending_agents_guardrail(&mut ctx) {
GuardrailAction::ForceTerminate(ids) => {
assert_eq!(ids, vec!["job_1".to_string()]);
}
_ => panic!("expected ForceTerminate action"),
}
assert_eq!(ctx.pending_agents_guardrail_count, 0);
assert!(!ctx.supervisor.as_ref().unwrap().read().has_job("job_1"));
}
} }
+82
View File
@@ -856,4 +856,86 @@ nodes:
); );
assert!(err.contains("sleeper"), "error should name frontier: {err}"); assert!(err.contains("sleeper"), "error should name frontier: {err}");
} }
#[cfg(unix)]
#[tokio::test]
async fn background_job_survives_graph_node_execution() {
if !cmd_available("bash") {
eprintln!("skipping: bash not available");
return;
}
let ws = TestWorkspace::new();
ws.write_script("noop.sh", "#!/bin/bash\necho '{}'\n");
let yaml = r#"
name: background_job_survival_test
start: noop
nodes:
noop:
type: script
script: noop.sh
state_updates: {}
next: done
done:
type: end
output: "done"
"#;
let graph: Graph = serde_yaml::from_str(yaml).unwrap();
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let join_handle = rt.spawn(async {
Ok(crate::supervisor::JobResult {
output: Value::Null,
exit_code: Some(0),
output_bytes_captured: 0,
})
});
std::mem::forget(rt);
let handle = crate::supervisor::JobHandle {
id: "job_bg".to_string(),
tool: "execute_command".to_string(),
started_at: Instant::now(),
join_handle,
abort_signal: create_abort_signal(),
state: Arc::new(parking_lot::Mutex::new(crate::supervisor::JobState {
status: crate::supervisor::JobStatus::Completed,
pgid: None,
})),
output_buf: Arc::new(parking_lot::Mutex::new(
crate::function::jobs::RingBuf::default(),
)),
no_change_checks: 0,
last_check_state: None,
};
let mut sup = crate::supervisor::Supervisor::new(0, 3).with_max_concurrent_jobs(4);
sup.register(handle).unwrap();
let mut ctx = make_ctx();
ctx.supervisor = Some(Arc::new(parking_lot::RwLock::new(sup)));
ctx.notification_queue
.push(crate::supervisor::notification::job_notification(
"job_bg",
"execute_command",
true,
));
let abort = create_abort_signal();
let result = GraphExecutor::new(graph, &ws.dir)
.execute(&mut ctx, abort)
.await
.unwrap_or_else(|e| panic!("executor failed: {e:#}"));
assert_eq!(result, "done");
assert!(
ctx.supervisor.as_ref().unwrap().read().has_job("job_bg"),
"graph execution must not touch registered job handles"
);
let events = ctx.notification_queue.drain();
assert_eq!(events.len(), 1, "queued notification must survive the run");
assert_eq!(events[0].id, "job_bg");
assert_eq!(events[0].event, "job_completed");
}
} }