feat(jobs): node-local job ownership and capability-gated job__* visibility

Graph LLM nodes now own the jobs they start, on every exit path. A new
node_job_scope on RequestContext records job ids started while a node
runs: the turn-end guardrail nags only about the node's own jobs
(parallel branches no longer see each other's), and the node executor
reaps — cancels and deregisters — anything left registered when the
node exits, including error, timeout, and retry-exhaustion paths.
Cross-node job handoff is no longer possible; a crashed node takes its
in-flight jobs with it.

With inheritance gone, job__* declarations are gated on capability:
the family is only declared when at least one declared tool would pass
job__start's whitelist (shared predicate: is_backgroundable_tool). One
carve-out — while a context still owns registered jobs (job started,
tool disabled mid-session), the lifecycle verbs stay declared so a
running job can never become unreachable; job__start alone disappears.
A graph node with tools: [] now sees no job__* tools at all.

Prompt instructions, tool declarations, and graph.example.yaml updated
to the node-local semantics; +7 tests, 8 visibility pins rewritten.
This commit is contained in:
2026-08-26 13:43:05 -06:00
parent bfb8105682
commit 404a45a311
7 changed files with 311 additions and 45 deletions
+99 -1
View File
@@ -140,7 +140,8 @@ pub fn job_function_declarations() -> Vec<FunctionDeclaration> {
tools cannot be backgrounded. The job runs against a snapshot of the current config and \
environment; later changes do not affect it. Process jobs honor COYOTE_TOOL_TIMEOUT; MCP \
jobs have NO timeout — cancel a hung one with `job__cancel`. Jobs do not survive coyote \
exiting.".to_string(),
exiting. In graph LLM nodes, jobs are node-local: collect or cancel every job you start \
before the node ends — leftovers are cancelled at node exit.".to_string(),
parameters: JsonSchema {
type_value: Some("object".to_string()),
properties: Some(IndexMap::from([
@@ -359,6 +360,13 @@ fn whitelist_rejection(tool: &str) -> Option<Value> {
})
}
/// Whether a declared tool could be run as a background job. This is the
/// declare-side twin of `whitelist_rejection`: a tool is backgroundable
/// exactly when `job__start` would not reject it by name.
pub fn is_backgroundable_tool(tool: &str) -> bool {
whitelist_rejection(tool).is_none()
}
async fn handle_start(ctx: &mut RequestContext, args: &Value) -> Result<Value> {
if !jobs_enabled(ctx.agent.as_ref(), &ctx.app.config) {
return Ok(json!({
@@ -511,6 +519,10 @@ async fn handle_start(ctx: &mut RequestContext, args: &Value) -> Result<Value> {
}));
}
if let Some(scope) = ctx.node_job_scope.as_mut() {
scope.push(job_id.clone());
}
Ok(json!({
"status": "ok",
"job_id": job_id,
@@ -803,6 +815,36 @@ async fn kill_job_with_grace(handle: &mut JobHandle) {
let _ = time::timeout(JOB_KILL_GRACE, &mut handle.join_handle).await;
}
/// Cancel and deregister the named jobs, if still registered. Used by graph
/// LLM nodes to enforce node-local job ownership: any job the node started
/// but did not collect or cancel by the time it exits is killed here, on
/// every exit path. Returns the ids actually reaped.
pub async fn reap_jobs(
supervisor: Option<&Arc<RwLock<Supervisor>>>,
ids: &[String],
) -> Vec<String> {
let Some(supervisor) = supervisor else {
return Vec::new();
};
let mut reaped = Vec::new();
for id in ids {
let handle = {
let mut sup = supervisor.write();
sup.take_job(id)
};
if let Some(mut handle) = handle {
handle.abort_signal.set_ctrlc();
kill_job_with_grace(&mut handle).await;
warn!(
"Reaped background job '{id}' ({}): left unreclaimed at graph node exit",
handle.tool
);
reaped.push(id.clone());
}
}
reaped
}
fn handle_list(ctx: &RequestContext) -> Result<Value> {
let Some(supervisor) = ctx.supervisor.as_ref() else {
return Ok(json!({
@@ -1552,6 +1594,62 @@ mod tests {
);
}
#[test]
fn is_backgroundable_tool_matches_start_whitelist() {
assert!(is_backgroundable_tool("execute_command"));
assert!(is_backgroundable_tool("my_custom_tool.sh"));
assert!(is_backgroundable_tool("mcp_invoke_github"));
assert!(!is_backgroundable_tool("job__start"));
assert!(!is_backgroundable_tool("agent__spawn"));
assert!(!is_backgroundable_tool("user__confirm"));
assert!(!is_backgroundable_tool("todo__add"));
assert!(!is_backgroundable_tool("fs_read"));
assert!(!is_backgroundable_tool("ast_grep"));
assert!(!is_backgroundable_tool("mcp_search_github"));
}
#[cfg(unix)]
#[test]
fn handle_start_records_job_in_node_scope() {
run_async(async {
let mut ctx = plain_ctx();
ctx.node_job_scope = Some(Vec::new());
ctx.declared_function_names.insert("echo".into());
let started = handle_start(&mut ctx, &json!({"tool": "echo", "arguments": {}}))
.await
.unwrap();
let job_id = started["job_id"].as_str().unwrap().to_string();
assert_eq!(ctx.node_job_scope.clone().unwrap(), vec![job_id.clone()]);
let collected = handle_collect(&ctx, &json!({"id": job_id})).await.unwrap();
assert_eq!(collected["status"], "completed");
});
}
#[test]
fn reap_jobs_kills_registered_jobs_and_reports_ids() {
run_async(async {
let ctx = ctx_with_job_supervisor(4);
ctx.supervisor
.as_ref()
.unwrap()
.write()
.register(make_running_job("j1"))
.unwrap();
let reaped = reap_jobs(
ctx.supervisor.as_ref(),
&["j1".to_string(), "missing".to_string()],
)
.await;
assert_eq!(reaped, vec!["j1".to_string()]);
assert!(!ctx.supervisor.as_ref().unwrap().read().has_job("j1"));
});
}
#[test]
fn handle_start_rejects_unconnected_mcp_server() {
let mut ctx = plain_ctx();