feat: generalize supervisor registry to TaskHandle enum with job scaffolding, kill discipline, and max_concurrent_jobs config

Implements T1 of plans/background-jobs-design.md (§6, R7/R8/R9):

- Supervisor.handles is now HashMap<String, TaskHandle> where
  TaskHandle = Agent(AgentHandle) | Job(JobHandle); agent-facing
  accessors (active_count, effective_active_count, is_finished, take,
  inbox, abort_signal_for, list_agents) match only Agent variants,
  preserving all existing external behavior byte-for-byte.
- New JobHandle/JobState/JobStatus/JobResult types with pgid-guarded
  process-group kill discipline: Drop and cancel_all/cancel_recursive
  kill the group only while state.pgid is still set (pid-reuse guard),
  via libc::killpg on unix and JoinHandle::abort elsewhere.
- Per-kind job capacity: Supervisor carries max_concurrent_jobs
  (builder-set, default 0); job registration rejects at capacity.
- Cross-kind teaching errors at the four agent-lookup miss sites
  (agent__check/collect/cancel/send_message) when the id is a
  registered job or job_-prefixed; genuinely-unknown ids keep their
  existing messages.
- Supervisor init condition is now can_spawn_agents || jobs_enabled in
  use_agent and both child-agent spawn paths, with agent capacity 0 in
  jobs-only contexts; use_agent cancels the old supervisor recursively
  before replacing it.
- max_concurrent_jobs config plumbing: global Config field, AgentConfig
  override + accessor, all four AppConfig touch points including the
  COYOTE_MAX_CONCURRENT_JOBS env override; shared
  effective_max_concurrent_jobs/jobs_enabled predicates
  (agent override -> global -> default 5; 0 disables).
- Stage dependency-free RingBuf (64 KiB default) in src/function/jobs.rs
  for the upcoming job output pump.
- New sanctioned dependency: libc 0.2 under cfg(unix).
This commit is contained in:
2026-08-25 15:17:57 -06:00
parent bfc3b7bfea
commit 7f3f95d89d
10 changed files with 942 additions and 60 deletions
+161
View File
@@ -0,0 +1,161 @@
use crate::supervisor::Supervisor;
use parking_lot::RwLock;
use std::sync::Arc;
#[allow(dead_code)]
pub fn is_agent_task(supervisor: Option<&Arc<RwLock<Supervisor>>>, id: &str) -> bool {
id.starts_with("agent_")
|| id.starts_with("graph_agent_")
|| supervisor.is_some_and(|sup| sup.read().has_agent(id))
}
pub struct RingBuf {
buf: Vec<u8>,
capacity: usize,
write_pos: usize,
total_written: u64,
}
#[allow(dead_code)]
impl RingBuf {
pub fn new(capacity: usize) -> Self {
Self {
buf: Vec::new(),
capacity,
write_pos: 0,
total_written: 0,
}
}
pub fn push(&mut self, bytes: &[u8]) {
self.total_written += bytes.len() as u64;
if self.capacity == 0 {
return;
}
let src = if bytes.len() > self.capacity {
&bytes[bytes.len() - self.capacity..]
} else {
bytes
};
for &byte in src {
if self.buf.len() < self.capacity {
self.buf.push(byte);
} else {
self.buf[self.write_pos] = byte;
}
self.write_pos = (self.write_pos + 1) % self.capacity;
}
}
pub fn total_written(&self) -> u64 {
self.total_written
}
pub fn tail(&self) -> Vec<u8> {
if self.buf.len() < self.capacity {
return self.buf.clone();
}
let mut out = Vec::with_capacity(self.capacity);
out.extend_from_slice(&self.buf[self.write_pos..]);
out.extend_from_slice(&self.buf[..self.write_pos]);
out
}
}
impl Default for RingBuf {
fn default() -> Self {
Self::new(64 * 1024)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn ring_buf_returns_contents_below_capacity() {
let mut buf = RingBuf::new(8);
buf.push(b"abc");
buf.push(b"de");
assert_eq!(buf.tail(), b"abcde");
assert_eq!(buf.total_written(), 5);
}
#[test]
fn ring_buf_exact_fit_keeps_everything() {
let mut buf = RingBuf::new(5);
buf.push(b"abcde");
assert_eq!(buf.tail(), b"abcde");
assert_eq!(buf.total_written(), 5);
}
#[test]
fn ring_buf_wrap_around_keeps_newest_bytes() {
let mut buf = RingBuf::new(5);
buf.push(b"abcde");
buf.push(b"fg");
assert_eq!(buf.tail(), b"cdefg");
assert_eq!(buf.total_written(), 7);
}
#[test]
fn ring_buf_oversize_push_keeps_last_capacity_bytes() {
let mut buf = RingBuf::new(4);
buf.push(b"abcdefghij");
assert_eq!(buf.tail(), b"ghij");
assert_eq!(buf.total_written(), 10);
}
#[test]
fn ring_buf_default_capacity_is_64_kib() {
let mut buf = RingBuf::default();
let payload = vec![b'x'; 64 * 1024 + 1];
buf.push(&payload);
assert_eq!(buf.tail().len(), 64 * 1024);
assert_eq!(buf.total_written(), 64 * 1024 + 1);
}
#[test]
fn is_agent_task_matches_agent_prefixes() {
assert!(is_agent_task(None, "agent_explore_a1b2c3d4"));
assert!(is_agent_task(None, "graph_agent_explore_a1b2c3d4"));
assert!(!is_agent_task(None, "job_deadbeef"));
}
#[test]
fn is_agent_task_matches_registered_agents() {
use crate::supervisor::mailbox::Inbox;
use crate::supervisor::{AgentExitStatus, AgentHandle, AgentResult};
use crate::utils::create_abort_signal;
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let join_handle = rt.spawn(async {
Ok(AgentResult {
id: "a1".into(),
agent_name: "explore".into(),
output: String::new(),
exit_status: AgentExitStatus::Completed,
})
});
std::mem::forget(rt);
let handle = AgentHandle {
id: "a1".to_string(),
agent_name: "explore".to_string(),
depth: 1,
inbox: Arc::new(Inbox::new()),
abort_signal: create_abort_signal(),
join_handle,
child_supervisor: None,
};
let mut sup = Supervisor::new(4, 3);
sup.register(handle).unwrap();
let sup = Arc::new(RwLock::new(sup));
assert!(is_agent_task(Some(&sup), "a1"));
assert!(!is_agent_task(Some(&sup), "missing"));
}
}
+1
View File
@@ -1,3 +1,4 @@
pub(crate) mod jobs;
pub(crate) mod memory;
pub(crate) mod rag_query;
pub(crate) mod skill;
+223 -25
View File
@@ -1,7 +1,8 @@
use super::{FunctionDeclaration, JsonSchema};
use crate::client::{Model, ModelType, call_chat_completions};
use crate::config::{
Agent, AppState, Input, RequestContext, Role, RoleLike, list_agents_with_descriptions,
Agent, AppState, Input, RequestContext, Role, RoleLike, effective_max_concurrent_jobs,
jobs_enabled, list_agents_with_descriptions,
};
use crate::supervisor::mailbox::{Envelope, EnvelopePayload, Inbox};
use crate::supervisor::{AgentExitStatus, AgentHandle, AgentResult, Supervisor};
@@ -32,6 +33,19 @@ fn agent_permitted(whitelist: Option<&[String]>, target: &str) -> bool {
}
}
fn is_job_task(supervisor: Option<&Arc<RwLock<Supervisor>>>, id: &str) -> bool {
id.starts_with("job_") || supervisor.is_some_and(|sup| sup.read().has_job(id))
}
fn job_id_teaching_error(id: &str) -> Value {
json!({
"status": "error",
"message": format!(
"'{id}' is a background job, not an agent — use job__check / job__collect / job__cancel"
),
})
}
pub enum GuardrailAction {
NoAction,
Inject(String),
@@ -557,9 +571,15 @@ pub async fn run_agent_for_graph(
let agent_mcp_servers = agent.mcp_server_names().to_vec();
let session = agent.agent_session().map(|v| v.to_string());
let should_init_supervisor = agent.can_spawn_agents();
let agent_max_concurrent = agent.max_concurrent_agents();
let child_jobs_enabled = jobs_enabled(Some(&agent), app_config.as_ref());
let should_init_supervisor = agent.can_spawn_agents() || child_jobs_enabled;
let agent_max_concurrent = if agent.can_spawn_agents() {
agent.max_concurrent_agents()
} else {
0
};
let agent_max_depth = agent.max_agent_depth();
let agent_max_jobs = effective_max_concurrent_jobs(Some(&agent), app_config.as_ref());
let mut child_ctx = RequestContext::new_for_child(
Arc::clone(&child_app_state),
@@ -571,10 +591,10 @@ pub async fn run_agent_for_graph(
child_ctx.rag = agent.rag();
child_ctx.agent = Some(agent);
if should_init_supervisor {
child_ctx.supervisor = Some(Arc::new(RwLock::new(Supervisor::new(
agent_max_concurrent,
agent_max_depth,
))));
child_ctx.supervisor = Some(Arc::new(RwLock::new(
Supervisor::new(agent_max_concurrent, agent_max_depth)
.with_max_concurrent_jobs(agent_max_jobs),
)));
}
if let Some(session) = session {
@@ -736,9 +756,15 @@ async fn handle_spawn(ctx: &mut RequestContext, args: &Value) -> Result<Value> {
let agent_mcp_servers = agent.mcp_server_names().to_vec();
let session = agent.agent_session().map(|v| v.to_string());
let should_init_supervisor = agent.can_spawn_agents();
let max_concurrent = agent.max_concurrent_agents();
let child_jobs_enabled = jobs_enabled(Some(&agent), app_config.as_ref());
let should_init_supervisor = agent.can_spawn_agents() || child_jobs_enabled;
let max_concurrent = if agent.can_spawn_agents() {
agent.max_concurrent_agents()
} else {
0
};
let max_depth = agent.max_agent_depth();
let max_jobs = effective_max_concurrent_jobs(Some(&agent), app_config.as_ref());
let mut child_ctx = RequestContext::new_for_child(
Arc::clone(&child_app_state),
ctx,
@@ -749,10 +775,9 @@ async fn handle_spawn(ctx: &mut RequestContext, args: &Value) -> Result<Value> {
child_ctx.rag = agent.rag();
child_ctx.agent = Some(agent);
if should_init_supervisor {
child_ctx.supervisor = Some(Arc::new(RwLock::new(Supervisor::new(
max_concurrent,
max_depth,
))));
child_ctx.supervisor = Some(Arc::new(RwLock::new(
Supervisor::new(max_concurrent, max_depth).with_max_concurrent_jobs(max_jobs),
)));
}
if let Some(session) = session {
@@ -861,10 +886,15 @@ async fn handle_check(ctx: &mut RequestContext, args: &Value) -> Result<Value> {
Ok(result)
}
None => Ok(json!({
"status": "error",
"message": format!("No agent found with id '{id}'")
})),
None => {
if is_job_task(ctx.supervisor.as_ref(), id) {
return Ok(job_id_teaching_error(id));
}
Ok(json!({
"status": "error",
"message": format!("No agent found with id '{id}'")
}))
}
}
}
@@ -883,6 +913,9 @@ async fn handle_collect(ctx: &mut RequestContext, args: &Value) -> Result<Value>
let target_abort = {
let sup = supervisor.read();
if sup.is_finished(id).is_none() {
if id.starts_with("job_") || sup.has_job(id) {
return Ok(job_id_teaching_error(id));
}
return Ok(json!({
"status": "error",
"message": format!("Agent '{id}' not found. Use agent__check to verify it exists and is finished.")
@@ -1065,10 +1098,15 @@ async fn handle_cancel(ctx: &mut RequestContext, args: &Value) -> Result<Value>
"message": message,
}))
}
None => Ok(json!({
"status": "error",
"message": format!("No agent found with id '{id}'"),
})),
None => {
if is_job_task(ctx.supervisor.as_ref(), id) {
return Ok(job_id_teaching_error(id));
}
Ok(json!({
"status": "error",
"message": format!("No agent found with id '{id}'"),
}))
}
}
}
@@ -1115,10 +1153,17 @@ fn handle_send_message(ctx: &mut RequestContext, args: &Value) -> Result<Value>
"message": format!("Message delivered to agent '{id}'"),
}))
}
None => Ok(json!({
"status": "error",
"message": format!("No agent found with id '{id}'. Agent may not exist or may have already completed."),
})),
None => {
if is_job_task(ctx.supervisor.as_ref(), id)
|| is_job_task(ctx.parent_supervisor.as_ref(), id)
{
return Ok(job_id_teaching_error(id));
}
Ok(json!({
"status": "error",
"message": format!("No agent found with id '{id}'. Agent may not exist or may have already completed."),
}))
}
}
}
@@ -1455,7 +1500,10 @@ mod tests {
use super::*;
use crate::config::test_fixtures::{FixtureServer, fixture_runtime};
use crate::config::{AgentConfig, AppState, WorkingMode};
use crate::function::jobs::RingBuf;
use crate::supervisor::escalation::{EscalationQueue, EscalationRequest};
use crate::supervisor::{JobHandle, JobResult, JobState, JobStatus};
use parking_lot::Mutex;
use serde_json::json;
use serial_test::serial;
@@ -1472,6 +1520,59 @@ mod tests {
ctx
}
fn ctx_with_job_capable_supervisor() -> RequestContext {
let mut ctx = RequestContext::new(default_app_state(), WorkingMode::Cmd);
ctx.supervisor = Some(Arc::new(RwLock::new(
Supervisor::new(4, 3).with_max_concurrent_jobs(4),
)));
ctx
}
fn make_fake_job(id: &str) -> JobHandle {
let rt = tokio::runtime::Runtime::new().unwrap();
let join_handle = rt.spawn(async {
Ok(JobResult {
output: json!(null),
exit_code: Some(0),
output_bytes_captured: 0,
})
});
std::mem::forget(rt);
JobHandle {
id: id.to_string(),
tool: "execute_command".to_string(),
started_at: std::time::Instant::now(),
join_handle,
abort_signal: create_abort_signal(),
state: Arc::new(Mutex::new(JobState {
status: JobStatus::Running,
pgid: None,
})),
output_buf: Arc::new(Mutex::new(RingBuf::default())),
no_change_checks: 0,
}
}
fn register_fake_job(ctx: &mut RequestContext, id: &str) {
ctx.supervisor
.as_ref()
.unwrap()
.write()
.register(make_fake_job(id))
.unwrap();
}
fn assert_job_teaching_error(result: &Value, id: &str) {
assert_eq!(result["status"], "error");
let message = result["message"].as_str().unwrap();
assert_eq!(
message,
format!(
"'{id}' is a background job, not an agent — use job__check / job__collect / job__cancel"
)
);
}
fn register_fake_agent(ctx: &mut RequestContext, id: &str, name: &str) {
register_fake_agent_with_output(ctx, id, name, "fake output");
}
@@ -2533,4 +2634,101 @@ mod tests {
assert_eq!(ctx.pending_agents_guardrail_count, 0);
});
}
#[test]
fn handle_check_registered_job_id_teaches_job_tools() {
run_async(async {
let mut ctx = ctx_with_job_capable_supervisor();
register_fake_job(&mut ctx, "bg_1");
let result = handle_check(&mut ctx, &json!({"id": "bg_1"}))
.await
.unwrap();
assert_job_teaching_error(&result, "bg_1");
});
}
#[test]
fn handle_check_job_prefixed_id_teaches_job_tools() {
run_async(async {
let mut ctx = ctx_with_supervisor(4, 3);
let result = handle_check(&mut ctx, &json!({"id": "job_deadbeef"}))
.await
.unwrap();
assert_job_teaching_error(&result, "job_deadbeef");
});
}
#[test]
fn handle_collect_registered_job_id_teaches_job_tools() {
run_async(async {
let mut ctx = ctx_with_job_capable_supervisor();
register_fake_job(&mut ctx, "bg_1");
let result = handle_collect(&mut ctx, &json!({"id": "bg_1"}))
.await
.unwrap();
assert_job_teaching_error(&result, "bg_1");
});
}
#[test]
fn handle_collect_job_prefixed_id_teaches_job_tools() {
run_async(async {
let mut ctx = ctx_with_supervisor(4, 3);
let result = handle_collect(&mut ctx, &json!({"id": "job_deadbeef"}))
.await
.unwrap();
assert_job_teaching_error(&result, "job_deadbeef");
});
}
#[test]
fn handle_cancel_registered_job_id_teaches_job_tools_and_keeps_job() {
run_async(async {
let mut ctx = ctx_with_job_capable_supervisor();
register_fake_job(&mut ctx, "bg_1");
let result = handle_cancel(&mut ctx, &json!({"id": "bg_1"}))
.await
.unwrap();
assert_job_teaching_error(&result, "bg_1");
assert!(ctx.supervisor.as_ref().unwrap().read().has_job("bg_1"));
});
}
#[test]
fn handle_send_message_registered_job_id_teaches_job_tools() {
run_async(async {
let mut ctx = ctx_with_job_capable_supervisor();
register_fake_job(&mut ctx, "bg_1");
let result =
handle_send_message(&mut ctx, &json!({"id": "bg_1", "message": "hi"})).unwrap();
assert_job_teaching_error(&result, "bg_1");
});
}
#[test]
fn handle_send_message_job_in_parent_supervisor_teaches_job_tools() {
run_async(async {
let mut ctx = RequestContext::new(default_app_state(), WorkingMode::Cmd);
let mut parent_sup = Supervisor::new(4, 3).with_max_concurrent_jobs(4);
parent_sup.register(make_fake_job("bg_p")).unwrap();
ctx.parent_supervisor = Some(Arc::new(RwLock::new(parent_sup)));
let result =
handle_send_message(&mut ctx, &json!({"id": "bg_p", "message": "hi"})).unwrap();
assert_job_teaching_error(&result, "bg_p");
});
}
}