feat: generalize supervisor registry to TaskHandle enum with job scaffolding, kill discipline, and max_concurrent_jobs config

Implements T1 of plans/background-jobs-design.md (§6, R7/R8/R9):

- Supervisor.handles is now HashMap<String, TaskHandle> where
  TaskHandle = Agent(AgentHandle) | Job(JobHandle); agent-facing
  accessors (active_count, effective_active_count, is_finished, take,
  inbox, abort_signal_for, list_agents) match only Agent variants,
  preserving all existing external behavior byte-for-byte.
- New JobHandle/JobState/JobStatus/JobResult types with pgid-guarded
  process-group kill discipline: Drop and cancel_all/cancel_recursive
  kill the group only while state.pgid is still set (pid-reuse guard),
  via libc::killpg on unix and JoinHandle::abort elsewhere.
- Per-kind job capacity: Supervisor carries max_concurrent_jobs
  (builder-set, default 0); job registration rejects at capacity.
- Cross-kind teaching errors at the four agent-lookup miss sites
  (agent__check/collect/cancel/send_message) when the id is a
  registered job or job_-prefixed; genuinely-unknown ids keep their
  existing messages.
- Supervisor init condition is now can_spawn_agents || jobs_enabled in
  use_agent and both child-agent spawn paths, with agent capacity 0 in
  jobs-only contexts; use_agent cancels the old supervisor recursively
  before replacing it.
- max_concurrent_jobs config plumbing: global Config field, AgentConfig
  override + accessor, all four AppConfig touch points including the
  COYOTE_MAX_CONCURRENT_JOBS env override; shared
  effective_max_concurrent_jobs/jobs_enabled predicates
  (agent override -> global -> default 5; 0 disables).
- Stage dependency-free RingBuf (64 KiB default) in src/function/jobs.rs
  for the upcoming job output pump.
- New sanctioned dependency: libc 0.2 under cfg(unix).

src/supervisor/mod.rs

@@ -2,16 +2,19 @@ pub mod escalation;
 pub mod mailbox;
 pub mod taskqueue;
 
+use crate::function::jobs::RingBuf;
 use crate::utils::AbortSignal;
 use fmt::{Debug, Formatter};
 use mailbox::Inbox;
-use parking_lot::RwLock;
+use parking_lot::{Mutex, RwLock};
 use taskqueue::TaskQueue;
 
 use anyhow::{Result, bail};
+use serde_json::Value;
 use std::collections::HashMap;
 use std::fmt;
 use std::sync::Arc;
+use std::time::Instant;
 use tokio::task::JoinHandle;
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -37,11 +40,85 @@ pub struct AgentHandle {
     pub child_supervisor: Option<Arc<RwLock<Supervisor>>>,
 }
 
+#[allow(dead_code)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum JobStatus {
+    Running,
+    Completed,
+    Failed,
+}
+
+pub struct JobState {
+    #[allow(dead_code)]
+    pub status: JobStatus,
+    pub pgid: Option<i32>,
+}
+
+#[allow(dead_code)]
+pub struct JobResult {
+    pub output: Value,
+    pub exit_code: Option<i32>,
+    pub output_bytes_captured: u64,
+}
+
+pub struct JobHandle {
+    pub id: String,
+    #[allow(dead_code)]
+    pub tool: String,
+    #[allow(dead_code)]
+    pub started_at: Instant,
+    pub join_handle: JoinHandle<Result<JobResult>>,
+    pub abort_signal: AbortSignal,
+    pub state: Arc<Mutex<JobState>>,
+    #[allow(dead_code)]
+    pub output_buf: Arc<Mutex<RingBuf>>,
+    #[allow(dead_code)]
+    pub no_change_checks: u32,
+}
+
+impl JobHandle {
+    // pgid == child pid under process_group(0); after wait() reaps the child
+    // the pid can be recycled, so never kill unless pgid is still set.
+    fn kill_process_group(&self) {
+        #[cfg(unix)]
+        if let Some(pgid) = self.state.lock().pgid {
+            unsafe {
+                libc::killpg(pgid, libc::SIGTERM);
+            }
+        }
+    }
+}
+
+impl Drop for JobHandle {
+    fn drop(&mut self) {
+        self.kill_process_group();
+        self.join_handle.abort();
+    }
+}
+
+pub enum TaskHandle {
+    Agent(AgentHandle),
+    Job(JobHandle),
+}
+
+impl From<AgentHandle> for TaskHandle {
+    fn from(handle: AgentHandle) -> Self {
+        Self::Agent(handle)
+    }
+}
+
+impl From<JobHandle> for TaskHandle {
+    fn from(handle: JobHandle) -> Self {
+        Self::Job(handle)
+    }
+}
+
 pub struct Supervisor {
-    handles: HashMap<String, AgentHandle>,
+    handles: HashMap<String, TaskHandle>,
     task_queue: TaskQueue,
     max_concurrent: usize,
     max_depth: usize,
+    max_concurrent_jobs: usize,
 }
 
 impl Supervisor {
@@ -51,17 +128,43 @@ impl Supervisor {
             task_queue: TaskQueue::new(),
             max_concurrent,
             max_depth,
+            max_concurrent_jobs: 0,
         }
     }
 
+    pub fn with_max_concurrent_jobs(mut self, max_concurrent_jobs: usize) -> Self {
+        self.max_concurrent_jobs = max_concurrent_jobs;
+        self
+    }
+
+    fn agent(&self, id: &str) -> Option<&AgentHandle> {
+        match self.handles.get(id) {
+            Some(TaskHandle::Agent(handle)) => Some(handle),
+            _ => None,
+        }
+    }
+
+    fn agents(&self) -> impl Iterator<Item = &AgentHandle> {
+        self.handles.values().filter_map(|handle| match handle {
+            TaskHandle::Agent(handle) => Some(handle),
+            TaskHandle::Job(_) => None,
+        })
+    }
+
     pub fn active_count(&self) -> usize {
-        self.handles.len()
+        self.agents().count()
     }
 
     pub fn effective_active_count(&self) -> usize {
+        self.agents()
+            .filter(|h| !h.join_handle.is_finished())
+            .count()
+    }
+
+    pub fn active_job_count(&self) -> usize {
         self.handles
             .values()
-            .filter(|h| !h.join_handle.is_finished())
+            .filter(|h| matches!(h, TaskHandle::Job(job) if !job.join_handle.is_finished()))
             .count()
     }
 
@@ -73,6 +176,11 @@ impl Supervisor {
         self.max_depth
     }
 
+    #[allow(dead_code)]
+    pub fn max_concurrent_jobs(&self) -> usize {
+        self.max_concurrent_jobs
+    }
+
     pub fn task_queue(&self) -> &TaskQueue {
         &self.task_queue
     }
@@ -81,59 +189,111 @@ impl Supervisor {
         &mut self.task_queue
     }
 
-    pub fn register(&mut self, handle: AgentHandle) -> Result<()> {
-        if self.effective_active_count() >= self.max_concurrent {
-            bail!(
-                "Cannot spawn agent: at capacity ({}/{})",
-                self.effective_active_count(),
-                self.max_concurrent
-            );
+    pub fn register(&mut self, handle: impl Into<TaskHandle>) -> Result<()> {
+        match handle.into() {
+            TaskHandle::Agent(handle) => {
+                if self.effective_active_count() >= self.max_concurrent {
+                    bail!(
+                        "Cannot spawn agent: at capacity ({}/{})",
+                        self.effective_active_count(),
+                        self.max_concurrent
+                    );
+                }
+                if handle.depth > self.max_depth {
+                    bail!(
+                        "Cannot spawn agent: max depth exceeded ({}/{})",
+                        handle.depth,
+                        self.max_depth
+                    );
+                }
+                self.handles
+                    .insert(handle.id.clone(), TaskHandle::Agent(handle));
+            }
+            TaskHandle::Job(handle) => {
+                if self.active_job_count() >= self.max_concurrent_jobs {
+                    bail!(
+                        "Cannot start job: at capacity ({}/{})",
+                        self.active_job_count(),
+                        self.max_concurrent_jobs
+                    );
+                }
+                self.handles
+                    .insert(handle.id.clone(), TaskHandle::Job(handle));
+            }
         }
-        if handle.depth > self.max_depth {
-            bail!(
-                "Cannot spawn agent: max depth exceeded ({}/{})",
-                handle.depth,
-                self.max_depth
-            );
-        }
-        self.handles.insert(handle.id.clone(), handle);
         Ok(())
     }
 
     pub fn is_finished(&self, id: &str) -> Option<bool> {
-        self.handles.get(id).map(|h| h.join_handle.is_finished())
+        self.agent(id).map(|h| h.join_handle.is_finished())
     }
 
     pub fn take(&mut self, id: &str) -> Option<AgentHandle> {
-        self.handles.remove(id)
+        self.agent(id)?;
+        match self.handles.remove(id) {
+            Some(TaskHandle::Agent(handle)) => Some(handle),
+            _ => None,
+        }
+    }
+
+    #[allow(dead_code)]
+    pub fn take_job(&mut self, id: &str) -> Option<JobHandle> {
+        if !self.has_job(id) {
+            return None;
+        }
+        match self.handles.remove(id) {
+            Some(TaskHandle::Job(handle)) => Some(handle),
+            _ => None,
+        }
+    }
+
+    pub fn has_job(&self, id: &str) -> bool {
+        matches!(self.handles.get(id), Some(TaskHandle::Job(_)))
+    }
+
+    pub fn has_agent(&self, id: &str) -> bool {
+        self.agent(id).is_some()
     }
 
     pub fn inbox(&self, id: &str) -> Option<&Arc<Inbox>> {
-        self.handles.get(id).map(|h| &h.inbox)
+        self.agent(id).map(|h| &h.inbox)
     }
 
     pub fn abort_signal_for(&self, id: &str) -> Option<AbortSignal> {
-        self.handles.get(id).map(|h| h.abort_signal.clone())
+        self.agent(id).map(|h| h.abort_signal.clone())
     }
 
     pub fn list_agents(&self) -> Vec<(&str, &str)> {
-        self.handles
-            .values()
+        self.agents()
             .map(|h| (h.id.as_str(), h.agent_name.as_str()))
             .collect()
     }
 
     pub fn cancel_all(&self) {
         for handle in self.handles.values() {
-            handle.abort_signal.set_ctrlc();
+            match handle {
+                TaskHandle::Agent(agent) => agent.abort_signal.set_ctrlc(),
+                TaskHandle::Job(job) => {
+                    job.abort_signal.set_ctrlc();
+                    job.kill_process_group();
+                }
+            }
         }
     }
 
     pub fn cancel_recursive(&self) {
         for handle in self.handles.values() {
-            handle.abort_signal.set_ctrlc();
-            if let Some(child_sup) = handle.child_supervisor.as_ref() {
-                child_sup.read().cancel_recursive();
+            match handle {
+                TaskHandle::Agent(agent) => {
+                    agent.abort_signal.set_ctrlc();
+                    if let Some(child_sup) = agent.child_supervisor.as_ref() {
+                        child_sup.read().cancel_recursive();
+                    }
+                }
+                TaskHandle::Job(job) => {
+                    job.abort_signal.set_ctrlc();
+                    job.kill_process_group();
+                }
             }
         }
     }
@@ -142,7 +302,7 @@ impl Supervisor {
 impl Debug for Supervisor {
     fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
         f.debug_struct("Supervisor")
-            .field("active_agents", &self.handles.len())
+            .field("active_agents", &self.active_count())
             .field("max_concurrent", &self.max_concurrent)
             .field("max_depth", &self.max_depth)
             .finish()
@@ -177,6 +337,33 @@ mod tests {
         }
     }
 
+    fn make_job(id: &str, abort_signal: AbortSignal) -> JobHandle {
+        // Keep the runtime alive so the spawned task is never polled and the
+        // job counts as running for capacity checks.
+        let rt = Builder::new_current_thread().enable_all().build().unwrap();
+        let join_handle = rt.spawn(async {
+            Ok(JobResult {
+                output: Value::Null,
+                exit_code: Some(0),
+                output_bytes_captured: 0,
+            })
+        });
+        std::mem::forget(rt);
+        JobHandle {
+            id: id.to_string(),
+            tool: "execute_command".to_string(),
+            started_at: Instant::now(),
+            join_handle,
+            abort_signal,
+            state: Arc::new(Mutex::new(JobState {
+                status: JobStatus::Running,
+                pgid: None,
+            })),
+            output_buf: Arc::new(Mutex::new(RingBuf::default())),
+            no_change_checks: 0,
+        }
+    }
+
     #[test]
     fn supervisor_new_empty() {
         let sup = Supervisor::new(4, 3);
@@ -315,4 +502,82 @@ mod tests {
         assert!(parent_sig.aborted());
         assert!(child_sig.aborted());
     }
+
+    #[test]
+    fn job_registration_rejects_when_job_capacity_zero() {
+        let mut sup = Supervisor::new(4, 3);
+        let result = sup.register(make_job("j1", create_abort_signal()));
+        assert!(result.is_err());
+        assert!(result.unwrap_err().to_string().contains("at capacity"));
+    }
+
+    #[test]
+    fn job_registration_rejects_at_job_capacity() {
+        let mut sup = Supervisor::new(4, 3).with_max_concurrent_jobs(1);
+        sup.register(make_job("j1", create_abort_signal())).unwrap();
+        let result = sup.register(make_job("j2", create_abort_signal()));
+        assert!(result.is_err());
+        assert!(result.unwrap_err().to_string().contains("at capacity"));
+    }
+
+    #[test]
+    fn job_capacity_is_independent_of_agent_capacity() {
+        let mut sup = Supervisor::new(1, 3).with_max_concurrent_jobs(1);
+        sup.register(make_job("j1", create_abort_signal())).unwrap();
+        sup.register(make_handle("a1", "explore", 1)).unwrap();
+        assert_eq!(sup.active_job_count(), 1);
+        assert_eq!(sup.active_count(), 1);
+        assert_eq!(sup.max_concurrent_jobs(), 1);
+    }
+
+    #[test]
+    fn agent_accessors_ignore_jobs() {
+        let mut sup = Supervisor::new(4, 3).with_max_concurrent_jobs(2);
+        sup.register(make_job("j1", create_abort_signal())).unwrap();
+
+        assert_eq!(sup.active_count(), 0);
+        assert_eq!(sup.effective_active_count(), 0);
+        assert!(sup.list_agents().is_empty());
+        assert_eq!(sup.is_finished("j1"), None);
+        assert!(sup.inbox("j1").is_none());
+        assert!(sup.abort_signal_for("j1").is_none());
+        assert!(sup.take("j1").is_none());
+        assert!(sup.has_job("j1"));
+        assert!(!sup.has_agent("j1"));
+        assert_eq!(sup.active_job_count(), 1);
+    }
+
+    #[test]
+    fn take_job_removes_job_but_not_agents() {
+        let mut sup = Supervisor::new(4, 3).with_max_concurrent_jobs(2);
+        sup.register(make_job("j1", create_abort_signal())).unwrap();
+        sup.register(make_handle("a1", "explore", 1)).unwrap();
+
+        assert!(sup.take_job("a1").is_none());
+        assert!(sup.has_agent("a1"));
+        assert!(sup.take_job("j1").is_some());
+        assert_eq!(sup.active_job_count(), 0);
+    }
+
+    #[test]
+    fn cancel_recursive_aborts_jobs() {
+        let sig = create_abort_signal();
+        let mut sup = Supervisor::new(4, 3).with_max_concurrent_jobs(1);
+        sup.register(make_job("j1", sig.clone())).unwrap();
+
+        sup.cancel_recursive();
+
+        assert!(sig.aborted());
+    }
+
+    #[test]
+    fn cancel_all_aborts_jobs() {
+        let sig = create_abort_signal();
+        let mut sup = Supervisor::new(4, 3).with_max_concurrent_jobs(1);
+        sup.register(make_job("j1", sig.clone())).unwrap();
+
+        sup.cancel_all();
+
+        assert!(sig.aborted());
+    }
 }