test(graph): prove per-attempt timeout budget and cover extraction-fault text
agent.rs: the retry-after-timeout test ran its second attempt with no
timeout bound, so it passed whether or not the retry owned a fresh
budget. Attempt 2 now runs under the same 1s bounded_attempt and sleeps
900ms under the paused clock; it only succeeds if the deadline is
per-attempt rather than shared with the attempt that already used it.
llm.rs: the `LLM node structured-extraction failed: {e:#}` branch had no
test because `run` cannot succeed without a model. Split the post-run
half of execute into `finish(node, state, ctx, result)` (byte-identical
body, mirrors agent.rs's outcome_from seam) and add tests feeding
Ok("not json") through it, with and without a fallback, asserting the
captured text prefix and the extractor's full context chain.
This commit is contained in:
2 files changed
+114
-27
No files matched your search
+8
-2
@@ -473,8 +473,11 @@ mod tests {
|
||||
assert!(chain.contains("connection error"), "{chain}");
|
||||
}
|
||||
|
||||
/// Attempt 2 is bounded by the same 1s wall clock as attempt 1 and
|
||||
/// sleeps for most of it; it can only succeed if the retry owns a fresh
|
||||
/// budget rather than the deadline attempt 1 already exhausted.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn run_with_retries_retries_a_timeout_and_succeeds() {
|
||||
async fn run_with_retries_gives_each_attempt_a_fresh_timeout_budget() {
|
||||
let mut ctx = plain_ctx();
|
||||
let node = retryable_node(2);
|
||||
let mut attempts = 0u32;
|
||||
@@ -493,7 +496,10 @@ mod tests {
|
||||
std::future::pending::<Result<String>>(),
|
||||
))
|
||||
} else {
|
||||
boxed_attempt(async move { Ok("recovered".to_string()) })
|
||||
boxed_attempt(bounded_attempt("test_agent", 1, async move {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(900)).await;
|
||||
Ok("recovered".to_string())
|
||||
}))
|
||||
}
|
||||
}),
|
||||
)
|
||||
|
||||
+106
-25
@@ -36,34 +36,45 @@ impl LlmNodeExecutor {
|
||||
abort: &AbortSignal,
|
||||
) -> Result<LlmExecutionOutcome> {
|
||||
let result = run(node_id, node, state_manager, parent_ctx, abort).await;
|
||||
let (output, failure_reason) = match result {
|
||||
Ok(raw) => match &node.output_schema {
|
||||
Some(schema) => match structured::extract(&raw, schema, parent_ctx).await {
|
||||
Ok(value) => (value, None),
|
||||
Err(e) => {
|
||||
warn!("llm node structured extraction failed: {e:#}");
|
||||
(
|
||||
Value::String(format!("LLM node structured-extraction failed: {e:#}")),
|
||||
Some(format!("structured-extraction failed: {e:#}")),
|
||||
)
|
||||
}
|
||||
},
|
||||
None => (Value::String(raw), None),
|
||||
},
|
||||
Err(e) => {
|
||||
warn!("llm node failed: {e:#}");
|
||||
(
|
||||
Value::String(format!("LLM node failed: {e:#}")),
|
||||
Some(format!("LLM call failed: {e:#}")),
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
apply_state_updates_with_output(node, state_manager, &output);
|
||||
outcome_from(failure_reason.as_deref(), node.fallback.as_deref())
|
||||
finish(node, state_manager, parent_ctx, result).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Extraction, state_updates, and the outcome for an already-finished run.
|
||||
/// Split from `execute` so tests can feed a raw run result without a model.
|
||||
async fn finish(
|
||||
node: &LlmNode,
|
||||
state_manager: &mut StateManager,
|
||||
parent_ctx: &mut RequestContext,
|
||||
result: Result<String>,
|
||||
) -> Result<LlmExecutionOutcome> {
|
||||
let (output, failure_reason) = match result {
|
||||
Ok(raw) => match &node.output_schema {
|
||||
Some(schema) => match structured::extract(&raw, schema, parent_ctx).await {
|
||||
Ok(value) => (value, None),
|
||||
Err(e) => {
|
||||
warn!("llm node structured extraction failed: {e:#}");
|
||||
(
|
||||
Value::String(format!("LLM node structured-extraction failed: {e:#}")),
|
||||
Some(format!("structured-extraction failed: {e:#}")),
|
||||
)
|
||||
}
|
||||
},
|
||||
None => (Value::String(raw), None),
|
||||
},
|
||||
Err(e) => {
|
||||
warn!("llm node failed: {e:#}");
|
||||
(
|
||||
Value::String(format!("LLM node failed: {e:#}")),
|
||||
Some(format!("LLM call failed: {e:#}")),
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
apply_state_updates_with_output(node, state_manager, &output);
|
||||
outcome_from(failure_reason.as_deref(), node.fallback.as_deref())
|
||||
}
|
||||
|
||||
fn outcome_from(
|
||||
failure_reason: Option<&str>,
|
||||
fallback: Option<&str>,
|
||||
@@ -852,4 +863,74 @@ mod tests {
|
||||
.to_string();
|
||||
assert!(captured.starts_with("LLM node failed: "), "{captured}");
|
||||
}
|
||||
|
||||
/// A run that succeeds but yields unparseable output must fault through
|
||||
/// the structured-extraction branch with the extractor's full chain, so
|
||||
/// fallback scripts can tell "model spoke prose" from "model call failed".
|
||||
#[tokio::test]
|
||||
async fn structured_extraction_failure_text_carries_full_error_chain() {
|
||||
let mut u = HashMap::new();
|
||||
u.insert("captured".into(), "{{output}}".into());
|
||||
let mut node = node_with_schema(Some(u), json!({"type": "object"}));
|
||||
node.fallback = Some("fb".into());
|
||||
let mut state = manager_with(&[]);
|
||||
let mut ctx = RequestContext::new(Arc::new(AppState::test_default()), WorkingMode::Cmd);
|
||||
ctx.agent = Some(Agent::test_new(AgentConfig::default()));
|
||||
|
||||
let outcome = finish(&node, &mut state, &mut ctx, Ok("not json".to_string()))
|
||||
.await
|
||||
.expect("a declared fallback turns the failure into a route");
|
||||
|
||||
assert_eq!(outcome, LlmExecutionOutcome::FellBack("fb".into()));
|
||||
let captured = state
|
||||
.state()
|
||||
.get("captured")
|
||||
.and_then(Value::as_str)
|
||||
.expect("captured failure text is a string")
|
||||
.to_string();
|
||||
assert!(
|
||||
captured.starts_with("LLM node structured-extraction failed: "),
|
||||
"{captured}"
|
||||
);
|
||||
assert!(
|
||||
captured.contains("Structured-output extractor LLM call failed: "),
|
||||
"{captured}"
|
||||
);
|
||||
assert!(captured.contains("Invalid model"), "{captured}");
|
||||
}
|
||||
|
||||
/// Without a fallback the extraction fault bails, wrapping the same
|
||||
/// extractor chain the state_updates text carries.
|
||||
#[tokio::test]
|
||||
async fn structured_extraction_failure_without_fallback_bails_with_full_chain() {
|
||||
let mut u = HashMap::new();
|
||||
u.insert("captured".into(), "{{output}}".into());
|
||||
let mut node = node_with_schema(Some(u), json!({"type": "object"}));
|
||||
node.fallback = None;
|
||||
let mut state = manager_with(&[]);
|
||||
let mut ctx = RequestContext::new(Arc::new(AppState::test_default()), WorkingMode::Cmd);
|
||||
ctx.agent = Some(Agent::test_new(AgentConfig::default()));
|
||||
|
||||
let err = finish(&node, &mut state, &mut ctx, Ok("not json".to_string()))
|
||||
.await
|
||||
.expect_err("no fallback means the failure bails");
|
||||
|
||||
let chain = format!("{err:#}");
|
||||
assert!(
|
||||
chain.contains(
|
||||
"LLM node failed and no fallback declared: structured-extraction failed: Structured-output extractor LLM call failed"
|
||||
),
|
||||
"{chain}"
|
||||
);
|
||||
let captured = state
|
||||
.state()
|
||||
.get("captured")
|
||||
.and_then(Value::as_str)
|
||||
.expect("state_updates still run before the bail")
|
||||
.to_string();
|
||||
assert!(
|
||||
captured.starts_with("LLM node structured-extraction failed: "),
|
||||
"{captured}"
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user