feat(TASK-029): review-gauntlet fault-only lane re-run loop
Give the review gauntlet a deterministic retry loop for lanes that FAULT (PIPELINE-FAULT marker, missing verdict sentinel, or a nested graph's own degraded-run wording) without ever re-running a lane that reached a real verdict. - graph.yaml: the four lane maps route to a new `retry_gate` script node (static `next: verdict_gate`; emits `_next: build_items` to retry). initial_state gains lane_attempts / kept_results / retry_lanes / review_incomplete / gauntlet_started_at. settings.timeout 24000 -> 69600 (3 passes x 2 attempts x 11400 + 1200 margin). - scripts/retry_gate.py (new): stashes every lane report that ran (completed and faulted alike) and restores reports a re-run pass overwrote with []; classifies with verdict_gate.py's sentinel regexes verbatim; counts attempts; retries while attempts < 3 and inside a 45600 s wall-clock budget; otherwise records the lane in review_incomplete and falls through. Crash guard appends to signals_error and never retries. - scripts/signals.py: stamps gauntlet_started_at before the git work so a failed diff still starts the clock. - scripts/build_items.py: retry pass emits items only for retry_lanes (with attempt numbers), adds a `retried:` note to lanes_summary, resets retry_lanes; falsy-guarded stamp fallback; crash guard appends to signals_error and preserves lanes_summary. - Tests: hermetic python3-guarded script tests for retry_gate / build_items / signals, graph wiring + initial_state pins, sentinel and LANE_KEYS cross-pins, retry-budget envelope vs settings.timeout, a first-pass golden payload, and an engine-level executor test proving a script `_next` back-edge re-fires a static fan-out and an empty map re-run clears its collect_into. Plan: PLAN-review-retry section A items 1-3 and 5. Verdict/report surface (section A items 4 and 6) is TASK-032.
This commit is contained in:
6 files changed
+976
-14
No files matched your search
@@ -1325,6 +1325,117 @@ nodes:
|
||||
);
|
||||
}
|
||||
|
||||
/// A script `_next` back-edge into a node with a static fan-out must
|
||||
/// re-fire every fan-out target on the next pass, and a map that
|
||||
/// re-runs over zero items must overwrite its `collect_into` with `[]`
|
||||
/// rather than leave the previous pass's results in place.
|
||||
#[tokio::test]
|
||||
async fn script_next_back_edge_refires_static_fan_out_and_empty_map_rerun_clears_collect_into()
|
||||
{
|
||||
if !cmd_available("python3") {
|
||||
eprintln!("skipping: python3 not available");
|
||||
return;
|
||||
}
|
||||
let ws = TestWorkspace::new();
|
||||
ws.write_script(
|
||||
"build.py",
|
||||
r#"#!/usr/bin/env python3
|
||||
import os, json
|
||||
state = json.loads(os.environ.get("GRAPH_STATE", "{}"))
|
||||
if state["pass"] == 0:
|
||||
print(json.dumps({"a_items": [1], "b_items": [1]}))
|
||||
else:
|
||||
print(json.dumps({"a_items": [], "b_items": [2]}))
|
||||
"#,
|
||||
);
|
||||
ws.write_script(
|
||||
"worker.py",
|
||||
r#"#!/usr/bin/env python3
|
||||
import os, json
|
||||
state = json.loads(os.environ.get("GRAPH_STATE", "{}"))
|
||||
print(json.dumps({"output": f"report-{state['item']}"}))
|
||||
"#,
|
||||
);
|
||||
ws.write_script(
|
||||
"gate.py",
|
||||
r#"#!/usr/bin/env python3
|
||||
import os, json
|
||||
state = json.loads(os.environ.get("GRAPH_STATE", "{}"))
|
||||
if state["pass"] == 0:
|
||||
print(json.dumps({"pass": 1, "_next": "build"}))
|
||||
else:
|
||||
print(json.dumps({}))
|
||||
"#,
|
||||
);
|
||||
|
||||
let yaml = r#"
|
||||
name: fan_out_rerun_test
|
||||
start: build
|
||||
initial_state:
|
||||
pass: 0
|
||||
a_items: []
|
||||
b_items: []
|
||||
a_results: []
|
||||
b_results: []
|
||||
nodes:
|
||||
build:
|
||||
type: script
|
||||
script: build.py
|
||||
next: [map_a, map_b]
|
||||
map_a:
|
||||
type: map
|
||||
over: "{{a_items}}"
|
||||
as: item
|
||||
branch: worker
|
||||
collect_into: a_results
|
||||
next: gate
|
||||
map_b:
|
||||
type: map
|
||||
over: "{{b_items}}"
|
||||
as: item
|
||||
branch: worker
|
||||
collect_into: b_results
|
||||
next: gate
|
||||
worker:
|
||||
type: script
|
||||
script: worker.py
|
||||
state_updates: {}
|
||||
timeout: 60
|
||||
gate:
|
||||
type: script
|
||||
script: gate.py
|
||||
next: done
|
||||
done:
|
||||
type: end
|
||||
output: "{{a_results}}|{{b_results}}"
|
||||
"#;
|
||||
let graph: Graph = serde_yaml::from_str(yaml).unwrap();
|
||||
let mut ctx = make_ctx();
|
||||
let abort = create_abort_signal();
|
||||
let result = GraphExecutor::new(graph, &ws.dir)
|
||||
.execute(&mut ctx, abort)
|
||||
.await
|
||||
.unwrap_or_else(|e| panic!("executor failed: {e:#}"));
|
||||
|
||||
let (a, b) = result
|
||||
.split_once('|')
|
||||
.unwrap_or_else(|| panic!("expected `a|b`, got: {result}"));
|
||||
let a: Value =
|
||||
serde_json::from_str(a).unwrap_or_else(|_| panic!("a_results not JSON: {result}"));
|
||||
let b: Value =
|
||||
serde_json::from_str(b).unwrap_or_else(|_| panic!("b_results not JSON: {result}"));
|
||||
assert_eq!(
|
||||
b,
|
||||
serde_json::json!(["report-2"]),
|
||||
"the back-edge into build must re-fire both fan-out targets: {result}"
|
||||
);
|
||||
assert_eq!(
|
||||
a,
|
||||
serde_json::json!([]),
|
||||
"map_a must re-fire on the back-edge pass and clear its collect_into over zero items: {result}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn parallel_branch_error_aborts_super_step() {
|
||||
if !cmd_available("bash") {
|
||||
|
||||
Reference in new issue
Block a user