feat(TASK-029): review-gauntlet fault-only lane re-run loop
Give the review gauntlet a deterministic retry loop for lanes that FAULT (PIPELINE-FAULT marker, missing verdict sentinel, or a nested graph's own degraded-run wording) without ever re-running a lane that reached a real verdict. - graph.yaml: the four lane maps route to a new `retry_gate` script node (static `next: verdict_gate`; emits `_next: build_items` to retry). initial_state gains lane_attempts / kept_results / retry_lanes / review_incomplete / gauntlet_started_at. settings.timeout 24000 -> 69600 (3 passes x 2 attempts x 11400 + 1200 margin). - scripts/retry_gate.py (new): stashes every lane report that ran (completed and faulted alike) and restores reports a re-run pass overwrote with []; classifies with verdict_gate.py's sentinel regexes verbatim; counts attempts; retries while attempts < 3 and inside a 45600 s wall-clock budget; otherwise records the lane in review_incomplete and falls through. Crash guard appends to signals_error and never retries. - scripts/signals.py: stamps gauntlet_started_at before the git work so a failed diff still starts the clock. - scripts/build_items.py: retry pass emits items only for retry_lanes (with attempt numbers), adds a `retried:` note to lanes_summary, resets retry_lanes; falsy-guarded stamp fallback; crash guard appends to signals_error and preserves lanes_summary. - Tests: hermetic python3-guarded script tests for retry_gate / build_items / signals, graph wiring + initial_state pins, sentinel and LANE_KEYS cross-pins, retry-budget envelope vs settings.timeout, a first-pass golden payload, and an engine-level executor test proving a script `_next` back-edge re-fires a static fan-out and an empty map re-run clears its collect_into. Plan: PLAN-review-retry section A items 1-3 and 5. Verdict/report surface (section A items 4 and 6) is TASK-032.
This commit is contained in:
6 files changed
+976
-14
No files matched your search
@@ -9,6 +9,10 @@ skip mechanism — no dynamic routing anywhere.
|
||||
The LLM's add_* flags can only ADD lanes. A flag that is false never
|
||||
removes a lane the hard rules selected.
|
||||
|
||||
A re-run pass (retry_gate emitted `retry_lanes`) skips selection: only the
|
||||
named lanes get an item, stamped with their attempt number, and every other
|
||||
lane maps over nothing — retry_gate restores their earlier reports.
|
||||
|
||||
Fail-closed: a builder crash emits all four lists empty plus a
|
||||
PIPELINE-FAULT in signals_error — verdict_gate blocks on it and the crash
|
||||
never kills the graph.
|
||||
@@ -16,6 +20,7 @@ never kills the graph.
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
# Keep in sync with default_lanes.ALIASES — a caller-forced lane must
|
||||
# canonicalize the same way on the degraded path as it does here.
|
||||
@@ -35,6 +40,18 @@ ALIASES = {
|
||||
"usage-pattern-testing": "probe",
|
||||
}
|
||||
|
||||
# Keep in sync with retry_gate.LANES
|
||||
LANE_KEYS = {
|
||||
"code-review": "code_review_items",
|
||||
"adversary": "adversary_items",
|
||||
"security": "security_items",
|
||||
"probe": "probe_items",
|
||||
}
|
||||
|
||||
# Filled by main() so the crash guard can append to an upstream fault
|
||||
# instead of overwriting it, and keep the first pass's lanes_summary.
|
||||
crash_context = {"signals_error": "", "lanes_summary": ""}
|
||||
|
||||
|
||||
def load_state():
|
||||
if path := os.environ.get("GRAPH_STATE_FILE"):
|
||||
@@ -45,11 +62,38 @@ def load_state():
|
||||
|
||||
def main():
|
||||
state = load_state()
|
||||
prior = state.get("signals_error")
|
||||
crash_context["signals_error"] = prior if isinstance(prior, str) else ""
|
||||
# retry_gate's wall-clock budget is measured from this stamp; signals.py
|
||||
# is the normal stamper, this is the defensive fallback.
|
||||
stamp = {} if state.get("gauntlet_started_at") else {"gauntlet_started_at": time.time()}
|
||||
|
||||
def text(key):
|
||||
v = state.get(key)
|
||||
return v.strip() if isinstance(v, str) else ""
|
||||
|
||||
crash_context["lanes_summary"] = text("lanes_summary")
|
||||
retry_lanes = [lane for lane in (state.get("retry_lanes") or []) if lane in LANE_KEYS]
|
||||
if retry_lanes:
|
||||
attempts = state.get("lane_attempts") or {}
|
||||
retried = [(lane, int(attempts.get(lane) or 0) + 1) for lane in retry_lanes]
|
||||
note = "- retried: [" + ", ".join(f"{lane} (attempt {n})" for lane, n in retried) + "]"
|
||||
summary = text("lanes_summary")
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
**{key: [] for key in LANE_KEYS.values()},
|
||||
**{
|
||||
LANE_KEYS[lane]: [{"lane": lane, "attempt": n}] for lane, n in retried
|
||||
},
|
||||
"lanes_summary": f"{summary}\n{note}" if summary else note,
|
||||
"retry_lanes": [],
|
||||
**stamp,
|
||||
}
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
forced_raw = state.get("forced_lanes") or []
|
||||
forced, unknown = [], []
|
||||
for lane in forced_raw:
|
||||
@@ -126,6 +170,7 @@ def main():
|
||||
"security_items": [{"lane": "security"}] if "security" in lanes else [],
|
||||
"probe_items": [{"lane": "probe"}] if "probe" in lanes else [],
|
||||
"lanes_summary": "\n".join(f"- {r}" for r in reasons),
|
||||
**stamp,
|
||||
}
|
||||
)
|
||||
)
|
||||
@@ -134,6 +179,9 @@ def main():
|
||||
try:
|
||||
main()
|
||||
except Exception as e: # noqa: BLE001 — a builder crash must never kill the graph
|
||||
fault = f"PIPELINE-FAULT: lane builder crashed — {e}; no lanes were run"
|
||||
if crash_context["signals_error"]:
|
||||
fault = f"{crash_context['signals_error']}; {fault}"
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
@@ -141,8 +189,8 @@ except Exception as e: # noqa: BLE001 — a builder crash must never kill the g
|
||||
"adversary_items": [],
|
||||
"security_items": [],
|
||||
"probe_items": [],
|
||||
"lanes_summary": "",
|
||||
"signals_error": f"PIPELINE-FAULT: lane builder crashed — {e}; no lanes were run",
|
||||
"lanes_summary": crash_context["lanes_summary"],
|
||||
"signals_error": fault,
|
||||
}
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,159 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deterministic fault-only re-run gate for the review gauntlet.
|
||||
|
||||
Sits between the four lane maps and the verdict gate. Every lane map runs
|
||||
on every pass — a lane that is not being re-run maps over [] — so a re-run
|
||||
pass overwrites the untouched lanes' collected results with []. The gate's
|
||||
hard rules:
|
||||
|
||||
- every lane that ran this pass is stashed in kept_results, completed
|
||||
and faulted alike, and a lane whose map ran nothing this pass gets
|
||||
its stashed report restored — the verdict gate always sees each
|
||||
lane's LAST report;
|
||||
- a lane is FAULTED only when its report is missing, non-string, the
|
||||
lane_fault.py marker, sentinel-less, or carries the nested graph's own
|
||||
degraded-run wording; a real verdict (DIVERGES, FAIL, INCONCLUSIVE,
|
||||
NEEDS-HUMAN, …) is never re-run — the review's judgment stands;
|
||||
- a faulted lane is re-run only while it has attempts left AND the
|
||||
gauntlet is inside its wall-clock budget; otherwise it is recorded in
|
||||
review_incomplete and its fault flows to the verdict gate as-is;
|
||||
- the gate itself must never retry by accident: any internal error
|
||||
records a PIPELINE-FAULT in signals_error and falls through to the
|
||||
verdict gate, which blocks on it.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
MAX_RETRY_ELAPSED_SECS = 45600
|
||||
MAX_ATTEMPTS = 3
|
||||
|
||||
# lane name -> (items key, results key)
|
||||
LANES = {
|
||||
"code-review": ("code_review_items", "code_review_results"),
|
||||
"adversary": ("adversary_items", "adversary_results"),
|
||||
"security": ("security_items", "security_results"),
|
||||
"probe": ("probe_items", "probe_results"),
|
||||
}
|
||||
|
||||
# Must stay identical to verdict_gate.py's sentinel regexes: a lane this
|
||||
# gate calls completed must be one the verdict gate can parse, or a drift
|
||||
# would re-run lanes the verdict gate accepts (or accept lanes it cannot
|
||||
# read). A cross-pin test compares the literals in both files.
|
||||
SENTINELS = {
|
||||
"code-review": r"Verdict:\**\s*\**\s*(MERGE-READY|NEEDS-HUMAN)",
|
||||
"adversary": r"ADVERSARIAL_REVIEW:\s*(CONFORMS|DIVERGES)",
|
||||
"security": r"SECURITY_REVIEW:\s*(PASS|FAIL)",
|
||||
"probe": r"USAGE_PROBE:\s*(PASS|FAIL|INCONCLUSIVE)",
|
||||
}
|
||||
|
||||
# Filled by main() so the crash guard can append to an upstream fault
|
||||
# instead of overwriting it, and report the lanes it had already declined.
|
||||
crash_context = {"signals_error": "", "review_incomplete": []}
|
||||
|
||||
|
||||
def load_state():
|
||||
if path := os.environ.get("GRAPH_STATE_FILE"):
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
return json.loads(os.environ.get("GRAPH_STATE", "{}"))
|
||||
|
||||
|
||||
def nested_degraded(lane, report):
|
||||
# The nested graphs complete with a sentinel even when a stage inside
|
||||
# them died; their own degraded-run wording marks a partial review,
|
||||
# which is not a completed one. Same anchoring as verdict_gate.py.
|
||||
if lane == "code-review":
|
||||
verdict_line = re.search(r"^\*\*Verdict: .*$", report, re.MULTILINE)
|
||||
return bool(
|
||||
verdict_line
|
||||
and re.search(
|
||||
r"^\*\*Verdict: NEEDS-HUMAN\*\* — .*"
|
||||
r"(pipeline fault\(s\) recorded|PIPELINE-FAULT:|report rendering error)",
|
||||
verdict_line.group(0),
|
||||
)
|
||||
)
|
||||
if lane == "adversary":
|
||||
return bool(
|
||||
re.search(
|
||||
r"^Criteria: .*degraded run: pipeline fault recorded",
|
||||
report,
|
||||
re.MULTILINE,
|
||||
)
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def is_faulted(lane, report):
|
||||
if not isinstance(report, str):
|
||||
return True
|
||||
if report.strip().startswith("PIPELINE-FAULT:"):
|
||||
return True
|
||||
if not re.search(SENTINELS[lane], report):
|
||||
return True
|
||||
return nested_degraded(lane, report)
|
||||
|
||||
|
||||
def main():
|
||||
state = load_state()
|
||||
prior = state.get("signals_error")
|
||||
crash_context["signals_error"] = prior if isinstance(prior, str) else ""
|
||||
incomplete = list(state.get("review_incomplete") or [])
|
||||
crash_context["review_incomplete"] = incomplete
|
||||
attempts = dict(state.get("lane_attempts") or {})
|
||||
kept = dict(state.get("kept_results") or {})
|
||||
out = {}
|
||||
faulted = []
|
||||
|
||||
for lane, (items_key, results_key) in LANES.items():
|
||||
results = state.get(results_key) or []
|
||||
if not state.get(items_key):
|
||||
if not results and lane in kept:
|
||||
out[results_key] = [kept[lane]]
|
||||
continue
|
||||
report = results[0] if results else None
|
||||
if report is not None:
|
||||
kept[lane] = report
|
||||
if is_faulted(lane, report):
|
||||
faulted.append(lane)
|
||||
attempts[lane] = int(attempts.get(lane) or 0) + 1
|
||||
|
||||
# A missing/zero stamp reads as an ancient start and declines every
|
||||
# retry — the safe direction when the clock's origin is unknown.
|
||||
started = state.get("gauntlet_started_at") or 0
|
||||
within_budget = (time.time() - float(started)) <= MAX_RETRY_ELAPSED_SECS
|
||||
retry = [lane for lane in faulted if within_budget and attempts[lane] < MAX_ATTEMPTS]
|
||||
declined = [lane for lane in faulted if lane not in retry]
|
||||
incomplete = sorted(set(incomplete) | set(declined))
|
||||
crash_context["review_incomplete"] = incomplete
|
||||
|
||||
out.update(
|
||||
{
|
||||
"lane_attempts": attempts,
|
||||
"kept_results": kept,
|
||||
"retry_lanes": retry,
|
||||
"review_incomplete": incomplete,
|
||||
}
|
||||
)
|
||||
if retry:
|
||||
out["_next"] = "build_items"
|
||||
print(json.dumps(out))
|
||||
|
||||
|
||||
try:
|
||||
main()
|
||||
except Exception as e: # noqa: BLE001 — a gate crash must fall through, never retry
|
||||
fault = f"PIPELINE-FAULT: retry gate crashed — {e}"
|
||||
if crash_context["signals_error"]:
|
||||
fault = f"{crash_context['signals_error']}; {fault}"
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"retry_lanes": [],
|
||||
"review_incomplete": crash_context["review_incomplete"],
|
||||
"signals_error": fault,
|
||||
}
|
||||
)
|
||||
)
|
||||
@@ -12,6 +12,7 @@ import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
def load_state():
|
||||
if path := os.environ.get("GRAPH_STATE_FILE"):
|
||||
@@ -37,6 +38,10 @@ def git(*args):
|
||||
|
||||
|
||||
out = {"project_dir": proj}
|
||||
# retry_gate measures its wall-clock budget from this stamp; it is set
|
||||
# before the git work so a failed diff still starts the clock.
|
||||
if not state.get("gauntlet_started_at"):
|
||||
out["gauntlet_started_at"] = time.time()
|
||||
try:
|
||||
if spec in ("", "worktree"):
|
||||
names = git("diff", "--name-only", "HEAD")
|
||||
|
||||
Reference in new issue
Block a user