feat(TASK-018): review-gauntlet lane degradation + verification passthrough
- lane_fault.py + fallback on all four run_* lane nodes: an agent-node failure is normalized into a PIPELINE-FAULT lane result instead of killing the graph - verdict_gate.py: PIPELINE-FAULT lane results block distinctly from SKIPPED and missing-sentinel; a PIPELINE-FAULT in signals_error blocks at the gauntlet level - build_items.py: restructured into main() + fail-closed guard (crash emits empty lanes + PIPELINE-FAULT) and folds lanes_degraded into lanes_summary - parse: state_updates/fallback -> parse_fault.py, which records the fault in signals_error and jumps to the verdict gate - select_lanes: state_updates/fallback -> default_lanes.py, a deterministic wider-never-narrower lane fallback - verification_commands threaded through parse into run_adversary's prompt (declared-only, never invented) - run_probe prompt gains the aborted-attempt reconcile line - 8 gauntlet_* R5 tests in src/config/request_context.rs (script harness + graph interpolation)
This commit is contained in:
7 files changed
+544
-21
No files matched your search
@@ -39,6 +39,7 @@ initial_state:
|
|||||||
security_posture: standard
|
security_posture: standard
|
||||||
probe_context: ''
|
probe_context: ''
|
||||||
extra_context: ''
|
extra_context: ''
|
||||||
|
verification_commands: []
|
||||||
forced_lanes: []
|
forced_lanes: []
|
||||||
changed_files: []
|
changed_files: []
|
||||||
file_count: 0
|
file_count: 0
|
||||||
@@ -56,6 +57,9 @@ initial_state:
|
|||||||
add_security: false
|
add_security: false
|
||||||
add_probe: false
|
add_probe: false
|
||||||
lane_reasons: ''
|
lane_reasons: ''
|
||||||
|
parse_failure: ''
|
||||||
|
select_lanes_failure: ''
|
||||||
|
lanes_degraded: ''
|
||||||
code_review_items: []
|
code_review_items: []
|
||||||
adversary_items: []
|
adversary_items: []
|
||||||
security_items: []
|
security_items: []
|
||||||
@@ -100,6 +104,10 @@ nodes:
|
|||||||
names). Empty array = automatic selection.
|
names). Empty array = automatic selection.
|
||||||
- `extra_context`: any remaining caller context reviewers should see,
|
- `extra_context`: any remaining caller context reviewers should see,
|
||||||
condensed.
|
condensed.
|
||||||
|
- `verification_commands`: the exact build/test/lint shell commands the
|
||||||
|
caller explicitly declares for verification (verbatim strings, in the
|
||||||
|
order given). Empty array when the caller declares none — NEVER invent,
|
||||||
|
guess, or auto-detect commands; undeclared stays undeclared.
|
||||||
prompt: |
|
prompt: |
|
||||||
Extract the review-request facts from this request:
|
Extract the review-request facts from this request:
|
||||||
|
|
||||||
@@ -126,11 +134,26 @@ nodes:
|
|||||||
type: string
|
type: string
|
||||||
project_dir_in:
|
project_dir_in:
|
||||||
type: string
|
type: string
|
||||||
required: [diff_spec, plan_context, rigor, security_posture, probe_context, forced_lanes, extra_context, project_dir_in]
|
verification_commands:
|
||||||
|
type: array
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
required: [diff_spec, plan_context, rigor, security_posture, probe_context, forced_lanes, extra_context, project_dir_in, verification_commands]
|
||||||
max_iterations: 3
|
max_iterations: 3
|
||||||
max_attempts: 2
|
max_attempts: 2
|
||||||
|
state_updates:
|
||||||
|
parse_failure: '{{output}}'
|
||||||
|
fallback: parse_fault
|
||||||
next: signals
|
next: signals
|
||||||
|
|
||||||
|
parse_fault:
|
||||||
|
id: parse_fault
|
||||||
|
type: script
|
||||||
|
description: Fail-closed marker for a dead parse stage (reached only via parse's fallback) — records a PIPELINE-FAULT in signals_error and jumps straight to the verdict gate, which blocks on it. Lane item lists stay their initial [] so every lane records SKIPPED.
|
||||||
|
script: scripts/parse_fault.py
|
||||||
|
timeout: 30
|
||||||
|
next: verdict_gate
|
||||||
|
|
||||||
signals:
|
signals:
|
||||||
id: signals
|
id: signals
|
||||||
type: script
|
type: script
|
||||||
@@ -204,6 +227,17 @@ nodes:
|
|||||||
required: [add_code_review, add_adversary, add_security, add_probe, lane_reasons]
|
required: [add_code_review, add_adversary, add_security, add_probe, lane_reasons]
|
||||||
max_iterations: 3
|
max_iterations: 3
|
||||||
max_attempts: 2
|
max_attempts: 2
|
||||||
|
state_updates:
|
||||||
|
select_lanes_failure: '{{output}}'
|
||||||
|
fallback: default_lanes
|
||||||
|
next: build_items
|
||||||
|
|
||||||
|
default_lanes:
|
||||||
|
id: default_lanes
|
||||||
|
type: script
|
||||||
|
description: Deterministic lane-selection fallback (reached only via select_lanes' fallback) — forces code-review + adversary, plus probe when consumer_surface, when the additive LLM judgment dies. Selection degrades WIDER, never narrower; the degradation note flows into lanes_summary via build_items.
|
||||||
|
script: scripts/default_lanes.py
|
||||||
|
timeout: 30
|
||||||
next: build_items
|
next: build_items
|
||||||
|
|
||||||
build_items:
|
build_items:
|
||||||
@@ -279,6 +313,7 @@ nodes:
|
|||||||
max_attempts: 2
|
max_attempts: 2
|
||||||
state_updates:
|
state_updates:
|
||||||
lane_out: '{{output}}'
|
lane_out: '{{output}}'
|
||||||
|
fallback: lane_fault
|
||||||
|
|
||||||
run_adversary:
|
run_adversary:
|
||||||
id: run_adversary
|
id: run_adversary
|
||||||
@@ -294,12 +329,15 @@ nodes:
|
|||||||
The spec / plan / acceptance criteria it must satisfy:
|
The spec / plan / acceptance criteria it must satisfy:
|
||||||
{{plan_context}}
|
{{plan_context}}
|
||||||
|
|
||||||
|
Verification commands (declared by the caller; empty = none declared): {{verification_commands}}
|
||||||
|
|
||||||
{{extra_context}}
|
{{extra_context}}
|
||||||
|
|
||||||
Return your ADVERSARIAL_REVIEW verdict.
|
Return your ADVERSARIAL_REVIEW verdict.
|
||||||
max_attempts: 2
|
max_attempts: 2
|
||||||
state_updates:
|
state_updates:
|
||||||
lane_out: '{{output}}'
|
lane_out: '{{output}}'
|
||||||
|
fallback: lane_fault
|
||||||
|
|
||||||
run_security:
|
run_security:
|
||||||
id: run_security
|
id: run_security
|
||||||
@@ -318,6 +356,7 @@ nodes:
|
|||||||
max_attempts: 2
|
max_attempts: 2
|
||||||
state_updates:
|
state_updates:
|
||||||
lane_out: '{{output}}'
|
lane_out: '{{output}}'
|
||||||
|
fallback: lane_fault
|
||||||
|
|
||||||
run_probe:
|
run_probe:
|
||||||
id: run_probe
|
id: run_probe
|
||||||
@@ -335,12 +374,23 @@ nodes:
|
|||||||
Plan / acceptance-criteria context:
|
Plan / acceptance-criteria context:
|
||||||
{{plan_context}}
|
{{plan_context}}
|
||||||
|
|
||||||
|
The workspace may contain test files from an aborted earlier attempt of
|
||||||
|
this lane; reconcile rather than duplicate.
|
||||||
|
|
||||||
Derive expected behaviors from the plan/contract BEFORE reading
|
Derive expected behaviors from the plan/contract BEFORE reading
|
||||||
implementation code, run existing suites first, then author spec-first
|
implementation code, run existing suites first, then author spec-first
|
||||||
tests for the uncovered delta. Return your USAGE_PROBE verdict.
|
tests for the uncovered delta. Return your USAGE_PROBE verdict.
|
||||||
max_attempts: 2
|
max_attempts: 2
|
||||||
state_updates:
|
state_updates:
|
||||||
lane_out: '{{output}}'
|
lane_out: '{{output}}'
|
||||||
|
fallback: lane_fault
|
||||||
|
|
||||||
|
lane_fault:
|
||||||
|
id: lane_fault
|
||||||
|
type: script
|
||||||
|
description: Fail-closed lane fault marker (reached only via the run_* lanes' fallback routes; shared across the four map branches). Normalizes the engine's "Agent node failed" text in lane_out into a PIPELINE-FAULT lane result the map collects; verdict_gate turns it into BLOCKED — a degraded lane is never a pass.
|
||||||
|
script: scripts/lane_fault.py
|
||||||
|
timeout: 30
|
||||||
|
|
||||||
verdict_gate:
|
verdict_gate:
|
||||||
id: verdict_gate
|
id: verdict_gate
|
||||||
|
|||||||
@@ -8,20 +8,15 @@ skip mechanism — no dynamic routing anywhere.
|
|||||||
|
|
||||||
The LLM's add_* flags can only ADD lanes. A flag that is false never
|
The LLM's add_* flags can only ADD lanes. A flag that is false never
|
||||||
removes a lane the hard rules selected.
|
removes a lane the hard rules selected.
|
||||||
|
|
||||||
|
Fail-closed: a builder crash emits all four lists empty plus a
|
||||||
|
PIPELINE-FAULT in signals_error — verdict_gate blocks on it and the crash
|
||||||
|
never kills the graph.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
|
||||||
def load_state():
|
|
||||||
if path := os.environ.get("GRAPH_STATE_FILE"):
|
|
||||||
with open(path) as f:
|
|
||||||
return json.load(f)
|
|
||||||
return json.loads(os.environ.get("GRAPH_STATE", "{}"))
|
|
||||||
|
|
||||||
|
|
||||||
state = load_state()
|
|
||||||
|
|
||||||
ALIASES = {
|
ALIASES = {
|
||||||
"code-review": "code-review",
|
"code-review": "code-review",
|
||||||
"code_review": "code-review",
|
"code_review": "code-review",
|
||||||
@@ -39,22 +34,31 @@ ALIASES = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def text(key):
|
def load_state():
|
||||||
|
if path := os.environ.get("GRAPH_STATE_FILE"):
|
||||||
|
with open(path) as f:
|
||||||
|
return json.load(f)
|
||||||
|
return json.loads(os.environ.get("GRAPH_STATE", "{}"))
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
state = load_state()
|
||||||
|
|
||||||
|
def text(key):
|
||||||
v = state.get(key)
|
v = state.get(key)
|
||||||
return v.strip() if isinstance(v, str) else ""
|
return v.strip() if isinstance(v, str) else ""
|
||||||
|
|
||||||
|
forced_raw = state.get("forced_lanes") or []
|
||||||
forced_raw = state.get("forced_lanes") or []
|
forced, unknown = [], []
|
||||||
forced, unknown = [], []
|
for lane in forced_raw:
|
||||||
for lane in forced_raw:
|
|
||||||
canon = ALIASES.get(str(lane).strip().lower())
|
canon = ALIASES.get(str(lane).strip().lower())
|
||||||
(forced.append(canon) if canon else unknown.append(str(lane)))
|
(forced.append(canon) if canon else unknown.append(str(lane)))
|
||||||
|
|
||||||
reasons = []
|
reasons = []
|
||||||
if forced:
|
if forced:
|
||||||
lanes = set(forced)
|
lanes = set(forced)
|
||||||
reasons.append(f"forced lanes honored exactly: {sorted(lanes)}")
|
reasons.append(f"forced lanes honored exactly: {sorted(lanes)}")
|
||||||
else:
|
else:
|
||||||
lanes = {"code-review"}
|
lanes = {"code-review"}
|
||||||
reasons.append(
|
reasons.append(
|
||||||
"code-review: always on — the gauntlet is only spawned for non-trivial work"
|
"code-review: always on — the gauntlet is only spawned for non-trivial work"
|
||||||
@@ -102,10 +106,17 @@ else:
|
|||||||
lanes.add(lane)
|
lanes.add(lane)
|
||||||
reasons.append(f"{lane}: added by lane-selection judgment — {llm_why}")
|
reasons.append(f"{lane}: added by lane-selection judgment — {llm_why}")
|
||||||
|
|
||||||
if unknown:
|
if unknown:
|
||||||
reasons.append(f"ignored unknown forced lane name(s): {unknown}")
|
reasons.append(f"ignored unknown forced lane name(s): {unknown}")
|
||||||
|
|
||||||
print(
|
# A degraded lane selection (default_lanes fallback) must survive into the
|
||||||
|
# final report; lanes_summary is the only channel this script doesn't
|
||||||
|
# overwrite silently.
|
||||||
|
degraded = text("lanes_degraded")
|
||||||
|
if degraded:
|
||||||
|
reasons.insert(0, degraded)
|
||||||
|
|
||||||
|
print(
|
||||||
json.dumps(
|
json.dumps(
|
||||||
{
|
{
|
||||||
"code_review_items": [{"lane": "code-review"}] if "code-review" in lanes else [],
|
"code_review_items": [{"lane": "code-review"}] if "code-review" in lanes else [],
|
||||||
@@ -115,4 +126,21 @@ print(
|
|||||||
"lanes_summary": "\n".join(f"- {r}" for r in reasons),
|
"lanes_summary": "\n".join(f"- {r}" for r in reasons),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
main()
|
||||||
|
except Exception as e: # noqa: BLE001 — a builder crash must never kill the graph
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"code_review_items": [],
|
||||||
|
"adversary_items": [],
|
||||||
|
"security_items": [],
|
||||||
|
"probe_items": [],
|
||||||
|
"lanes_summary": "",
|
||||||
|
"signals_error": f"PIPELINE-FAULT: lane builder crashed — {e}; no lanes were run",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Deterministic lane-selection fallback for the review gauntlet.
|
||||||
|
|
||||||
|
Reached ONLY via select_lanes' `fallback:` route, when the additive LLM
|
||||||
|
judgment dies after retries. Emits `forced_lanes` — code-review + adversary,
|
||||||
|
plus probe when the deterministic signals flagged a consumer-facing
|
||||||
|
surface — which build_items honors exactly, so selection degrades WIDER,
|
||||||
|
never narrower. The degradation note goes into `lanes_degraded`; build_items
|
||||||
|
folds it into lanes_summary so the final report names the degradation.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
|
||||||
|
MAX_DETAIL_CHARS = 500
|
||||||
|
|
||||||
|
|
||||||
|
def load_state():
|
||||||
|
if path := os.environ.get("GRAPH_STATE_FILE"):
|
||||||
|
with open(path) as f:
|
||||||
|
return json.load(f)
|
||||||
|
return json.loads(os.environ.get("GRAPH_STATE", "{}"))
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
state = load_state()
|
||||||
|
forced = ["code-review", "adversary"]
|
||||||
|
if state.get("consumer_surface"):
|
||||||
|
forced.append("probe")
|
||||||
|
raw = state.get("select_lanes_failure")
|
||||||
|
detail = raw.strip().replace("\n", " ") if isinstance(raw, str) else ""
|
||||||
|
if len(detail) > MAX_DETAIL_CHARS:
|
||||||
|
detail = detail[: MAX_DETAIL_CHARS - 1] + "…"
|
||||||
|
note = "lane selection degraded to deterministic defaults"
|
||||||
|
if detail:
|
||||||
|
note += f" — {detail}"
|
||||||
|
print(json.dumps({"forced_lanes": forced, "lanes_degraded": note}))
|
||||||
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
main()
|
||||||
|
except Exception as e: # noqa: BLE001 — the fallback itself must never crash
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"forced_lanes": ["code-review", "adversary"],
|
||||||
|
"lanes_degraded": "lane selection degraded to deterministic "
|
||||||
|
f"defaults (fallback script error: {e})",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Fail-closed lane fault marker for the review gauntlet.
|
||||||
|
|
||||||
|
Reached ONLY via the run_* lane nodes' `fallback:` routes inside their map
|
||||||
|
branches. Grounded in engine behavior: on agent-node failure the engine
|
||||||
|
writes "Agent node failed: {e:#}" into `lane_out` via the node's
|
||||||
|
state_updates BEFORE routing to the fallback, so the raw failure text is
|
||||||
|
already in state. This node normalizes it into a
|
||||||
|
"PIPELINE-FAULT: <lane> lane failed after retries — …" marker that the map
|
||||||
|
collects as the lane result; verdict_gate turns any PIPELINE-FAULT lane
|
||||||
|
result into BLOCKED — a degraded lane can never read as a pass.
|
||||||
|
|
||||||
|
Lane identity comes from `lane_ctx` (the map item build_items emitted, which
|
||||||
|
carries a `lane` field); a missing/malformed item degrades to "unknown".
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
|
||||||
|
MAX_DETAIL_CHARS = 500
|
||||||
|
|
||||||
|
|
||||||
|
def load_state():
|
||||||
|
if path := os.environ.get("GRAPH_STATE_FILE"):
|
||||||
|
with open(path) as f:
|
||||||
|
return json.load(f)
|
||||||
|
return json.loads(os.environ.get("GRAPH_STATE", "{}"))
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
state = load_state()
|
||||||
|
lane = "unknown"
|
||||||
|
lane_ctx = state.get("lane_ctx")
|
||||||
|
if isinstance(lane_ctx, dict):
|
||||||
|
name = lane_ctx.get("lane")
|
||||||
|
if isinstance(name, str) and name.strip():
|
||||||
|
lane = name.strip()
|
||||||
|
raw = state.get("lane_out")
|
||||||
|
detail = raw.strip().replace("\n", " ") if isinstance(raw, str) else ""
|
||||||
|
if len(detail) > MAX_DETAIL_CHARS:
|
||||||
|
detail = detail[: MAX_DETAIL_CHARS - 1] + "…"
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{"lane_out": f"PIPELINE-FAULT: {lane} lane failed after retries — {detail}"}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
main()
|
||||||
|
except Exception as e: # noqa: BLE001 — the fault marker itself must never crash
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"lane_out": "PIPELINE-FAULT: lane failed after retries — "
|
||||||
|
f"fault-marker script error: {e}"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Fail-closed marker for a dead parse stage of the review gauntlet.
|
||||||
|
|
||||||
|
Reached ONLY via parse's `fallback:` route. parse's state_updates wrote an
|
||||||
|
"LLM node …failed: <chain>" string into `parse_failure` before routing here;
|
||||||
|
this node records the failure as a PIPELINE-FAULT in `signals_error` and the
|
||||||
|
graph jumps straight to the verdict gate. Every lane item list is still its
|
||||||
|
initial [] so every lane records SKIPPED, and verdict_gate's signals_error
|
||||||
|
rule makes the verdict BLOCKED — a gauntlet that could not even parse its
|
||||||
|
request never reads as a pass.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
|
||||||
|
MAX_DETAIL_CHARS = 500
|
||||||
|
|
||||||
|
|
||||||
|
def load_state():
|
||||||
|
if path := os.environ.get("GRAPH_STATE_FILE"):
|
||||||
|
with open(path) as f:
|
||||||
|
return json.load(f)
|
||||||
|
return json.loads(os.environ.get("GRAPH_STATE", "{}"))
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
state = load_state()
|
||||||
|
raw = state.get("parse_failure")
|
||||||
|
detail = raw.strip().replace("\n", " ") if isinstance(raw, str) else ""
|
||||||
|
if len(detail) > MAX_DETAIL_CHARS:
|
||||||
|
detail = detail[: MAX_DETAIL_CHARS - 1] + "…"
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"signals_error": "PIPELINE-FAULT: parse failed — cannot run the "
|
||||||
|
f"gauntlet: {detail}"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
main()
|
||||||
|
except Exception as e: # noqa: BLE001 — the fault marker itself must never crash
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"signals_error": "PIPELINE-FAULT: parse failed — cannot run the "
|
||||||
|
f"gauntlet: fault-marker script error: {e}"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
@@ -5,6 +5,9 @@ Parses each configured lane's verdict SENTINEL with a regex — no LLM
|
|||||||
re-reads, no judgment. The gate's hard rules:
|
re-reads, no judgment. The gate's hard rules:
|
||||||
|
|
||||||
- a missing sentinel is a LANE FAILURE and blocks — never a pass;
|
- a missing sentinel is a LANE FAILURE and blocks — never a pass;
|
||||||
|
- a PIPELINE-FAULT lane result (the lane's agent died after retries) or a
|
||||||
|
PIPELINE-FAULT in signals_error (builder/parse fault) blocks — a
|
||||||
|
degraded pipeline is never a pass;
|
||||||
- any 🔴 finding in the code-review report blocks regardless of the
|
- any 🔴 finding in the code-review report blocks regardless of the
|
||||||
lane's own verdict line;
|
lane's own verdict line;
|
||||||
- NEEDS-HUMAN without 🔴 passes but is surfaced as attention required;
|
- NEEDS-HUMAN without 🔴 passes but is surfaced as attention required;
|
||||||
@@ -40,10 +43,26 @@ def main():
|
|||||||
def record(lane, status, detail=""):
|
def record(lane, status, detail=""):
|
||||||
lane_lines.append(f"| {lane} | {status} | {detail} |")
|
lane_lines.append(f"| {lane} | {status} | {detail} |")
|
||||||
|
|
||||||
|
def fault_excerpt(report):
|
||||||
|
line = next(
|
||||||
|
(ln for ln in report.splitlines() if "PIPELINE-FAULT:" in ln), report
|
||||||
|
)
|
||||||
|
return line.strip()[:200]
|
||||||
|
|
||||||
|
def lane_fault_blocker(lane, report):
|
||||||
|
blockers.append(
|
||||||
|
f"{lane}: PIPELINE-FAULT — the lane failed after retries and produced "
|
||||||
|
f"no verdict; a degraded lane is never a pass ({fault_excerpt(report)})"
|
||||||
|
)
|
||||||
|
record(lane, "BLOCKED", "PIPELINE-FAULT (lane failed)")
|
||||||
|
|
||||||
# --- code-review ------------------------------------------------------
|
# --- code-review ------------------------------------------------------
|
||||||
cr = lane_report("code_review_results")
|
cr = lane_report("code_review_results")
|
||||||
if cr is None:
|
if cr is None:
|
||||||
record("code-review", "SKIPPED", "not selected")
|
record("code-review", "SKIPPED", "not selected")
|
||||||
|
elif "PIPELINE-FAULT:" in cr:
|
||||||
|
lane_fault_blocker("code-review", cr)
|
||||||
|
reports.append(("code-review", cr))
|
||||||
else:
|
else:
|
||||||
reports.append(("code-review", cr))
|
reports.append(("code-review", cr))
|
||||||
m = re.search(r"Verdict:\**\s*\**\s*(MERGE-READY|NEEDS-HUMAN)", cr)
|
m = re.search(r"Verdict:\**\s*\**\s*(MERGE-READY|NEEDS-HUMAN)", cr)
|
||||||
@@ -72,6 +91,9 @@ def main():
|
|||||||
adv = lane_report("adversary_results")
|
adv = lane_report("adversary_results")
|
||||||
if adv is None:
|
if adv is None:
|
||||||
record("adversary", "SKIPPED", "not selected")
|
record("adversary", "SKIPPED", "not selected")
|
||||||
|
elif "PIPELINE-FAULT:" in adv:
|
||||||
|
lane_fault_blocker("adversary", adv)
|
||||||
|
reports.append(("adversary", adv))
|
||||||
else:
|
else:
|
||||||
reports.append(("adversary", adv))
|
reports.append(("adversary", adv))
|
||||||
m = re.search(r"ADVERSARIAL_REVIEW:\s*(CONFORMS|DIVERGES)", adv)
|
m = re.search(r"ADVERSARIAL_REVIEW:\s*(CONFORMS|DIVERGES)", adv)
|
||||||
@@ -92,6 +114,9 @@ def main():
|
|||||||
sec = lane_report("security_results")
|
sec = lane_report("security_results")
|
||||||
if sec is None:
|
if sec is None:
|
||||||
record("security", "SKIPPED", "not selected")
|
record("security", "SKIPPED", "not selected")
|
||||||
|
elif "PIPELINE-FAULT:" in sec:
|
||||||
|
lane_fault_blocker("security", sec)
|
||||||
|
reports.append(("security", sec))
|
||||||
else:
|
else:
|
||||||
reports.append(("security", sec))
|
reports.append(("security", sec))
|
||||||
m = re.search(r"SECURITY_REVIEW:\s*(PASS|FAIL)", sec)
|
m = re.search(r"SECURITY_REVIEW:\s*(PASS|FAIL)", sec)
|
||||||
@@ -110,6 +135,9 @@ def main():
|
|||||||
pb = lane_report("probe_results")
|
pb = lane_report("probe_results")
|
||||||
if pb is None:
|
if pb is None:
|
||||||
record("probe", "SKIPPED", "not selected")
|
record("probe", "SKIPPED", "not selected")
|
||||||
|
elif "PIPELINE-FAULT:" in pb:
|
||||||
|
lane_fault_blocker("probe", pb)
|
||||||
|
reports.append(("probe", pb))
|
||||||
else:
|
else:
|
||||||
reports.append(("probe", pb))
|
reports.append(("probe", pb))
|
||||||
m = re.search(r"USAGE_PROBE:\s*(PASS|FAIL|INCONCLUSIVE)", pb)
|
m = re.search(r"USAGE_PROBE:\s*(PASS|FAIL|INCONCLUSIVE)", pb)
|
||||||
@@ -131,6 +159,12 @@ def main():
|
|||||||
)
|
)
|
||||||
record("probe", "BLOCKED", "missing verdict sentinel")
|
record("probe", "BLOCKED", "missing verdict sentinel")
|
||||||
|
|
||||||
|
# A gauntlet-level fault (build_items crash, dead parse stage) means the
|
||||||
|
# lanes above were never built — SKIPPED rows alone must not read as PASS.
|
||||||
|
signals_error = state.get("signals_error")
|
||||||
|
if isinstance(signals_error, str) and "PIPELINE-FAULT:" in signals_error:
|
||||||
|
blockers.append(f"pipeline: {signals_error}")
|
||||||
|
|
||||||
verdict = "BLOCKED" if blockers else "PASS"
|
verdict = "BLOCKED" if blockers else "PASS"
|
||||||
|
|
||||||
md = "\n".join(
|
md = "\n".join(
|
||||||
|
|||||||
@@ -8762,6 +8762,255 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---- review-gauntlet suite-script regression tests (R5) ----
|
||||||
|
//
|
||||||
|
// Degradation paths of the gauntlet's lane/builder/gate scripts,
|
||||||
|
// exercised the same way as the adversary suite above: `python3 <script>`
|
||||||
|
// with a synthetic GRAPH_STATE env.
|
||||||
|
|
||||||
|
fn run_gauntlet_script(script: &str, state: &serde_json::Value) -> serde_json::Value {
|
||||||
|
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||||
|
.join("assets/agents/review-gauntlet/scripts")
|
||||||
|
.join(script);
|
||||||
|
let out = std::process::Command::new("python3")
|
||||||
|
.arg(&path)
|
||||||
|
.env("GRAPH_STATE", state.to_string())
|
||||||
|
.output()
|
||||||
|
.unwrap_or_else(|e| panic!("failed to invoke python3 {script}: {e}"));
|
||||||
|
assert!(
|
||||||
|
out.status.success(),
|
||||||
|
"{script} exited nonzero: {}",
|
||||||
|
String::from_utf8_lossy(&out.stderr)
|
||||||
|
);
|
||||||
|
serde_json::from_slice(&out.stdout).unwrap_or_else(|e| {
|
||||||
|
panic!(
|
||||||
|
"{script} stdout is not JSON ({e}): {}",
|
||||||
|
String::from_utf8_lossy(&out.stdout)
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn gauntlet_verdict_gate_blocks_lane_fault_distinctly() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// One run exercising all three failure classes at once — a faulted
|
||||||
|
// lane, an unselected lane, and a lane that returned no sentinel —
|
||||||
|
// each must be reported distinctly.
|
||||||
|
let state = json!({
|
||||||
|
"code_review_results": ["...**Verdict: MERGE-READY**..."],
|
||||||
|
"adversary_results": ["PIPELINE-FAULT: adversary lane failed after retries — Agent node failed: boom"],
|
||||||
|
"security_results": [],
|
||||||
|
"probe_results": ["some text with no sentinel"]
|
||||||
|
});
|
||||||
|
let out = run_gauntlet_script("verdict_gate.py", &state);
|
||||||
|
assert_eq!(out["gauntlet_verdict"], "BLOCKED");
|
||||||
|
let report = out["gauntlet_report"].as_str().unwrap();
|
||||||
|
assert!(
|
||||||
|
report.contains("adversary: PIPELINE-FAULT — the lane failed after retries"),
|
||||||
|
"the faulted lane must be named as a blocker: {report}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
report.contains("| adversary | BLOCKED | PIPELINE-FAULT (lane failed) |"),
|
||||||
|
"the lane table must carry the fault detail: {report}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
report.contains("| security | SKIPPED | not selected |"),
|
||||||
|
"an unselected lane must stay SKIPPED: {report}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
report.contains("| probe | BLOCKED | missing verdict sentinel |"),
|
||||||
|
"a sentinel-less lane must stay a missing-sentinel failure: {report}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn gauntlet_verdict_gate_signals_error_fault_blocks() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let state = json!({
|
||||||
|
"code_review_results": [],
|
||||||
|
"adversary_results": [],
|
||||||
|
"security_results": [],
|
||||||
|
"probe_results": [],
|
||||||
|
"signals_error": "PIPELINE-FAULT: lane builder crashed — boom; no lanes were run"
|
||||||
|
});
|
||||||
|
let out = run_gauntlet_script("verdict_gate.py", &state);
|
||||||
|
assert_eq!(out["gauntlet_verdict"], "BLOCKED");
|
||||||
|
let report = out["gauntlet_report"].as_str().unwrap();
|
||||||
|
assert!(
|
||||||
|
report.contains(
|
||||||
|
"## Blockers\n- pipeline: PIPELINE-FAULT: lane builder crashed — boom; no lanes were run"
|
||||||
|
),
|
||||||
|
"the pipeline fault must appear in the Blockers section: {report}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn gauntlet_verdict_gate_happy_path_regression() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let state = json!({
|
||||||
|
"code_review_results": ["**Verdict: MERGE-READY**"],
|
||||||
|
"adversary_results": ["ADVERSARIAL_REVIEW: CONFORMS"],
|
||||||
|
"security_results": [],
|
||||||
|
"probe_results": []
|
||||||
|
});
|
||||||
|
let out = run_gauntlet_script("verdict_gate.py", &state);
|
||||||
|
assert_eq!(
|
||||||
|
out["gauntlet_verdict"], "PASS",
|
||||||
|
"non-degraded semantics must be unchanged: {out}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn gauntlet_build_items_crash_fails_closed() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// A truthy non-iterable forced_lanes raises inside main; the guard
|
||||||
|
// must exit 0 (asserted in the helper) with empty lanes and a
|
||||||
|
// PIPELINE-FAULT for the gate.
|
||||||
|
let out = run_gauntlet_script("build_items.py", &json!({"forced_lanes": 42}));
|
||||||
|
for key in [
|
||||||
|
"code_review_items",
|
||||||
|
"adversary_items",
|
||||||
|
"security_items",
|
||||||
|
"probe_items",
|
||||||
|
] {
|
||||||
|
assert_eq!(
|
||||||
|
out[key],
|
||||||
|
json!([]),
|
||||||
|
"{key} must be empty on a builder crash: {out}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
out["signals_error"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.contains("PIPELINE-FAULT: lane builder crashed"),
|
||||||
|
"the crash must surface as a PIPELINE-FAULT: {out}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn gauntlet_build_items_folds_degradation_note_into_summary() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let state = json!({
|
||||||
|
"forced_lanes": ["code-review", "adversary"],
|
||||||
|
"lanes_degraded": "lane selection degraded to deterministic defaults"
|
||||||
|
});
|
||||||
|
let out = run_gauntlet_script("build_items.py", &state);
|
||||||
|
let summary = out["lanes_summary"].as_str().unwrap();
|
||||||
|
assert!(
|
||||||
|
summary.contains("lane selection degraded to deterministic defaults"),
|
||||||
|
"the degradation note must survive into lanes_summary: {summary}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
summary.contains("forced lanes honored exactly"),
|
||||||
|
"the forced-lanes reason must still be recorded: {summary}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn gauntlet_lane_fault_normalizes_engine_failure_text() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let out = run_gauntlet_script(
|
||||||
|
"lane_fault.py",
|
||||||
|
&json!({"lane_ctx": {"lane": "security"}, "lane_out": "Agent node failed: kaboom"}),
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
out["lane_out"],
|
||||||
|
"PIPELINE-FAULT: security lane failed after retries — Agent node failed: kaboom"
|
||||||
|
);
|
||||||
|
|
||||||
|
let out = run_gauntlet_script(
|
||||||
|
"lane_fault.py",
|
||||||
|
&json!({"lane_out": "Agent node failed: kaboom"}),
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
out["lane_out"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.starts_with("PIPELINE-FAULT: unknown lane failed after retries"),
|
||||||
|
"a missing lane_ctx must degrade to the unknown lane: {out}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn gauntlet_default_lanes_is_deterministic() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let out = run_gauntlet_script("default_lanes.py", &json!({}));
|
||||||
|
assert_eq!(out["forced_lanes"], json!(["code-review", "adversary"]));
|
||||||
|
assert!(
|
||||||
|
out["lanes_degraded"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.contains("deterministic defaults"),
|
||||||
|
"the degradation note must name the deterministic defaults: {out}"
|
||||||
|
);
|
||||||
|
|
||||||
|
let probe = run_gauntlet_script("default_lanes.py", &json!({"consumer_surface": true}));
|
||||||
|
assert_eq!(
|
||||||
|
probe["forced_lanes"],
|
||||||
|
json!(["code-review", "adversary", "probe"]),
|
||||||
|
"a consumer surface must widen selection to include probe: {probe}"
|
||||||
|
);
|
||||||
|
|
||||||
|
let again = run_gauntlet_script("default_lanes.py", &json!({}));
|
||||||
|
assert_eq!(out, again, "same input must produce identical output");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn gauntlet_lane_prompts_carry_passthroughs() {
|
||||||
|
use crate::graph::{GraphParser, NodeType};
|
||||||
|
let dir =
|
||||||
|
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("assets/agents/review-gauntlet");
|
||||||
|
let graph = GraphParser::new(&dir)
|
||||||
|
.load_from_file(dir.join("graph.yaml"))
|
||||||
|
.expect("review-gauntlet graph.yaml must parse");
|
||||||
|
let node = graph
|
||||||
|
.nodes
|
||||||
|
.get("run_adversary")
|
||||||
|
.expect("review-gauntlet graph must have a run_adversary node");
|
||||||
|
let NodeType::Agent(adv) = &node.node_type else {
|
||||||
|
panic!("run_adversary must be an agent node");
|
||||||
|
};
|
||||||
|
assert!(
|
||||||
|
adv.prompt.contains("{{verification_commands}}"),
|
||||||
|
"run_adversary's prompt must pass the declared verification commands through: {}",
|
||||||
|
adv.prompt
|
||||||
|
);
|
||||||
|
let node = graph
|
||||||
|
.nodes
|
||||||
|
.get("run_probe")
|
||||||
|
.expect("review-gauntlet graph must have a run_probe node");
|
||||||
|
let NodeType::Agent(probe) = &node.node_type else {
|
||||||
|
panic!("run_probe must be an agent node");
|
||||||
|
};
|
||||||
|
assert!(
|
||||||
|
probe.prompt.contains("reconcile rather than duplicate"),
|
||||||
|
"run_probe's prompt must carry the reconcile line: {}",
|
||||||
|
probe.prompt
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
#[serial]
|
||||||
fn install_functions_force_preserves_user_mcp_json() {
|
fn install_functions_force_preserves_user_mcp_json() {
|
||||||
|
|||||||
Reference in new issue
Block a user