feat(TASK-017): adversary execution evidence + fail-closed routing
- parse extracts caller-declared verification_commands (schema + initial_state);
never invents commands. parse gains fallback + state_updates capturing the
failure string (TASK-016 fallback-capture warning stays silent).
- new run_checks script node (facts -> run_checks -> map_criteria, timeout
3600): runs each declared command in the project dir (900s per-command
timeout), records exec_results {cmd, exit, duration_s, tail~50}; 'none
declared' / ENVIRONMENT markers instead of guessing or failing the node.
- check_criterion judges execution criteria against {{exec_results}} only:
recorded green run = MET citing it, recorded failure = UNMET, markers =
PARTIAL with an ENVIRONMENT note.
- fail-closed routing (R1/R2): parse/facts fallback -> pipeline_fault marker
script -> verdict; holistic fallback -> verdict with holistic_failure
captured. verdict.py: any PIPELINE-FAULT forces DIVERGES with the fault as
complaint #1, per-criterion results still reported; report gains a
'Verification runs:' section. Non-degraded semantics unchanged.
- R5 rust tests (python3-guarded, local cmd_available): verdict fault path,
holistic-failure fault, CONFORMS regression, run_checks markers + recorded
runs, crit_gate contract regression.
This commit is contained in:
5 files changed
+564
-26
No files matched your search
@@ -12,7 +12,11 @@ description: |
|
|||||||
tests — ADDITIVE-only: it can add complaints, never flip a criterion verdict.
|
tests — ADDITIVE-only: it can add complaints, never flip a criterion verdict.
|
||||||
The final CONFORMS/DIVERGES sentinel is computed by a script: CONFORMS
|
The final CONFORMS/DIVERGES sentinel is computed by a script: CONFORMS
|
||||||
requires every criterion MET and zero holistic complaints. Missing plan =
|
requires every criterion MET and zero holistic complaints. Missing plan =
|
||||||
DIVERGES (fail-closed) naming the missing spec.
|
DIVERGES (fail-closed) naming the missing spec. Caller-declared verification
|
||||||
|
commands are executed ONCE by a deterministic stage and recorded — criterion
|
||||||
|
branches cite the record and never run anything themselves. Any pipeline
|
||||||
|
fault (parse, diff resolution, or holistic failure) lands a PIPELINE-FAULT
|
||||||
|
marker and fails closed into DIVERGES: degraded is never passing.
|
||||||
|
|
||||||
version: '1.0'
|
version: '1.0'
|
||||||
|
|
||||||
@@ -42,12 +46,17 @@ initial_state:
|
|||||||
changed_files: []
|
changed_files: []
|
||||||
diff_text: ''
|
diff_text: ''
|
||||||
diff_note: ''
|
diff_note: ''
|
||||||
|
verification_commands: []
|
||||||
|
exec_results: ''
|
||||||
crit_verdicts: []
|
crit_verdicts: []
|
||||||
extra_complaints: []
|
extra_complaints: []
|
||||||
observations: ''
|
observations: ''
|
||||||
adv_report: ''
|
adv_report: ''
|
||||||
gate_feedback: ''
|
gate_feedback: ''
|
||||||
gate_attempts: 0
|
gate_attempts: 0
|
||||||
|
parse_failure: ''
|
||||||
|
holistic_failure: ''
|
||||||
|
pipeline_faults: []
|
||||||
|
|
||||||
start: parse
|
start: parse
|
||||||
|
|
||||||
@@ -75,6 +84,10 @@ nodes:
|
|||||||
verbatim — the holistic pass checks the diff didn't silently override them.
|
verbatim — the holistic pass checks the diff didn't silently override them.
|
||||||
- `scope_notes`: what the plan says NOT to touch / explicit boundaries.
|
- `scope_notes`: what the plan says NOT to touch / explicit boundaries.
|
||||||
- `extra_context`: anything else conformance-relevant, condensed.
|
- `extra_context`: anything else conformance-relevant, condensed.
|
||||||
|
- `verification_commands`: the exact build/test/lint shell commands the
|
||||||
|
caller explicitly declares for verification (verbatim strings, in the
|
||||||
|
order given). Empty array when the caller declares none — NEVER invent,
|
||||||
|
guess, or auto-detect commands; undeclared stays undeclared.
|
||||||
prompt: |
|
prompt: |
|
||||||
Extract the conformance-review facts from this request:
|
Extract the conformance-review facts from this request:
|
||||||
|
|
||||||
@@ -103,9 +116,16 @@ nodes:
|
|||||||
type: string
|
type: string
|
||||||
project_dir_in:
|
project_dir_in:
|
||||||
type: string
|
type: string
|
||||||
required: [criteria, diff_spec, plan_notes, scope_notes, extra_context, project_dir_in]
|
verification_commands:
|
||||||
|
type: array
|
||||||
|
items:
|
||||||
|
type: string
|
||||||
|
required: [criteria, diff_spec, plan_notes, scope_notes, extra_context, project_dir_in, verification_commands]
|
||||||
max_iterations: 3
|
max_iterations: 3
|
||||||
max_attempts: 2
|
max_attempts: 2
|
||||||
|
state_updates:
|
||||||
|
parse_failure: '{{output}}'
|
||||||
|
fallback: pipeline_fault
|
||||||
next: facts
|
next: facts
|
||||||
|
|
||||||
facts:
|
facts:
|
||||||
@@ -114,6 +134,15 @@ nodes:
|
|||||||
description: Deterministic diff resolution — changed files and the (bounded) diff text every criterion branch verifies against.
|
description: Deterministic diff resolution — changed files and the (bounded) diff text every criterion branch verifies against.
|
||||||
script: scripts/diff_facts.py
|
script: scripts/diff_facts.py
|
||||||
timeout: 120
|
timeout: 120
|
||||||
|
fallback: pipeline_fault
|
||||||
|
next: run_checks
|
||||||
|
|
||||||
|
run_checks:
|
||||||
|
id: run_checks
|
||||||
|
type: script
|
||||||
|
description: Deterministic execution evidence — runs ONLY the caller-declared verification commands in the project dir and records exec_results; criterion branches cite the record and never execute anything. Undeclared commands degrade to a "none declared" marker, runner errors to an ENVIRONMENT marker; never fails the node.
|
||||||
|
script: scripts/run_checks.py
|
||||||
|
timeout: 3600
|
||||||
next: map_criteria
|
next: map_criteria
|
||||||
|
|
||||||
map_criteria:
|
map_criteria:
|
||||||
@@ -158,6 +187,18 @@ nodes:
|
|||||||
values shaped to the criterion's example, TODO/stub presented as done —
|
values shaped to the criterion's example, TODO/stub presented as done —
|
||||||
those are PARTIAL or DIVERGED with the gaming named in the complaint.
|
those are PARTIAL or DIVERGED with the gaming named in the complaint.
|
||||||
|
|
||||||
|
Execution criteria ("tests pass", "lint clean", "build succeeds") are
|
||||||
|
judged ONLY against the recorded verification runs in your prompt — you
|
||||||
|
have no execution tools BY DESIGN and must never claim to have run
|
||||||
|
anything yourself:
|
||||||
|
- a recorded green run (exit 0) of a command covering the criterion is
|
||||||
|
the proving evidence — MET, citing that run (the command + exit 0).
|
||||||
|
- a recorded failing run covering the criterion ⇒ UNMET, citing the
|
||||||
|
recorded exit code and output tail.
|
||||||
|
- "none declared" or an ENVIRONMENT marker ⇒ at best PARTIAL, with an
|
||||||
|
ENVIRONMENT note in the complaint naming the exact command the caller
|
||||||
|
should declare via verification_commands (e.g. "cargo test --all").
|
||||||
|
|
||||||
Output format — respond with ONLY this JSON object, no prose around it:
|
Output format — respond with ONLY this JSON object, no prose around it:
|
||||||
{"id": "<the criterion's id, echoed>",
|
{"id": "<the criterion's id, echoed>",
|
||||||
"status": "MET" | "PARTIAL" | "UNMET" | "DIVERGED",
|
"status": "MET" | "PARTIAL" | "UNMET" | "DIVERGED",
|
||||||
@@ -181,6 +222,9 @@ nodes:
|
|||||||
|
|
||||||
The diff{{diff_note}}:
|
The diff{{diff_note}}:
|
||||||
{{diff_text}}
|
{{diff_text}}
|
||||||
|
|
||||||
|
Verification runs (recorded by the deterministic run_checks stage):
|
||||||
|
{{exec_results}}
|
||||||
{{gate_feedback}}
|
{{gate_feedback}}
|
||||||
tools:
|
tools:
|
||||||
- fs_read
|
- fs_read
|
||||||
@@ -273,12 +317,23 @@ nodes:
|
|||||||
required: [extra_complaints, observations]
|
required: [extra_complaints, observations]
|
||||||
max_iterations: 10
|
max_iterations: 10
|
||||||
max_attempts: 2
|
max_attempts: 2
|
||||||
|
state_updates:
|
||||||
|
holistic_failure: '{{output}}'
|
||||||
|
fallback: verdict
|
||||||
|
next: verdict
|
||||||
|
|
||||||
|
pipeline_fault:
|
||||||
|
id: pipeline_fault
|
||||||
|
type: script
|
||||||
|
description: Fail-closed fault marker (reached only via fallback routes from parse/facts) — records a PIPELINE-FAULT in state and continues to verdict, which turns any fault into DIVERGES. The graph never dies without its sentinel.
|
||||||
|
script: scripts/pipeline_fault.py
|
||||||
|
timeout: 30
|
||||||
next: verdict
|
next: verdict
|
||||||
|
|
||||||
verdict:
|
verdict:
|
||||||
id: verdict
|
id: verdict
|
||||||
type: script
|
type: script
|
||||||
description: Deterministic verdict — CONFORMS requires every criterion MET and zero holistic complaints; no criteria at all fails closed. Assembles the exact sentinel format.
|
description: Deterministic verdict — CONFORMS requires every criterion MET, zero holistic complaints, and zero pipeline faults; no criteria at all fails closed and any recorded PIPELINE-FAULT forces DIVERGES (degraded is never passing). Assembles the exact sentinel format.
|
||||||
script: scripts/verdict.py
|
script: scripts/verdict.py
|
||||||
timeout: 30
|
timeout: 30
|
||||||
next: done
|
next: done
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Fail-closed fault marker for the adversary graph.
|
||||||
|
|
||||||
|
Reached ONLY via `fallback:` routes (parse, facts). Appends a machine-readable
|
||||||
|
"PIPELINE-FAULT:" marker to `pipeline_faults` and continues to `verdict`,
|
||||||
|
which turns any recorded fault into DIVERGES — the graph never dies without
|
||||||
|
its sentinel, and a degraded run can never read as CONFORMS.
|
||||||
|
|
||||||
|
Stage attribution: an llm-node failure writes an "LLM node …failed: <chain>"
|
||||||
|
string into `parse_failure` via parse's `state_updates`; anything else means
|
||||||
|
the deterministic facts (diff resolution) script failed, which leaves no
|
||||||
|
failure text in state by design.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
|
||||||
|
MAX_DETAIL_CHARS = 500
|
||||||
|
|
||||||
|
|
||||||
|
def load_state():
|
||||||
|
if path := os.environ.get("GRAPH_STATE_FILE"):
|
||||||
|
with open(path) as f:
|
||||||
|
return json.load(f)
|
||||||
|
return json.loads(os.environ.get("GRAPH_STATE", "{}"))
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
state = load_state()
|
||||||
|
faults = [f for f in (state.get("pipeline_faults") or []) if isinstance(f, str)]
|
||||||
|
parse_failure = state.get("parse_failure")
|
||||||
|
detail = parse_failure.strip() if isinstance(parse_failure, str) else ""
|
||||||
|
if detail.startswith("LLM node"):
|
||||||
|
detail = detail.replace("\n", " ")
|
||||||
|
if len(detail) > MAX_DETAIL_CHARS:
|
||||||
|
detail = detail[: MAX_DETAIL_CHARS - 1] + "…"
|
||||||
|
faults.append(f"PIPELINE-FAULT: parse failed — cannot review: {detail}")
|
||||||
|
else:
|
||||||
|
faults.append(
|
||||||
|
"PIPELINE-FAULT: diff resolution failed — changed files and diff text "
|
||||||
|
"could not be resolved; criterion verification never ran"
|
||||||
|
)
|
||||||
|
print(json.dumps({"pipeline_faults": faults}))
|
||||||
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
main()
|
||||||
|
except Exception as e: # noqa: BLE001 — the fault marker itself must never crash
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"pipeline_faults": [
|
||||||
|
f"PIPELINE-FAULT: fault-marker script error: {e} — treat the "
|
||||||
|
"run as degraded"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
@@ -0,0 +1,113 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Deterministic execution-evidence recorder for the adversary graph.
|
||||||
|
|
||||||
|
Runs ONLY the caller-declared verification commands (verification_commands)
|
||||||
|
in the project dir and records the outcomes as `exec_results` — a list of
|
||||||
|
{cmd, exit, duration_s, tail} records. Criterion branches cite this record;
|
||||||
|
they never execute anything themselves.
|
||||||
|
|
||||||
|
NO auto-detection: undeclared commands mean a "none declared" marker
|
||||||
|
(fail-visible, not fail-guessed). Any runner error degrades into an
|
||||||
|
ENVIRONMENT marker — this script never fails the node.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
|
||||||
|
PER_COMMAND_TIMEOUT_SECS = 900
|
||||||
|
TAIL_LINES = 50
|
||||||
|
MAX_TAIL_CHARS = 8000
|
||||||
|
|
||||||
|
|
||||||
|
def load_state():
|
||||||
|
if path := os.environ.get("GRAPH_STATE_FILE"):
|
||||||
|
with open(path) as f:
|
||||||
|
return json.load(f)
|
||||||
|
return json.loads(os.environ.get("GRAPH_STATE", "{}"))
|
||||||
|
|
||||||
|
|
||||||
|
def tail_of(text):
|
||||||
|
tail = "\n".join(text.splitlines()[-TAIL_LINES:])
|
||||||
|
if len(tail) > MAX_TAIL_CHARS:
|
||||||
|
tail = "…" + tail[-MAX_TAIL_CHARS:]
|
||||||
|
return tail
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
state = load_state()
|
||||||
|
declared = state.get("verification_commands")
|
||||||
|
cmds = (
|
||||||
|
[c.strip() for c in declared if isinstance(c, str) and c.strip()]
|
||||||
|
if isinstance(declared, list)
|
||||||
|
else []
|
||||||
|
)
|
||||||
|
if not cmds:
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"exec_results": (
|
||||||
|
"none declared — the caller supplied no verification_commands; "
|
||||||
|
"execution criteria (build/test/lint) cannot be independently "
|
||||||
|
"confirmed. Declare the exact commands in the spawn prompt to "
|
||||||
|
"get recorded runs."
|
||||||
|
)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
proj = os.path.expanduser(
|
||||||
|
(state.get("project_dir") or "").strip() or "."
|
||||||
|
)
|
||||||
|
results = []
|
||||||
|
for cmd in cmds:
|
||||||
|
start = time.monotonic()
|
||||||
|
try:
|
||||||
|
r = subprocess.run(
|
||||||
|
cmd,
|
||||||
|
shell=True,
|
||||||
|
cwd=proj,
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=PER_COMMAND_TIMEOUT_SECS,
|
||||||
|
)
|
||||||
|
combined = (r.stdout or "") + (("\n" + r.stderr) if r.stderr else "")
|
||||||
|
results.append(
|
||||||
|
{
|
||||||
|
"cmd": cmd,
|
||||||
|
"exit": r.returncode,
|
||||||
|
"duration_s": round(time.monotonic() - start, 1),
|
||||||
|
"tail": tail_of(combined),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
results.append(
|
||||||
|
{
|
||||||
|
"cmd": cmd,
|
||||||
|
"exit": -1,
|
||||||
|
"duration_s": round(time.monotonic() - start, 1),
|
||||||
|
"tail": (
|
||||||
|
f"TIMEOUT: no exit within {PER_COMMAND_TIMEOUT_SECS}s — "
|
||||||
|
"recorded as a failing run"
|
||||||
|
),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
print(json.dumps({"exec_results": results}))
|
||||||
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
main()
|
||||||
|
except Exception as e: # noqa: BLE001 — never fail the node; degrade visibly
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"exec_results": (
|
||||||
|
f"ENVIRONMENT: verification runner error: {e} — the declared "
|
||||||
|
"commands could not be executed; treat execution criteria as "
|
||||||
|
"PARTIAL (unproven), not as failed"
|
||||||
|
)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
@@ -1,15 +1,21 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""Deterministic conformance verdict for the adversary graph.
|
"""Deterministic conformance verdict for the adversary graph.
|
||||||
|
|
||||||
CONFORMS requires: at least one criterion, every criterion MET, and zero
|
CONFORMS requires: at least one criterion, every criterion MET, zero
|
||||||
holistic complaints. Everything else — including a missing plan — is
|
holistic complaints, and zero pipeline faults. Everything else — including
|
||||||
DIVERGES (fail-closed). Assembles the exact sentinel format the callers
|
a missing plan or a degraded (faulted) pipeline — is DIVERGES (fail-closed).
|
||||||
route on. Never crashes into a silent verdict.
|
Any "PIPELINE-FAULT:" marker recorded in `pipeline_faults` (or a holistic
|
||||||
|
llm-node failure captured in `holistic_failure`) becomes complaint #1;
|
||||||
|
per-criterion results are still reported. Assembles the exact sentinel
|
||||||
|
format the callers route on. Never crashes into a silent verdict.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
|
||||||
|
MAX_FAULT_DETAIL_CHARS = 300
|
||||||
|
|
||||||
|
|
||||||
def load_state():
|
def load_state():
|
||||||
if path := os.environ.get("GRAPH_STATE_FILE"):
|
if path := os.environ.get("GRAPH_STATE_FILE"):
|
||||||
with open(path) as f:
|
with open(path) as f:
|
||||||
@@ -17,11 +23,85 @@ def load_state():
|
|||||||
return json.loads(os.environ.get("GRAPH_STATE", "{}"))
|
return json.loads(os.environ.get("GRAPH_STATE", "{}"))
|
||||||
|
|
||||||
|
|
||||||
|
def collect_pipeline_faults(state):
|
||||||
|
"""Recorded fault markers, plus a holistic llm-node failure if one occurred.
|
||||||
|
|
||||||
|
`holistic_failure` holds the "LLM node …failed: <chain>" string when the
|
||||||
|
holistic pass fell back to verdict; on a successful pass it holds the
|
||||||
|
rendered schema output (never that prefix).
|
||||||
|
"""
|
||||||
|
faults = [
|
||||||
|
f.strip()
|
||||||
|
for f in (state.get("pipeline_faults") or [])
|
||||||
|
if isinstance(f, str) and f.strip()
|
||||||
|
]
|
||||||
|
hf = state.get("holistic_failure")
|
||||||
|
if isinstance(hf, str) and hf.strip().startswith("LLM node"):
|
||||||
|
detail = hf.strip().replace("\n", " ")
|
||||||
|
if len(detail) > MAX_FAULT_DETAIL_CHARS:
|
||||||
|
detail = detail[: MAX_FAULT_DETAIL_CHARS - 1] + "…"
|
||||||
|
faults.append(
|
||||||
|
"PIPELINE-FAULT: holistic pass failed — criterion verdicts stand "
|
||||||
|
f"but cross-cutting hunt did not run ({detail})"
|
||||||
|
)
|
||||||
|
return faults
|
||||||
|
|
||||||
|
|
||||||
|
def append_criterion_complaints(lines, verdicts, extra, start):
|
||||||
|
"""Numbered complaints for non-MET verdicts and holistic extras."""
|
||||||
|
i = start
|
||||||
|
for v in verdicts:
|
||||||
|
if v.get("status") == "MET":
|
||||||
|
continue
|
||||||
|
i += 1
|
||||||
|
text = (v.get("text") or "").strip().replace("\n", " ")
|
||||||
|
if len(text) > 140:
|
||||||
|
text = text[:137] + "…"
|
||||||
|
lines.append(
|
||||||
|
f'{i}. Acceptance criterion "{text}" — {v.get("status", "?").title()} — '
|
||||||
|
f'{(v.get("complaint") or "").strip()}'
|
||||||
|
)
|
||||||
|
for c in extra:
|
||||||
|
i += 1
|
||||||
|
lines.append(
|
||||||
|
f"{i}. {c.get('kind', 'violation')} — {c.get('location', '?')} — "
|
||||||
|
f"{c.get('violation', '')} — {c.get('fix', '')}"
|
||||||
|
)
|
||||||
|
return i
|
||||||
|
|
||||||
|
|
||||||
|
def render_exec_results(exec_results):
|
||||||
|
"""Lines for the "Verification runs:" report section.
|
||||||
|
|
||||||
|
exec_results is a list of {cmd, exit, duration_s, tail} records from
|
||||||
|
run_checks.py, or a string marker ("none declared —…" / "ENVIRONMENT: …"),
|
||||||
|
or empty when the run never reached the verification stage.
|
||||||
|
"""
|
||||||
|
if isinstance(exec_results, list):
|
||||||
|
out = []
|
||||||
|
for r in exec_results:
|
||||||
|
if not isinstance(r, dict):
|
||||||
|
continue
|
||||||
|
status = "PASS" if r.get("exit") == 0 else "FAIL"
|
||||||
|
out.append(
|
||||||
|
f"- [{status}] `{r.get('cmd', '?')}` — exit {r.get('exit', '?')}, "
|
||||||
|
f"{r.get('duration_s', '?')}s"
|
||||||
|
)
|
||||||
|
if status == "FAIL":
|
||||||
|
tail = (r.get("tail") or "").strip()
|
||||||
|
out.extend(" " + t for t in tail.splitlines()[-15:])
|
||||||
|
return out or ["- (no verification commands were run)"]
|
||||||
|
if isinstance(exec_results, str) and exec_results.strip():
|
||||||
|
return [f"- {exec_results.strip()}"]
|
||||||
|
return ["- none recorded (the run did not reach the verification stage)"]
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
state = load_state()
|
state = load_state()
|
||||||
verdicts = [v for v in (state.get("crit_verdicts") or []) if isinstance(v, dict)]
|
verdicts = [v for v in (state.get("crit_verdicts") or []) if isinstance(v, dict)]
|
||||||
extra = [c for c in (state.get("extra_complaints") or []) if isinstance(c, dict)]
|
extra = [c for c in (state.get("extra_complaints") or []) if isinstance(c, dict)]
|
||||||
observations = (state.get("observations") or "").strip()
|
observations = (state.get("observations") or "").strip()
|
||||||
|
faults = collect_pipeline_faults(state)
|
||||||
n = len(verdicts)
|
n = len(verdicts)
|
||||||
|
|
||||||
met = [v for v in verdicts if v.get("status") == "MET"]
|
met = [v for v in verdicts if v.get("status") == "MET"]
|
||||||
@@ -29,7 +109,30 @@ def main():
|
|||||||
bad = [v for v in verdicts if v.get("status") in ("UNMET", "DIVERGED")]
|
bad = [v for v in verdicts if v.get("status") in ("UNMET", "DIVERGED")]
|
||||||
|
|
||||||
lines = []
|
lines = []
|
||||||
if n == 0:
|
if faults:
|
||||||
|
lines.append("ADVERSARIAL_REVIEW: DIVERGES")
|
||||||
|
if n:
|
||||||
|
lines.append(
|
||||||
|
f"Criteria: {len(met)}/{n} met, {len(partial)} partial, "
|
||||||
|
f"{len(bad)} unmet/diverged — degraded run: pipeline fault recorded "
|
||||||
|
"(fail-closed)."
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
lines.append(
|
||||||
|
"Criteria: none verified — degraded run: pipeline fault recorded "
|
||||||
|
"(fail-closed)."
|
||||||
|
)
|
||||||
|
lines.append("Complaints:")
|
||||||
|
i = 0
|
||||||
|
for f in faults:
|
||||||
|
i += 1
|
||||||
|
lines.append(f"{i}. {f}")
|
||||||
|
append_criterion_complaints(lines, verdicts, extra, i)
|
||||||
|
if observations:
|
||||||
|
lines.append("")
|
||||||
|
lines.append("Non-blocking observations:")
|
||||||
|
lines.append(observations)
|
||||||
|
elif n == 0:
|
||||||
lines.append("ADVERSARIAL_REVIEW: DIVERGES")
|
lines.append("ADVERSARIAL_REVIEW: DIVERGES")
|
||||||
lines.append("Criteria: none provided.")
|
lines.append("Criteria: none provided.")
|
||||||
lines.append("Complaints:")
|
lines.append("Complaints:")
|
||||||
@@ -51,24 +154,7 @@ def main():
|
|||||||
f"Criteria: {len(met)}/{n} met, {len(partial)} partial, {len(bad)} unmet/diverged."
|
f"Criteria: {len(met)}/{n} met, {len(partial)} partial, {len(bad)} unmet/diverged."
|
||||||
)
|
)
|
||||||
lines.append("Complaints:")
|
lines.append("Complaints:")
|
||||||
i = 0
|
append_criterion_complaints(lines, verdicts, extra, 0)
|
||||||
for v in verdicts:
|
|
||||||
if v.get("status") == "MET":
|
|
||||||
continue
|
|
||||||
i += 1
|
|
||||||
text = (v.get("text") or "").strip().replace("\n", " ")
|
|
||||||
if len(text) > 140:
|
|
||||||
text = text[:137] + "…"
|
|
||||||
lines.append(
|
|
||||||
f'{i}. Acceptance criterion "{text}" — {v.get("status", "?").title()} — '
|
|
||||||
f'{(v.get("complaint") or "").strip()}'
|
|
||||||
)
|
|
||||||
for c in extra:
|
|
||||||
i += 1
|
|
||||||
lines.append(
|
|
||||||
f"{i}. {c.get('kind', 'violation')} — {c.get('location', '?')} — "
|
|
||||||
f"{c.get('violation', '')} — {c.get('fix', '')}"
|
|
||||||
)
|
|
||||||
if observations:
|
if observations:
|
||||||
lines.append("")
|
lines.append("")
|
||||||
lines.append("Non-blocking observations:")
|
lines.append("Non-blocking observations:")
|
||||||
@@ -84,6 +170,10 @@ def main():
|
|||||||
text = text[:97] + "…"
|
text = text[:97] + "…"
|
||||||
lines.append(f'- "{text}" — {(v.get("evidence") or "").strip()}')
|
lines.append(f'- "{text}" — {(v.get("evidence") or "").strip()}')
|
||||||
|
|
||||||
|
lines.append("")
|
||||||
|
lines.append("Verification runs:")
|
||||||
|
lines.extend(render_exec_results(state.get("exec_results")))
|
||||||
|
|
||||||
print(json.dumps({"adv_report": "\n".join(lines)}))
|
print(json.dumps({"adv_report": "\n".join(lines)}))
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -8348,6 +8348,228 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---- adversary suite-script regression tests (R5) ----
|
||||||
|
//
|
||||||
|
// Fault paths of the adversary's verdict/gate scripts, exercised by
|
||||||
|
// invoking `python3 <script>` with a synthetic GRAPH_STATE env — the same
|
||||||
|
// contract the graph's script executor uses. Guarded on python3 being
|
||||||
|
// available, mirroring src/graph/script.rs's local helper.
|
||||||
|
|
||||||
|
fn cmd_available(name: &str) -> bool {
|
||||||
|
which::which(name).is_ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run_adversary_script(script: &str, state: &serde_json::Value) -> serde_json::Value {
|
||||||
|
let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||||
|
.join("assets/agents/adversary/scripts")
|
||||||
|
.join(script);
|
||||||
|
let out = std::process::Command::new("python3")
|
||||||
|
.arg(&path)
|
||||||
|
.env("GRAPH_STATE", state.to_string())
|
||||||
|
.output()
|
||||||
|
.unwrap_or_else(|e| panic!("failed to invoke python3 {script}: {e}"));
|
||||||
|
assert!(
|
||||||
|
out.status.success(),
|
||||||
|
"{script} exited nonzero: {}",
|
||||||
|
String::from_utf8_lossy(&out.stderr)
|
||||||
|
);
|
||||||
|
serde_json::from_slice(&out.stdout).unwrap_or_else(|e| {
|
||||||
|
panic!(
|
||||||
|
"{script} stdout is not JSON ({e}): {}",
|
||||||
|
String::from_utf8_lossy(&out.stdout)
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn adversary_verdict_pipeline_fault_forces_diverges() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let state = json!({
|
||||||
|
"pipeline_faults": ["PIPELINE-FAULT: parse failed — cannot review: LLM node failed: boom"],
|
||||||
|
"crit_verdicts": [
|
||||||
|
{"id": "c1", "text": "does X", "status": "MET",
|
||||||
|
"evidence": "src/x.rs:1 + test src/x.rs:99", "complaint": ""},
|
||||||
|
{"id": "c2", "text": "does Y", "status": "UNMET",
|
||||||
|
"evidence": "", "complaint": "nothing in the diff does Y"}
|
||||||
|
],
|
||||||
|
"extra_complaints": [],
|
||||||
|
"observations": "",
|
||||||
|
"exec_results": ""
|
||||||
|
});
|
||||||
|
let out = run_adversary_script("verdict.py", &state);
|
||||||
|
let report = out["adv_report"].as_str().unwrap();
|
||||||
|
assert!(
|
||||||
|
report.starts_with("ADVERSARIAL_REVIEW: DIVERGES"),
|
||||||
|
"a pipeline fault must force DIVERGES: {report}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
report.contains("1. PIPELINE-FAULT: parse failed — cannot review"),
|
||||||
|
"the fault must be complaint #1: {report}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
report.contains("2. Acceptance criterion \"does Y\""),
|
||||||
|
"per-criterion results must still be reported after the fault: {report}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
report.contains("Verification runs:"),
|
||||||
|
"the report must carry the Verification runs section: {report}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn adversary_verdict_holistic_failure_is_a_pipeline_fault() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let state = json!({
|
||||||
|
"pipeline_faults": [],
|
||||||
|
"crit_verdicts": [
|
||||||
|
{"id": "c1", "text": "does X", "status": "MET",
|
||||||
|
"evidence": "src/x.rs:1 + test src/x.rs:99", "complaint": ""}
|
||||||
|
],
|
||||||
|
"extra_complaints": [],
|
||||||
|
"observations": "",
|
||||||
|
"holistic_failure": "LLM node failed: provider exploded",
|
||||||
|
"exec_results": ""
|
||||||
|
});
|
||||||
|
let out = run_adversary_script("verdict.py", &state);
|
||||||
|
let report = out["adv_report"].as_str().unwrap();
|
||||||
|
assert!(
|
||||||
|
report.starts_with("ADVERSARIAL_REVIEW: DIVERGES"),
|
||||||
|
"a holistic-pass failure must force DIVERGES: {report}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
report.contains(
|
||||||
|
"PIPELINE-FAULT: holistic pass failed — criterion verdicts stand \
|
||||||
|
but cross-cutting hunt did not run"
|
||||||
|
),
|
||||||
|
"the holistic failure must become a PIPELINE-FAULT complaint: {report}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn adversary_verdict_conforms_is_unaffected_without_faults() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// holistic_failure carries the rendered SUCCESS output here — it must
|
||||||
|
// not be mistaken for a failure, and a recorded green run must render
|
||||||
|
// as [PASS] in the Verification runs section.
|
||||||
|
let state = json!({
|
||||||
|
"pipeline_faults": [],
|
||||||
|
"crit_verdicts": [
|
||||||
|
{"id": "c1", "text": "does X", "status": "MET",
|
||||||
|
"evidence": "src/x.rs:1 + test src/x.rs:99", "complaint": ""}
|
||||||
|
],
|
||||||
|
"extra_complaints": [],
|
||||||
|
"observations": "",
|
||||||
|
"holistic_failure": "{\"extra_complaints\": [], \"observations\": \"\"}",
|
||||||
|
"exec_results": [{"cmd": "cargo test --all", "exit": 0, "duration_s": 42.0, "tail": "ok"}]
|
||||||
|
});
|
||||||
|
let out = run_adversary_script("verdict.py", &state);
|
||||||
|
let report = out["adv_report"].as_str().unwrap();
|
||||||
|
assert!(
|
||||||
|
report.starts_with("ADVERSARIAL_REVIEW: CONFORMS\nCriteria: 1/1 met"),
|
||||||
|
"non-degraded all-MET semantics must be unchanged: {report}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
report.contains("Verification runs:") && report.contains("- [PASS] `cargo test --all`"),
|
||||||
|
"a recorded green run must render as PASS evidence: {report}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn adversary_run_checks_none_declared_marker() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let out = run_adversary_script("run_checks.py", &json!({}));
|
||||||
|
let marker = out["exec_results"].as_str().unwrap();
|
||||||
|
assert!(
|
||||||
|
marker.contains("none declared"),
|
||||||
|
"no verification_commands must degrade to a 'none declared' marker: {marker}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn adversary_run_checks_records_green_and_failing_runs() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let state = json!({"verification_commands": ["echo ok", "false"]});
|
||||||
|
let out = run_adversary_script("run_checks.py", &state);
|
||||||
|
let results = out["exec_results"].as_array().unwrap();
|
||||||
|
assert_eq!(results.len(), 2, "one record per declared command: {results:?}");
|
||||||
|
assert_eq!(results[0]["cmd"], "echo ok");
|
||||||
|
assert_eq!(results[0]["exit"], 0, "green command must record exit 0");
|
||||||
|
assert!(
|
||||||
|
results[0]["tail"].as_str().unwrap().contains("ok"),
|
||||||
|
"the output tail must be recorded: {results:?}"
|
||||||
|
);
|
||||||
|
assert!(results[0]["duration_s"].is_number());
|
||||||
|
assert_ne!(
|
||||||
|
results[1]["exit"], 0,
|
||||||
|
"failing command must record its nonzero exit: {results:?}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn adversary_crit_gate_contract_regression() {
|
||||||
|
if !cmd_available("python3") {
|
||||||
|
eprintln!("skipping: python3 not available");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Valid MET verdict passes through with the id stamped from the criterion.
|
||||||
|
let out = run_adversary_script(
|
||||||
|
"crit_gate.py",
|
||||||
|
&json!({
|
||||||
|
"criterion": {"id": "c7", "text": "the spec"},
|
||||||
|
"gate_attempts": 0,
|
||||||
|
"crit_verdict": "{\"id\": \"wrong\", \"status\": \"MET\", \
|
||||||
|
\"evidence\": \"src/a.rs:1 + test tests/a.rs:5\", \"complaint\": \"\"}"
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
assert_eq!(out["crit_verdict"]["id"], "c7", "id must be stamped: {out}");
|
||||||
|
assert_eq!(out["crit_verdict"]["status"], "MET");
|
||||||
|
|
||||||
|
// Malformed output with retry budget left → reject back to check_criterion.
|
||||||
|
let out = run_adversary_script(
|
||||||
|
"crit_gate.py",
|
||||||
|
&json!({
|
||||||
|
"criterion": {"id": "c7", "text": "the spec"},
|
||||||
|
"gate_attempts": 0,
|
||||||
|
"crit_verdict": "not json at all"
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
assert_eq!(out["_next"], "check_criterion", "first failure must retry: {out}");
|
||||||
|
assert_eq!(out["gate_attempts"], 1);
|
||||||
|
|
||||||
|
// Malformed output with the retry exhausted → recorded PARTIAL (unproven).
|
||||||
|
let out = run_adversary_script(
|
||||||
|
"crit_gate.py",
|
||||||
|
&json!({
|
||||||
|
"criterion": {"id": "c7", "text": "the spec"},
|
||||||
|
"gate_attempts": 1,
|
||||||
|
"crit_verdict": "still not json"
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
assert_eq!(out["crit_verdict"]["status"], "PARTIAL");
|
||||||
|
assert!(
|
||||||
|
out["crit_verdict"]["complaint"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.contains("failed machine validation"),
|
||||||
|
"exhausted retries must record the unproven complaint: {out}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[serial]
|
#[serial]
|
||||||
fn install_functions_force_preserves_user_mcp_json() {
|
fn install_functions_force_preserves_user_mcp_json() {
|
||||||
|
|||||||
Reference in new issue
Block a user