feat: Significant code review suite rewrite with improved graph agent features
This commit is contained in:
41 files changed
+3572
-709
No files matched your search
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deterministic lane builder for the review gauntlet.
|
||||
|
||||
Selection = hard rules (the floor) ∪ LLM additions; forced lanes override
|
||||
everything exactly. Emits one 0-or-1-item list per lane: an empty list makes
|
||||
that lane's map node run zero branches (map-over-nothing), which IS the
|
||||
skip mechanism — no dynamic routing anywhere.
|
||||
|
||||
The LLM's add_* flags can only ADD lanes. A flag that is false never
|
||||
removes a lane the hard rules selected.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
def load_state():
|
||||
if path := os.environ.get("GRAPH_STATE_FILE"):
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
return json.loads(os.environ.get("GRAPH_STATE", "{}"))
|
||||
|
||||
|
||||
state = load_state()
|
||||
|
||||
ALIASES = {
|
||||
"code-review": "code-review",
|
||||
"code_review": "code-review",
|
||||
"code-reviewer": "code-review",
|
||||
"adversary": "adversary",
|
||||
"adversarial": "adversary",
|
||||
"adversarial-review": "adversary",
|
||||
"security": "security",
|
||||
"security-review": "security",
|
||||
"security-reviewer": "security",
|
||||
"probe": "probe",
|
||||
"usage": "probe",
|
||||
"usage-pattern": "probe",
|
||||
"usage-pattern-testing": "probe",
|
||||
}
|
||||
|
||||
|
||||
def text(key):
|
||||
v = state.get(key)
|
||||
return v.strip() if isinstance(v, str) else ""
|
||||
|
||||
|
||||
forced_raw = state.get("forced_lanes") or []
|
||||
forced, unknown = [], []
|
||||
for lane in forced_raw:
|
||||
canon = ALIASES.get(str(lane).strip().lower())
|
||||
(forced.append(canon) if canon else unknown.append(str(lane)))
|
||||
|
||||
reasons = []
|
||||
if forced:
|
||||
lanes = set(forced)
|
||||
reasons.append(f"forced lanes honored exactly: {sorted(lanes)}")
|
||||
else:
|
||||
lanes = {"code-review"}
|
||||
reasons.append(
|
||||
"code-review: always on — the gauntlet is only spawned for non-trivial work"
|
||||
)
|
||||
if text("plan_context"):
|
||||
lanes.add("adversary")
|
||||
reasons.append("adversary: a spec/plan/acceptance-criteria context is present")
|
||||
if (
|
||||
state.get("touches_auth")
|
||||
or state.get("touches_deps")
|
||||
or state.get("touches_exec")
|
||||
or state.get("security_posture") == "hardened"
|
||||
):
|
||||
lanes.add("security")
|
||||
reasons.append(
|
||||
"security: attack-surface signals (auth/deps/exec) or hardened posture"
|
||||
)
|
||||
if state.get("consumer_surface") and text("probe_context"):
|
||||
lanes.add("probe")
|
||||
reasons.append(
|
||||
"probe: consumer-facing surface changed and a local-run recipe/suite pointer exists"
|
||||
)
|
||||
|
||||
# LLM judgment is additive-only.
|
||||
llm_why = text("lane_reasons")[:300]
|
||||
for flag, lane in (
|
||||
("add_code_review", "code-review"),
|
||||
("add_adversary", "adversary"),
|
||||
("add_security", "security"),
|
||||
("add_probe", "probe"),
|
||||
):
|
||||
if state.get(flag) and lane not in lanes:
|
||||
if lane == "probe" and not text("probe_context"):
|
||||
reasons.append(
|
||||
"probe: requested by lane judgment but skipped — no local-run "
|
||||
"recipe/suite pointer (a probe without one is guaranteed INCONCLUSIVE)"
|
||||
)
|
||||
continue
|
||||
if lane == "adversary" and not text("plan_context"):
|
||||
reasons.append(
|
||||
"adversary: requested by lane judgment but skipped — no spec/plan "
|
||||
"to check conformance against"
|
||||
)
|
||||
continue
|
||||
lanes.add(lane)
|
||||
reasons.append(f"{lane}: added by lane-selection judgment — {llm_why}")
|
||||
|
||||
if unknown:
|
||||
reasons.append(f"ignored unknown forced lane name(s): {unknown}")
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"code_review_items": [{"lane": "code-review"}] if "code-review" in lanes else [],
|
||||
"adversary_items": [{"lane": "adversary"}] if "adversary" in lanes else [],
|
||||
"security_items": [{"lane": "security"}] if "security" in lanes else [],
|
||||
"probe_items": [{"lane": "probe"}] if "probe" in lanes else [],
|
||||
"lanes_summary": "\n".join(f"- {r}" for r in reasons),
|
||||
}
|
||||
)
|
||||
)
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deterministic diff signals for the review gauntlet.
|
||||
|
||||
Pure git — no LLM judgment. Every signal is computed from the changed-file
|
||||
list and the added lines of the diff. Failures never sink the gauntlet:
|
||||
on any error the script reports `signals_error` and leaves the defaults
|
||||
(false/empty) in place, which biases lane selection toward the caller's
|
||||
context and forced lanes.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
def load_state():
|
||||
if path := os.environ.get("GRAPH_STATE_FILE"):
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
return json.loads(os.environ.get("GRAPH_STATE", "{}"))
|
||||
|
||||
|
||||
state = load_state()
|
||||
proj = state.get("project_dir") or "."
|
||||
spec = (state.get("diff_spec") or "worktree").strip()
|
||||
|
||||
|
||||
def git(*args):
|
||||
r = subprocess.run(
|
||||
["git", "-C", proj, *args], capture_output=True, text=True, timeout=90
|
||||
)
|
||||
if r.returncode != 0:
|
||||
raise RuntimeError((r.stderr or r.stdout).strip()[:400])
|
||||
return r.stdout
|
||||
|
||||
|
||||
out = {}
|
||||
try:
|
||||
if spec in ("", "worktree"):
|
||||
names = git("diff", "--name-only", "HEAD")
|
||||
difftext = git("diff", "HEAD")
|
||||
else:
|
||||
names = git("diff", "--name-only", spec)
|
||||
difftext = git("diff", spec)
|
||||
files = [f.strip() for f in names.splitlines() if f.strip()]
|
||||
added = "\n".join(
|
||||
l for l in difftext.splitlines() if l.startswith("+") and not l.startswith("+++")
|
||||
)
|
||||
|
||||
def any_path(pattern):
|
||||
return any(re.search(pattern, f, re.I) for f in files)
|
||||
|
||||
out.update(
|
||||
{
|
||||
"changed_files": files[:200],
|
||||
"file_count": len(files),
|
||||
"touches_proto": any_path(r"\.proto$|buf\.(yaml|gen\.yaml)$"),
|
||||
"touches_migrations": any_path(r"migrations?/|\.sql$"),
|
||||
# auth(?!or) admits auth/authn/authz/oauth but not "author".
|
||||
# Over-firing here is the safe direction: it only ADDS a lane.
|
||||
"touches_auth": any_path(
|
||||
r"auth(?!or)|login|logout|token|session|permission|rbac|\bacl\b|credential|secret"
|
||||
)
|
||||
or bool(
|
||||
re.search(
|
||||
r"auth(?!or)|password|token|secret|credential|jwt|session",
|
||||
added,
|
||||
re.I,
|
||||
)
|
||||
),
|
||||
"touches_deps": any_path(
|
||||
r"go\.(mod|sum)$|package(-lock)?\.json$|yarn\.lock$|pnpm-lock\.yaml$"
|
||||
r"|Cargo\.(toml|lock)$|requirements[^/]*\.txt$|pyproject\.toml$"
|
||||
r"|Gemfile(\.lock)?$|pom\.xml$|build\.gradle|composer\.(json|lock)$"
|
||||
),
|
||||
"touches_exec": bool(
|
||||
re.search(
|
||||
r"exec\.Command|subprocess|os/exec|Runtime\.getRuntime"
|
||||
r"|shell_exec|\beval\(|\bsystem\(|child_process",
|
||||
added,
|
||||
)
|
||||
),
|
||||
"consumer_surface": any_path(
|
||||
r"routes?|handlers?|\bapi\b|endpoints?|\bcmd/|\bcli\b|\.proto$"
|
||||
r"|openapi|swagger|controllers?|graphql"
|
||||
),
|
||||
"docs_only": bool(files)
|
||||
and all(re.search(r"\.(md|rst|txt|adoc)$", f, re.I) for f in files),
|
||||
"signals_summary": f"{len(files)} file(s) changed (diff: {spec})",
|
||||
}
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 — degraded signals must not sink the gauntlet
|
||||
out["signals_error"] = (
|
||||
f"NOTE: diff signals could not be computed ({e}); "
|
||||
"lane selection falls back to caller context and forced lanes."
|
||||
)
|
||||
out["signals_summary"] = "signals unavailable"
|
||||
|
||||
print(json.dumps(out))
|
||||
@@ -0,0 +1,163 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deterministic verdict gate for the review gauntlet.
|
||||
|
||||
Parses each configured lane's verdict SENTINEL with a regex — no LLM
|
||||
re-reads, no judgment. The gate's hard rules:
|
||||
|
||||
- a missing sentinel is a LANE FAILURE and blocks — never a pass;
|
||||
- any 🔴 finding in the code-review report blocks regardless of the
|
||||
lane's own verdict line;
|
||||
- NEEDS-HUMAN without 🔴 passes but is surfaced as attention required;
|
||||
- probe INCONCLUSIVE blocks with an ENVIRONMENT note — it is never
|
||||
treated as PASS or FAIL (fix the environment per the plan's local-run
|
||||
recipe, then re-run);
|
||||
- the gate itself must never crash into a silent pass: any internal
|
||||
error emits GAUNTLET: BLOCKED naming the error.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
|
||||
def load_state():
|
||||
if path := os.environ.get("GRAPH_STATE_FILE"):
|
||||
with open(path) as f:
|
||||
return json.load(f)
|
||||
return json.loads(os.environ.get("GRAPH_STATE", "{}"))
|
||||
|
||||
|
||||
def main():
|
||||
state = load_state()
|
||||
blockers, attention, lane_lines, reports = [], [], [], []
|
||||
|
||||
def lane_report(key):
|
||||
arr = state.get(key) or []
|
||||
if not arr:
|
||||
return None
|
||||
item = arr[0]
|
||||
return item if isinstance(item, str) else json.dumps(item)
|
||||
|
||||
def record(lane, status, detail=""):
|
||||
lane_lines.append(f"| {lane} | {status} | {detail} |")
|
||||
|
||||
# --- code-review ------------------------------------------------------
|
||||
cr = lane_report("code_review_results")
|
||||
if cr is None:
|
||||
record("code-review", "SKIPPED", "not selected")
|
||||
else:
|
||||
reports.append(("code-review", cr))
|
||||
m = re.search(r"Verdict:\**\s*\**\s*(MERGE-READY|NEEDS-HUMAN)", cr)
|
||||
reds = cr.count("🔴")
|
||||
if reds:
|
||||
blockers.append(
|
||||
f"code-review: {reds} 🔴 CRITICAL finding(s) — fix before claiming done"
|
||||
)
|
||||
record("code-review", "BLOCKED", f"{reds} 🔴 finding(s)")
|
||||
elif m and m.group(1) == "MERGE-READY":
|
||||
record("code-review", "GREEN", "MERGE-READY")
|
||||
elif m:
|
||||
attention.append(
|
||||
"code-review: NEEDS-HUMAN — see the lane report's "
|
||||
"'Human attention required' section"
|
||||
)
|
||||
record("code-review", "GREEN (attention)", "NEEDS-HUMAN, no 🔴")
|
||||
else:
|
||||
blockers.append(
|
||||
"code-review: no Review Verdict sentinel found — the lane failed; "
|
||||
"a missing verdict is never a pass"
|
||||
)
|
||||
record("code-review", "BLOCKED", "missing verdict sentinel")
|
||||
|
||||
# --- adversary --------------------------------------------------------
|
||||
adv = lane_report("adversary_results")
|
||||
if adv is None:
|
||||
record("adversary", "SKIPPED", "not selected")
|
||||
else:
|
||||
reports.append(("adversary", adv))
|
||||
m = re.search(r"ADVERSARIAL_REVIEW:\s*(CONFORMS|DIVERGES)", adv)
|
||||
if m and m.group(1) == "CONFORMS":
|
||||
record("adversary", "GREEN", "CONFORMS")
|
||||
elif m:
|
||||
blockers.append(
|
||||
"adversary: DIVERGES — the implementation does not conform to the plan"
|
||||
)
|
||||
record("adversary", "BLOCKED", "DIVERGES")
|
||||
else:
|
||||
blockers.append(
|
||||
"adversary: no ADVERSARIAL_REVIEW sentinel found — lane failed; never a pass"
|
||||
)
|
||||
record("adversary", "BLOCKED", "missing verdict sentinel")
|
||||
|
||||
# --- security ---------------------------------------------------------
|
||||
sec = lane_report("security_results")
|
||||
if sec is None:
|
||||
record("security", "SKIPPED", "not selected")
|
||||
else:
|
||||
reports.append(("security", sec))
|
||||
m = re.search(r"SECURITY_REVIEW:\s*(PASS|FAIL)", sec)
|
||||
if m and m.group(1) == "PASS":
|
||||
record("security", "GREEN", "PASS")
|
||||
elif m:
|
||||
blockers.append("security: SECURITY_REVIEW: FAIL — exploitable finding(s)")
|
||||
record("security", "BLOCKED", "FAIL")
|
||||
else:
|
||||
blockers.append(
|
||||
"security: no SECURITY_REVIEW sentinel found — lane failed; never a pass"
|
||||
)
|
||||
record("security", "BLOCKED", "missing verdict sentinel")
|
||||
|
||||
# --- probe ------------------------------------------------------------
|
||||
pb = lane_report("probe_results")
|
||||
if pb is None:
|
||||
record("probe", "SKIPPED", "not selected")
|
||||
else:
|
||||
reports.append(("probe", pb))
|
||||
m = re.search(r"USAGE_PROBE:\s*(PASS|FAIL|INCONCLUSIVE)", pb)
|
||||
if m and m.group(1) == "PASS":
|
||||
record("probe", "GREEN", "PASS")
|
||||
elif m and m.group(1) == "FAIL":
|
||||
blockers.append("probe: USAGE_PROBE: FAIL — observed behavior contradicts the contract")
|
||||
record("probe", "BLOCKED", "FAIL")
|
||||
elif m:
|
||||
blockers.append(
|
||||
"probe: INCONCLUSIVE — the ENVIRONMENT could not be established; this is "
|
||||
"not a code verdict. Follow the plan's local-run recipe to fix the "
|
||||
"environment and re-run. INCONCLUSIVE is never treated as PASS or FAIL."
|
||||
)
|
||||
record("probe", "BLOCKED", "INCONCLUSIVE (environment)")
|
||||
else:
|
||||
blockers.append(
|
||||
"probe: no USAGE_PROBE sentinel found — lane failed; never a pass"
|
||||
)
|
||||
record("probe", "BLOCKED", "missing verdict sentinel")
|
||||
|
||||
verdict = "BLOCKED" if blockers else "PASS"
|
||||
|
||||
md = "\n".join(
|
||||
["| lane | status | detail |", "|------|--------|--------|"] + lane_lines
|
||||
)
|
||||
md += "\n\n## Lane selection\n" + (state.get("lanes_summary") or "(none recorded)")
|
||||
if blockers:
|
||||
md += "\n\n## Blockers\n" + "\n".join(f"- {b}" for b in blockers)
|
||||
if attention:
|
||||
md += "\n\n## Human attention required\n" + "\n".join(f"- {a}" for a in attention)
|
||||
if state.get("signals_error"):
|
||||
md += f"\n\n> {state['signals_error']}"
|
||||
md += "\n\n## Full lane reports\n"
|
||||
for lane, rep in reports:
|
||||
md += f"\n<details><summary>{lane}</summary>\n\n{rep}\n\n</details>\n"
|
||||
|
||||
print(json.dumps({"gauntlet_verdict": verdict, "gauntlet_report": md}))
|
||||
|
||||
|
||||
try:
|
||||
main()
|
||||
except Exception as e: # noqa: BLE001 — the gate must never crash into a silent pass
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"gauntlet_verdict": "BLOCKED",
|
||||
"gauntlet_report": f"verdict gate error: {e} — treat as a failed gate, not a pass.",
|
||||
}
|
||||
)
|
||||
)
|
||||
Reference in new issue
Block a user