fix(TASK-017): bound run_checks runtime and fail-close its node-level death
Adversary conformance-review fixes for the TASK-017 exec-evidence stage:
- run_checks.py: total-runtime deadline (3300s = node timeout minus margin);
each command gets min(900s, remaining budget), commands past the deadline
are recorded as {cmd, skipped: deadline} entries so in-script handling
stays authoritative instead of the node timeout killing the script from
outside. POSIX hardening: per-command process group (start_new_session)
+ killpg on timeout so hung grandchildren cannot linger; non-POSIX
limitation documented in the docstring.
- graph.yaml: run_checks gains fallback: pipeline_fault; check_criterion
treats SKIPPED records as at-best PARTIAL (unproven, never failed).
- pipeline_fault.py: run_checks-stage attribution grounded in real engine
behavior (script-node fallback writes nothing into state, executor.rs
script arm) — nonempty diff_text proves facts completed, so the fault is
the verification runner; emits 'PIPELINE-FAULT: verification runner
killed'. Empty diff_text still attributes to diff resolution.
- verdict.py: renders skipped records as [SKIPPED] instead of a bogus
[FAIL] exit-? line; non-degraded verdict semantics untouched.
- tests: outer-except ENVIRONMENT degradation, deadline timeout+skip (env
seam ADVERSARY_RUN_CHECKS_DEADLINE_SECS), pipeline_fault stage
attribution (parse / facts / run_checks), check_criterion prompt cites
{{exec_results}}, verdict SKIPPED rendering; run_adversary_script_env
helper for env-carrying invocations.
- cargo fmt violations in 3d7dea9 test code fixed (fmt --check exit 0).
This commit is contained in:
5 files changed
+303
-51
No files matched your search
@@ -9,14 +9,32 @@ they never execute anything themselves.
|
||||
NO auto-detection: undeclared commands mean a "none declared" marker
|
||||
(fail-visible, not fail-guessed). Any runner error degrades into an
|
||||
ENVIRONMENT marker — this script never fails the node.
|
||||
|
||||
Budget: commands run sequentially, so TOTAL runtime is bounded by a deadline
|
||||
(TOTAL_DEADLINE_SECS = 3300s, the run_checks node's 3600s timeout minus
|
||||
margin). Without it, a handful of hanging commands (4 x 900s) would blow the
|
||||
NODE timeout and the executor would kill this script from OUTSIDE — the
|
||||
in-script degradation would never fire. Each command gets
|
||||
min(PER_COMMAND_TIMEOUT_SECS, remaining budget); commands the deadline
|
||||
leaves no budget for are recorded as {"cmd", "skipped": "deadline"} entries,
|
||||
so the in-script handling stays authoritative. The env var
|
||||
ADVERSARY_RUN_CHECKS_DEADLINE_SECS overrides the deadline (test seam).
|
||||
|
||||
Process hygiene: commands run through the shell. On POSIX each command gets
|
||||
its own session (start_new_session) and a timeout SIGKILLs the whole process
|
||||
group, so grandchildren of a hung command cannot linger. On non-POSIX
|
||||
platforms only the direct shell child is killed — grandchildren of a
|
||||
timed-out command may survive (documented limitation).
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
PER_COMMAND_TIMEOUT_SECS = 900
|
||||
TOTAL_DEADLINE_SECS = float(os.environ.get("ADVERSARY_RUN_CHECKS_DEADLINE_SECS") or 3300)
|
||||
TAIL_LINES = 50
|
||||
MAX_TAIL_CHARS = 8000
|
||||
|
||||
@@ -35,6 +53,45 @@ def tail_of(text):
|
||||
return tail
|
||||
|
||||
|
||||
def run_one(cmd, proj, budget):
|
||||
"""Run one command, hard-bounded by `budget` seconds; return its record."""
|
||||
start = time.monotonic()
|
||||
popen_kwargs = {"start_new_session": True} if os.name == "posix" else {}
|
||||
p = subprocess.Popen(
|
||||
cmd,
|
||||
shell=True,
|
||||
cwd=proj,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
**popen_kwargs,
|
||||
)
|
||||
try:
|
||||
out_s, err_s = p.communicate(timeout=budget)
|
||||
except subprocess.TimeoutExpired:
|
||||
if os.name == "posix":
|
||||
try:
|
||||
os.killpg(os.getpgid(p.pid), signal.SIGKILL)
|
||||
except (ProcessLookupError, PermissionError):
|
||||
pass
|
||||
else:
|
||||
p.kill()
|
||||
p.communicate() # reap the child and drain the pipes
|
||||
return {
|
||||
"cmd": cmd,
|
||||
"exit": -1,
|
||||
"duration_s": round(time.monotonic() - start, 1),
|
||||
"tail": f"TIMEOUT: no exit within {budget:.0f}s — recorded as a failing run",
|
||||
}
|
||||
combined = (out_s or "") + (("\n" + err_s) if err_s else "")
|
||||
return {
|
||||
"cmd": cmd,
|
||||
"exit": p.returncode,
|
||||
"duration_s": round(time.monotonic() - start, 1),
|
||||
"tail": tail_of(combined),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
state = load_state()
|
||||
declared = state.get("verification_commands")
|
||||
@@ -61,39 +118,14 @@ def main():
|
||||
proj = os.path.expanduser(
|
||||
(state.get("project_dir") or "").strip() or "."
|
||||
)
|
||||
deadline = time.monotonic() + TOTAL_DEADLINE_SECS
|
||||
results = []
|
||||
for cmd in cmds:
|
||||
start = time.monotonic()
|
||||
try:
|
||||
r = subprocess.run(
|
||||
cmd,
|
||||
shell=True,
|
||||
cwd=proj,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=PER_COMMAND_TIMEOUT_SECS,
|
||||
)
|
||||
combined = (r.stdout or "") + (("\n" + r.stderr) if r.stderr else "")
|
||||
results.append(
|
||||
{
|
||||
"cmd": cmd,
|
||||
"exit": r.returncode,
|
||||
"duration_s": round(time.monotonic() - start, 1),
|
||||
"tail": tail_of(combined),
|
||||
}
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
results.append(
|
||||
{
|
||||
"cmd": cmd,
|
||||
"exit": -1,
|
||||
"duration_s": round(time.monotonic() - start, 1),
|
||||
"tail": (
|
||||
f"TIMEOUT: no exit within {PER_COMMAND_TIMEOUT_SECS}s — "
|
||||
"recorded as a failing run"
|
||||
),
|
||||
}
|
||||
)
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
results.append({"cmd": cmd, "skipped": "deadline"})
|
||||
continue
|
||||
results.append(run_one(cmd, proj, min(PER_COMMAND_TIMEOUT_SECS, remaining)))
|
||||
print(json.dumps({"exec_results": results}))
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user