fix(review-suite): mark example verification commands as placeholders; pin variable description and prompt note

Adversary re-review follow-ups to 665fee9/402d25a:

- architect/sisyphus spawn templates: the pinned cargo commands are only an
  EXAMPLE in a language-agnostic orchestrator — a literal follower in a
  non-Rust repo would record 'cargo: command not found' and yield a false
  DIVERGES. Each --variables line now carries a substitution cue (pinned).
- run_checks.py: a non-list prior pipeline_faults value is no longer iterated
  character by character (test case added).
- Pins: both graphs' verification_commands description wording; the
  run_adversary prompt states where the commands come from; the replacement
  sentence sits on one physical line.
This commit is contained in:
Dark-Alex-17 authored and Dark-Alex-17 committed 2026-09-14 10:50:34 -06:00
1 parent 27101909e0
commit 84b5a72eac
5 files changed
+46 -3

No files matched your search

@@ -147,7 +147,8 @@ def main():
cmds = declared_commands(state.get("verification_commands"))
except InvalidDeclaration as e:
msg = f"PIPELINE-FAULT: verification_commands declaration invalid — {e}"
faults = [f for f in (state.get("pipeline_faults") or []) if isinstance(f, str)]
prior = state.get("pipeline_faults")
faults = [f for f in prior if isinstance(f, str)] if isinstance(prior, list) else []
faults.append(msg)
print(json.dumps({"exec_results": msg, "pipeline_faults": faults}))
return
+2
View File
@@ -334,6 +334,7 @@ instructions: |
Plan / acceptance criteria: <the task's acceptance criteria, pasted verbatim>
Local-run recipe / usage suites: <from the plan, when present>"
--variables {"verification_commands": "[\"cargo test --all\", \"cargo clippy -- -D warnings\"]"}
<— the cargo commands above are only an EXAMPLE: replace them with THIS project's exact build/test/lint commands; never guess, and pass "[]" when none apply>
```
Verification commands go in `--variables` (a JSON-array string), never in the prompt: the
@@ -357,6 +358,7 @@ instructions: |
<paste the plan's ## Quality bar section VERBATIM — recorded dropped practices are
conformance facts, not divergences>"
--variables {"verification_commands": "[\"cargo test --all\", \"cargo clippy -- -D warnings\"]"}
<— the cargo commands above are only an EXAMPLE: replace them with THIS project's exact build/test/lint commands; never guess, and pass "[]" when none apply>
```
Same trust boundary as the gauntlet spawn: verification commands are declared via
+1 -2
View File
@@ -333,8 +333,7 @@ nodes:
The spec / plan / acceptance criteria it must satisfy:
{{plan_context}}
Verification commands arrive as a structured input (the
verification_commands variable), not in this prompt.
Verification commands arrive as a structured input (the verification_commands variable), not in this prompt.
{{extra_context}}
+2
View File
@@ -316,6 +316,7 @@ instructions: |
Plan / acceptance criteria: <pasted verbatim — enables the adversary lane; omit when none>
Local-run recipe / usage suites: <from the plan — enables the probe lane; omit when none>"
--variables {"verification_commands": "[\"cargo test --all\", \"cargo clippy -- -D warnings\"]"}
<— the cargo commands above are only an EXAMPLE: replace them with THIS project's exact build/test/lint commands; never guess, and pass "[]" when none apply>
```
Verification commands go in `--variables` (a JSON-array string), never in the prompt: the prompt
@@ -398,6 +399,7 @@ instructions: |
PLAN — acceptance criteria to check against:
<paste the task/step spec + acceptance criteria VERBATIM — not a summary>"
--variables {"verification_commands": "[\"cargo test --all\", \"cargo clippy -- -D warnings\"]"}
<— the cargo commands above are only an EXAMPLE: replace them with THIS project's exact build/test/lint commands; never guess, and pass "[]" when none apply>
```
Same trust boundary as the gauntlet spawn: verification commands are declared via `--variables`,
+39
View File
@@ -8233,6 +8233,8 @@ mod tests {
const VARIABLES_FORM: &str =
"--variables {\"verification_commands\": \"[\\\"cargo test --all\\\"";
const EXAMPLE_CUE: &str =
"only an EXAMPLE: replace them with THIS project's exact build/test/lint commands";
for name in ["architect", "sisyphus"] {
let config =
read_to_string(paths::agents_data_dir().join(name).join("config.yaml")).unwrap();
@@ -8254,6 +8256,11 @@ mod tests {
config.contains("untrusted pasted text"),
"{name} config must name the untrusted prompt text as the reason"
);
assert!(
config.matches(EXAMPLE_CUE).count() >= 2,
"{name} config must mark the cargo commands as an example to substitute on both \
spawn templates, or a literal follower records `cargo: command not found` in non-Rust repos"
);
}
let readmes = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("assets/agents");
@@ -9094,6 +9101,20 @@ mod tests {
!pwned.exists(),
"an invalid declaration must execute nothing, even its string items"
);
// A non-list prior `pipeline_faults` is never iterated character by
// character; only the new marker is recorded.
let out = run_adversary_script(
"run_checks.py",
&json!({"verification_commands": "not json", "pipeline_faults": "oops"}),
);
let marker = out["exec_results"].as_str().unwrap();
assert!(marker.starts_with(FAULT), "{marker}");
assert_eq!(
out["pipeline_faults"],
json!([marker]),
"a string-valued prior pipeline_faults must not be spread into characters"
);
}
#[test]
@@ -10614,6 +10635,11 @@ mod tests {
"verification commands are a structured input, never lane-prompt prose: {}",
adv.prompt
);
assert!(
adv.prompt.contains("arrive as a structured input"),
"run_adversary's prompt must say where the commands come from instead: {}",
adv.prompt
);
let node = graph
.nodes
.get("run_probe")
@@ -10648,6 +10674,19 @@ mod tests {
Some("[]"),
"{name}: variables land as strings, so the default is the JSON-encoded empty list"
);
// The YAML block scalar wraps mid-sentence; pin the words, not the wrap.
let description = var
.description
.split_whitespace()
.collect::<Vec<_>>()
.join(" ");
assert!(
description.contains("JSON array of shell commands the CALLER declares")
&& description.contains("Never inferred from the prompt"),
"{name}: the variable must tell the caller what it is and that it is never \
extracted from the prompt: {}",
description
);
assert!(
!graph.initial_state.contains_key("verification_commands"),
"{name}: an initial_state key would shadow the declared variable"