fix(review-suite): mark example verification commands as placeholders; pin variable description and prompt note
Adversary re-review follow-ups to 665fee9/402d25a: - architect/sisyphus spawn templates: the pinned cargo commands are only an EXAMPLE in a language-agnostic orchestrator — a literal follower in a non-Rust repo would record 'cargo: command not found' and yield a false DIVERGES. Each --variables line now carries a substitution cue (pinned). - run_checks.py: a non-list prior pipeline_faults value is no longer iterated character by character (test case added). - Pins: both graphs' verification_commands description wording; the run_adversary prompt states where the commands come from; the replacement sentence sits on one physical line.
This commit is contained in:
1 parent
27101909e0
commit
84b5a72eac
5 files changed
+46
-3
No files matched your search
@@ -147,7 +147,8 @@ def main():
|
|||||||
cmds = declared_commands(state.get("verification_commands"))
|
cmds = declared_commands(state.get("verification_commands"))
|
||||||
except InvalidDeclaration as e:
|
except InvalidDeclaration as e:
|
||||||
msg = f"PIPELINE-FAULT: verification_commands declaration invalid — {e}"
|
msg = f"PIPELINE-FAULT: verification_commands declaration invalid — {e}"
|
||||||
faults = [f for f in (state.get("pipeline_faults") or []) if isinstance(f, str)]
|
prior = state.get("pipeline_faults")
|
||||||
|
faults = [f for f in prior if isinstance(f, str)] if isinstance(prior, list) else []
|
||||||
faults.append(msg)
|
faults.append(msg)
|
||||||
print(json.dumps({"exec_results": msg, "pipeline_faults": faults}))
|
print(json.dumps({"exec_results": msg, "pipeline_faults": faults}))
|
||||||
return
|
return
|
||||||
|
|||||||
@@ -334,6 +334,7 @@ instructions: |
|
|||||||
Plan / acceptance criteria: <the task's acceptance criteria, pasted verbatim>
|
Plan / acceptance criteria: <the task's acceptance criteria, pasted verbatim>
|
||||||
Local-run recipe / usage suites: <from the plan, when present>"
|
Local-run recipe / usage suites: <from the plan, when present>"
|
||||||
--variables {"verification_commands": "[\"cargo test --all\", \"cargo clippy -- -D warnings\"]"}
|
--variables {"verification_commands": "[\"cargo test --all\", \"cargo clippy -- -D warnings\"]"}
|
||||||
|
<— the cargo commands above are only an EXAMPLE: replace them with THIS project's exact build/test/lint commands; never guess, and pass "[]" when none apply>
|
||||||
```
|
```
|
||||||
|
|
||||||
Verification commands go in `--variables` (a JSON-array string), never in the prompt: the
|
Verification commands go in `--variables` (a JSON-array string), never in the prompt: the
|
||||||
@@ -357,6 +358,7 @@ instructions: |
|
|||||||
<paste the plan's ## Quality bar section VERBATIM — recorded dropped practices are
|
<paste the plan's ## Quality bar section VERBATIM — recorded dropped practices are
|
||||||
conformance facts, not divergences>"
|
conformance facts, not divergences>"
|
||||||
--variables {"verification_commands": "[\"cargo test --all\", \"cargo clippy -- -D warnings\"]"}
|
--variables {"verification_commands": "[\"cargo test --all\", \"cargo clippy -- -D warnings\"]"}
|
||||||
|
<— the cargo commands above are only an EXAMPLE: replace them with THIS project's exact build/test/lint commands; never guess, and pass "[]" when none apply>
|
||||||
```
|
```
|
||||||
|
|
||||||
Same trust boundary as the gauntlet spawn: verification commands are declared via
|
Same trust boundary as the gauntlet spawn: verification commands are declared via
|
||||||
|
|||||||
@@ -333,8 +333,7 @@ nodes:
|
|||||||
The spec / plan / acceptance criteria it must satisfy:
|
The spec / plan / acceptance criteria it must satisfy:
|
||||||
{{plan_context}}
|
{{plan_context}}
|
||||||
|
|
||||||
Verification commands arrive as a structured input (the
|
Verification commands arrive as a structured input (the verification_commands variable), not in this prompt.
|
||||||
verification_commands variable), not in this prompt.
|
|
||||||
|
|
||||||
{{extra_context}}
|
{{extra_context}}
|
||||||
|
|
||||||
|
|||||||
@@ -316,6 +316,7 @@ instructions: |
|
|||||||
Plan / acceptance criteria: <pasted verbatim — enables the adversary lane; omit when none>
|
Plan / acceptance criteria: <pasted verbatim — enables the adversary lane; omit when none>
|
||||||
Local-run recipe / usage suites: <from the plan — enables the probe lane; omit when none>"
|
Local-run recipe / usage suites: <from the plan — enables the probe lane; omit when none>"
|
||||||
--variables {"verification_commands": "[\"cargo test --all\", \"cargo clippy -- -D warnings\"]"}
|
--variables {"verification_commands": "[\"cargo test --all\", \"cargo clippy -- -D warnings\"]"}
|
||||||
|
<— the cargo commands above are only an EXAMPLE: replace them with THIS project's exact build/test/lint commands; never guess, and pass "[]" when none apply>
|
||||||
```
|
```
|
||||||
|
|
||||||
Verification commands go in `--variables` (a JSON-array string), never in the prompt: the prompt
|
Verification commands go in `--variables` (a JSON-array string), never in the prompt: the prompt
|
||||||
@@ -398,6 +399,7 @@ instructions: |
|
|||||||
PLAN — acceptance criteria to check against:
|
PLAN — acceptance criteria to check against:
|
||||||
<paste the task/step spec + acceptance criteria VERBATIM — not a summary>"
|
<paste the task/step spec + acceptance criteria VERBATIM — not a summary>"
|
||||||
--variables {"verification_commands": "[\"cargo test --all\", \"cargo clippy -- -D warnings\"]"}
|
--variables {"verification_commands": "[\"cargo test --all\", \"cargo clippy -- -D warnings\"]"}
|
||||||
|
<— the cargo commands above are only an EXAMPLE: replace them with THIS project's exact build/test/lint commands; never guess, and pass "[]" when none apply>
|
||||||
```
|
```
|
||||||
|
|
||||||
Same trust boundary as the gauntlet spawn: verification commands are declared via `--variables`,
|
Same trust boundary as the gauntlet spawn: verification commands are declared via `--variables`,
|
||||||
|
|||||||
@@ -8233,6 +8233,8 @@ mod tests {
|
|||||||
|
|
||||||
const VARIABLES_FORM: &str =
|
const VARIABLES_FORM: &str =
|
||||||
"--variables {\"verification_commands\": \"[\\\"cargo test --all\\\"";
|
"--variables {\"verification_commands\": \"[\\\"cargo test --all\\\"";
|
||||||
|
const EXAMPLE_CUE: &str =
|
||||||
|
"only an EXAMPLE: replace them with THIS project's exact build/test/lint commands";
|
||||||
for name in ["architect", "sisyphus"] {
|
for name in ["architect", "sisyphus"] {
|
||||||
let config =
|
let config =
|
||||||
read_to_string(paths::agents_data_dir().join(name).join("config.yaml")).unwrap();
|
read_to_string(paths::agents_data_dir().join(name).join("config.yaml")).unwrap();
|
||||||
@@ -8254,6 +8256,11 @@ mod tests {
|
|||||||
config.contains("untrusted pasted text"),
|
config.contains("untrusted pasted text"),
|
||||||
"{name} config must name the untrusted prompt text as the reason"
|
"{name} config must name the untrusted prompt text as the reason"
|
||||||
);
|
);
|
||||||
|
assert!(
|
||||||
|
config.matches(EXAMPLE_CUE).count() >= 2,
|
||||||
|
"{name} config must mark the cargo commands as an example to substitute on both \
|
||||||
|
spawn templates, or a literal follower records `cargo: command not found` in non-Rust repos"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
let readmes = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("assets/agents");
|
let readmes = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("assets/agents");
|
||||||
@@ -9094,6 +9101,20 @@ mod tests {
|
|||||||
!pwned.exists(),
|
!pwned.exists(),
|
||||||
"an invalid declaration must execute nothing, even its string items"
|
"an invalid declaration must execute nothing, even its string items"
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// A non-list prior `pipeline_faults` is never iterated character by
|
||||||
|
// character; only the new marker is recorded.
|
||||||
|
let out = run_adversary_script(
|
||||||
|
"run_checks.py",
|
||||||
|
&json!({"verification_commands": "not json", "pipeline_faults": "oops"}),
|
||||||
|
);
|
||||||
|
let marker = out["exec_results"].as_str().unwrap();
|
||||||
|
assert!(marker.starts_with(FAULT), "{marker}");
|
||||||
|
assert_eq!(
|
||||||
|
out["pipeline_faults"],
|
||||||
|
json!([marker]),
|
||||||
|
"a string-valued prior pipeline_faults must not be spread into characters"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -10614,6 +10635,11 @@ mod tests {
|
|||||||
"verification commands are a structured input, never lane-prompt prose: {}",
|
"verification commands are a structured input, never lane-prompt prose: {}",
|
||||||
adv.prompt
|
adv.prompt
|
||||||
);
|
);
|
||||||
|
assert!(
|
||||||
|
adv.prompt.contains("arrive as a structured input"),
|
||||||
|
"run_adversary's prompt must say where the commands come from instead: {}",
|
||||||
|
adv.prompt
|
||||||
|
);
|
||||||
let node = graph
|
let node = graph
|
||||||
.nodes
|
.nodes
|
||||||
.get("run_probe")
|
.get("run_probe")
|
||||||
@@ -10648,6 +10674,19 @@ mod tests {
|
|||||||
Some("[]"),
|
Some("[]"),
|
||||||
"{name}: variables land as strings, so the default is the JSON-encoded empty list"
|
"{name}: variables land as strings, so the default is the JSON-encoded empty list"
|
||||||
);
|
);
|
||||||
|
// The YAML block scalar wraps mid-sentence; pin the words, not the wrap.
|
||||||
|
let description = var
|
||||||
|
.description
|
||||||
|
.split_whitespace()
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
.join(" ");
|
||||||
|
assert!(
|
||||||
|
description.contains("JSON array of shell commands the CALLER declares")
|
||||||
|
&& description.contains("Never inferred from the prompt"),
|
||||||
|
"{name}: the variable must tell the caller what it is and that it is never \
|
||||||
|
extracted from the prompt: {}",
|
||||||
|
description
|
||||||
|
);
|
||||||
assert!(
|
assert!(
|
||||||
!graph.initial_state.contains_key("verification_commands"),
|
!graph.initial_state.contains_key("verification_commands"),
|
||||||
"{name}: an initial_state key would shadow the declared variable"
|
"{name}: an initial_state key would shadow the declared variable"
|
||||||
|
|||||||
Reference in new issue
Block a user