From 2b45e3a9b83c42b9da1541c4b1beffd519352a75 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Mon, 10 Aug 2026 15:37:53 -0600 Subject: [PATCH] feat: improved wording and heuristic detection for sisyphus suite of agents --- assets/agents/.shared/utils.sh | 118 ++++++++++++++---- assets/agents/coder/graph.yaml | 7 +- assets/agents/coder/scripts/verify_build.sh | 3 +- assets/agents/coder/scripts/verify_tests.sh | 3 +- assets/agents/sisyphus/config.yaml | 6 + assets/agents/step-runner/graph.yaml | 6 + .../step-runner/scripts/verify_build.sh | 3 +- .../step-runner/scripts/verify_format_lint.sh | 21 ++-- .../step-runner/scripts/verify_tests.sh | 3 +- 9 files changed, 136 insertions(+), 34 deletions(-) diff --git a/assets/agents/.shared/utils.sh b/assets/agents/.shared/utils.sh index 773f92d..ba3bc15 100755 --- a/assets/agents/.shared/utils.sh +++ b/assets/agents/.shared/utils.sh @@ -40,15 +40,57 @@ _write_project_cache() { _detect_heuristic() { local dir="$1" + local runner="" runner_type="" runner_targets="" + if [[ -f "${dir}/Taskfile.yml" || -f "${dir}/Taskfile.yaml" || -f "${dir}/taskfile.yml" || -f "${dir}/taskfile.yaml" ]]; then + runner="task" runner_type="taskfile" + runner_targets=$( (cd "${dir}" && task --list-all 2>/dev/null | sed -n 's/^\* \([^:[:space:]]*\):.*/\1/p') || true) + elif [[ -f "${dir}/justfile" || -f "${dir}/Justfile" ]]; then + runner="just" runner_type="just" + runner_targets=$( (cd "${dir}" && just --summary 2>/dev/null | tr ' ' '\n') || true) + elif [[ -f "${dir}/Makefile" || -f "${dir}/makefile" || -f "${dir}/GNUmakefile" ]]; then + runner="make" runner_type="make" + local mk mkfiles=() + for mk in Makefile makefile GNUmakefile; do + [[ -f "${dir}/${mk}" ]] && mkfiles+=("${dir}/${mk}") + done + runner_targets=$(sed -n 's/^\([A-Za-z0-9_][A-Za-z0-9_.-]*\):\([^=].*\|\)$/\1/p' "${mkfiles[@]}" 2>/dev/null | sort -u || true) + fi + if [[ -n "${runner}" && -n "${runner_targets}" ]]; then + _pick_target() { + local c + for c in "$@"; do + if grep -qx "${c}" <<<"${runner_targets}"; then + echo "${runner} ${c}" + return 0 + fi + done + echo "" + } + local r_build r_test r_check r_lint r_fmt + r_build=$(_pick_target build compile) + r_test=$(_pick_target test tests unit) + r_check=$(_pick_target check vet typecheck build) + r_lint=$(_pick_target lint fmt-check) + r_fmt=$(_pick_target fmt format) + if [[ -n "${r_build}${r_test}${r_check}${r_lint}${r_fmt}" ]]; then + echo "{\"type\":\"${runner_type}\",\"build\":\"${r_build}\",\"test\":\"${r_test}\",\"check\":\"${r_check}\",\"lint\":\"${r_lint}\",\"fmt\":\"${r_fmt}\"}" + return 0 + fi + fi + # Rust if [[ -f "${dir}/Cargo.toml" ]]; then - echo '{"type":"rust","build":"cargo build","test":"cargo test","check":"cargo check"}' + echo '{"type":"rust","build":"cargo build","test":"cargo test","check":"cargo check","lint":"cargo clippy --no-deps -- -D warnings","fmt":"cargo fmt"}' return 0 fi # Go if [[ -f "${dir}/go.mod" ]]; then - echo '{"type":"go","build":"go build ./...","test":"go test ./...","check":"go vet ./..."}' + local go_lint="" + if compgen -G "${dir}/.golangci.*" &>/dev/null && command -v golangci-lint &>/dev/null; then + go_lint="golangci-lint run" + fi + echo "{\"type\":\"go\",\"build\":\"go build ./...\",\"test\":\"go test ./...\",\"check\":\"go vet ./...\",\"lint\":\"${go_lint}\",\"fmt\":\"gofmt -w .\"}" return 0 fi @@ -65,7 +107,25 @@ _detect_heuristic() { [[ -f "${dir}/pnpm-lock.yaml" ]] && pm="pnpm" [[ -f "${dir}/yarn.lock" ]] && pm="yarn" - echo "{\"type\":\"nodejs\",\"build\":\"${pm} run build\",\"test\":\"${pm} test\",\"check\":\"${pm} run lint\"}" + # Emit only scripts the manifest actually declares (same introspection + # contract as the runner tier: never guess a target into existence). + _pkg_script() { + local s + for s in "$@"; do + if jq -e --arg s "$s" '.scripts[$s] // empty' "${dir}/package.json" &>/dev/null; then + echo "${pm} run ${s}" + return 0 + fi + done + echo "" + } + local p_build p_test p_check p_lint p_fmt + p_build=$(_pkg_script build compile) + p_test=$(_pkg_script test) + p_check=$(_pkg_script check typecheck tsc) + p_lint=$(_pkg_script lint) + p_fmt=$(_pkg_script fmt format prettier) + echo "{\"type\":\"nodejs\",\"build\":\"${p_build}\",\"test\":\"${p_test}\",\"check\":\"${p_check}\",\"lint\":\"${p_lint}\",\"fmt\":\"${p_fmt}\"}" return 0 fi @@ -82,7 +142,7 @@ _detect_heuristic() { check_cmd="uv run ruff check ." fi - echo "{\"type\":\"python\",\"build\":\"\",\"test\":\"${test_cmd}\",\"check\":\"${check_cmd}\"}" + echo "{\"type\":\"python\",\"build\":\"\",\"test\":\"${test_cmd}\",\"check\":\"${check_cmd}\",\"lint\":\"${check_cmd}\",\"fmt\":\"ruff format .\"}" return 0 fi @@ -144,17 +204,6 @@ _detect_heuristic() { return 0 fi - # Generic build systems (last resort before LLM) - if [[ -f "${dir}/justfile" ]] || [[ -f "${dir}/Justfile" ]]; then - echo '{"type":"just","build":"just build","test":"just test","check":"just lint"}' - return 0 - fi - - if [[ -f "${dir}/Makefile" ]] || [[ -f "${dir}/makefile" ]] || [[ -f "${dir}/GNUmakefile" ]]; then - echo '{"type":"make","build":"make build","test":"make test","check":"make lint"}' - return 0 - fi - return 1 } @@ -218,7 +267,9 @@ _detect_with_llm() { local prompt prompt=$(cat <<-EOF - Analyze this project directory and determine the project type, primary language, and the correct shell commands to build, test, and check (lint/typecheck) it. + Analyze this project directory and determine the project type, primary language, and the correct shell commands to build, test, check (typecheck/vet), lint, and format it. + + PRIORITY RULE: if the project declares its own task-runner interface (a Taskfile, justfile, Makefile, package.json scripts, or similar), those declared targets ARE the correct commands — prefer them over generic ecosystem defaults, and never invent a target the interface does not declare. EOF ) @@ -226,12 +277,12 @@ _detect_with_llm() { prompt+=$(cat <<-EOF Respond with ONLY a valid JSON object. No markdown fences, no explanation, no extra text. - The JSON must have exactly these 4 keys: - {"type":"","build":"","test":"","check":""} + The JSON must have exactly these 6 keys: + {"type":"","build":"","test":"","check":"","lint":"","fmt":""} Rules: - "type" must be a single lowercase word (e.g. rust, go, python, nodejs, java, ruby, elixir, cpp, c, zig, haskell, scala, kotlin, dart, swift, php, dotnet, etc.) - - If a command doesn't apply to this project, use an empty string, "" + - If a command doesn't apply to this project, use an empty string, "" — NEVER guess a command that might not exist; a wrongly-guessed command is worse than an empty one - Use the most standard/common commands for the detected ecosystem - If you detect a package manager lockfile, use that package manager (e.g. pnpm over npm) EOF @@ -244,7 +295,7 @@ _detect_with_llm() { llm_response=$(echo "${llm_response}" | grep -o '{[^}]*}' | head -1) if echo "${llm_response}" | jq -e '.type and .build != null and .test != null and .check != null' &>/dev/null; then - echo "${llm_response}" | jq -c '{type: (.type // "unknown"), build: (.build // ""), test: (.test // ""), check: (.check // "")}' + echo "${llm_response}" | jq -c '{type: (.type // "unknown"), build: (.build // ""), test: (.test // ""), check: (.check // ""), lint: (.lint // ""), fmt: (.fmt // "")}' return 0 fi @@ -258,7 +309,7 @@ detect_project() { local cached if cached=$(_read_project_cache "${dir}"); then - echo "${cached}" | jq -c '{type, build, test, check}' + echo "${cached}" | jq -c '{type, build, test, check, lint: (.lint // ""), fmt: (.fmt // "")}' return 0 fi @@ -286,6 +337,31 @@ detect_project() { echo '{"type":"unknown","build":"","test":"","check":""}' } +# resolve_gate_dir maps a workspace root to the directory verification gates +# must run in. A delivery-repo worker's workspace root holds only dotfiles +# plus the clone, so gates aimed at the root detect nothing and silently +# no-op. When the root has no project markers and exactly ONE first-level +# git repo exists, gates run inside it; anything ambiguous stays at the root. +resolve_gate_dir() { + local dir="${1:-.}" + local m + for m in Taskfile.yml Taskfile.yaml taskfile.yml Cargo.toml go.mod package.json pyproject.toml setup.py pom.xml build.gradle mix.exs Gemfile composer.json Makefile justfile Justfile CMakeLists.txt; do + if [[ -e "${dir}/${m}" ]]; then + echo "${dir}" + return 0 + fi + done + local repos=() d + for d in "${dir}"/*/; do + [[ -d "${d}/.git" ]] && repos+=("${d}") + done + if [[ ${#repos[@]} -eq 1 ]]; then + echo "${repos[0]%/}" + return 0 + fi + echo "${dir}" +} + ########################### ## FILE SEARCH UTILITIES ## ########################### diff --git a/assets/agents/coder/graph.yaml b/assets/agents/coder/graph.yaml index d3dd5a8..06e5697 100644 --- a/assets/agents/coder/graph.yaml +++ b/assets/agents/coder/graph.yaml @@ -227,6 +227,11 @@ nodes: on unfamiliar lints, etc.). 4. No dead code, no commented-out blocks, no premature abstractions. 5. End your turn when editing is done. The graph runs verification next. + 6. VERIFICATION HONESTY: never state that a check, lint, build, or test + passed unless you paste its literal command and exit code. A gate + that did not run is UNVERIFIED — say so. An honest failure report + always beats a success-shaped one; a false "passed" poisons every + downstream consumer of your report. Project directory: {{project_dir}} prompt: | @@ -248,7 +253,7 @@ nodes: - fs_write - fs_patch - execute_command - max_iterations: 30 + max_iterations: 100 state_updates: last_node_output: '{{output}}' fallback: end_failure diff --git a/assets/agents/coder/scripts/verify_build.sh b/assets/agents/coder/scripts/verify_build.sh index f9b9d65..c67a7d1 100644 --- a/assets/agents/coder/scripts/verify_build.sh +++ b/assets/agents/coder/scripts/verify_build.sh @@ -13,6 +13,7 @@ else fi project_dir=$(echo "$state" | jq -r '.project_dir // "."') +project_dir=$(resolve_gate_dir "$project_dir") if [[ -n "${BUILD_CMD:-}" ]]; then cmd="$BUILD_CMD" @@ -24,7 +25,7 @@ fi if [[ -z "$cmd" || "$cmd" == "null" ]]; then jq -nc '{ "build_ok": true, - "build_output": "(no build/check command available for this project type)", + "build_output": "(GATE NOT RUN: no build/check command configured or detected. This is NOT evidence that the build passed — set BUILD_CMD, and never report the build as verified.)", "_next": "verify_tests" }' exit 0 diff --git a/assets/agents/coder/scripts/verify_tests.sh b/assets/agents/coder/scripts/verify_tests.sh index a72de94..80edeb8 100644 --- a/assets/agents/coder/scripts/verify_tests.sh +++ b/assets/agents/coder/scripts/verify_tests.sh @@ -13,6 +13,7 @@ else fi project_dir=$(echo "$state" | jq -r '.project_dir // "."') +project_dir=$(resolve_gate_dir "$project_dir") if [[ -n "${TEST_CMD:-}" ]]; then cmd="$TEST_CMD" @@ -24,7 +25,7 @@ fi if [[ -z "$cmd" || "$cmd" == "null" ]]; then jq -nc '{ "tests_ok": true, - "tests_output": "(no test command available for this project type)", + "tests_output": "(GATE NOT RUN: no test command configured or detected. This is NOT evidence that tests passed — set TEST_CMD, and never report the suite as green.)", "_next": "self_review" }' exit 0 diff --git a/assets/agents/sisyphus/config.yaml b/assets/agents/sisyphus/config.yaml index 89a2a05..a0be536 100644 --- a/assets/agents/sisyphus/config.yaml +++ b/assets/agents/sisyphus/config.yaml @@ -266,6 +266,12 @@ instructions: | **No evidence = not complete.** Mark a todo `completed` only after evidence is collected. + ### Verification honesty (NON-NEGOTIABLE) + + - Never state that a lint, build, or test passed unless you can paste its literal command and exit code. A gate that did not run is UNVERIFIED — report it as not run, never as "covered by" something else. + - Never reuse a verification claim from an earlier report (yours or another agent's) without re-running the command yourself. Prior reports are unverified context, not evidence. + - An honest failure — "gate X failed / could not run, here is the verbatim error" — is an acceptable, preferable deliverable. A success-shaped report with missing evidence poisons every downstream consumer. + ### Independent code review (post-coder, non-trivial work) After completing delegated `coder` work, spawn `code-reviewer` for an independent review pass if ANY of these are true: diff --git a/assets/agents/step-runner/graph.yaml b/assets/agents/step-runner/graph.yaml index 08365ce..dc8cbe0 100644 --- a/assets/agents/step-runner/graph.yaml +++ b/assets/agents/step-runner/graph.yaml @@ -439,6 +439,12 @@ nodes: staleness report, gate decisions, and fix loop history. Downstream plan updates come from the sweep results. + VERIFICATION HONESTY: evidence marked "GATE NOT RUN" means that gate + is UNVERIFIED — record it as not run; never paraphrase a skipped gate + as covered, passing, or handled elsewhere. A handoff that admits an + unverified gate is correct; one that dresses it up as verified poisons + every downstream reader. + Then append durable, step-independent facts (if any) to {{notes_path}} - create the file if missing, never rewrite existing entries. diff --git a/assets/agents/step-runner/scripts/verify_build.sh b/assets/agents/step-runner/scripts/verify_build.sh index 23704f3..fdb2bf8 100755 --- a/assets/agents/step-runner/scripts/verify_build.sh +++ b/assets/agents/step-runner/scripts/verify_build.sh @@ -13,6 +13,7 @@ else fi project_dir=$(echo "$state" | jq -r '.project_dir // "."') +project_dir=$(resolve_gate_dir "$project_dir") if [[ -n "${BUILD_CMD:-}" ]]; then cmd="$BUILD_CMD" @@ -24,7 +25,7 @@ fi if [[ -z "$cmd" || "$cmd" == "null" ]]; then jq -nc '{ "build_ok": true, - "build_output": "(no build/check command available for this project type)", + "build_output": "(GATE NOT RUN: no build/check command configured or detected. This is NOT evidence that the build passed — set BUILD_CMD, and never report the build as verified.)", "_next": "verify_tests" }' exit 0 diff --git a/assets/agents/step-runner/scripts/verify_format_lint.sh b/assets/agents/step-runner/scripts/verify_format_lint.sh index c20e2f3..45f5b9d 100755 --- a/assets/agents/step-runner/scripts/verify_format_lint.sh +++ b/assets/agents/step-runner/scripts/verify_format_lint.sh @@ -13,19 +13,18 @@ else fi project_dir=$(echo "$state" | jq -r '.project_dir // "."') -project_type=$(detect_project "$project_dir" | jq -r '.type // "unknown"') +project_dir=$(resolve_gate_dir "$project_dir") +project_info=$(detect_project "$project_dir") +project_type=$(echo "$project_info" | jq -r '.type // "unknown"') format_cmd="${FORMAT_CMD:-}" if [[ -z "$format_cmd" ]]; then - case "$project_type" in - rust) format_cmd="cargo fmt" ;; - go) format_cmd="gofmt -w ." ;; - python) command -v ruff &>/dev/null && format_cmd="ruff format ." ;; - esac + format_cmd=$(echo "$project_info" | jq -r '.fmt // ""') fi +if [[ "$format_cmd" == "null" ]]; then format_cmd=""; fi if [[ -z "$format_cmd" ]]; then - format_output="(no format command configured for project type '$project_type'; skipped. Set FORMAT_CMD to enable.)" + format_output="(GATE NOT RUN: no format command configured or detected for project type '$project_type'. This is NOT evidence that formatting is clean. Set FORMAT_CMD to enable.)" else fmt_rc=0 fmt_out=$(cd "$project_dir" && eval "$format_cmd" 2>&1) || fmt_rc=$? @@ -37,12 +36,18 @@ fi lint_cmd="${LINT_CMD:-}" if [[ -z "$lint_cmd" ]]; then + lint_cmd=$(echo "$project_info" | jq -r '.lint // ""') +fi +# The skip message must read as a WARNING, never a reassurance: the previous +# wording ("linting is covered by the build/check command") was quoted +# verbatim by workers as false evidence that linting passed +if [[ -z "$lint_cmd" || "$lint_cmd" == "null" ]]; then jq -nc \ --arg fo "$format_output" \ '{ "format_output": $fo, "lint_ok": true, - "lint_output": "(no LINT_CMD configured; linting is covered by the build/check command)", + "lint_output": "(GATE NOT RUN: no lint command configured or detected. This is NOT evidence that linting passed — set LINT_CMD or add a Taskfile lint target, and never report linting as covered.)", "_next": "verify_build" }' exit 0 diff --git a/assets/agents/step-runner/scripts/verify_tests.sh b/assets/agents/step-runner/scripts/verify_tests.sh index 481e126..0235909 100755 --- a/assets/agents/step-runner/scripts/verify_tests.sh +++ b/assets/agents/step-runner/scripts/verify_tests.sh @@ -13,6 +13,7 @@ else fi project_dir=$(echo "$state" | jq -r '.project_dir // "."') +project_dir=$(resolve_gate_dir "$project_dir") if [[ -n "${TEST_CMD:-}" ]]; then cmd="$TEST_CMD" @@ -24,7 +25,7 @@ fi if [[ -z "$cmd" || "$cmd" == "null" ]]; then jq -nc '{ "tests_ok": true, - "tests_output": "(no test command available for this project type)", + "tests_output": "(GATE NOT RUN: no test command configured or detected. This is NOT evidence that tests passed — set TEST_CMD, and never report the suite as green.)", "_next": "edge_case_sweep" }' exit 0