From 5495813baf7ac02f8d7388b379eed19bda18af75 Mon Sep 17 00:00:00 2001 From: Alex Clarke Date: Wed, 16 Sep 2026 14:41:05 -0600 Subject: [PATCH] feat(TASK-057): web_search_coyote works on both OpenAI wire APIs - openai searches run an attempt chain: the chat-completions mechanism (search-preview model + web_search_options patch) first, then the Responses-native web_search tool on a responses-capable model; the search fails only when both attempts fail - both wire-keyed patches (COYOTE_PATCH_OPENAI_CHAT_COMPLETIONS / COYOTE_PATCH_OPENAI_RESPONSES) are exported per attempt; the client's resolved wire applies exactly one - supports_native_search accepts responses-capable openai families (gpt-4o*/gpt-4.1*/gpt-5*/o3*/o4*) - stub-coyote harness pins attempt order, patch export, model mapping, hard failure, and untouched non-openai paths (scripts/test-web-search-tool.sh) - CHANGELOG: responses-default breakage caveat replaced with the both-wire feature note --- CHANGELOG.md | 4 +- assets/functions/tools/web_search_coyote.sh | 76 ++++++- scripts/test-web-search-tool.sh | 222 ++++++++++++++++++++ 3 files changed, 289 insertions(+), 13 deletions(-) create mode 100755 scripts/test-web-search-tool.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 1aac5f8..255ddb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,12 +12,10 @@ migrate them to the sibling `patch.responses` key / the model patch's `responses (env override: `COYOTE_PATCH__RESPONSES`). - Sessions saved after this change may contain the new `reasoning` thinking blocks (encrypted reasoning items from the Responses API); older coyote binaries cannot load those sessions. -- OpenAI native web search (the bundled `web_search_coyote` tool) uses the chat-only -`gpt-4o-*search-preview` mechanism, which the responses wire does not serve: until the tool is -updated, pin `wire_api: chat` on the openai client when relying on it. ### Feat +- the bundled `web_search_coyote` tool serves openai native web search on both wires: it tries the chat-completions mechanism (the chat-only `gpt-4o-*search-preview` models + `web_search_options`) first and falls back to the Responses-native `web_search` tool on responses-capable models (`gpt-4o*`/`gpt-4.1*`/`gpt-5*`/`o3*`/`o4*`), failing only when both attempts fail — the stock responses default needs no `wire_api: chat` pin - support `wire_api: responses | chat` on the openai and openai-compatible clients, with wire-aware request patching on both patch layers - report token usage (including cached input tokens) on the openai responses path - streaming responses that end with `response.incomplete` now surface the truncation reason (e.g. `max_output_tokens`) when no visible output (text or tool calls) was delivered — matching the non-streaming behavior — and record token usage for truncated turns instead of ending silently diff --git a/assets/functions/tools/web_search_coyote.sh b/assets/functions/tools/web_search_coyote.sh index 1ee3ace..68e51c4 100755 --- a/assets/functions/tools/web_search_coyote.sh +++ b/assets/functions/tools/web_search_coyote.sh @@ -23,10 +23,16 @@ set -e # - perplexity:* # - ernie:* # - claude:* (Anthropic native web_search server tool) -# - openai:gpt-4o-search-preview (and -mini-; requires an api-key openai -# client — the codex OAuth path uses the -# Responses API where this parameter -# does not exist) +# - openai:gpt-4o*/gpt-4.1*/gpt-5*/o3*/o4* +# OpenAI serves web search through two different mechanisms depending on +# the wire API the client resolves (the script cannot pick the wire): +# Chat Completions serves it via `web_search_options` on the chat-only +# *-search-preview models, while the Responses API serves it via its +# native `web_search` tool on regular models. Both request patches are +# exported (request patching is wire-aware, so the resolved wire applies +# exactly one) and openai searches run as an attempt chain: the +# chat-completions mechanism first, then the Responses mechanism, and +# the search fails only when both attempts fail. # @env WEB_SEARCH_USE_CURRENT_MODEL=true Prefer the running session's model when it natively supports web search. # @env LLM_OUTPUT=/dev/stdout The output path @@ -34,7 +40,7 @@ set -e supports_native_search() { case "$1" in gemini:* | vertexai:gemini-* | perplexity:* | ernie:* | claude:*) return 0 ;; - openai:gpt-4o*search-preview*) return 0 ;; + openai:gpt-4o* | openai:gpt-4.1* | openai:gpt-5* | openai:o3* | openai:o4*) return 0 ;; *) return 1 ;; esac } @@ -46,7 +52,8 @@ export_search_patch() { COYOTE_PATCH_VERTEXAI_CHAT_COMPLETIONS \ COYOTE_PATCH_ERNIE_CHAT_COMPLETIONS \ COYOTE_PATCH_CLAUDE_CHAT_COMPLETIONS \ - COYOTE_PATCH_OPENAI_CHAT_COMPLETIONS + COYOTE_PATCH_OPENAI_CHAT_COMPLETIONS \ + COYOTE_PATCH_OPENAI_RESPONSES case "${1%%:*}" in gemini) @@ -65,18 +72,67 @@ export_search_patch() { export COYOTE_PATCH_CLAUDE_CHAT_COMPLETIONS='{".*":{"body":{"tools":[{"type":"web_search_20250305","name":"web_search","max_uses":5}]}}}' ;; openai) - # Chat Completions native search exists only on the search-preview - # models; the regex scopes the patch so other OpenAI models run - # unpatched instead of erroring on an unsupported parameter. + # Both patches are exported; request patching is wire-aware, so the + # client's resolved wire applies exactly one of them. Each regex + # scopes its patch to the models that actually support that + # mechanism, so other OpenAI models run unpatched instead of + # erroring on an unsupported parameter: + # - Chat Completions native search exists only on the chat-only + # *-search-preview models (`web_search_options`). + # - The Responses API serves search through its native + # `web_search` tool, which only some model families support. export COYOTE_PATCH_OPENAI_CHAT_COMPLETIONS='{"gpt-4o.*search-preview.*":{"body":{"web_search_options":{}}}}' + export COYOTE_PATCH_OPENAI_RESPONSES='{"(gpt-4o|gpt-4\\.1|gpt-5|o3|o4).*":{"body":{"tools":[{"type":"web_search"}]}}}' ;; esac } +# Maps an openai model to the one used for the chat-completions attempt: +# Chat Completions web search is served only by the *-search-preview models. +openai_chat_search_model() { + case "${1#openai:}" in + gpt-4o*search-preview*) echo "$1" ;; + gpt-4o-mini*) echo "openai:gpt-4o-mini-search-preview" ;; + *) echo "openai:gpt-4o-search-preview" ;; + esac +} + +# Maps an openai model to the one used for the Responses attempt: the +# *-search-preview models are chat-only and do not exist on /v1/responses, +# so they map back to their base models. +openai_responses_search_model() { + case "${1#openai:}" in + gpt-4o-mini-search-preview*) echo "openai:gpt-4o-mini" ;; + gpt-4o*search-preview*) echo "openai:gpt-4o" ;; + *) echo "$1" ;; + esac +} + +# OpenAI attempt chain: try the chat-completions mechanism first (works when +# the client's wire is chat); if that fails — e.g. the client resolved the +# responses wire, where the search-preview models do not exist — retry with +# the Responses mechanism on a responses-capable model. Fails when both +# attempts fail. +# shellcheck disable=SC2154 +run_openai_search() { + if coyote -m "$(openai_chat_search_model "$1")" "$argc_query" >>"$LLM_OUTPUT"; then + return 0 + fi + echo "web_search: openai chat-completions attempt failed; retrying via the Responses API" >&2 + coyote -m "$(openai_responses_search_model "$1")" "$argc_query" >>"$LLM_OUTPUT" +} + # shellcheck disable=SC2154 run_search() { export_search_patch "$1" - coyote -m "$1" "$argc_query" >>"$LLM_OUTPUT" + case "$1" in + openai:*) + run_openai_search "$1" + ;; + *) + coyote -m "$1" "$argc_query" >>"$LLM_OUTPUT" + ;; + esac } main() { diff --git a/scripts/test-web-search-tool.sh b/scripts/test-web-search-tool.sh new file mode 100755 index 0000000..5f9ca57 --- /dev/null +++ b/scripts/test-web-search-tool.sh @@ -0,0 +1,222 @@ +#!/usr/bin/env bash +# Regression harness for assets/functions/tools/web_search_coyote.sh. +# +# Runs the tool against a fake `coyote` executable on PATH that records every +# invocation (model + the search-patch env vars it received) and returns +# scripted per-call exit codes, pinning: +# A) openai attempt chain order: the chat-completions mechanism (search-preview +# model) runs first, the Responses mechanism (responses-capable model) +# runs second, and the run fails only when both attempts fail; +# B) both openai patch env vars are exported with the expected values so the +# client's resolved wire (chat vs responses) picks the matching one; +# C) model mapping between the two attempts (search-preview <-> base models); +# D) non-openai providers are untouched: single invocation, their own patch, +# openai patch keys cleared; +# E) the WEB_SEARCH_MODEL fallback and WEB_SEARCH_USE_CURRENT_MODEL +# semantics are unchanged. +# +# No network or credentials required. Usage: scripts/test-web-search-tool.sh + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TOOL="$REPO_ROOT/assets/functions/tools/web_search_coyote.sh" + +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "$WORK_DIR"' EXIT +export STUB_DIR="$WORK_DIR" + +# Expected patch values (must stay in sync with the tool script). +EXPECTED_OPENAI_CHAT_PATCH='{"gpt-4o.*search-preview.*":{"body":{"web_search_options":{}}}}' +EXPECTED_OPENAI_RESPONSES_PATCH='{"(gpt-4o|gpt-4\\.1|gpt-5|o3|o4).*":{"body":{"tools":[{"type":"web_search"}]}}}' +EXPECTED_CLAUDE_PATCH='{".*":{"body":{"tools":[{"type":"web_search_20250305","name":"web_search","max_uses":5}]}}}' + +# Fake coyote: logs "call=N model=M chat=... responses=... own=..." per +# invocation and exits with the N-th code from COYOTE_STUB_RESULTS. +cat >"$WORK_DIR/coyote" <<'STUB' +#!/usr/bin/env bash +n=$(( $(cat "$STUB_DIR/count" 2>/dev/null || echo 0) + 1 )) +echo "$n" >"$STUB_DIR/count" +model="" +[[ "${1:-}" == "-m" ]] && model="${2:-}" +{ + printf 'call=%s model=%s\n' "$n" "$model" + printf 'call=%s chat_patch=%s\n' "$n" "${COYOTE_PATCH_OPENAI_CHAT_COMPLETIONS-}" + printf 'call=%s responses_patch=%s\n' "$n" "${COYOTE_PATCH_OPENAI_RESPONSES-}" + printf 'call=%s gemini_patch=%s\n' "$n" "${COYOTE_PATCH_GEMINI_CHAT_COMPLETIONS-}" + printf 'call=%s claude_patch=%s\n' "$n" "${COYOTE_PATCH_CLAUDE_CHAT_COMPLETIONS-}" +} >>"$STUB_DIR/log" +read -r -a results <<<"${COYOTE_STUB_RESULTS:-}" +rc="${results[$((n - 1))]:-0}" +[[ "$rc" == "0" ]] && echo "stub-output-$n" +exit "$rc" +STUB +chmod +x "$WORK_DIR/coyote" + +FAILED=0 +CASE="" + +fail() { + echo "FAIL [$CASE]: $*" >&2 + echo "--- invocation log ---" >&2 + cat "$STUB_DIR/log" 2>/dev/null >&2 || true + echo "----------------------" >&2 + FAILED=1 +} + +# run_tool [VAR=value ...] +# Sources the tool with the argc runtime emulated (argc_query/LLM_OUTPUT set) +# and runs main(). Sets TOOL_RC to the tool's exit code. +run_tool() { + local results="$1" + shift + rm -f "$STUB_DIR/log" "$STUB_DIR/count" + : >"$STUB_DIR/out" + TOOL_RC=0 + ( + export PATH="$STUB_DIR:$PATH" + export COYOTE_STUB_RESULTS="$results" + export argc_query="test query" + export LLM_OUTPUT="$STUB_DIR/out" + unset WEB_SEARCH_MODEL COYOTE_CURRENT_MODEL WEB_SEARCH_USE_CURRENT_MODEL + local kv + for kv in "$@"; do + # shellcheck disable=SC2163 + export "${kv?}" + done + # shellcheck disable=SC1090 + source "$TOOL" + main + ) || TOOL_RC=$? +} + +assert_calls() { + local expected="$1" actual + actual="$(cat "$STUB_DIR/count" 2>/dev/null || echo 0)" + [[ "$actual" == "$expected" ]] || fail "expected $expected coyote invocation(s), got $actual" +} + +assert_log() { + local line="$1" + grep -qF -- "$line" "$STUB_DIR/log" || fail "missing log line: $line" +} + +assert_rc() { + local expected="$1" + if [[ "$expected" == "0" ]]; then + [[ "$TOOL_RC" -eq 0 ]] || fail "expected success, got exit code $TOOL_RC" + else + [[ "$TOOL_RC" -ne 0 ]] || fail "expected failure, tool exited 0" + fi +} + +# --- A/B: stock success — chat mechanism runs first with both patches set --- +CASE="openai current model, first attempt succeeds" +run_tool "0" COYOTE_CURRENT_MODEL=openai:gpt-4o +assert_rc 0 +assert_calls 1 +assert_log "call=1 model=openai:gpt-4o-search-preview" +assert_log "call=1 chat_patch=$EXPECTED_OPENAI_CHAT_PATCH" +assert_log "call=1 responses_patch=$EXPECTED_OPENAI_RESPONSES_PATCH" + +CASE="search-preview current model kept as-is on the chat attempt (pinned wire_api: chat behavior)" +run_tool "0" COYOTE_CURRENT_MODEL=openai:gpt-4o-search-preview +assert_rc 0 +assert_calls 1 +assert_log "call=1 model=openai:gpt-4o-search-preview" +assert_log "call=1 chat_patch=$EXPECTED_OPENAI_CHAT_PATCH" + +# --- A/C: chat attempt fails -> Responses attempt on a responses-capable model --- +CASE="openai chain: chat fails, Responses succeeds (current model responses-capable)" +run_tool "1 0" COYOTE_CURRENT_MODEL=openai:gpt-4o +assert_rc 0 +assert_calls 2 +assert_log "call=1 model=openai:gpt-4o-search-preview" +assert_log "call=2 model=openai:gpt-4o" +assert_log "call=2 responses_patch=$EXPECTED_OPENAI_RESPONSES_PATCH" + +CASE="openai chain: search-preview model maps to its base model on the Responses attempt" +run_tool "1 0" COYOTE_CURRENT_MODEL=openai:gpt-4o-search-preview +assert_rc 0 +assert_calls 2 +assert_log "call=1 model=openai:gpt-4o-search-preview" +assert_log "call=2 model=openai:gpt-4o" + +CASE="openai chain: mini variants map both ways" +run_tool "1 0" COYOTE_CURRENT_MODEL=openai:gpt-4o-mini-search-preview +assert_rc 0 +assert_calls 2 +assert_log "call=1 model=openai:gpt-4o-mini-search-preview" +assert_log "call=2 model=openai:gpt-4o-mini" + +CASE="openai chain: gpt-5 current model runs the Responses attempt on itself" +run_tool "1 0" COYOTE_CURRENT_MODEL=openai:gpt-5 +assert_rc 0 +assert_calls 2 +assert_log "call=1 model=openai:gpt-4o-search-preview" +assert_log "call=2 model=openai:gpt-5" + +# --- A/E: both openai attempts fail -> existing WEB_SEARCH_MODEL fallback --- +CASE="openai chain exhausted falls back to WEB_SEARCH_MODEL with openai patches cleared" +run_tool "1 1 0" COYOTE_CURRENT_MODEL=openai:gpt-4o +assert_rc 0 +assert_calls 3 +assert_log "call=3 model=gemini:gemini-2.5-flash" +assert_log "call=3 chat_patch=" +assert_log "call=3 responses_patch=" +assert_log 'call=3 gemini_patch={".*":{"body":{"tools":[{"google_search":{}}]}}}' + +# --- A: hard failure when both attempts of the last resort fail --- +CASE="openai fallback model: both attempts fail -> tool fails" +run_tool "1 1" WEB_SEARCH_MODEL=openai:gpt-4o +assert_rc 1 +assert_calls 2 +assert_log "call=1 model=openai:gpt-4o-search-preview" +assert_log "call=2 model=openai:gpt-4o" + +# --- D: non-openai providers untouched --- +CASE="claude current model: single invocation, own patch, openai keys cleared" +run_tool "0" COYOTE_CURRENT_MODEL=claude:claude-sonnet-4-20250514 \ + COYOTE_PATCH_OPENAI_CHAT_COMPLETIONS=stale COYOTE_PATCH_OPENAI_RESPONSES=stale +assert_rc 0 +assert_calls 1 +assert_log "call=1 model=claude:claude-sonnet-4-20250514" +assert_log "call=1 claude_patch=$EXPECTED_CLAUDE_PATCH" +assert_log "call=1 chat_patch=" +assert_log "call=1 responses_patch=" + +CASE="default fallback (gemini): single invocation" +run_tool "0" +assert_rc 0 +assert_calls 1 +assert_log "call=1 model=gemini:gemini-2.5-flash" +assert_log 'call=1 gemini_patch={".*":{"body":{"tools":[{"google_search":{}}]}}}' + +# --- E: unsupported / opted-out current models skip straight to fallback --- +CASE="unsupported openai model (o1) goes straight to fallback" +run_tool "0" COYOTE_CURRENT_MODEL=openai:o1-mini +assert_rc 0 +assert_calls 1 +assert_log "call=1 model=gemini:gemini-2.5-flash" + +CASE="WEB_SEARCH_USE_CURRENT_MODEL=false ignores the current model" +run_tool "0" COYOTE_CURRENT_MODEL=openai:gpt-4o WEB_SEARCH_USE_CURRENT_MODEL=false +assert_rc 0 +assert_calls 1 +assert_log "call=1 model=gemini:gemini-2.5-flash" + +# --- E: current-model failure still falls back (chain counts as one failure) --- +CASE="openai current chain exhausted, non-openai fallback succeeds, output appended" +run_tool "1 1 0" COYOTE_CURRENT_MODEL=openai:gpt-4.1 WEB_SEARCH_MODEL=gemini:gemini-2.5-pro +assert_rc 0 +assert_calls 3 +assert_log "call=1 model=openai:gpt-4o-search-preview" +assert_log "call=2 model=openai:gpt-4.1" +assert_log "call=3 model=gemini:gemini-2.5-pro" +grep -qF "stub-output-3" "$STUB_DIR/out" || fail "fallback output missing from LLM_OUTPUT" + +if [[ "$FAILED" -ne 0 ]]; then + echo "web_search_coyote harness: FAILED" >&2 + exit 1 +fi +echo "web_search_coyote harness: all cases passed"