Dark-Alex-17/llm-functions
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add web_search_* tools, rm some search_* tools (#78)

Browse Source
This commit is contained in:
sigoden
2024-07-29 07:32:26 +08:00
committed by GitHub
parent 4d18d6346d
commit a7342a1cbf
11 changed files with 44 additions and 177 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/search_bing.sh
-20
View File
@@ -1,20 +0,0 @@
#!/usr/bin/env bash
set -e
# @describe Perform a web search using Bing Web Search API to get up-to-date information or additional context.
# Use this when you need current information or feel a search could provide a better answer.
# @env BING_API_KEY! The api key
# @env SEARCH_MAX_RESULTS=5 The max results to return.
# @option --query! The query to search for.
main() {
encoded_query="$(jq -nr --arg q "$argc_query" '$q|@uri')"
url="https://api.bing.microsoft.com/v7.0/search?q=$encoded_query&mkt=en-us&textdecorations=true&textformat=raw&count=$SEARCH_MAX_RESULTS&offset=0"
curl -fsSL "$url" \
-H "Ocp-Apim-Subscription-Key: $BING_API_KEY" | \
jq '[.webPages.value[] | {link: .url, title: .name, snippet: .snippet}]' \
>> "$LLM_OUTPUT"
}
eval "$(argc --argc-eval "$0" "$@")"
tools/search_brave.sh
-21
View File
@@ -1,21 +0,0 @@
#!/usr/bin/env bash
set -e
# @describe Perform a web search using Brave Search API to get up-to-date information or additional context.
# Use this when you need current information or feel a search could provide a better answer.
# @env BRAVE_API_KEY! The api key
# @env SEARCH_MAX_RESULTS=5 The max results to return.
# @option --query! The query to search for.
main() {
encoded_query="$(jq -nr --arg q "$argc_query" '$q|@uri')"
url="https://api.search.brave.com/res/v1/web/search?q=$encoded_query&count=$SEARCH_MAX_RESULTS"
curl -fsSL "$url" \
-H "Accept: application/json" \
-H "X-Subscription-Token: $BRAVE_API_KEY" | \
jq '[.web.results[] | {link: .url, title: .title, snippet: .description}]' \
>> "$LLM_OUTPUT"
}
eval "$(argc --argc-eval "$0" "$@")"
tools/search_duckduckgo.sh
-35
View File
@@ -1,35 +0,0 @@
#!/usr/bin/env bash
set -e
# @describe Perform a web search using DuckDuckGo API to get up-to-date information or additional context.
# Use this when you need current information or feel a search could provide a better answer.
# @env SEARCH_MAX_RESULTS=5 The max results to return.
# @option --query! The query to search for.
main() {
encoded_query="$(jq -nr --arg q "$argc_query" '$q|@uri')"
vqd="$(curl -fsSL -X POST https://duckduckgo.com -d "q=$encoded_query" | sed -En 's/.*vqd=([0-9-]+)&.*/\1/p')"
url="https://links.duckduckgo.com/d.js?q=$encoded_query&kl=wt-wt&l=wt-wt&p=&s=0&df=&vqd=$vqd&bing_market=wt-WT&ex=-1"
data="$(curl -fsSL "$url" | sed -En 's/.*DDG.pageLayout.load\(\x27d\x27,\[(.*)\]\);DDG.duckbar.load\(.*/\1/p')"
echo "[$data]" | jq '
def strip_tags:
gsub("<[^>]*>"; "");
def unescape_html_entities:
gsub("&amp;"; "&") |
gsub("&lt;"; "<") |
gsub("&gt;"; ">") |
gsub("&quot;"; "\"") |
gsub("&apos;"; "'\''") |
gsub("&#x27;"; "'\''") |
gsub("&nbsp;"; " ");
def normalize: strip_tags | unescape_html_entities;
[.[:'"$SEARCH_MAX_RESULTS"'] | .[] | select(has("u")) | {link: .u, title: (.t | normalize), snippet: (.a | normalize)}]
' >> "$LLM_OUTPUT"
}
eval "$(argc --argc-eval "$0" "$@")"
tools/search_exa.sh
-30
View File
@@ -1,30 +0,0 @@
#!/usr/bin/env bash
set -e
# @describe Perform a web search using Exa API to get up-to-date information or additional context.
# Use this when you need current information or feel a search could provide a better answer.
# @env EXA_API_KEY! The api key
# @env SEARCH_MAX_RESULTS=5 The max results to return.
# @option --query! The query to search for.
main() {
curl -fsSL -X POST https://api.exa.ai/search \
-H "content-type: application/json" \
-H "x-api-key: $EXA_API_KEY" \
-d '
{
"query": "'"$argc_query"'",
"numResults": '"$SEARCH_MAX_RESULTS"',
"type": "keyword",
"contents": {
"text": {
"maxCharacters": 200
}
}
}' | \
jq '[.results[] | {link: .url, title: .title, snippet: .text}]' \
>> "$LLM_OUTPUT"
}
eval "$(argc --argc-eval "$0" "$@")"
tools/search_google.sh
-20
View File
@@ -1,20 +0,0 @@
#!/usr/bin/env bash
set -e
# @describe Perform a web search using Google Search API to get up-to-date information or additional context.
# Use this when you need current information or feel a search could provide a better answer.
# @env GOOGLE_API_KEY! The api key
# @env GOOGLE_CSE_ID! The id of google search engine
# @env SEARCH_MAX_RESULTS=5 The max results to return.
# @option --query! The query to search for.
main() {
encoded_query="$(jq -nr --arg q "$argc_query" '$q|@uri')"
url="https://www.googleapis.com/customsearch/v1?key=$GOOGLE_API_KEY&cx=$GOOGLE_CSE_ID&q=$encoded_query"
curl -fsSL "$url" | \
jq '[.items[:'"$SEARCH_MAX_RESULTS"'] | .[] | {link: .link, title: .title, snippet: .snippet}]' \
>> "$LLM_OUTPUT"
}
eval "$(argc --argc-eval "$0" "$@")"
tools/search_jina.sh
-22
View File
@@ -1,22 +0,0 @@
#!/usr/bin/env bash
set -e
# @describe Perform a web search using Jina API to get up-to-date information or additional context.
# Use this when you need current information or feel a search could provide a better answer.
# @env JINA_API_KEY The api key
# @env SEARCH_MAX_RESULTS=5 The max results to return.
# @option --query! The query to search for.
main() {
curl_args=("-H" "Accept: application/json")
if [[ -n "$JINA_API_KEY" ]]; then
curl_args+=("-H" "Authorization: Bearer $JINA_API_KEY")
fi
encoded_query="$(jq -nr --arg q "$argc_query" '$q|@uri')"
curl -fsSL "${curl_args[@]}" "https://s.jina.ai/$encoded_query" | \
jq '[.data[:'"$SEARCH_MAX_RESULTS"'] | .[] | {link: .url, title: .title, snippet: .description}]' \
>> "$LLM_OUTPUT"
}
eval "$(argc --argc-eval "$0" "$@")"
tools/search_searxng.sh
-20
View File
@@ -1,20 +0,0 @@
#!/usr/bin/env bash
set -e
# @describe Perform a web search using SearXNG API to get up-to-date information or additional context.
# Use this when you need current information or feel a search could provide a better answer.
# @env SEARXNG_API_BASE! The api url
# @env SEARCH_MAX_RESULTS=5 The max results to return.
# @option --query! The query to search for.
main() {
encoded_query="$(jq -nr --arg q "$argc_query" '$q|@uri')"
url="$SEARXNG_API_BASE/search?q=$encoded_query&categories=general&language=en-US&format=json"
curl -fsSL "$url" | \
jq '[.results[:'"$SEARCH_MAX_RESULTS"'] | .[] | {link: .url, title: .title, snippet: .content}]' \
>> "$LLM_OUTPUT"
}
eval "$(argc --argc-eval "$0" "$@")"
tools/web_search_perplexity.sh
Executable
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env bash
set -e
# @describe Perform a web search using Perplexity API to get up-to-date information or additional context.
# Use this when you need current information or feel a search could provide a better answer.
# @env PERPLEXITY_API_KEY! The api key
# @option --query! The query to search for.
main() {
curl -fsS -X POST https://api.perplexity.ai/chat/completions \
-H "authorization: Bearer $PERPLEXITY_API_KEY" \
-H "accept: application/json" \
-H "content-type: application/json" \
--data '
{
"model": "llama-3-sonar-small-32k-online",
"messages": [
{
"role": "user",
"content": "'"$argc_query"'"
}
]
}
' | \
jq -r '.choices[0].message.content' \
>> "$LLM_OUTPUT"
}
eval "$(argc --argc-eval "$0" "$@")"
tools/search_tavily.sh → tools/web_search_tavily.sh
+2 -5
View File
@@ -5,7 +5,6 @@ set -e
# Use this when you need current information or feel a search could provide a better answer.
# @env TAVILY_API_KEY! The api key
# @env SEARCH_MAX_RESULTS=5 The max results to return.
# @option --query! The query to search for.
main() {
@@ -15,11 +14,9 @@ main() {
{
"api_key": "'"$TAVILY_API_KEY"'",
"query": "'"$argc_query"'",
"search_depth": "advanced",
"max_results": "'"$SEARCH_MAX_RESULTS"'"
"include_answer": true
}' | \
jq '[.results[] | {link: .url, title: .title, snippet: .content}]' \
>> "$LLM_OUTPUT"
jq -r '.answer' >> "$LLM_OUTPUT"
}
eval "$(argc --argc-eval "$0" "$@")"