Dark-Alex-17/llm-functions
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: search tools (#76)

Browse Source
This commit is contained in:
sigoden
2024-07-17 09:23:52 +08:00
committed by GitHub
parent a4087408a2
commit 2ee24731dc
10 changed files with 43 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/search_arxiv.sh
+1 -1
View File
@@ -3,7 +3,7 @@ set -e
# @describe Search arXiv for a query and return the top papers.
# @env ARXIV_MAX_RESULTS=5 The max results to return.
# @env ARXIV_MAX_RESULTS=3 The max results to return.
# @option --query! The query to search for.
main() {
tools/search_bing.sh
+3 -3
View File
@@ -5,15 +5,15 @@ set -e
# Use this when you need current information or feel a search could provide a better answer.
# @env BING_API_KEY! The api key
# @env BING_MAX_RESULTS=5 The max results to return.
# @env SEARCH_MAX_RESULTS=5 The max results to return.
# @option --query! The query to search for.
main() {
encoded_query="$(jq -nr --arg q "$argc_query" '$q|@uri')"
url="https://api.bing.microsoft.com/v7.0/search?q=$encoded_query&mkt=en-us&textdecorations=true&textformat=raw&count=$BING_MAX_RESULTS&offset=0"
url="https://api.bing.microsoft.com/v7.0/search?q=$encoded_query&mkt=en-us&textdecorations=true&textformat=raw&count=$SEARCH_MAX_RESULTS&offset=0"
curl -fsSL "$url" \
-H "Ocp-Apim-Subscription-Key: $BING_API_KEY" | \
jq '[.webPages.value[] | {name: .name, url: .url, snippet: .snippet}]' \
jq '[.webPages.value[] | {link: .url, title: .name, snippet: .snippet}]' \
>> "$LLM_OUTPUT"
}
tools/search_brave.sh
+3 -3
View File
@@ -5,16 +5,16 @@ set -e
# Use this when you need current information or feel a search could provide a better answer.
# @env BRAVE_API_KEY! The api key
# @env BRAVE_MAX_RESULTS=5 The max results to return.
# @env SEARCH_MAX_RESULTS=5 The max results to return.
# @option --query! The query to search for.
main() {
encoded_query="$(jq -nr --arg q "$argc_query" '$q|@uri')"
url="https://api.search.brave.com/res/v1/web/search?q=$encoded_query&count=$BRAVE_MAX_RESULTS"
url="https://api.search.brave.com/res/v1/web/search?q=$encoded_query&count=$SEARCH_MAX_RESULTS"
curl -fsSL "$url" \
-H "Accept: application/json" \
-H "X-Subscription-Token: $BRAVE_API_KEY" | \
jq '[.web.results[] | {title: .title, url: .url, description: .description}]' \
jq '[.web.results[] | {link: .url, title: .title, snippet: .description}]' \
>> "$LLM_OUTPUT"
}
tools/search_duckduckgo.sh
+23 -3
View File
@@ -4,12 +4,32 @@ set -e
# @describe Perform a web search using DuckDuckGo API to get up-to-date information or additional context.
# Use this when you need current information or feel a search could provide a better answer.
# @meta require-tools ddgr
# @env DDG_MAX_RESULTS=5 The max results to return.
# @env SEARCH_MAX_RESULTS=5 The max results to return.
# @option --query! The query to search for.
main() {
ddgr -n $DDG_MAX_RESULTS --json "$argc_query" >> "$LLM_OUTPUT"
encoded_query="$(jq -nr --arg q "$argc_query" '$q|@uri')"
vqd="$(curl -fsSL -X POST https://duckduckgo.com -d "q=$encoded_query" | sed -En 's/.*vqd=([0-9-]+)&.*/\1/p')"
url="https://links.duckduckgo.com/d.js?q=$encoded_query&kl=wt-wt&l=wt-wt&p=&s=0&df=&vqd=$vqd&bing_market=wt-WT&ex=-1"
data="$(curl -fsSL "$url" | sed -En 's/.*DDG.pageLayout.load\(\x27d\x27,\[(.*)\]\);DDG.duckbar.load\(.*/\1/p')"
echo "[$data]" | jq '
def strip_tags:
gsub("<[^>]*>"; "");
def unescape_html_entities:
gsub("&amp;"; "&") |
gsub("&lt;"; "<") |
gsub("&gt;"; ">") |
gsub("&quot;"; "\"") |
gsub("&apos;"; "'\''") |
gsub("&#x27;"; "'\''") |
gsub("&nbsp;"; " ");
def normalize: strip_tags | unescape_html_entities;
[.[:'"$SEARCH_MAX_RESULTS"'] | .[] | select(has("u")) | {link: .u, title: (.t | normalize), snippet: (.a | normalize)}]
' >> "$LLM_OUTPUT"
}
eval "$(argc --argc-eval "$0" "$@")"
tools/search_exa.sh
+3 -3
View File
@@ -5,7 +5,7 @@ set -e
# Use this when you need current information or feel a search could provide a better answer.
# @env EXA_API_KEY! The api key
# @env EXA_MAX_RESULTS=5 The max results to return.
# @env SEARCH_MAX_RESULTS=5 The max results to return.
# @option --query! The query to search for.
main() {
@@ -15,7 +15,7 @@ main() {
-d '
{
"query": "'"$argc_query"'",
"numResults": '"$EXA_MAX_RESULTS"',
"numResults": '"$SEARCH_MAX_RESULTS"',
"type": "keyword",
"contents": {
"text": {
@@ -23,7 +23,7 @@ main() {
}
}
}' | \
jq '[.results[] | {title: .title, url: .url, text: .text}]' \
jq '[.results[] | {link: .url, title: .title, snippet: .text}]' \
>> "$LLM_OUTPUT"
}
tools/search_google.sh
+2 -2
View File
@@ -6,14 +6,14 @@ set -e
# @env GOOGLE_API_KEY! The api key
# @env GOOGLE_CSE_ID! The id of google search engine
# @env GOOGLE_MAX_RESULTS=5 The max results to return.
# @env SEARCH_MAX_RESULTS=5 The max results to return.
# @option --query! The query to search for.
main() {
encoded_query="$(jq -nr --arg q "$argc_query" '$q|@uri')"
url="https://www.googleapis.com/customsearch/v1?key=$GOOGLE_API_KEY&cx=$GOOGLE_CSE_ID&q=$encoded_query"
curl -fsSL "$url" | \
jq '[.items[:'"$GOOGLE_MAX_RESULTS"'] | .[] | {title: .title, link: .link, snippet: .snippet}]' \
jq '[.items[:'"$SEARCH_MAX_RESULTS"'] | .[] | {link: .link, title: .title, snippet: .snippet}]' \
>> "$LLM_OUTPUT"
}
tools/search_jina.sh
+2 -2
View File
@@ -5,7 +5,7 @@ set -e
# Use this when you need current information or feel a search could provide a better answer.
# @env JINA_API_KEY The api key
# @env JINA_MAX_RESULTS=5 The max results to return.
# @env SEARCH_MAX_RESULTS=5 The max results to return.
# @option --query! The query to search for.
main() {
@@ -15,7 +15,7 @@ main() {
fi
encoded_query="$(jq -nr --arg q "$argc_query" '$q|@uri')"
curl -fsSL "${curl_args[@]}" "https://s.jina.ai/$encoded_query" | \
jq '[.data[:'"$JINA_MAX_RESULTS"'] | .[] | {title: .title, url: .url, description: .description}]' \
jq '[.data[:'"$SEARCH_MAX_RESULTS"'] | .[] | {link: .url, title: .title, snippet: .description}]' \
>> "$LLM_OUTPUT"
}
tools/search_searxng.sh
+2 -2
View File
@@ -5,14 +5,14 @@ set -e
# Use this when you need current information or feel a search could provide a better answer.
# @env SEARXNG_API_BASE! The api url
# @env SEARXNG_MAX_RESULTS=5 The max results to return.
# @env SEARCH_MAX_RESULTS=5 The max results to return.
# @option --query! The query to search for.
main() {
encoded_query="$(jq -nr --arg q "$argc_query" '$q|@uri')"
url="$SEARXNG_API_BASE/search?q=$encoded_query&categories=general&language=en-US&format=json"
curl -fsSL "$url" | \
jq '[.results[:'"$SEARXNG_MAX_RESULTS"'] | .[] | {url: .url, title: .title, content: .content}]' \
jq '[.results[:'"$SEARCH_MAX_RESULTS"'] | .[] | {link: .url, title: .title, snippet: .content}]' \
>> "$LLM_OUTPUT"
}
tools/search_tavily.sh
+3 -3
View File
@@ -5,7 +5,7 @@ set -e
# Use this when you need current information or feel a search could provide a better answer.
# @env TAVILY_API_KEY! The api key
# @env TAVILY_MAX_RESULTS=5 The max results to return.
# @env SEARCH_MAX_RESULTS=5 The max results to return.
# @option --query! The query to search for.
main() {
@@ -16,9 +16,9 @@ main() {
"api_key": "'"$TAVILY_API_KEY"'",
"query": "'"$argc_query"'",
"search_depth": "advanced",
"max_results": "'"$TAVILY_MAX_RESULTS"'"
"max_results": "'"$SEARCH_MAX_RESULTS"'"
}' | \
jq '[.results[] | {title: .title, url: .url, content: .content}]' \
jq '[.results[] | {link: .url, title: .title, snippet: .content}]' \
>> "$LLM_OUTPUT"
}