Dark-Alex-17/llm-functions
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: improve Argcfile.sh and scripts/run-tool* (#34)

Browse Source 
* refactor: improve Argcfile and scripts/run-tool*

* fix run-tool.js on windows
This commit is contained in:
sigoden
2024-06-08 10:30:34 +08:00
committed by GitHub
parent 0c6b609c26
commit 2b5b0f6502
4 changed files with 238 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Argcfile.sh
+38 -28
View File
@@ -12,7 +12,8 @@ LANG_CMDS=( \
)
# @cmd Run the tool
# @arg cmd![`_choice_cmd`] The function command
# @alias call
# @arg cmd![`_choice_cmd`] The tool command
# @arg json The json data
run-tool() {
if _is_win; then
@@ -22,44 +23,52 @@ run-tool() {
}
# @cmd Build the project
build() {
argc build-tools
}
# @cmd Build tools
# @option --names-file=functions.txt Path to a file containing tool filenames, one per line.
# This file specifies which tools will be used.
# @option --declarations-file=functions.json <FILE> Path to a json file to save function declarations
# This file specifies which function files to build.
# Example:
# get_current_weather.sh
# may_execute_js_code.js
build() {
argc build-declarations-json --names-file "${argc_names_file}" --declarations-file "${argc_declarations_file}"
argc build-bin --names-file "${argc_names_file}"
build-tools() {
argc build-tools-json --names-file "${argc_names_file}" --declarations-file "${argc_declarations_file}"
argc build-tools-bin --names-file "${argc_names_file}"
}
# @cmd Build tool binaries
# @cmd Build tools to bin
# @option --names-file=functions.txt Path to a file containing tool filenames, one per line.
# @arg tools*[`_choice_tool`] The tool filenames
build-bin() {
build-tools-bin() {
mkdir -p "$BIN_DIR"
if [[ "${#argc_tools[@]}" -gt 0 ]]; then
names=("${argc_tools[@]}" )
elif [[ -f "$argc_names_file" ]]; then
names=($(cat "$argc_names_file"))
if [[ "${#names[@]}" -gt 0 ]]; then
(cd "$BIN_DIR" && rm -rf "${names[@]}")
fi
fi
if [[ -z "$names" ]]; then
_die "error: no tools selected"
_die "error: not input tools, not found '$argc_names_file', please create it add some tools."
fi
mkdir -p "$BIN_DIR"
rm -rf "$BIN_DIR"/*
not_found_tools=()
for name in "${names[@]}"; do
basename="${name%.*}"
lang="${name##*.}"
func_file="tools/$name"
if [[ -f "$func_file" ]]; then
tool_path="tools/$name"
if [[ -f "$tool_path" ]]; then
if _is_win; then
bin_file="$BIN_DIR/$basename.cmd"
_build_win_shim $lang > "$bin_file"
_build_win_shim_tool $lang > "$bin_file"
else
bin_file="$BIN_DIR/$basename"
ln -s -f "$PWD/scripts/run-tool.$lang" "$bin_file"
fi
echo "Build tool $name"
else
not_found_tools+=("$name")
fi
@@ -67,31 +76,28 @@ build-bin() {
if [[ -n "$not_found_tools" ]]; then
_die "error: not found tools: ${not_found_tools[*]}"
fi
for name in "$BIN_DIR"/*; do
echo "Build $name"
done
}
# @cmd Build declarations.json
# @cmd Build tool functions.json
# @option --names-file=functions.txt Path to a file containing tool filenames, one per line.
# @option --declarations-file=functions.json <FILE> Path to a json file to save function declarations
# @arg tools*[`_choice_tool`] The tool filenames
build-declarations-json() {
build-tools-json() {
if [[ "${#argc_tools[@]}" -gt 0 ]]; then
names=("${argc_tools[@]}" )
elif [[ -f "$argc_names_file" ]]; then
names=($(cat "$argc_names_file"))
fi
if [[ -z "$names" ]]; then
_die "error: no tools selected"
_die "error: not input tools, not found '$argc_names_file', please create it add some tools."
fi
json_list=()
not_found_tools=()
build_failed_tools=()
for name in "${names[@]}"; do
lang="${name##*.}"
func_file="tools/$name"
if [[ ! -f "$func_file" ]]; then
tool_path="tools/$name"
if [[ ! -f "$tool_path" ]]; then
not_found_tools+=("$name")
continue;
fi
@@ -131,15 +137,19 @@ test() {
test-tools
}
# @cmd Test call functions
# @cmd Test tools
test-tools() {
tmp_dir="cache/tmp"
mkdir -p "$tmp_dir"
names_file="$tmp_dir/functions.txt"
declarations_file="$tmp_dir/functions.json"
argc list-tools > "$names_file"
argc build --names-file "$names_file" --declarations-file "$declarations_file"
argc build-tools --names-file "$names_file" --declarations-file "$declarations_file"
test-tools-execute-lang
}
# @cmd Test maybe_execute_* tools
test-tools-execute-lang() {
if _is_win; then
ext=".cmd"
fi
@@ -152,8 +162,8 @@ test-tools() {
for test_case in "${test_cases[@]}"; do
IFS='#' read -r lang tool_name data <<<"${test_case}"
cmd="$(_lang_to_cmd "$lang")"
cmd_path="$BIN_DIR/$tool_name$ext"
if command -v "$cmd" &> /dev/null; then
cmd_path="$BIN_DIR/$tool_name$ext"
echo -n "Test $cmd_path: "
"$cmd_path" "$data"
if ! _is_win; then
@@ -164,12 +174,12 @@ test-tools() {
done
}
# @cmd Test all demo tools
test-demo-tools() {
# @cmd Test demo tools
test-tools-demo() {
for item in "${LANG_CMDS[@]}"; do
lang="${item%:*}"
echo "---- Test demo_tool.$lang ---"
argc build-bin "demo_tool.$lang"
argc build-tools-bin "demo_tool.$lang"
argc run-tool demo_tool '{
"boolean": true,
"string": "Hello",
@@ -236,7 +246,7 @@ _lang_to_cmd() {
done
}
_build_win_shim() {
_build_win_shim_tool() {
lang="$1"
cmd="$(_lang_to_cmd "$lang")"
if [[ "$lang" == "sh" ]]; then
scripts/run-tool.js
+67 -38
View File
@@ -2,12 +2,24 @@
const path = require("path");
const fs = require("fs");
const os = require("os");
function parseArgv() {
async function main() {
const [toolName, rawData] = parseArgv("run-tool.js");
const toolData = parseRawData(rawData);
const rootDir = path.resolve(__dirname, "..");
setupEnv(rootDir, toolName);
const toolPath = path.resolve(rootDir, `tools/${toolName}.js`);
await run(toolPath, "run", toolData);
}
function parseArgv(thisFileName) {
let toolName = process.argv[1];
let toolData = null;
if (toolName.endsWith("run-tool.js")) {
if (toolName.endsWith(thisFileName)) {
toolName = process.argv[2];
toolData = process.argv[3];
} else {
@@ -22,18 +34,22 @@ function parseArgv() {
return [toolName, toolData];
}
function loadModule(toolName) {
const toolFileName = `${toolName}.js`;
const toolPath = path.resolve(
process.env["LLM_ROOT_DIR"],
`tools/${toolFileName}`,
);
try {
return require(toolPath);
} catch {
console.log(`Invalid tooltion: ${toolFileName}`);
process.exit(1);
function parseRawData(data) {
if (!data) {
throw new Error("No JSON data");
}
try {
return JSON.parse(data);
} catch {
throw new Error("Invalid JSON data");
}
}
function setupEnv(rootDir, toolName) {
process.env["LLM_ROOT_DIR"] = rootDir;
loadEnv(path.resolve(rootDir, ".env"));
process.env["LLM_TOOL_NAME"] = toolName;
process.env["LLM_TOOL_CACHE_DIR"] = path.resolve(rootDir, "cache", toolName);
}
function loadEnv(filePath) {
@@ -50,32 +66,45 @@ function loadEnv(filePath) {
} catch {}
}
const LLM_ROOT_DIR = path.resolve(__dirname, "..");
process.env["LLM_ROOT_DIR"] = LLM_ROOT_DIR;
loadEnv(path.resolve(LLM_ROOT_DIR, ".env"));
const [toolName, toolData] = parseArgv();
process.env["LLM_TOOL_NAME"] = toolName;
process.env["LLM_TOOL_CACHE_DIR"] = path.resolve(
LLM_ROOT_DIR,
"cache",
toolName,
);
if (!toolData) {
console.log("No json data");
process.exit(1);
async function run(toolPath, toolFunc, toolData) {
let mod;
if (os.platform() === "win32") {
toolPath = `file://${toolPath}`;
}
try {
mod = await import(toolPath);
} catch {
throw new Error(`Unable to load tool at '${toolPath}'`);
}
if (!mod || !mod[toolFunc]) {
throw new Error(`Not module function '${toolFunc}' at '${toolPath}'`);
}
const value = await mod[toolFunc](toolData);
dumpValue(value);
}
let data = null;
try {
data = JSON.parse(toolData);
} catch {
console.log("Invalid json data");
process.exit(1);
function dumpValue(value) {
if (value === null || value === undefined) {
return;
}
const type = typeof value;
if (type === "string" || type === "number" || type === "boolean") {
console.log(value);
} else if (type === "object") {
const proto = Object.prototype.toString.call(value);
if (proto === "[object Object]" || proto === "[object Array]") {
const valueStr = JSON.stringify(value, null, 2);
require("assert").deepStrictEqual(value, JSON.parse(valueStr));
console.log(valueStr);
}
}
}
const { run } = loadModule(toolName);
run(data);
(async () => {
try {
await main();
} catch (err) {
console.error(err?.message || err);
process.exit(1);
}
})();
scripts/run-tool.py
+65 -36
View File
@@ -6,16 +6,39 @@ import sys
import importlib.util
def parse_argv():
tool_name = sys.argv[0]
def main():
(tool_name, raw_data) = parse_argv("run-tool.py")
tool_data = parse_raw_data(raw_data)
root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
setup_env(root_dir, tool_name)
tool_path = os.path.join(root_dir, f"tools/{tool_name}.py")
run(tool_path, "run", tool_data)
def parse_raw_data(data):
if not data:
raise ValueError("No JSON data")
try:
return json.loads(data)
except Exception:
raise ValueError("Invalid JSON data")
def parse_argv(this_file_name):
argv = sys.argv[:] + [None] * max(0, 3 - len(sys.argv))
tool_name = argv[0]
tool_data = None
if tool_name.endswith("run-tool.py"):
tool_name = sys.argv[1] if len(sys.argv) > 1 else None
tool_data = sys.argv[2] if len(sys.argv) > 2 else None
if tool_name.endswith(this_file_name):
tool_name = argv[1]
tool_data = argv[2]
else:
tool_name = os.path.basename(tool_name)
tool_data = sys.argv[1] if len(sys.argv) > 1 else None
tool_data = sys.argv[1]
if tool_name.endswith(".py"):
tool_name = tool_name[:-3]
@@ -23,17 +46,11 @@ def parse_argv():
return tool_name, tool_data
def load_module(tool_name):
tool_file_name = f"{tool_name}.py"
tool_path = os.path.join(os.environ["LLM_ROOT_DIR"], f"tools/{tool_file_name}")
if os.path.exists(tool_path):
spec = importlib.util.spec_from_file_location(f"{tool_file_name}", tool_path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
else:
print(f"Invalid function: {tool_file_name}")
sys.exit(1)
def setup_env(root_dir, tool_name):
os.environ["LLM_ROOT_DIR"] = root_dir
load_env(os.path.join(root_dir, ".env"))
os.environ["LLM_TOOL_NAME"] = tool_name
os.environ["LLM_TOOL_CACHE_DIR"] = os.path.join(root_dir, "cache", tool_name)
def load_env(file_path):
@@ -50,27 +67,39 @@ def load_env(file_path):
pass
LLM_ROOT_DIR = os.environ["LLM_ROOT_DIR"] = os.path.abspath(
os.path.join(os.path.dirname(__file__), "..")
)
def run(tool_path, tool_func, tool_data):
try:
spec = importlib.util.spec_from_file_location(
os.path.basename(tool_path), tool_path
)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
except:
raise Exception(f"Unable to load tool at '{tool_path}'")
load_env(os.path.join(LLM_ROOT_DIR, ".env"))
if not hasattr(mod, tool_func):
raise Exception(f"Not module function '{tool_func}' at '{tool_path}'")
tool_name, tool_data = parse_argv()
value = getattr(mod, tool_func)(**tool_data)
dump_value(value)
os.environ["LLM_TOOL_NAME"] = tool_name
os.environ["LLM_TOOL_CACHE_DIR"] = os.path.join(LLM_ROOT_DIR, "cache", tool_name)
if not tool_data:
print("No json data")
def dump_value(value):
if value is None:
return
value_type = type(value).__name__
if value_type in ("str", "int", "float", "bool"):
print(value)
elif value_type == "dict" or value_type == "list":
value_str = json.dumps(value, indent=2)
assert value == json.loads(value_str)
print(value_str)
if __name__ == "__main__":
try:
main()
except Exception as e:
print(e, file=sys.stderr)
sys.exit(1)
data = None
try:
data = json.loads(tool_data)
except (json.JSONDecodeError, TypeError):
print("Invalid json data")
sys.exit(1)
module = load_module(tool_name)
module.run(**data)
scripts/run-tool.sh
+46 -30
View File
@@ -1,40 +1,49 @@
#!/usr/bin/env bash
set -e
export LLM_ROOT_DIR="$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )/.." &> /dev/null && pwd)"
main() {
this_file_name=run-tool.sh
parse_argv "$@"
root_dir="$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )/.." &> /dev/null && pwd)"
setup_env
tool_path="$root_dir/tools/$tool_name.sh"
run
}
if [[ -f "$LLM_ROOT_DIR/.env" ]]; then
source "$LLM_ROOT_DIR/.env"
fi
if [[ "$0" == *run-tool.sh ]]; then
parse_argv() {
if [[ "$0" == *"$this_file_name" ]]; then
tool_name="$1"
tool_data="$2"
else
else
tool_name="$(basename "$0")"
tool_data="$1"
fi
if [[ "$tool_name" == *.sh ]]; then
fi
if [[ "$tool_name" == *.sh ]]; then
tool_name="${tool_name:0:$((${#tool_name}-3))}"
fi
fi
}
export LLM_TOOL_NAME="$tool_name"
export LLM_TOOL_CACHE_DIR="$LLM_ROOT_DIR/cache/$tool_name"
setup_env() {
export LLM_ROOT_DIR="$root_dir"
if [[ -f "$LLM_ROOT_DIR/.env" ]]; then
source "$LLM_ROOT_DIR/.env"
fi
export LLM_TOOL_NAME="$tool_name"
export LLM_TOOL_CACHE_DIR="$LLM_ROOT_DIR/cache/$tool_name"
}
tool_file="$LLM_ROOT_DIR/tools/$tool_name.sh"
run() {
if [[ -z "$tool_data" ]]; then
die "No JSON data"
fi
_jq=jq
if [[ "$OS" == "Windows_NT" ]]; then
_jq=jq
if [[ "$OS" == "Windows_NT" ]]; then
_jq="jq -b"
tool_file="$(cygpath -w "$tool_file")"
fi
tool_path="$(cygpath -w "$tool_path")"
fi
if [[ -z "$tool_data" ]]; then
echo "No json data"
exit 1
fi
data="$(
data="$(
echo "$tool_data" | \
$_jq -r '
to_entries | .[] |
@@ -46,15 +55,22 @@ data="$(
else
"--\($key)\n\(.value | @json)"
end'
)" || {
echo "Invalid json data"
exit 1
}
while IFS= read -r line; do
)" || {
die "Invalid JSON data"
}
while IFS= read -r line; do
if [[ "$line" == '--'* ]]; then
args+=("$line")
else
args+=("$(echo "$line" | $_jq -r '.')")
fi
done <<< "$data"
"$tool_file" "${args[@]}"
done <<< "$data"
"$tool_path" "${args[@]}"
}
die() {
echo "$*" >&2
exit 1
}
main "$@"