Dark-Alex-17/llm-functions
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: numerous improvements (#32)

Browse Source
This commit is contained in:
sigoden
2024-06-07 21:36:34 +08:00
committed by GitHub
parent e1d895cc9a
commit 63df67acea
11 changed files with 159 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+3 -3
View File
@@ -1,8 +1,8 @@
/tmp /tmp
/functions.txt functions.txt
/functions.txt.test functions.json
/functions.json
/bin /bin
/cache
/tools/test.* /tools/test.*
/.env /.env
*.cmd *.cmd
Argcfile.sh
+5 -5
View File
@@ -148,15 +148,15 @@ test-tools() {
) )
for test_case in "${test_cases[@]}"; do for test_case in "${test_cases[@]}"; do
IFS='#' read -r lang func data <<<"${test_case}" IFS='#' read -r lang tool_name data <<<"${test_case}"
cmd="$(_lang_to_cmd "$lang")" cmd="$(_lang_to_cmd "$lang")"
cmd_path="$BIN_DIR/$func$ext" cmd_path="$BIN_DIR/$tool_name$ext"
if command -v "$cmd" &> /dev/null; then if command -v "$cmd" &> /dev/null; then
echo -n "Test $cmd_path: " echo -n "Test $cmd_path: "
"$cmd_path" "$data" "$cmd_path" "$data"
if ! _is_win; then if ! _is_win; then
echo -n "Test $cmd scripts/run-tool.$lang $func: " echo -n "Test $cmd scripts/run-tool.$lang $tool_name: "
"$cmd" "scripts/run-tool.$lang" "$func" "$data" "$cmd" "scripts/run-tool.$lang" "$tool_name" "$data"
fi fi
fi fi
done done
@@ -258,7 +258,7 @@ _choice_cmd() {
} }
_die() { _die() {
echo "$*" echo "$*" >&2
exit 1 exit 1
} }
scripts/build-declarations.js
+29 -20
View File
@@ -1,11 +1,13 @@
#!/usr/bin/env node #!/usr/bin/env node
const fs = require("fs"); const fs = require("fs");
const path = require("path");
const TOOL_ENTRY_FUNC = "run"; const TOOL_ENTRY_FUNC = "run";
function main(isTool = true) { function main() {
const scriptfile = process.argv[2]; const scriptfile = process.argv[2];
const isTool = path.dirname(scriptfile) == "tools";
const contents = fs.readFileSync(process.argv[2], "utf8"); const contents = fs.readFileSync(process.argv[2], "utf8");
const functions = extractFunctions(contents, isTool); const functions = extractFunctions(contents, isTool);
let declarations = functions.map(({ funcName, jsdoc }) => { let declarations = functions.map(({ funcName, jsdoc }) => {
@@ -56,13 +58,20 @@ function extractFunctions(contents, isTool) {
}); });
} }
} else { } else {
const match = /function *([_A-Za-z]+)/.exec(line); let match = /^export (async )?function ([A-Za-z0-9_]+)/.exec(line);
let funcName = null;
if (match) { if (match) {
const funcName = match[1]; funcName = match[2];
if (!funcName.startsWith("_")) { }
output.push({ funcName, jsdoc }); if (!funcName) {
match = /^exports\.([A-Za-z0-9_]+) = (async )?function /.exec(line);
if (match) {
funcName = match[1];
} }
} }
if (funcName) {
output.push({ funcName, jsdoc });
}
} }
jsdoc = ""; jsdoc = "";
} }
@@ -165,21 +174,6 @@ function buildProperty(type, description) {
return property; return property;
} }
/**
* @param {string} filePath
*/
function getBasename(filePath) {
const filenameWithExt = filePath.split(/[/\\]/).pop();
const lastDotIndex = filenameWithExt.lastIndexOf(".");
if (lastDotIndex === -1) {
return filenameWithExt;
}
return filenameWithExt.substring(0, lastDotIndex);
}
/** /**
* @param {string} name * @param {string} name
* @param {string} description * @param {string} description
@@ -204,4 +198,19 @@ function buildDeclaration(name, description, params) {
return schema; return schema;
} }
/**
* @param {string} filePath
*/
function getBasename(filePath) {
const filenameWithExt = filePath.split(/[/\\]/).pop();
const lastDotIndex = filenameWithExt.lastIndexOf(".");
if (lastDotIndex === -1) {
return filenameWithExt;
}
return filenameWithExt.substring(0, lastDotIndex);
}
main(); main();
scripts/build-declarations.py
+6 -3
View File
@@ -9,8 +9,11 @@ from collections import OrderedDict
TOOL_ENTRY_FUNC = "run" TOOL_ENTRY_FUNC = "run"
def main(is_tool = True):
def main(is_tool=True):
scriptfile = sys.argv[1] scriptfile = sys.argv[1]
is_tool = os.path.dirname(scriptfile) == "tools"
with open(scriptfile, "r", encoding="utf-8") as f: with open(scriptfile, "r", encoding="utf-8") as f:
contents = f.read() contents = f.read()
@@ -92,8 +95,8 @@ def parse_docstring(docstring: str):
break break
params = {} params = {}
for rawParam in rawParams: for rawParam in rawParams:
name, type_, description = parse_param(rawParam) name, type_, param_description = parse_param(rawParam)
params[name] = (type_, description) params[name] = (type_, param_description)
return (description.strip(), params) return (description.strip(), params)
scripts/build-declarations.sh
+30 -7
View File
@@ -1,7 +1,23 @@
#!/usr/bin/env bash #!/usr/bin/env bash
argc --argc-export "$1" | \ main() {
jq -r ' scriptfile="$1"
is_tool=false
if [[ "$(dirname "$scriptfile")" == tools ]]; then
is_tool=true
fi
if [[ "$is_tool" == "true" ]]; then
expr='[.]'
else
expr='.subcommands'
fi
argc --argc-export "$scriptfile" | \
jq "$expr" | \
build_declarations
}
build_declarations() {
jq -r '
def parse_description(flag_option): def parse_description(flag_option):
if flag_option.describe == "" then if flag_option.describe == "" then
{} {}
@@ -36,8 +52,15 @@ jq -r '
required: [flag_options[] | select(.required == true) | .id | sub("-"; "_"; "g")], required: [flag_options[] | select(.required == true) | .id | sub("-"; "_"; "g")],
}; };
[{ def parse_declaration:
name: (.name | sub("-"; "_"; "g")), {
description: .describe, name: (.name | sub("-"; "_"; "g")),
parameters: parse_parameter([.flag_options[] | select(.id != "help" and .id != "version")]) description: .describe,
}]' parameters: parse_parameter([.flag_options[] | select(.id != "help" and .id != "version")])
};
[
.[] | parse_declaration
]'
}
main "$@"
scripts/create-tool.sh
+4 -4
View File
@@ -20,15 +20,15 @@ main() {
ext="${argc_name##*.}" ext="${argc_name##*.}"
support_exts=('.sh' '.js' '.py') support_exts=('.sh' '.js' '.py')
if [[ "$ext" == "$argc_name" ]]; then if [[ "$ext" == "$argc_name" ]]; then
_die "No extension name, pelease add one of ${support_exts[*]}" _die "error: no extension name, pelease add one of ${support_exts[*]}"
fi fi
case $ext in case $ext in
sh) create_sh ;; sh) create_sh ;;
js) create_js ;; js) create_js ;;
py) create_py ;; py) create_py ;;
*) _die "Invalid extension name: $ext, must be one of ${support_exts[*]}" ;; *) _die "error: invalid extension name: $ext, must be one of ${support_exts[*]}" ;;
esac esac
_die "$output generated" echo "$output generated"
} }
create_sh() { create_sh() {
@@ -187,7 +187,7 @@ build_properties() {
} }
_die() { _die() {
echo "$*" echo "$*" >&2
exit 1 exit 1
} }
scripts/run-tool.js
+32 -26
View File
@@ -4,34 +4,34 @@ const path = require("path");
const fs = require("fs"); const fs = require("fs");
function parseArgv() { function parseArgv() {
let funcName = process.argv[1]; let toolName = process.argv[1];
let funcData = null; let toolData = null;
if (funcName.endsWith("run-tool.js")) { if (toolName.endsWith("run-tool.js")) {
funcName = process.argv[2]; toolName = process.argv[2];
funcData = process.argv[3]; toolData = process.argv[3];
} else { } else {
funcName = path.basename(funcName); toolName = path.basename(toolName);
funcData = process.argv[2]; toolData = process.argv[2];
} }
if (funcName.endsWith(".js")) { if (toolName.endsWith(".js")) {
funcName = funcName.slice(0, -3); toolName = toolName.slice(0, -3);
} }
return [funcName, funcData]; return [toolName, toolData];
} }
function loadFunc(funcName) { function loadModule(toolName) {
const funcFileName = `${funcName}.js`; const toolFileName = `${toolName}.js`;
const funcPath = path.resolve( const toolPath = path.resolve(
process.env["LLM_FUNCTIONS_DIR"], process.env["LLM_ROOT_DIR"],
`tools/${funcFileName}`, `tools/${toolFileName}`,
); );
try { try {
return require(funcPath); return require(toolPath);
} catch { } catch {
console.log(`Invalid function: ${funcFileName}`); console.log(`Invalid tooltion: ${toolFileName}`);
process.exit(1); process.exit(1);
} }
} }
@@ -50,26 +50,32 @@ function loadEnv(filePath) {
} catch {} } catch {}
} }
process.env["LLM_FUNCTIONS_DIR"] = path.resolve(__dirname, ".."); const LLM_ROOT_DIR = path.resolve(__dirname, "..");
process.env["LLM_ROOT_DIR"] = LLM_ROOT_DIR;
loadEnv(path.resolve(process.env["LLM_FUNCTIONS_DIR"], ".env")); loadEnv(path.resolve(LLM_ROOT_DIR, ".env"));
const [funcName, funcData] = parseArgv(); const [toolName, toolData] = parseArgv();
process.env["LLM_FUNCTION_NAME"] = funcName; process.env["LLM_TOOL_NAME"] = toolName;
process.env["LLM_TOOL_CACHE_DIR"] = path.resolve(
LLM_ROOT_DIR,
"cache",
toolName,
);
if (!funcData) { if (!toolData) {
console.log("No json data"); console.log("No json data");
process.exit(1); process.exit(1);
} }
let args; let data = null;
try { try {
args = JSON.parse(funcData); data = JSON.parse(toolData);
} catch { } catch {
console.log("Invalid json data"); console.log("Invalid json data");
process.exit(1); process.exit(1);
} }
const { run } = loadFunc(funcName); const { run } = loadModule(toolName);
run(args); run(data);
scripts/run-tool.py
+26 -25
View File
@@ -7,32 +7,32 @@ import importlib.util
def parse_argv(): def parse_argv():
func_name = sys.argv[0] tool_name = sys.argv[0]
func_data = None tool_data = None
if func_name.endswith("run-tool.py"): if tool_name.endswith("run-tool.py"):
func_name = sys.argv[1] if len(sys.argv) > 1 else None tool_name = sys.argv[1] if len(sys.argv) > 1 else None
func_data = sys.argv[2] if len(sys.argv) > 2 else None tool_data = sys.argv[2] if len(sys.argv) > 2 else None
else: else:
func_name = os.path.basename(func_name) tool_name = os.path.basename(tool_name)
func_data = sys.argv[1] if len(sys.argv) > 1 else None tool_data = sys.argv[1] if len(sys.argv) > 1 else None
if func_name.endswith(".py"): if tool_name.endswith(".py"):
func_name = func_name[:-3] tool_name = tool_name[:-3]
return func_name, func_data return tool_name, tool_data
def load_func(func_name): def load_module(tool_name):
func_file_name = f"{func_name}.py" tool_file_name = f"{tool_name}.py"
func_path = os.path.join(os.environ["LLM_FUNCTIONS_DIR"], f"tools/{func_file_name}") tool_path = os.path.join(os.environ["LLM_ROOT_DIR"], f"tools/{tool_file_name}")
if os.path.exists(func_path): if os.path.exists(tool_path):
spec = importlib.util.spec_from_file_location(f"{func_file_name}", func_path) spec = importlib.util.spec_from_file_location(f"{tool_file_name}", tool_path)
module = importlib.util.module_from_spec(spec) module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module) spec.loader.exec_module(module)
return module return module
else: else:
print(f"Invalid function: {func_file_name}") print(f"Invalid function: {tool_file_name}")
sys.exit(1) sys.exit(1)
@@ -50,26 +50,27 @@ def load_env(file_path):
pass pass
os.environ["LLM_FUNCTIONS_DIR"] = os.path.abspath( LLM_ROOT_DIR = os.environ["LLM_ROOT_DIR"] = os.path.abspath(
os.path.join(os.path.dirname(__file__), "..") os.path.join(os.path.dirname(__file__), "..")
) )
load_env(os.path.join(os.environ["LLM_FUNCTIONS_DIR"], ".env")) load_env(os.path.join(LLM_ROOT_DIR, ".env"))
func_name, func_data = parse_argv() tool_name, tool_data = parse_argv()
os.environ["LLM_FUNCTION_NAME"] = func_name os.environ["LLM_TOOL_NAME"] = tool_name
os.environ["LLM_TOOL_CACHE_DIR"] = os.path.join(LLM_ROOT_DIR, "cache", tool_name)
if not func_data: if not tool_data:
print("No json data") print("No json data")
sys.exit(1) sys.exit(1)
args = None data = None
try: try:
args = json.loads(func_data) data = json.loads(tool_data)
except (json.JSONDecodeError, TypeError): except (json.JSONDecodeError, TypeError):
print("Invalid json data") print("Invalid json data")
sys.exit(1) sys.exit(1)
module = load_func(func_name) module = load_module(tool_name)
module.run(**args) module.run(**data)
scripts/run-tool.sh
+21 -19
View File
@@ -1,40 +1,42 @@
#!/usr/bin/env bash #!/usr/bin/env bash
set -e set -e
export LLM_FUNCTIONS_DIR="$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )/.." &> /dev/null && pwd)" export LLM_ROOT_DIR="$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )/.." &> /dev/null && pwd)"
if [[ -f "$LLM_FUNCTIONS_DIR/.env" ]]; then if [[ -f "$LLM_ROOT_DIR/.env" ]]; then
source "$LLM_FUNCTIONS_DIR/.env" source "$LLM_ROOT_DIR/.env"
fi fi
if [[ "$0" == *run-tool.sh ]]; then if [[ "$0" == *run-tool.sh ]]; then
func_name="$1" tool_name="$1"
func_data="$2" tool_data="$2"
else else
func_name="$(basename "$0")" tool_name="$(basename "$0")"
func_data="$1" tool_data="$1"
fi fi
if [[ "$func_name" == *.sh ]]; then if [[ "$tool_name" == *.sh ]]; then
func_name="${func_name:0:$((${#func_name}-3))}" tool_name="${tool_name:0:$((${#tool_name}-3))}"
fi fi
export LLM_FUNCTION_NAME="$func_name" export LLM_TOOL_NAME="$tool_name"
func_file="$LLM_FUNCTIONS_DIR/tools/$func_name.sh" export LLM_TOOL_CACHE_DIR="$LLM_ROOT_DIR/cache/$tool_name"
export JQ=jq tool_file="$LLM_ROOT_DIR/tools/$tool_name.sh"
_jq=jq
if [[ "$OS" == "Windows_NT" ]]; then if [[ "$OS" == "Windows_NT" ]]; then
export JQ="jq -b" _jq="jq -b"
func_file="$(cygpath -w "$func_file")" tool_file="$(cygpath -w "$tool_file")"
fi fi
if [[ -z "$func_data" ]]; then if [[ -z "$tool_data" ]]; then
echo "No json data" echo "No json data"
exit 1 exit 1
fi fi
data="$( data="$(
echo "$func_data" | \ echo "$tool_data" | \
$JQ -r ' $_jq -r '
to_entries | .[] | to_entries | .[] |
(.key | split("_") | join("-")) as $key | (.key | split("_") | join("-")) as $key |
if .value | type == "array" then if .value | type == "array" then
@@ -52,7 +54,7 @@ while IFS= read -r line; do
if [[ "$line" == '--'* ]]; then if [[ "$line" == '--'* ]]; then
args+=("$line") args+=("$line")
else else
args+=("$(echo "$line" | $JQ -r '.')") args+=("$(echo "$line" | $_jq -r '.')")
fi fi
done <<< "$data" done <<< "$data"
"$func_file" "${args[@]}" "$tool_file" "${args[@]}"
tools/demo_tool.js
+1 -1
View File
@@ -12,5 +12,5 @@
* @param {Args} args * @param {Args} args
*/ */
exports.run = function run(args) { exports.run = function run(args) {
console.log(args); console.log(JSON.stringify(args, null, 2));
} }
tools/save_file.sh
+2 -2
View File
@@ -6,9 +6,9 @@ set -e
# @option --contents! The contents to save. # @option --contents! The contents to save.
main() { main() {
base_dir="$LLM_FUNCTIONS_DIR/tmp/files" base_dir="${LLM_TOOL_CACHE_DIR:-/tmp}"
output_file="$base_dir/$argc_file_name"
mkdir -p "$base_dir" mkdir -p "$base_dir"
output_file="$base_dir/$argc_file_name"
echo "$argc_contents" > "$output_file" echo "$argc_contents" > "$output_file"
echo "$output_file" echo "$output_file"
} }