refactor: improve Argcfile.sh and scripts/run-tool* (#34)

* refactor: improve Argcfile and scripts/run-tool*

* fix run-tool.js on windows

Argcfile.sh

@@ -12,7 +12,8 @@ LANG_CMDS=( \
 )
 
 # @cmd Run the tool
-# @arg cmd![`_choice_cmd`] The function command
+# @alias call
+# @arg cmd![`_choice_cmd`] The tool command
 # @arg json The json data
 run-tool() {
     if _is_win; then
@@ -22,44 +23,52 @@ run-tool() {
 }
 
 # @cmd Build the project
+build() {
+    argc build-tools
+}
+
+# @cmd Build tools
 # @option --names-file=functions.txt Path to a file containing tool filenames, one per line.
+# This file specifies which tools will be used.
 # @option --declarations-file=functions.json <FILE> Path to a json file to save function declarations
-# This file specifies which function files to build. 
 # Example:
 #   get_current_weather.sh
 #   may_execute_js_code.js
-build() {
+build-tools() {
-    argc build-declarations-json --names-file "${argc_names_file}" --declarations-file "${argc_declarations_file}"
+    argc build-tools-json --names-file "${argc_names_file}" --declarations-file "${argc_declarations_file}"
-    argc build-bin --names-file "${argc_names_file}"
+    argc build-tools-bin --names-file "${argc_names_file}"
 }
 
-# @cmd Build tool binaries
+# @cmd Build tools to bin
 # @option --names-file=functions.txt Path to a file containing tool filenames, one per line.
 # @arg tools*[`_choice_tool`] The tool filenames
-build-bin() {
+build-tools-bin() {
+    mkdir -p "$BIN_DIR"
     if [[ "${#argc_tools[@]}" -gt 0 ]]; then
         names=("${argc_tools[@]}" )
     elif [[ -f "$argc_names_file" ]]; then
         names=($(cat "$argc_names_file"))
+        if [[ "${#names[@]}" -gt 0 ]]; then
+            (cd "$BIN_DIR" && rm -rf "${names[@]}")
+        fi
     fi
     if [[ -z "$names" ]]; then
-        _die "error: no tools selected"
+        _die "error: not input tools, not found '$argc_names_file', please create it add some tools."
     fi
-    mkdir -p "$BIN_DIR"
-    rm -rf "$BIN_DIR"/*
     not_found_tools=()
     for name in "${names[@]}"; do
         basename="${name%.*}"
         lang="${name##*.}"
-        func_file="tools/$name"
+        tool_path="tools/$name"
-        if [[  -f "$func_file" ]]; then
+        if [[  -f "$tool_path" ]]; then
             if _is_win; then
                 bin_file="$BIN_DIR/$basename.cmd" 
-                _build_win_shim $lang > "$bin_file"
+                _build_win_shim_tool $lang > "$bin_file"
             else
                 bin_file="$BIN_DIR/$basename" 
                 ln -s -f "$PWD/scripts/run-tool.$lang" "$bin_file"
             fi
+            echo "Build tool $name"
         else
             not_found_tools+=("$name")
         fi
@@ -67,31 +76,28 @@ build-bin() {
     if [[ -n "$not_found_tools" ]]; then
         _die "error: not found tools: ${not_found_tools[*]}"
     fi
-    for name in "$BIN_DIR"/*; do
-        echo "Build $name"
-    done
 }
 
-# @cmd Build declarations.json
+# @cmd Build tool functions.json
 # @option --names-file=functions.txt Path to a file containing tool filenames, one per line.
 # @option --declarations-file=functions.json <FILE> Path to a json file to save function declarations
 # @arg tools*[`_choice_tool`] The tool filenames
-build-declarations-json() {
+build-tools-json() {
     if [[ "${#argc_tools[@]}" -gt 0 ]]; then
         names=("${argc_tools[@]}" )
     elif [[ -f "$argc_names_file" ]]; then
         names=($(cat "$argc_names_file"))
     fi
     if [[ -z "$names" ]]; then
-        _die "error: no tools selected"
+        _die "error: not input tools, not found '$argc_names_file', please create it add some tools."
     fi
     json_list=()
     not_found_tools=()
     build_failed_tools=()
     for name in "${names[@]}"; do
         lang="${name##*.}"
-        func_file="tools/$name"
+        tool_path="tools/$name"
-        if [[ ! -f "$func_file" ]]; then
+        if [[ ! -f "$tool_path" ]]; then
             not_found_tools+=("$name")
             continue;
         fi
@@ -131,15 +137,19 @@ test() {
     test-tools
 }
 
-# @cmd Test call functions
+# @cmd Test tools
 test-tools() {
     tmp_dir="cache/tmp"
     mkdir -p "$tmp_dir"
     names_file="$tmp_dir/functions.txt"
     declarations_file="$tmp_dir/functions.json"
     argc list-tools > "$names_file"
-    argc build --names-file "$names_file" --declarations-file "$declarations_file"
+    argc build-tools --names-file "$names_file" --declarations-file "$declarations_file"
+    test-tools-execute-lang
+}
 
+# @cmd Test maybe_execute_* tools
+test-tools-execute-lang() {
     if _is_win; then
         ext=".cmd"
     fi
@@ -152,8 +162,8 @@ test-tools() {
     for test_case in "${test_cases[@]}"; do
         IFS='#' read -r lang tool_name data <<<"${test_case}"
         cmd="$(_lang_to_cmd "$lang")"
-        cmd_path="$BIN_DIR/$tool_name$ext"
         if command -v "$cmd" &> /dev/null; then
+            cmd_path="$BIN_DIR/$tool_name$ext"
             echo -n "Test $cmd_path: "
             "$cmd_path" "$data"
             if ! _is_win; then
@@ -164,12 +174,12 @@ test-tools() {
     done
 }
 
-# @cmd Test all demo tools
+# @cmd Test demo tools
-test-demo-tools() {
+test-tools-demo() {
     for item in "${LANG_CMDS[@]}"; do
         lang="${item%:*}"
         echo "---- Test demo_tool.$lang ---"
-        argc build-bin "demo_tool.$lang"
+        argc build-tools-bin "demo_tool.$lang"
         argc run-tool demo_tool '{
      "boolean": true,
      "string": "Hello",
@@ -236,7 +246,7 @@ _lang_to_cmd() {
     done
 }
 
-_build_win_shim() {
+_build_win_shim_tool() {
     lang="$1"
     cmd="$(_lang_to_cmd "$lang")"
     if [[ "$lang" == "sh" ]]; then

scripts/run-tool.js

@@ -2,12 +2,24 @@
 
 const path = require("path");
 const fs = require("fs");
+const os = require("os");
 
-function parseArgv() {
+async function main() {
+  const [toolName, rawData] = parseArgv("run-tool.js");
+  const toolData = parseRawData(rawData);
+
+  const rootDir = path.resolve(__dirname, "..");
+  setupEnv(rootDir, toolName);
+
+  const toolPath = path.resolve(rootDir, `tools/${toolName}.js`);
+  await run(toolPath, "run", toolData);
+}
+
+function parseArgv(thisFileName) {
   let toolName = process.argv[1];
   let toolData = null;
 
-  if (toolName.endsWith("run-tool.js")) {
+  if (toolName.endsWith(thisFileName)) {
     toolName = process.argv[2];
     toolData = process.argv[3];
   } else {
@@ -22,18 +34,22 @@ function parseArgv() {
   return [toolName, toolData];
 }
 
-function loadModule(toolName) {
+function parseRawData(data) {
-  const toolFileName = `${toolName}.js`;
+  if (!data) {
-  const toolPath = path.resolve(
+    throw new Error("No JSON data");
-    process.env["LLM_ROOT_DIR"],
-    `tools/${toolFileName}`,
-  );
-  try {
-    return require(toolPath);
-  } catch {
-    console.log(`Invalid tooltion: ${toolFileName}`);
-    process.exit(1);
   }
+  try {
+    return JSON.parse(data);
+  } catch {
+    throw new Error("Invalid JSON data");
+  }
+}
+
+function setupEnv(rootDir, toolName) {
+  process.env["LLM_ROOT_DIR"] = rootDir;
+  loadEnv(path.resolve(rootDir, ".env"));
+  process.env["LLM_TOOL_NAME"] = toolName;
+  process.env["LLM_TOOL_CACHE_DIR"] = path.resolve(rootDir, "cache", toolName);
 }
 
 function loadEnv(filePath) {
@@ -50,32 +66,45 @@ function loadEnv(filePath) {
   } catch {}
 }
 
-const LLM_ROOT_DIR = path.resolve(__dirname, "..");
+async function run(toolPath, toolFunc, toolData) {
-process.env["LLM_ROOT_DIR"] = LLM_ROOT_DIR;
+  let mod;
-
+  if (os.platform() === "win32") {  
-loadEnv(path.resolve(LLM_ROOT_DIR, ".env"));
+    toolPath = `file://${toolPath}`;
-
+  }
-const [toolName, toolData] = parseArgv();
+  try {
-
+    mod = await import(toolPath);
-process.env["LLM_TOOL_NAME"] = toolName;
+  } catch {
-process.env["LLM_TOOL_CACHE_DIR"] = path.resolve(
+    throw new Error(`Unable to load tool at '${toolPath}'`);
-  LLM_ROOT_DIR,
+  }
-  "cache",
+  if (!mod || !mod[toolFunc]) {
-  toolName,
+    throw new Error(`Not module function '${toolFunc}' at '${toolPath}'`);
-);
+  }
-
+  const value = await mod[toolFunc](toolData);
-if (!toolData) {
+  dumpValue(value);
-  console.log("No json data");
-  process.exit(1);
 }
 
-let data = null;
+function dumpValue(value) {
-try {
+  if (value === null || value === undefined) {
-  data = JSON.parse(toolData);
+    return;
-} catch {
+  }
-  console.log("Invalid json data");
+  const type = typeof value;
-  process.exit(1);
+  if (type === "string" || type === "number" || type === "boolean") {
+    console.log(value);
+  } else if (type === "object") {
+    const proto = Object.prototype.toString.call(value);
+    if (proto === "[object Object]" || proto === "[object Array]") {
+      const valueStr = JSON.stringify(value, null, 2);
+      require("assert").deepStrictEqual(value, JSON.parse(valueStr));
+      console.log(valueStr);
+    }
+  }
 }
 
-const { run } = loadModule(toolName);
+(async () => {
-run(data);
+  try {
+    await main();
+  } catch (err) {
+    console.error(err?.message || err);
+    process.exit(1);
+  }
+})();

scripts/run-tool.py

@@ -6,16 +6,39 @@ import sys
 import importlib.util
 
 
-def parse_argv():
+def main():
-    tool_name = sys.argv[0]
+    (tool_name, raw_data) = parse_argv("run-tool.py")
+    tool_data = parse_raw_data(raw_data)
+
+    root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+    setup_env(root_dir, tool_name)
+
+    tool_path = os.path.join(root_dir, f"tools/{tool_name}.py")
+    run(tool_path, "run", tool_data)
+
+
+def parse_raw_data(data):
+    if not data:
+        raise ValueError("No JSON data")
+
+    try:
+        return json.loads(data)
+    except Exception:
+        raise ValueError("Invalid JSON data")
+
+
+def parse_argv(this_file_name):
+    argv = sys.argv[:] + [None] * max(0, 3 - len(sys.argv))
+
+    tool_name = argv[0]
     tool_data = None
 
-    if tool_name.endswith("run-tool.py"):
+    if tool_name.endswith(this_file_name):
-        tool_name = sys.argv[1] if len(sys.argv) > 1 else None
+        tool_name = argv[1]
-        tool_data = sys.argv[2] if len(sys.argv) > 2 else None
+        tool_data = argv[2]
     else:
         tool_name = os.path.basename(tool_name)
-        tool_data = sys.argv[1] if len(sys.argv) > 1 else None
+        tool_data = sys.argv[1]
 
     if tool_name.endswith(".py"):
         tool_name = tool_name[:-3]
@@ -23,17 +46,11 @@ def parse_argv():
     return tool_name, tool_data
 
 
-def load_module(tool_name):
+def setup_env(root_dir, tool_name):
-    tool_file_name = f"{tool_name}.py"
+    os.environ["LLM_ROOT_DIR"] = root_dir
-    tool_path = os.path.join(os.environ["LLM_ROOT_DIR"], f"tools/{tool_file_name}")
+    load_env(os.path.join(root_dir, ".env"))
-    if os.path.exists(tool_path):
+    os.environ["LLM_TOOL_NAME"] = tool_name
-        spec = importlib.util.spec_from_file_location(f"{tool_file_name}", tool_path)
+    os.environ["LLM_TOOL_CACHE_DIR"] = os.path.join(root_dir, "cache", tool_name)
-        module = importlib.util.module_from_spec(spec)
-        spec.loader.exec_module(module)
-        return module
-    else:
-        print(f"Invalid function: {tool_file_name}")
-        sys.exit(1)
 
 
 def load_env(file_path):
@@ -50,27 +67,39 @@ def load_env(file_path):
         pass
 
 
-LLM_ROOT_DIR = os.environ["LLM_ROOT_DIR"] = os.path.abspath(
+def run(tool_path, tool_func, tool_data):
-    os.path.join(os.path.dirname(__file__), "..")
+    try:
-)
+        spec = importlib.util.spec_from_file_location(
+            os.path.basename(tool_path), tool_path
+        )
+        mod = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(mod)
+    except:
+        raise Exception(f"Unable to load tool at '{tool_path}'")
 
-load_env(os.path.join(LLM_ROOT_DIR, ".env"))
+    if not hasattr(mod, tool_func):
+        raise Exception(f"Not module function '{tool_func}' at '{tool_path}'")
 
-tool_name, tool_data = parse_argv()
+    value = getattr(mod, tool_func)(**tool_data)
+    dump_value(value)
 
-os.environ["LLM_TOOL_NAME"] = tool_name
-os.environ["LLM_TOOL_CACHE_DIR"] = os.path.join(LLM_ROOT_DIR, "cache", tool_name)
 
-if not tool_data:
+def dump_value(value):
-    print("No json data")
+    if value is None:
-    sys.exit(1)
+        return
 
-data = None
+    value_type = type(value).__name__
-try:
+    if value_type in ("str", "int", "float", "bool"):
-    data = json.loads(tool_data)
+        print(value)
-except (json.JSONDecodeError, TypeError):
+    elif value_type == "dict" or value_type == "list":
-    print("Invalid json data")
+        value_str = json.dumps(value, indent=2)
-    sys.exit(1)
+        assert value == json.loads(value_str)
+        print(value_str)
 
-module = load_module(tool_name)
+
-module.run(**data)
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as e:
+        print(e, file=sys.stderr)
+        sys.exit(1)

scripts/run-tool.sh

@@ -1,60 +1,76 @@
 #!/usr/bin/env bash
 set -e
 
-export LLM_ROOT_DIR="$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )/.." &> /dev/null && pwd)"
+main() {
+    this_file_name=run-tool.sh
+    parse_argv "$@"
+    root_dir="$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )/.." &> /dev/null && pwd)"
+    setup_env
+    tool_path="$root_dir/tools/$tool_name.sh"
+    run
+}
 
-if [[ -f "$LLM_ROOT_DIR/.env" ]]; then
+parse_argv() {
-    source "$LLM_ROOT_DIR/.env"
+    if [[ "$0" == *"$this_file_name" ]]; then
-fi
+        tool_name="$1"
-
+        tool_data="$2"
-if [[ "$0" == *run-tool.sh ]]; then
-    tool_name="$1"
-    tool_data="$2"
-else
-    tool_name="$(basename "$0")"
-    tool_data="$1"
-fi
-if [[ "$tool_name" == *.sh ]]; then
-    tool_name="${tool_name:0:$((${#tool_name}-3))}"
-fi
-
-export LLM_TOOL_NAME="$tool_name"
-export LLM_TOOL_CACHE_DIR="$LLM_ROOT_DIR/cache/$tool_name"
-
-tool_file="$LLM_ROOT_DIR/tools/$tool_name.sh"
-
-_jq=jq
-if [[ "$OS" == "Windows_NT" ]]; then
-    _jq="jq -b"
-    tool_file="$(cygpath -w "$tool_file")"
-fi
-
-if [[ -z "$tool_data" ]]; then
-    echo "No json data"
-    exit 1
-fi
-
-data="$(
-    echo "$tool_data" | \
-    $_jq -r '
-    to_entries | .[] | 
-    (.key | split("_") | join("-")) as $key |
-    if .value | type == "array" then
-        .value | .[] | "--\($key)\n\(. | @json)"
-    elif .value | type == "boolean" then
-        if .value then "--\($key)" else "" end
     else
-        "--\($key)\n\(.value | @json)"
+        tool_name="$(basename "$0")"
-    end'
+        tool_data="$1"
-)" || {
+    fi
-    echo "Invalid json data"
+    if [[ "$tool_name" == *.sh ]]; then
+        tool_name="${tool_name:0:$((${#tool_name}-3))}"
+    fi
+}
+
+setup_env() {
+    export LLM_ROOT_DIR="$root_dir"
+    if [[ -f "$LLM_ROOT_DIR/.env" ]]; then
+        source "$LLM_ROOT_DIR/.env"
+    fi
+    export LLM_TOOL_NAME="$tool_name"
+    export LLM_TOOL_CACHE_DIR="$LLM_ROOT_DIR/cache/$tool_name"
+}
+
+run() {
+    if [[ -z "$tool_data" ]]; then
+        die "No JSON data"
+    fi
+
+    _jq=jq
+    if [[ "$OS" == "Windows_NT" ]]; then
+        _jq="jq -b"
+        tool_path="$(cygpath -w "$tool_path")"
+    fi
+
+    data="$(
+        echo "$tool_data" | \
+        $_jq -r '
+        to_entries | .[] | 
+        (.key | split("_") | join("-")) as $key |
+        if .value | type == "array" then
+            .value | .[] | "--\($key)\n\(. | @json)"
+        elif .value | type == "boolean" then
+            if .value then "--\($key)" else "" end
+        else
+            "--\($key)\n\(.value | @json)"
+        end'
+    )" || {
+        die "Invalid JSON data"
+    }
+    while IFS= read -r line; do
+        if [[ "$line" == '--'* ]]; then
+            args+=("$line")
+        else
+            args+=("$(echo "$line" | $_jq -r '.')")
+        fi
+    done <<< "$data"
+    "$tool_path" "${args[@]}"
+}
+
+die() {
+    echo "$*" >&2
     exit 1
 }
-while IFS= read -r line; do
+
-    if [[ "$line" == '--'* ]]; then
+main "$@"
-        args+=("$line")
-    else
-        args+=("$(echo "$line" | $_jq -r '.')")
-    fi
-done <<< "$data"
-"$tool_file" "${args[@]}"