Dark-Alex-17/llm-functions
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: numerous improvements (#32)

Browse Source
This commit is contained in:
sigoden
2024-06-07 21:36:34 +08:00
committed by GitHub
parent e1d895cc9a
commit 63df67acea
11 changed files with 159 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/build-declarations.js
+29 -20
View File
@@ -1,11 +1,13 @@
#!/usr/bin/env node
const fs = require("fs");
const path = require("path");
const TOOL_ENTRY_FUNC = "run";
function main(isTool = true) {
function main() {
const scriptfile = process.argv[2];
const isTool = path.dirname(scriptfile) == "tools";
const contents = fs.readFileSync(process.argv[2], "utf8");
const functions = extractFunctions(contents, isTool);
let declarations = functions.map(({ funcName, jsdoc }) => {
@@ -56,13 +58,20 @@ function extractFunctions(contents, isTool) {
});
}
} else {
const match = /function *([_A-Za-z]+)/.exec(line);
let match = /^export (async )?function ([A-Za-z0-9_]+)/.exec(line);
let funcName = null;
if (match) {
const funcName = match[1];
if (!funcName.startsWith("_")) {
output.push({ funcName, jsdoc });
funcName = match[2];
}
if (!funcName) {
match = /^exports\.([A-Za-z0-9_]+) = (async )?function /.exec(line);
if (match) {
funcName = match[1];
}
}
if (funcName) {
output.push({ funcName, jsdoc });
}
}
jsdoc = "";
}
@@ -165,21 +174,6 @@ function buildProperty(type, description) {
return property;
}
/**
* @param {string} filePath
*/
function getBasename(filePath) {
const filenameWithExt = filePath.split(/[/\\]/).pop();
const lastDotIndex = filenameWithExt.lastIndexOf(".");
if (lastDotIndex === -1) {
return filenameWithExt;
}
return filenameWithExt.substring(0, lastDotIndex);
}
/**
* @param {string} name
* @param {string} description
@@ -204,4 +198,19 @@ function buildDeclaration(name, description, params) {
return schema;
}
/**
* @param {string} filePath
*/
function getBasename(filePath) {
const filenameWithExt = filePath.split(/[/\\]/).pop();
const lastDotIndex = filenameWithExt.lastIndexOf(".");
if (lastDotIndex === -1) {
return filenameWithExt;
}
return filenameWithExt.substring(0, lastDotIndex);
}
main();
scripts/build-declarations.py
+6 -3
View File
@@ -9,8 +9,11 @@ from collections import OrderedDict
TOOL_ENTRY_FUNC = "run"
def main(is_tool = True):
def main(is_tool=True):
scriptfile = sys.argv[1]
is_tool = os.path.dirname(scriptfile) == "tools"
with open(scriptfile, "r", encoding="utf-8") as f:
contents = f.read()
@@ -92,8 +95,8 @@ def parse_docstring(docstring: str):
break
params = {}
for rawParam in rawParams:
name, type_, description = parse_param(rawParam)
params[name] = (type_, description)
name, type_, param_description = parse_param(rawParam)
params[name] = (type_, param_description)
return (description.strip(), params)
scripts/build-declarations.sh
+30 -7
View File
@@ -1,7 +1,23 @@
#!/usr/bin/env bash
argc --argc-export "$1" | \
jq -r '
main() {
scriptfile="$1"
is_tool=false
if [[ "$(dirname "$scriptfile")" == tools ]]; then
is_tool=true
fi
if [[ "$is_tool" == "true" ]]; then
expr='[.]'
else
expr='.subcommands'
fi
argc --argc-export "$scriptfile" | \
jq "$expr" | \
build_declarations
}
build_declarations() {
jq -r '
def parse_description(flag_option):
if flag_option.describe == "" then
{}
@@ -36,8 +52,15 @@ jq -r '
required: [flag_options[] | select(.required == true) | .id | sub("-"; "_"; "g")],
};
[{
name: (.name | sub("-"; "_"; "g")),
description: .describe,
parameters: parse_parameter([.flag_options[] | select(.id != "help" and .id != "version")])
}]'
def parse_declaration:
{
name: (.name | sub("-"; "_"; "g")),
description: .describe,
parameters: parse_parameter([.flag_options[] | select(.id != "help" and .id != "version")])
};
[
.[] | parse_declaration
]'
}
main "$@"
scripts/create-tool.sh
+4 -4
View File
@@ -20,15 +20,15 @@ main() {
ext="${argc_name##*.}"
support_exts=('.sh' '.js' '.py')
if [[ "$ext" == "$argc_name" ]]; then
_die "No extension name, pelease add one of ${support_exts[*]}"
_die "error: no extension name, pelease add one of ${support_exts[*]}"
fi
case $ext in
sh) create_sh ;;
js) create_js ;;
py) create_py ;;
*) _die "Invalid extension name: $ext, must be one of ${support_exts[*]}" ;;
*) _die "error: invalid extension name: $ext, must be one of ${support_exts[*]}" ;;
esac
_die "$output generated"
echo "$output generated"
}
create_sh() {
@@ -187,7 +187,7 @@ build_properties() {
}
_die() {
echo "$*"
echo "$*" >&2
exit 1
}
scripts/run-tool.js
+32 -26
View File
@@ -4,34 +4,34 @@ const path = require("path");
const fs = require("fs");
function parseArgv() {
let funcName = process.argv[1];
let funcData = null;
let toolName = process.argv[1];
let toolData = null;
if (funcName.endsWith("run-tool.js")) {
funcName = process.argv[2];
funcData = process.argv[3];
if (toolName.endsWith("run-tool.js")) {
toolName = process.argv[2];
toolData = process.argv[3];
} else {
funcName = path.basename(funcName);
funcData = process.argv[2];
toolName = path.basename(toolName);
toolData = process.argv[2];
}
if (funcName.endsWith(".js")) {
funcName = funcName.slice(0, -3);
if (toolName.endsWith(".js")) {
toolName = toolName.slice(0, -3);
}
return [funcName, funcData];
return [toolName, toolData];
}
function loadFunc(funcName) {
const funcFileName = `${funcName}.js`;
const funcPath = path.resolve(
process.env["LLM_FUNCTIONS_DIR"],
`tools/${funcFileName}`,
function loadModule(toolName) {
const toolFileName = `${toolName}.js`;
const toolPath = path.resolve(
process.env["LLM_ROOT_DIR"],
`tools/${toolFileName}`,
);
try {
return require(funcPath);
return require(toolPath);
} catch {
console.log(`Invalid function: ${funcFileName}`);
console.log(`Invalid tooltion: ${toolFileName}`);
process.exit(1);
}
}
@@ -50,26 +50,32 @@ function loadEnv(filePath) {
} catch {}
}
process.env["LLM_FUNCTIONS_DIR"] = path.resolve(__dirname, "..");
const LLM_ROOT_DIR = path.resolve(__dirname, "..");
process.env["LLM_ROOT_DIR"] = LLM_ROOT_DIR;
loadEnv(path.resolve(process.env["LLM_FUNCTIONS_DIR"], ".env"));
loadEnv(path.resolve(LLM_ROOT_DIR, ".env"));
const [funcName, funcData] = parseArgv();
const [toolName, toolData] = parseArgv();
process.env["LLM_FUNCTION_NAME"] = funcName;
process.env["LLM_TOOL_NAME"] = toolName;
process.env["LLM_TOOL_CACHE_DIR"] = path.resolve(
LLM_ROOT_DIR,
"cache",
toolName,
);
if (!funcData) {
if (!toolData) {
console.log("No json data");
process.exit(1);
}
let args;
let data = null;
try {
args = JSON.parse(funcData);
data = JSON.parse(toolData);
} catch {
console.log("Invalid json data");
process.exit(1);
}
const { run } = loadFunc(funcName);
run(args);
const { run } = loadModule(toolName);
run(data);
scripts/run-tool.py
+26 -25
View File
@@ -7,32 +7,32 @@ import importlib.util
def parse_argv():
func_name = sys.argv[0]
func_data = None
tool_name = sys.argv[0]
tool_data = None
if func_name.endswith("run-tool.py"):
func_name = sys.argv[1] if len(sys.argv) > 1 else None
func_data = sys.argv[2] if len(sys.argv) > 2 else None
if tool_name.endswith("run-tool.py"):
tool_name = sys.argv[1] if len(sys.argv) > 1 else None
tool_data = sys.argv[2] if len(sys.argv) > 2 else None
else:
func_name = os.path.basename(func_name)
func_data = sys.argv[1] if len(sys.argv) > 1 else None
tool_name = os.path.basename(tool_name)
tool_data = sys.argv[1] if len(sys.argv) > 1 else None
if func_name.endswith(".py"):
func_name = func_name[:-3]
if tool_name.endswith(".py"):
tool_name = tool_name[:-3]
return func_name, func_data
return tool_name, tool_data
def load_func(func_name):
func_file_name = f"{func_name}.py"
func_path = os.path.join(os.environ["LLM_FUNCTIONS_DIR"], f"tools/{func_file_name}")
if os.path.exists(func_path):
spec = importlib.util.spec_from_file_location(f"{func_file_name}", func_path)
def load_module(tool_name):
tool_file_name = f"{tool_name}.py"
tool_path = os.path.join(os.environ["LLM_ROOT_DIR"], f"tools/{tool_file_name}")
if os.path.exists(tool_path):
spec = importlib.util.spec_from_file_location(f"{tool_file_name}", tool_path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
else:
print(f"Invalid function: {func_file_name}")
print(f"Invalid function: {tool_file_name}")
sys.exit(1)
@@ -50,26 +50,27 @@ def load_env(file_path):
pass
os.environ["LLM_FUNCTIONS_DIR"] = os.path.abspath(
LLM_ROOT_DIR = os.environ["LLM_ROOT_DIR"] = os.path.abspath(
os.path.join(os.path.dirname(__file__), "..")
)
load_env(os.path.join(os.environ["LLM_FUNCTIONS_DIR"], ".env"))
load_env(os.path.join(LLM_ROOT_DIR, ".env"))
func_name, func_data = parse_argv()
tool_name, tool_data = parse_argv()
os.environ["LLM_FUNCTION_NAME"] = func_name
os.environ["LLM_TOOL_NAME"] = tool_name
os.environ["LLM_TOOL_CACHE_DIR"] = os.path.join(LLM_ROOT_DIR, "cache", tool_name)
if not func_data:
if not tool_data:
print("No json data")
sys.exit(1)
args = None
data = None
try:
args = json.loads(func_data)
data = json.loads(tool_data)
except (json.JSONDecodeError, TypeError):
print("Invalid json data")
sys.exit(1)
module = load_func(func_name)
module.run(**args)
module = load_module(tool_name)
module.run(**data)
scripts/run-tool.sh
+21 -19
View File
@@ -1,40 +1,42 @@
#!/usr/bin/env bash
set -e
export LLM_FUNCTIONS_DIR="$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )/.." &> /dev/null && pwd)"
export LLM_ROOT_DIR="$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )/.." &> /dev/null && pwd)"
if [[ -f "$LLM_FUNCTIONS_DIR/.env" ]]; then
source "$LLM_FUNCTIONS_DIR/.env"
if [[ -f "$LLM_ROOT_DIR/.env" ]]; then
source "$LLM_ROOT_DIR/.env"
fi
if [[ "$0" == *run-tool.sh ]]; then
func_name="$1"
func_data="$2"
tool_name="$1"
tool_data="$2"
else
func_name="$(basename "$0")"
func_data="$1"
tool_name="$(basename "$0")"
tool_data="$1"
fi
if [[ "$func_name" == *.sh ]]; then
func_name="${func_name:0:$((${#func_name}-3))}"
if [[ "$tool_name" == *.sh ]]; then
tool_name="${tool_name:0:$((${#tool_name}-3))}"
fi
export LLM_FUNCTION_NAME="$func_name"
func_file="$LLM_FUNCTIONS_DIR/tools/$func_name.sh"
export LLM_TOOL_NAME="$tool_name"
export LLM_TOOL_CACHE_DIR="$LLM_ROOT_DIR/cache/$tool_name"
export JQ=jq
tool_file="$LLM_ROOT_DIR/tools/$tool_name.sh"
_jq=jq
if [[ "$OS" == "Windows_NT" ]]; then
export JQ="jq -b"
func_file="$(cygpath -w "$func_file")"
_jq="jq -b"
tool_file="$(cygpath -w "$tool_file")"
fi
if [[ -z "$func_data" ]]; then
if [[ -z "$tool_data" ]]; then
echo "No json data"
exit 1
fi
data="$(
echo "$func_data" | \
$JQ -r '
echo "$tool_data" | \
$_jq -r '
to_entries | .[] |
(.key | split("_") | join("-")) as $key |
if .value | type == "array" then
@@ -52,7 +54,7 @@ while IFS= read -r line; do
if [[ "$line" == '--'* ]]; then
args+=("$line")
else
args+=("$(echo "$line" | $JQ -r '.')")
args+=("$(echo "$line" | $_jq -r '.')")
fi
done <<< "$data"
"$func_file" "${args[@]}"
"$tool_file" "${args[@]}"