refactor: Extracted common Python parser logic into a common.rs module

Cargo.lock

@@ -859,7 +859,7 @@ dependencies = [
  "proc-macro2",
  "quote",
  "regex",
- "rustc-hash 2.1.2",
+ "rustc-hash",
  "shlex",
  "syn",
 ]
@@ -1355,7 +1355,6 @@ dependencies = [
  "mio",
  "parking_lot",
  "rustix 0.38.44",
- "serde",
  "signal-hook",
  "signal-hook-mio",
  "winapi",
@@ -1374,6 +1373,7 @@ dependencies = [
  "mio",
  "parking_lot",
  "rustix 1.1.4",
+ "serde",
  "signal-hook",
  "signal-hook-mio",
  "winapi",
@@ -1388,12 +1388,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "crunchy"
-version = "0.2.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
-
 [[package]]
 name = "crypto-common"
 version = "0.1.7"
@@ -1518,34 +1512,13 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "derive_more"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05"
-dependencies = [
- "derive_more-impl 1.0.0",
-]
-
 [[package]]
 name = "derive_more"
 version = "2.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134"
 dependencies = [
- "derive_more-impl 2.1.1",
+ "derive_more-impl",
-]
-
-[[package]]
-name = "derive_more-impl"
-version = "1.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn",
- "unicode-xid",
 ]
 
 [[package]]
@@ -2105,15 +2078,6 @@ dependencies = [
  "windows-link",
 ]
 
-[[package]]
-name = "getopts"
-version = "0.2.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df"
-dependencies = [
- "unicode-width",
-]
-
 [[package]]
 name = "getrandom"
 version = "0.2.17"
@@ -2258,15 +2222,6 @@ version = "0.12.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
 
-[[package]]
-name = "hashbrown"
-version = "0.14.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
-dependencies = [
- "ahash",
-]
-
 [[package]]
 name = "hashbrown"
 version = "0.15.5"
@@ -2822,18 +2777,6 @@ dependencies = [
  "once_cell",
 ]
 
-[[package]]
-name = "is-macro"
-version = "0.3.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d57a3e447e24c22647738e4607f1df1e0ec6f72e16182c4cd199f647cdfb0e4"
-dependencies = [
- "heck",
- "proc-macro2",
- "quote",
- "syn",
-]
-
 [[package]]
 name = "is-terminal"
 version = "0.4.17"
@@ -2870,15 +2813,6 @@ version = "1.70.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
 
-[[package]]
-name = "itertools"
-version = "0.11.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
-dependencies = [
- "either",
-]
-
 [[package]]
 name = "itertools"
 version = "0.13.0"
@@ -2987,12 +2921,6 @@ dependencies = [
  "simple_asn1",
 ]
 
-[[package]]
-name = "lalrpop-util"
-version = "0.20.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553"
-
 [[package]]
 name = "lazy_static"
 version = "1.5.0"
@@ -3021,12 +2949,6 @@ dependencies = [
  "windows-link",
 ]
 
-[[package]]
-name = "libm"
-version = "0.2.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981"
-
 [[package]]
 name = "libredox"
 version = "0.1.15"
@@ -3135,8 +3057,10 @@ dependencies = [
  "clap_complete_nushell",
  "colored",
  "crossterm 0.28.1",
+ "crossterm 0.29.0",
  "dirs",
  "duct",
+ "dunce",
  "fancy-regex",
  "futures-util",
  "fuzzy-matcher",
@@ -3169,8 +3093,6 @@ dependencies = [
  "reqwest-eventsource",
  "rmcp",
  "rust-embed",
- "rustpython-ast",
- "rustpython-parser",
  "scraper",
  "serde",
  "serde_json",
@@ -3186,6 +3108,8 @@ dependencies = [
  "tokio",
  "tokio-graceful",
  "tokio-stream",
+ "tree-sitter",
+ "tree-sitter-python",
  "unicode-segmentation",
  "unicode-width",
  "url",
@@ -3231,64 +3155,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "malachite"
-version = "0.4.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fbdf9cb251732db30a7200ebb6ae5d22fe8e11397364416617d2c2cf0c51cb5"
-dependencies = [
- "malachite-base",
- "malachite-nz",
- "malachite-q",
-]
-
-[[package]]
-name = "malachite-base"
-version = "0.4.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ea0ed76adf7defc1a92240b5c36d5368cfe9251640dcce5bd2d0b7c1fd87aeb"
-dependencies = [
- "hashbrown 0.14.5",
- "itertools 0.11.0",
- "libm",
- "ryu",
-]
-
-[[package]]
-name = "malachite-bigint"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d149aaa2965d70381709d9df4c7ee1fc0de1c614a4efc2ee356f5e43d68749f8"
-dependencies = [
- "derive_more 1.0.0",
- "malachite",
- "num-integer",
- "num-traits",
- "paste",
-]
-
-[[package]]
-name = "malachite-nz"
-version = "0.4.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "34a79feebb2bc9aa7762047c8e5495269a367da6b5a90a99882a0aeeac1841f7"
-dependencies = [
- "itertools 0.11.0",
- "libm",
- "malachite-base",
-]
-
-[[package]]
-name = "malachite-q"
-version = "0.4.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50f235d5747b1256b47620f5640c2a17a88c7569eebdf27cd9cb130e1a619191"
-dependencies = [
- "itertools 0.11.0",
- "malachite-base",
- "malachite-nz",
-]
-
 [[package]]
 name = "markup5ever"
 version = "0.12.1"
@@ -3943,12 +3809,6 @@ dependencies = [
  "subtle",
 ]
 
-[[package]]
-name = "paste"
-version = "1.0.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
-
 [[package]]
 name = "pastey"
 version = "0.2.1"
@@ -4291,7 +4151,7 @@ dependencies = [
  "pin-project-lite",
  "quinn-proto",
  "quinn-udp",
- "rustc-hash 2.1.2",
+ "rustc-hash",
  "rustls 0.23.37",
  "socket2 0.6.3",
  "thiserror 2.0.18",
@@ -4311,7 +4171,7 @@ dependencies = [
  "lru-slab",
  "rand 0.9.2",
  "ring",
- "rustc-hash 2.1.2",
+ "rustc-hash",
  "rustls 0.23.37",
  "rustls-pki-types",
  "slab",
@@ -4362,8 +4222,6 @@ version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
 dependencies = [
- "libc",
- "rand_chacha 0.3.1",
  "rand_core 0.6.4",
 ]
 
@@ -4373,20 +4231,10 @@ version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
 dependencies = [
- "rand_chacha 0.9.0",
+ "rand_chacha",
  "rand_core 0.9.5",
 ]
 
-[[package]]
-name = "rand_chacha"
-version = "0.3.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
-dependencies = [
- "ppv-lite86",
- "rand_core 0.6.4",
-]
-
 [[package]]
 name = "rand_chacha"
 version = "0.9.0"
@@ -4457,12 +4305,12 @@ dependencies = [
 
 [[package]]
 name = "reedline"
-version = "0.40.0"
+version = "0.46.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5cdfab7494d13ebfb6ce64828648518205d3ce8541ef1f94a27887f29d2d50b"
+checksum = "fe9e7c532bfc2759bc8a28902c04e8b993fc13ebd085ee4292eb1b230fa9beef"
 dependencies = [
  "chrono",
- "crossterm 0.28.1",
+ "crossterm 0.29.0",
  "fd-lock",
  "itertools 0.13.0",
  "nu-ansi-term",
@@ -4471,6 +4319,7 @@ dependencies = [
  "strum",
  "strum_macros 0.26.4",
  "thiserror 2.0.18",
+ "unicase",
  "unicode-segmentation",
  "unicode-width",
 ]
@@ -4729,12 +4578,6 @@ version = "0.1.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d"
 
-[[package]]
-name = "rustc-hash"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
-
 [[package]]
 name = "rustc-hash"
 version = "2.1.2"
@@ -4848,63 +4691,6 @@ dependencies = [
  "untrusted",
 ]
 
-[[package]]
-name = "rustpython-ast"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4cdaf8ee5c1473b993b398c174641d3aa9da847af36e8d5eb8291930b72f31a5"
-dependencies = [
- "is-macro",
- "malachite-bigint",
- "rustpython-parser-core",
- "static_assertions",
-]
-
-[[package]]
-name = "rustpython-parser"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "868f724daac0caf9bd36d38caf45819905193a901e8f1c983345a68e18fb2abb"
-dependencies = [
- "anyhow",
- "is-macro",
- "itertools 0.11.0",
- "lalrpop-util",
- "log",
- "malachite-bigint",
- "num-traits",
- "phf",
- "phf_codegen",
- "rustc-hash 1.1.0",
- "rustpython-ast",
- "rustpython-parser-core",
- "tiny-keccak",
- "unic-emoji-char",
- "unic-ucd-ident",
- "unicode_names2",
-]
-
-[[package]]
-name = "rustpython-parser-core"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b4b6c12fa273825edc7bccd9a734f0ad5ba4b8a2f4da5ff7efe946f066d0f4ad"
-dependencies = [
- "is-macro",
- "memchr",
- "rustpython-parser-vendored",
-]
-
-[[package]]
-name = "rustpython-parser-vendored"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "04fcea49a4630a3a5d940f4d514dc4f575ed63c14c3e3ed07146634aed7f67a6"
-dependencies = [
- "memchr",
- "once_cell",
-]
-
 [[package]]
 name = "rustversion"
 version = "1.0.22"
@@ -5385,12 +5171,6 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
 
-[[package]]
-name = "static_assertions"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
-
 [[package]]
 name = "stop-words"
 version = "0.9.0"
@@ -5400,6 +5180,12 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "streaming-iterator"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520"
+
 [[package]]
 name = "string_cache"
 version = "0.8.9"
@@ -5736,15 +5522,6 @@ dependencies = [
  "time-core",
 ]
 
-[[package]]
-name = "tiny-keccak"
-version = "2.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
-dependencies = [
- "crunchy",
-]
-
 [[package]]
 name = "tinystr"
 version = "0.8.3"
@@ -6050,6 +5827,35 @@ dependencies = [
  "tracing-log",
 ]
 
+[[package]]
+name = "tree-sitter"
+version = "0.24.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5387dffa7ffc7d2dae12b50c6f7aab8ff79d6210147c6613561fc3d474c6f75"
+dependencies = [
+ "cc",
+ "regex",
+ "regex-syntax",
+ "streaming-iterator",
+ "tree-sitter-language",
+]
+
+[[package]]
+name = "tree-sitter-language"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782"
+
+[[package]]
+name = "tree-sitter-python"
+version = "0.23.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d065aaa27f3aaceaf60c1f0e0ac09e1cb9eb8ed28e7bcdaa52129cffc7f4b04"
+dependencies = [
+ "cc",
+ "tree-sitter-language",
+]
+
 [[package]]
 name = "tree_magic_mini"
 version = "3.2.2"
@@ -6133,58 +5939,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "unic-char-property"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221"
-dependencies = [
- "unic-char-range",
-]
-
-[[package]]
-name = "unic-char-range"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc"
-
-[[package]]
-name = "unic-common"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc"
-
-[[package]]
-name = "unic-emoji-char"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d"
-dependencies = [
- "unic-char-property",
- "unic-char-range",
- "unic-ucd-version",
-]
-
-[[package]]
-name = "unic-ucd-ident"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e230a37c0381caa9219d67cf063aa3a375ffed5bf541a452db16e744bdab6987"
-dependencies = [
- "unic-char-property",
- "unic-char-range",
- "unic-ucd-version",
-]
-
-[[package]]
-name = "unic-ucd-version"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4"
-dependencies = [
- "unic-common",
-]
-
 [[package]]
 name = "unicase"
 version = "2.9.0"
@@ -6221,28 +5975,6 @@ version = "0.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853"
 
-[[package]]
-name = "unicode_names2"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd"
-dependencies = [
- "phf",
- "unicode_names2_generator",
-]
-
-[[package]]
-name = "unicode_names2_generator"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b91e5b84611016120197efd7dc93ef76774f4e084cd73c9fb3ea4a86c570c56e"
-dependencies = [
- "getopts",
- "log",
- "phf_codegen",
- "rand 0.8.5",
-]
-
 [[package]]
 name = "universal-hash"
 version = "0.5.1"

Cargo.toml

@@ -18,10 +18,11 @@ anyhow = "1.0.69"
 bytes = "1.4.0"
 clap = { version = "4.5.40", features = ["cargo", "derive", "wrap_help"] }
 dirs = "6.0.0"
+dunce = "1.0.5"
 futures-util = "0.3.29"
 inquire = "0.9.4"
 is-terminal = "0.4.9"
-reedline = "0.40.0"
+reedline = "0.46.0"
 serde = { version = "1.0.152", features = ["derive"] }
 serde_json = { version = "1.0.93", features = ["preserve_order"] }
 serde_yaml = "0.9.17"
@@ -37,7 +38,7 @@ tokio-graceful = "0.2.2"
 tokio-stream = { version = "0.1.15", default-features = false, features = [
   "sync",
 ] }
-crossterm = "0.28.1"
+crossterm = "0.29.0"
 chrono = "0.4.23"
 bincode = { version = "2.0.0", features = [
   "serde",
@@ -90,8 +91,8 @@ strum_macros = "0.27.2"
 indoc = "2.0.6"
 rmcp = { version = "0.16.0", features = ["client", "transport-child-process"] }
 num_cpus = "1.17.0"
-rustpython-parser = "0.4.0"
+tree-sitter = "0.24"
-rustpython-ast = "0.4.0"
+tree-sitter-python = "0.23"
 colored = "3.0.0"
 clap_complete = { version = "4.5.58", features = ["unstable-dynamic"] }
 gman = "0.3.0"

assets/functions/scripts/run-agent.py

@@ -50,7 +50,13 @@ def parse_raw_data(data):
 
 def parse_argv():
     agent_func = sys.argv[1]
-    agent_data = sys.argv[2]
+
+    tool_data_file = os.environ.get("LLM_TOOL_DATA_FILE")
+    if tool_data_file and os.path.isfile(tool_data_file):
+        with open(tool_data_file, "r", encoding="utf-8") as f:
+            agent_data = f.read()
+    else:
+        agent_data = sys.argv[2]
 
     if (not agent_data) or (not agent_func):
         print("Usage: ./{agent_name}.py <agent-func> <agent-data>", file=sys.stderr)

assets/functions/scripts/run-agent.sh

@@ -14,7 +14,11 @@ main() {
 
 parse_argv() {
 		agent_func="$1"
-		agent_data="$2"
+    if [[ -n "$LLM_TOOL_DATA_FILE" ]] && [[ -f "$LLM_TOOL_DATA_FILE" ]]; then
+      	agent_data="$(cat "$LLM_TOOL_DATA_FILE")"
+    else
+				agent_data="$2"
+    fi
     if [[ -z "$agent_data" ]] || [[ -z "$agent_func" ]]; then
         die "usage: ./{agent_name}.sh <agent-func> <agent-data>"
     fi
@@ -57,7 +61,6 @@ run() {
     if [[ "$OS" == "Windows_NT" ]]; then
         set -o igncr
         tools_path="$(cygpath -w "$tools_path")"
-        tool_data="$(echo "$tool_data" | sed 's/\\/\\\\/g')"
     fi
 
     jq_script="$(cat <<-'EOF'

assets/functions/scripts/run-tool.py

@@ -49,6 +49,11 @@ def parse_raw_data(data):
 
 
 def parse_argv():
+    tool_data_file = os.environ.get("LLM_TOOL_DATA_FILE")
+    if tool_data_file and os.path.isfile(tool_data_file):
+        with open(tool_data_file, "r", encoding="utf-8") as f:
+            return f.read()
+
     argv = sys.argv[:] + [None] * max(0, 2 - len(sys.argv))
 
     tool_data = argv[1]

assets/functions/scripts/run-tool.sh

@@ -13,7 +13,11 @@ main() {
 }
 
 parse_argv() {
-		tool_data="$1"
+    if [[ -n "$LLM_TOOL_DATA_FILE" ]] && [[ -f "$LLM_TOOL_DATA_FILE" ]]; then
+        tool_data="$(cat "$LLM_TOOL_DATA_FILE")"
+    else
+				tool_data="$1"
+    fi
     if [[ -z "$tool_data" ]]; then
         die "usage: ./{function_name}.sh <tool-data>"
     fi
@@ -54,7 +58,6 @@ run() {
     if [[ "$OS" == "Windows_NT" ]]; then
         set -o igncr
         tool_path="$(cygpath -w "$tool_path")"
-        tool_data="$(echo "$tool_data" | sed 's/\\/\\\\/g')"
     fi
 
     jq_script="$(cat <<-'EOF'

docs/ENVIRONMENT-VARIABLES.md

@@ -107,6 +107,7 @@ The following variables can be used to change the log level of Loki or the locat
 can also pass the `--disable-log-colors` flag as well.
 
 ## Miscellaneous Variables
-| Environment Variable | Description                                                                                      | Default Value |
+| Environment Variable | Description                                                                                                                                                                                                                                                                                                                    | Default Value |
-|----------------------|--------------------------------------------------------------------------------------------------|---------------|
+|----------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|
-| `AUTO_CONFIRM`       | Bypass all `guard_*` checks in the bash prompt helpers; useful for agent composition and routing |               |
+| `AUTO_CONFIRM`       | Bypass all `guard_*` checks in the bash prompt helpers; useful for agent composition and routing                                                                                                                                                                                                                               |               |
+| `LLM_TOOL_DATA_FILE` | Set automatically by Loki on Windows. Points to a temporary file containing the JSON tool call data. <br>Tool scripts (`run-tool.sh`, `run-agent.sh`, etc.) read from this file instead of command-line args <br>to avoid JSON escaping issues when data passes through `cmd.exe` → bash. **Not intended to be set by users.** |               |

src/function/mod.rs

@@ -613,6 +613,7 @@ impl Functions {
             )
         })?;
         let content_template = unsafe { std::str::from_utf8_unchecked(&embedded_file.data) };
+        let to_script_path = |p: &str| -> String { p.replace('\\', "/") };
         let content = match binary_type {
             BinaryType::Tool(None) => {
                 let root_dir = Config::functions_dir();
@@ -622,8 +623,8 @@ impl Functions {
                 );
                 content_template
                     .replace("{function_name}", binary_name)
-                    .replace("{root_dir}", &root_dir.to_string_lossy())
+                    .replace("{root_dir}", &to_script_path(&root_dir.to_string_lossy()))
-                    .replace("{tool_path}", &tool_path)
+                    .replace("{tool_path}", &to_script_path(&tool_path))
             }
             BinaryType::Tool(Some(agent_name)) => {
                 let root_dir = Config::agent_data_dir(agent_name);
@@ -633,16 +634,19 @@ impl Functions {
                 );
                 content_template
                     .replace("{function_name}", binary_name)
-                    .replace("{root_dir}", &root_dir.to_string_lossy())
+                    .replace("{root_dir}", &to_script_path(&root_dir.to_string_lossy()))
-                    .replace("{tool_path}", &tool_path)
+                    .replace("{tool_path}", &to_script_path(&tool_path))
             }
             BinaryType::Agent => content_template
                 .replace("{agent_name}", binary_name)
-                .replace("{config_dir}", &Config::config_dir().to_string_lossy()),
+                .replace(
+                    "{config_dir}",
+                    &to_script_path(&Config::config_dir().to_string_lossy()),
+                ),
         }
         .replace(
             "{prompt_utils_file}",
-            &Config::bash_prompt_utils_file().to_string_lossy(),
+            &to_script_path(&Config::bash_prompt_utils_file().to_string_lossy()),
         );
         if binary_script_file.exists() {
             fs::remove_file(&binary_script_file)?;
@@ -666,7 +670,7 @@ impl Functions {
                     .join(".venv")
                     .join("Scripts")
                     .join("activate.bat");
-                let canonicalized_path = fs::canonicalize(&executable_path)?;
+                let canonicalized_path = dunce::canonicalize(&executable_path)?;
                 format!(
                     "call \"{}\" && {}",
                     canonicalized_path.to_string_lossy(),
@@ -677,19 +681,16 @@ impl Functions {
                 let executable_path = which::which("python")
                     .or_else(|_| which::which("python3"))
                     .map_err(|_| anyhow!("Python executable not found in PATH"))?;
-                let canonicalized_path = fs::canonicalize(&executable_path)?;
+                let canonicalized_path = dunce::canonicalize(&executable_path)?;
                 canonicalized_path.to_string_lossy().into_owned()
             }
             _ => bail!("Unsupported language: {}", language.as_ref()),
         };
         let bin_dir = binary_file
             .parent()
-            .expect("Failed to get parent directory of binary file")
+            .expect("Failed to get parent directory of binary file");
-            .canonicalize()?
+        let canonical_bin_dir = dunce::canonicalize(bin_dir)?.to_string_lossy().into_owned();
-            .to_string_lossy()
+        let wrapper_binary = dunce::canonicalize(&binary_script_file)?
-            .into_owned();
-        let wrapper_binary = binary_script_file
-            .canonicalize()?
             .to_string_lossy()
             .into_owned();
         let content = formatdoc!(
@@ -697,7 +698,7 @@ impl Functions {
 						@echo off
 						setlocal
 
-						set "bin_dir={bin_dir}"
+						set "bin_dir={canonical_bin_dir}"
 
 						{run} "{wrapper_binary}" %*"#,
         );
@@ -1117,6 +1118,20 @@ pub fn run_llm_function(
     #[cfg(windows)]
     let cmd_name = polyfill_cmd_name(&cmd_name, &bin_dirs);
 
+    #[cfg(windows)]
+    let cmd_args = {
+        let mut args = cmd_args;
+        if let Some(json_data) = args.pop() {
+            let tool_data_file = temp_file("-tool-data-", ".json");
+            fs::write(&tool_data_file, &json_data)?;
+            envs.insert(
+                "LLM_TOOL_DATA_FILE".into(),
+                tool_data_file.display().to_string(),
+            );
+        }
+        args
+    };
+
     envs.insert("CLICOLOR_FORCE".into(), "1".into());
     envs.insert("FORCE_COLOR".into(), "1".into());
 

src/parsers/common.rs

@@ -0,0 +1,269 @@
+use crate::function::{FunctionDeclaration, JsonSchema};
+use anyhow::{Context, Result, anyhow, bail};
+use indexmap::IndexMap;
+use serde_json::Value;
+use tree_sitter::Node;
+
+#[derive(Debug)]
+pub(crate) struct Param {
+    pub name: String,
+    pub ty_hint: String,
+    pub required: bool,
+    pub default: Option<Value>,
+    pub doc_type: Option<String>,
+    pub doc_desc: Option<String>,
+}
+
+pub(crate) trait ScriptedLanguage {
+    fn ts_language(&self) -> tree_sitter::Language;
+
+    fn default_runtime(&self) -> &str;
+
+    fn lang_name(&self) -> &str;
+
+    fn find_functions<'a>(
+        &self,
+        root: Node<'a>,
+        src: &str,
+    ) -> Vec<(Node<'a>, Node<'a>)>;
+
+    fn function_name<'a>(&self, func_node: Node<'a>, src: &'a str) -> Result<&'a str>;
+
+    fn extract_description(
+        &self,
+        wrapper_node: Node<'_>,
+        func_node: Node<'_>,
+        src: &str,
+    ) -> Option<String>;
+
+    fn extract_params(
+        &self,
+        func_node: Node<'_>,
+        src: &str,
+        description: &str,
+    ) -> Result<Vec<Param>>;
+}
+
+pub(crate) fn build_param(
+    name: &str,
+    mut ty: String,
+    mut required: bool,
+    default: Option<Value>,
+) -> Param {
+    if ty.ends_with('?') {
+        ty.pop();
+        required = false;
+    }
+
+    Param {
+        name: name.to_string(),
+        ty_hint: ty,
+        required,
+        default,
+        doc_type: None,
+        doc_desc: None,
+    }
+}
+
+pub(crate) fn build_parameters_schema(params: &[Param], _description: &str) -> JsonSchema {
+    let mut props: IndexMap<String, JsonSchema> = IndexMap::new();
+    let mut req: Vec<String> = Vec::new();
+
+    for p in params {
+        let name = p.name.replace('-', "_");
+        let mut schema = JsonSchema::default();
+
+        let ty = if !p.ty_hint.is_empty() {
+            p.ty_hint.as_str()
+        } else if let Some(t) = &p.doc_type {
+            t.as_str()
+        } else {
+            "str"
+        };
+
+        if let Some(d) = &p.doc_desc
+            && !d.is_empty()
+        {
+            schema.description = Some(d.clone());
+        }
+
+        apply_type_to_schema(ty, &mut schema);
+
+        if p.default.is_none() && p.required {
+            req.push(name.clone());
+        }
+
+        props.insert(name, schema);
+    }
+
+    JsonSchema {
+        type_value: Some("object".into()),
+        description: None,
+        properties: Some(props),
+        items: None,
+        any_of: None,
+        enum_value: None,
+        default: None,
+        required: if req.is_empty() { None } else { Some(req) },
+    }
+}
+
+pub(crate) fn apply_type_to_schema(ty: &str, s: &mut JsonSchema) {
+    let t = ty.trim_end_matches('?');
+    if let Some(rest) = t.strip_prefix("list[") {
+        s.type_value = Some("array".into());
+        let inner = rest.trim_end_matches(']');
+        let mut item = JsonSchema::default();
+
+        apply_type_to_schema(inner, &mut item);
+
+        if item.type_value.is_none() {
+            item.type_value = Some("string".into());
+        }
+        s.items = Some(Box::new(item));
+
+        return;
+    }
+
+    if let Some(rest) = t.strip_prefix("literal:") {
+        s.type_value = Some("string".into());
+        let vals = rest
+            .split('|')
+            .map(|x| x.trim().trim_matches('"').trim_matches('\'').to_string())
+            .collect::<Vec<_>>();
+        if !vals.is_empty() {
+            s.enum_value = Some(vals);
+        }
+        return;
+    }
+
+    s.type_value = Some(
+        match t {
+            "bool" => "boolean",
+            "int" => "integer",
+            "float" => "number",
+            "str" | "any" | "" => "string",
+            _ => "string",
+        }
+        .into(),
+    );
+}
+
+pub(crate) fn underscore(s: &str) -> String {
+    s.chars()
+        .map(|c| {
+            if c.is_ascii_alphanumeric() {
+                c.to_ascii_lowercase()
+            } else {
+                '_'
+            }
+        })
+        .collect::<String>()
+        .split('_')
+        .filter(|t| !t.is_empty())
+        .collect::<Vec<_>>()
+        .join("_")
+}
+
+pub(crate) fn node_text<'a>(node: Node<'_>, src: &'a str) -> Result<&'a str> {
+    node.utf8_text(src.as_bytes())
+        .map_err(|err| anyhow!("invalid utf-8 in source: {err}"))
+}
+
+pub(crate) fn named_child(node: Node<'_>, index: usize) -> Option<Node<'_>> {
+    let mut cursor = node.walk();
+    node.named_children(&mut cursor).nth(index)
+}
+
+pub(crate) fn extract_runtime(tree: &tree_sitter::Tree, src: &str, default: &str) -> String {
+    let root = tree.root_node();
+    let mut cursor = root.walk();
+    for child in root.named_children(&mut cursor) {
+        let text = match child.kind() {
+            "hash_bang_line" | "comment" => match child.utf8_text(src.as_bytes()) {
+                Ok(t) => t,
+                Err(_) => continue,
+            },
+            _ => break,
+        };
+
+        if let Some(cmd) = text.strip_prefix("#!") {
+            let cmd = cmd.trim();
+            if let Some(after_env) = cmd.strip_prefix("/usr/bin/env ") {
+                return after_env.trim().to_string();
+            }
+            return cmd.to_string();
+        }
+
+        break;
+    }
+    default.to_string()
+}
+
+pub(crate) fn generate_declarations<L: ScriptedLanguage>(
+    lang: &L,
+    src: &str,
+    file_name: &str,
+    is_tool: bool,
+) -> Result<Vec<FunctionDeclaration>> {
+    let mut parser = tree_sitter::Parser::new();
+    let language = lang.ts_language();
+    parser.set_language(&language).with_context(|| {
+        format!(
+            "failed to initialize {} tree-sitter parser",
+            lang.lang_name()
+        )
+    })?;
+
+    let tree = parser
+        .parse(src.as_bytes(), None)
+        .ok_or_else(|| anyhow!("failed to parse {}: {file_name}", lang.lang_name()))?;
+
+    if tree.root_node().has_error() {
+        bail!(
+            "failed to parse {}: syntax error in {file_name}",
+            lang.lang_name()
+        );
+    }
+
+    let _runtime = extract_runtime(&tree, src, lang.default_runtime());
+
+    let mut out = Vec::new();
+    for (wrapper, func) in lang.find_functions(tree.root_node(), src) {
+        let func_name = lang.function_name(func, src)?;
+
+        if func_name.starts_with('_') && func_name != "_instructions" {
+            continue;
+        }
+        if is_tool && func_name != "run" {
+            continue;
+        }
+
+        let description = lang
+            .extract_description(wrapper, func, src)
+            .unwrap_or_default();
+        let params = lang
+            .extract_params(func, src, &description)
+            .with_context(|| format!("in function '{func_name}' in {file_name}"))?;
+        let schema = build_parameters_schema(&params, &description);
+
+        let name = if is_tool && func_name == "run" {
+            underscore(file_name)
+        } else {
+            underscore(func_name)
+        };
+
+        let desc_trim = description.trim().to_string();
+        if desc_trim.is_empty() {
+            bail!("Missing or empty description on function: {func_name}");
+        }
+
+        out.push(FunctionDeclaration {
+            name,
+            description: desc_trim,
+            parameters: schema,
+            agent: !is_tool,
+        });
+    }
+    Ok(out)
+}

src/parsers/mod.rs

@@ -1,2 +1,3 @@
 pub(crate) mod bash;
+pub(crate) mod common;
 pub(crate) mod python;

src/repl/completer.rs

@@ -111,12 +111,14 @@ fn create_suggestion(value: &str, description: &str, span: Span) -> Suggestion {
         Some(description.to_string())
     };
     Suggestion {
+        display_override: None,
         value: value.to_string(),
         description,
         style: None,
         extra: None,
         span,
         append_whitespace: false,
+        match_indices: None,
     }
 }