diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1f06695..fc98eba 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -36,6 +36,17 @@ jobs: - uses: Swatinem/rust-cache@v2 + - name: Cache DuckDB Extensions + id: duckdb-extensions + uses: actions/cache@v4 + with: + path: ~/.duckdb/extensions + key: duckdb-ext-${{ matrix.os }}-${{ hashFiles('Cargo.lock') }} + + - name: Install DuckDB Extensions + if: steps.duckdb-extensions.outputs.cache-hit != 'true' + run: cargo test --all duckdb + - name: Test run: cargo test --all diff --git a/.gitignore b/.gitignore index 54413f1..87207b0 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ .idea/ /coyote.iml /.idea/ +.coyote diff --git a/Cargo.lock b/Cargo.lock index 5ce821f..a71758f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -34,6 +34,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "const-random", + "getrandom 0.3.4", "once_cell", "version_check", "zerocopy", @@ -56,9 +58,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android_system_properties" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" dependencies = [ "libc", ] @@ -207,7 +209,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17060e608fbc0809d62a996a65cdee9e7c441a979f40f2d1d2fbdce9eef60dad" dependencies = [ "anyhow", - "base64", + "base64 0.22.1", "convert_case 0.11.0", "dirs", "either", @@ -244,6 +246,169 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" +[[package]] +name = "arrow" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.17.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64 0.22.1", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-data" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ord" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" +dependencies = [ + "bitflags 2.13.1", +] + +[[package]] +name = "arrow-select" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "58.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + [[package]] name = "async-compression" version = "0.4.43" @@ -280,15 +445,24 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.91" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae36dc4177970ef04fde5178d3e2429882def40e57a451f919c098f72baa6cec" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", "syn 3.0.3", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -346,11 +520,11 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.17.3" +version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00bdb5da18dac48ca2cc7cd4a98e533e8635a58e2361d13a1a4ee3888e0d72f1" +checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" dependencies = [ - "aws-lc-sys 0.43.0", + "aws-lc-sys 0.44.0", "zeroize", ] @@ -369,9 +543,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.43.0" +version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43103168cc76fe62678a375e722fc9cb3a0146159ac5828bc4f0dfd755c2224c" +checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483" dependencies = [ "cc", "cmake", @@ -878,6 +1052,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64-simd" version = "0.8.0" @@ -1074,7 +1254,7 @@ dependencies = [ "cached_proc_macro_types", "hashbrown 0.15.5", "once_cell", - "thiserror 2.0.19", + "thiserror 2.0.20", "web-time", ] @@ -1097,10 +1277,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ade8366b8bd5ba243f0a58f036cc0ca8a2f069cff1a2351ef1cac6b083e16fc0" [[package]] -name = "cc" -version = "1.4.0" +name = "cast" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" dependencies = [ "find-msvc-tools", "jobserver", @@ -1202,9 +1388,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.5" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" dependencies = [ "clap_builder", "clap_derive", @@ -1212,9 +1398,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.5" +version = "4.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" dependencies = [ "anstream", "anstyle", @@ -1225,9 +1411,9 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.6.8" +version = "4.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f84a88507dbd05c695f2cb5e8558e747179134005e9893882dec964190ed89" +checksum = "3be2ad0423bdbbb0e25bc89add796f3559706d4a95e1bc98e4d9662a957b6a19" dependencies = [ "clap", "clap_lex", @@ -1237,9 +1423,9 @@ dependencies = [ [[package]] name = "clap_complete_nushell" -version = "4.6.1" +version = "4.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "933b05d5d83ff65fd7eaf5d106c792f2264908790a2642aca57429767b762ce2" +checksum = "ffb66bc82eb9c92b1727310ae2c5868df22ae7cf46185bc5c544a4fa71955e49" dependencies = [ "clap", "clap_complete", @@ -1314,13 +1500,13 @@ dependencies = [ [[package]] name = "comfy-table" -version = "7.2.2" +version = "7.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +checksum = "4a65ebfec4fb190b6f90e944a817d60499ee0744e582530e2c9900a22e591d9a" dependencies = [ "ansi-str", - "console", - "crossterm", + "console 0.15.11", + "crossterm 0.28.1", "unicode-segmentation", "unicode-width", ] @@ -1352,7 +1538,20 @@ dependencies = [ "lazy_static", "serde", "serde_yaml", - "thiserror 2.0.19", + "thiserror 2.0.20", +] + +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.59.0", ] [[package]] @@ -1379,6 +1578,26 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + [[package]] name = "convert_case" version = "0.10.0" @@ -1399,9 +1618,9 @@ dependencies = [ [[package]] name = "cookie" -version = "0.18.1" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +checksum = "1a373e3602691c3cdea496d2f0ee5935151e6168fe87739483c463db1b2f2f87" dependencies = [ "percent-encoding", "time", @@ -1454,7 +1673,7 @@ dependencies = [ "async-trait", "aws-smithy-eventstream", "aws-smithy-types", - "base64", + "base64 0.22.1", "bincode 2.0.1", "bitflags 2.13.1", "bm25", @@ -1465,8 +1684,9 @@ dependencies = [ "clap_complete_nushell", "colored", "comfy-table", - "crossterm", + "crossterm 0.29.0", "dirs", + "duckdb", "duct", "dunce", "eventsource-stream", @@ -1595,6 +1815,19 @@ version = "0.8.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags 2.13.1", + "crossterm_winapi", + "parking_lot", + "rustix 0.38.44", + "winapi", +] + [[package]] name = "crossterm" version = "0.29.0" @@ -1608,7 +1841,7 @@ dependencies = [ "filedescriptor", "mio", "parking_lot", - "rustix", + "rustix 1.1.4", "serde", "signal-hook", "signal-hook-mio", @@ -1624,6 +1857,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -1800,7 +2039,7 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" dependencies = [ - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -1886,7 +2125,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25f104b501bf2364e78d0d3974cbc774f738f5865306ed128e1e0d7499c0ad96" dependencies = [ - "console", + "console 0.16.4", "shell-words", "tempfile", "zeroize", @@ -1993,6 +2232,23 @@ dependencies = [ "dtoa", ] +[[package]] +name = "duckdb" +version = "1.10505.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "970e05eedd3f55c435194d9104f90a9b4a79a80d6e73251bc9ff43e178130c4e" +dependencies = [ + "arrow", + "cast", + "comfy-table", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libduckdb-sys", + "num-integer", + "strum", +] + [[package]] name = "duct" version = "1.1.1" @@ -2168,6 +2424,18 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fancy-regex" version = "0.14.0" @@ -2192,7 +2460,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce92ff622d6dadf7349484f42c93271a0d49b7cc4d466a936405bacbe10aa78" dependencies = [ "cfg-if", - "rustix", + "rustix 1.1.4", "windows-sys 0.59.0", ] @@ -2225,9 +2493,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" [[package]] name = "fixedbitset" @@ -2300,9 +2568,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" dependencies = [ "futures-channel", "futures-core", @@ -2315,9 +2583,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", "futures-sink", @@ -2325,15 +2593,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-executor" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" dependencies = [ "futures-core", "futures-task", @@ -2342,38 +2610,38 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" [[package]] name = "futures-macro" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.3", ] [[package]] name = "futures-sink" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-util" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-channel", "futures-core", @@ -2448,7 +2716,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" dependencies = [ - "rustix", + "rustix 1.1.4", "windows-link", ] @@ -2519,14 +2787,14 @@ dependencies = [ "azure_identity", "azure_security_keyvault_secrets", "backtrace", - "base64", + "base64 0.22.1", "chacha20poly1305", "chrono", "clap", "clap_complete", "confy", "crc32c", - "crossterm", + "crossterm 0.29.0", "dialoguer", "dirs", "futures", @@ -2545,7 +2813,7 @@ dependencies = [ "serde_with", "serde_yaml", "tempfile", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "validator", "which", @@ -2590,6 +2858,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -2613,6 +2893,15 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "heck" version = "0.5.0" @@ -2931,7 +3220,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -3122,7 +3411,7 @@ version = "0.18.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9433806cd6b4ec1aba79c021c7e4c58fb4c3b9977c085062e611ac929998fb0c" dependencies = [ - "console", + "console 0.16.4", "portable-atomic", "unicode-width", "unit-prefix", @@ -3154,7 +3443,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6654738b8024300cf062d04a1c13c10c8e2cea598ec1c47dc9b6641159429756" dependencies = [ "bitflags 2.13.1", - "crossterm", + "crossterm 0.29.0", "dyn-clone", "fuzzy-matcher", "unicode-segmentation", @@ -3245,10 +3534,12 @@ dependencies = [ "defmt", "jiff-core", "jiff-static", + "jiff-tzdb-platform", "log", "portable-atomic", "portable-atomic-util", "serde_core", + "windows-link", ] [[package]] @@ -3272,6 +3563,21 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "jiff-tzdb" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bd39932ad231f10513df9ab62661fead8719872150b7ad02a2df79f4e141e" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jni" version = "0.22.4" @@ -3284,7 +3590,7 @@ dependencies = [ "jni-sys", "log", "simd_cesu8", - "thiserror 2.0.19", + "thiserror 2.0.20", "walkdir", "windows-link", ] @@ -3333,9 +3639,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.103" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" dependencies = [ "cfg-if", "futures-util", @@ -3351,7 +3657,7 @@ dependencies = [ "jsonptr", "serde", "serde_json", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -3370,7 +3676,7 @@ version = "10.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eba32bfb4ffdeaca3e34431072faf01745c9b26d25504aa7a6cf5684334fc4fc" dependencies = [ - "base64", + "base64 0.22.1", "getrandom 0.2.17", "js-sys", "pem", @@ -3387,12 +3693,86 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + [[package]] name = "libc" version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +[[package]] +name = "libduckdb-sys" +version = "1.10505.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb514dab5e271e849235c1cb98bd65a2ae107fbd619a6740219319c54a71d95" +dependencies = [ + "cc", + "flate2", + "pkg-config", + "serde", + "serde_json", + "tar", + "ureq", + "vcpkg", + "zip", +] + [[package]] name = "libloading" version = "0.8.9" @@ -3403,6 +3783,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "libredox" version = "0.1.19" @@ -3412,6 +3798,12 @@ dependencies = [ "libc", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -3476,7 +3868,7 @@ dependencies = [ "serde-value", "serde_json", "serde_yaml", - "thiserror 2.0.19", + "thiserror 2.0.20", "thread-id", "typemap-ors", "unicode-segmentation", @@ -3623,7 +4015,7 @@ dependencies = [ "mach2", "nix 0.30.1", "sysctl", - "thiserror 2.0.19", + "thiserror 2.0.20", "widestring", "windows 0.48.0", ] @@ -3744,6 +4136,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" @@ -3766,6 +4167,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -4010,9 +4412,9 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "open" -version = "5.4.0" +version = "5.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0b3d059e795d52b8a72fef45658620edd4d9c359b338564aa14391ffa511ed5" +checksum = "f9cfef937e9c486488c7e3d949ae31c0f1d06bdacd75b99c086cb35356e30408" dependencies = [ "is-wsl", "libc", @@ -4187,7 +4589,7 @@ version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64", + "base64 0.22.1", "serde_core", ] @@ -4326,7 +4728,7 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da1d65da6dd5d1e44199ac0f58712d241c0f439f80adea8924d832384087f85" dependencies = [ - "base64", + "base64 0.22.1", "indexmap 2.14.0", "quick-xml 0.41.0", "serde", @@ -4346,9 +4748,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.14.0" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d20d5497ef88037a52ff98267d066e7f11fcc5e99bbfbd58a42336193aacec3" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "portable-atomic-util" @@ -4533,7 +4935,7 @@ dependencies = [ "rustc-hash", "rustls 0.23.43", "socket2 0.6.5", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tracing", "web-time", @@ -4556,7 +4958,7 @@ dependencies = [ "rustls 0.23.43", "rustls-pki-types", "slab", - "thiserror 2.0.19", + "thiserror 2.0.20", "tinyvec", "tracing", "web-time", @@ -4707,7 +5109,7 @@ checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" dependencies = [ "getrandom 0.2.17", "libredox", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] @@ -4717,14 +5119,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2066729dce9fecd28d1c6850a159ee68719130f149b22467c362353e16994e90" dependencies = [ "chrono", - "crossterm", + "crossterm 0.29.0", "fd-lock", "itertools 0.13.0", "nu-ansi-term", "serde", "strip-ansi-escapes", "strum", - "thiserror 2.0.19", + "thiserror 2.0.20", "unicase", "unicode-segmentation", "unicode-width", @@ -4764,9 +5166,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.16" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -4791,7 +5193,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "futures-util", @@ -4835,7 +5237,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-core", @@ -4906,7 +5308,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d1f571c72940a19d9532fe52dbea8bc9912bf1d766c2970bb824056b86f3f59" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "chrono", "futures", "http 1.5.0", @@ -4919,7 +5321,7 @@ dependencies = [ "serde", "serde_json", "sse-stream", - "thiserror 2.0.19", + "thiserror 2.0.20", "tokio", "tokio-stream", "tokio-util", @@ -5032,6 +5434,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.1", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + [[package]] name = "rustix" version = "1.1.4" @@ -5041,7 +5456,7 @@ dependencies = [ "bitflags 2.13.1", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.12.1", "windows-sys 0.61.2", ] @@ -5068,7 +5483,7 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.13", + "rustls-webpki 0.103.14", "subtle", "zeroize", ] @@ -5109,7 +5524,7 @@ dependencies = [ "rustls 0.23.43", "rustls-native-certs", "rustls-platform-verifier-android", - "rustls-webpki 0.103.13", + "rustls-webpki 0.103.14", "security-framework", "security-framework-sys", "webpki-root-certs", @@ -5134,9 +5549,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a" dependencies = [ "aws-lc-rs", "ring", @@ -5436,16 +5851,17 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.21.0" +version = "3.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76a5c54c7310e7b8b9577c286d7e399ddd876c3e12b3ed917a8aabc4b96e9e8c" +checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a" dependencies = [ - "base64", + "base64 0.22.1", "bs58", "chrono", "hex", "indexmap 1.9.3", "indexmap 2.14.0", + "jiff", "schemars 0.9.0", "schemars 1.2.2", "serde_core", @@ -5456,9 +5872,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.21.0" +version = "3.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84d57bc0c8b9a17920c178daa6bb924850d54a9c97ab45194bb8c17ad66bb660" +checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -5663,7 +6079,7 @@ checksum = "0d585997b0ac10be3c5ee635f1bab02d512760d14b7c468801ac8a01d9ae5f1d" dependencies = [ "num-bigint", "num-traits", - "thiserror 2.0.19", + "thiserror 2.0.20", "time", ] @@ -5891,7 +6307,7 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "thiserror 2.0.19", + "thiserror 2.0.20", "walkdir", ] @@ -5952,7 +6368,7 @@ dependencies = [ "fastrand", "getrandom 0.4.3", "once_cell", - "rustix", + "rustix 1.1.4", "windows-sys 0.61.2", ] @@ -5999,7 +6415,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ - "rustix", + "rustix 1.1.4", "windows-sys 0.61.2", ] @@ -6025,11 +6441,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" dependencies = [ - "thiserror-impl 2.0.19", + "thiserror-impl 2.0.20", ] [[package]] @@ -6045,9 +6461,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.19" +version = "2.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", @@ -6114,6 +6530,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" version = "0.8.3" @@ -6257,7 +6682,7 @@ checksum = "ac2a5518c70fa84342385732db33fb3f44bc4cc748936eb5833d2df34d6445ef" dependencies = [ "async-trait", "axum", - "base64", + "base64 0.22.1", "bytes", "h2 0.4.15", "http 1.5.0", @@ -6378,9 +6803,9 @@ dependencies = [ [[package]] name = "tree-sitter" -version = "0.26.11" +version = "0.26.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af1c71c1c4cc0920b20d6b0f6572e7682cd07a6a2faec71067a31fa394c586df" +checksum = "83c567a8e18ae93f20982c90370b16fd24023aeaf52f6052b96957ab253a0fec" dependencies = [ "cc", "regex", @@ -6454,7 +6879,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dd1eb4a538c1ab3d5c05437129bc16891296146b23c9b0bb3f5df99f5b3a18d" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures", "serde", @@ -6469,7 +6894,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e632235c99ae896a3c451d1ead00cea11a2219aeda1b35a74027fe99ea3f3b72" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "dyn-clone", "futures", "getrandom 0.3.4", @@ -6580,11 +7005,11 @@ checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" [[package]] name = "ureq" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dea7109cdcd5864d4eeb1b58a1648dc9bf520360d7af16ec26d0a9354bafcfc0" +checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d" dependencies = [ - "base64", + "base64 0.23.1", "cookie_store", "encoding_rs", "flate2", @@ -6602,11 +7027,11 @@ dependencies = [ [[package]] name = "ureq-proto" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e994ba84b0bd1b1b0cf92878b7ef898a5c1760108fe7b6010327e274917a808c" +checksum = "da5f78b09e6941e1a0f2e30e695e4b120377b54d5e0aec11b594bb57b3971613" dependencies = [ - "base64", + "base64 0.23.1", "http 1.5.0", "httparse", "log", @@ -6758,9 +7183,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" dependencies = [ "cfg-if", "once_cell", @@ -6771,9 +7196,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.76" +version = "0.4.77" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" dependencies = [ "js-sys", "wasm-bindgen", @@ -6781,9 +7206,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6791,9 +7216,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" dependencies = [ "bumpalo", "proc-macro2", @@ -6804,9 +7229,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.126" +version = "0.2.127" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" dependencies = [ "unicode-ident", ] @@ -6845,7 +7270,7 @@ checksum = "016ccf01d1c58b6f8999612813e17c9b2390f7d70671428869913310f83f54b8" dependencies = [ "cc", "downcast-rs", - "rustix", + "rustix 1.1.4", "smallvec", "wayland-sys", ] @@ -6857,7 +7282,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c36a0f861ad76d0901f2800b46321410d9f73f2ea88aac0650d86c32688073" dependencies = [ "bitflags 2.13.1", - "rustix", + "rustix 1.1.4", "wayland-backend", "wayland-scanner", ] @@ -6909,9 +7334,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.103" +version = "0.3.104" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" dependencies = [ "js-sys", "wasm-bindgen", @@ -7347,8 +7772,8 @@ dependencies = [ "libc", "log", "os_pipe", - "rustix", - "thiserror 2.0.19", + "rustix 1.1.4", + "thiserror 2.0.20", "tree_magic_mini", "wayland-backend", "wayland-client", @@ -7369,7 +7794,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" dependencies = [ "gethostname", - "rustix", + "rustix 1.1.4", "x11rb-protocol", ] @@ -7386,7 +7811,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", - "rustix", + "rustix 1.1.4", ] [[package]] @@ -7443,18 +7868,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.55" +version = "0.8.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" dependencies = [ "proc-macro2", "quote", @@ -7556,9 +7981,9 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dba6063ff82cdbd9a765add16d369abe81e520f836054e997c2db217ceca40c0" dependencies = [ - "base64", + "base64 0.22.1", "ed25519-dalek", - "thiserror 2.0.19", + "thiserror 2.0.20", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 32c56cb..d51421c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,8 +17,9 @@ exclude = [".github", "CONTRIBUTING.md"] anyhow = "1.0.69" bytes = "1.4.0" clap = { version = "4.5.40", features = ["cargo", "derive", "wrap_help"] } -comfy-table = { version = "7.2.2", features = ["custom_styling"] } +comfy-table = { version = "7.1.4", features = ["custom_styling"] } dirs = "6.0.0" +duckdb = { version = "1.10505.0", features = ["bundled"] } dunce = "1.0.5" futures-util = "0.3.29" inquire = "0.9.4" diff --git a/src/config/agent.rs b/src/config/agent.rs index d9d95ea..6b993dc 100644 --- a/src/config/agent.rs +++ b/src/config/agent.rs @@ -4,6 +4,7 @@ use crate::{ client::Model, config::memory, function::{Functions, run_llm_function}, + graph, rag, }; use super::rag_cache::RagKey; @@ -12,6 +13,7 @@ use crate::config::prompts::{ DEFAULT_SPAWN_INSTRUCTIONS, DEFAULT_TEAMMATE_INSTRUCTIONS, DEFAULT_TODO_INSTRUCTIONS, DEFAULT_USER_INTERACTION_INSTRUCTIONS, }; +use crate::graph::types::RagNode; use crate::graph::{Graph, GraphParser, NodeType}; use crate::rag::RagInitConfig; use crate::vault::SECRET_RE; @@ -146,11 +148,18 @@ impl Agent { let rag = if rag_path.exists() { let key = RagKey::Agent(name.to_string()); let app_clone = app.clone(); + let vault_clone = app_state.vault.clone(); let rag_path_clone = rag_path.clone(); let rag = app_state .rag_cache .load_with(key, || async move { - Rag::load(&app_clone, DEFAULT_AGENT_NAME, &rag_path_clone) + Rag::load_async( + &app_clone, + &vault_clone, + DEFAULT_AGENT_NAME, + &rag_path_clone, + ) + .await }) .await?; Some(rag) @@ -171,7 +180,15 @@ impl Agent { let rag = app_state .rag_cache .load_with(key, || async move { - Rag::init(&app_clone, "rag", &rag_path_clone, &document_paths, abort).await + Rag::init( + &app_clone, + "rag", + &rag_path_clone, + &document_paths, + abort, + true, + ) + .await }) .await?; Some(rag) @@ -937,6 +954,30 @@ fn resolve_document_paths( Ok(document_paths) } +/// How a graph rag node describes the knowledge base it wants built. +/// +/// `driver` is forwarded as-is: `None` means the node did not ask for one, which +/// `RagInitConfig` resolves to yaml, so workflows written before drivers existed +/// keep their current storage. +/// +/// Every field is now named explicitly, so adding one to `RagInitConfig` breaks +/// this literal. That is deliberate: the new field then gets a decision about +/// whether a rag node can drive it, instead of silently taking its default. +fn rag_init_config(rag_node: &RagNode) -> RagInitConfig { + RagInitConfig { + embedding_model: rag_node.embedding_model.clone(), + chunk_size: rag_node.chunk_size, + chunk_overlap: rag_node.chunk_overlap, + reranker_model: rag_node.reranker_model.clone(), + top_k: rag_node.top_k, + batch_size: rag_node.batch_size, + extractor_model: rag_node.extractor_model.clone(), + extractor_prompt: rag_node.extractor_prompt.clone(), + graph_hops: rag_node.graph_hops, + driver: rag_node.driver.clone(), + } +} + #[allow(clippy::too_many_arguments)] async fn init_graph_rags( app: &AppConfig, @@ -964,26 +1005,28 @@ async fn init_graph_rags( }; let rag = if rag_path.exists() { let app_clone = app.clone(); + let vault_clone = app_state.vault.clone(); let path_clone = rag_path.clone(); let name_clone = node_id.clone(); app_state .rag_cache .load_with(key, || async move { - Rag::load(&app_clone, &name_clone, &path_clone) + Rag::load_async(&app_clone, &vault_clone, &name_clone, &path_clone).await }) .await? } else { - let config = RagInitConfig { - embedding_model: rag_node.embedding_model.clone(), - chunk_size: rag_node.chunk_size, - chunk_overlap: rag_node.chunk_overlap, - reranker_model: rag_node.reranker_model.clone(), - top_k: rag_node.top_k, - batch_size: rag_node.batch_size, - extractor_model: rag_node.extractor_model.clone(), - extractor_prompt: rag_node.extractor_prompt.clone(), - graph_hops: rag_node.graph_hops, - }; + // Checked before anything is built: an unknown driver would otherwise + // fall through `Rag::create`'s catch-all to a yaml store, embed every + // document, and persist the bogus driver string. The RAG would then be + // rejected on every subsequent load, leaving the agent unstartable. + // Graph validation catches this too, but it is skipped when + // `validate_before_run` is off, so this guard is the load-bearing one. + if let Some(driver) = &rag_node.driver + && let Some(message) = graph::validator::rag_driver_error(driver) + { + bail!("rag node '{node_id}': {message}"); + } + let mut config = rag_init_config(rag_node); let fully_specified = config.embedding_model.is_some() && config.chunk_size.is_some() && config.chunk_overlap.is_some(); @@ -1009,6 +1052,10 @@ async fn init_graph_rags( initialized. RAG initialization is required for this agent." ); } + + if config.driver.is_none() { + config.driver = Some(rag::select_rag_driver()?); + } } let document_paths = @@ -1317,4 +1364,39 @@ version: "1.0" assert_eq!(meta.description, ""); } + + #[test] + fn rag_init_config_forwards_an_explicit_driver() { + let node: RagNode = + serde_yaml::from_str("documents: [\"./docs\"]\ndriver: duckdb\n").unwrap(); + + assert_eq!(rag_init_config(&node).driver.as_deref(), Some("duckdb")); + } + + /// A node that names no driver must forward `None`, which `RagInitConfig` + /// documents as "yaml". Existing workflows therefore keep their yaml store. + #[test] + fn rag_init_config_leaves_the_driver_unset_by_default() { + let node: RagNode = serde_yaml::from_str("documents: [\"./docs\"]\n").unwrap(); + + assert_eq!(rag_init_config(&node).driver, None); + } + + /// The driver must ride alongside the rest of the node's settings, not + /// replace them. + #[test] + fn rag_init_config_forwards_the_other_settings_too() { + let node: RagNode = serde_yaml::from_str( + "documents: [\"./docs\"]\ndriver: duckdb\nchunk_size: 512\nchunk_overlap: 64\ntop_k: 7\nembedding_model: some:model\n", + ) + .unwrap(); + + let config = rag_init_config(&node); + + assert_eq!(config.driver.as_deref(), Some("duckdb")); + assert_eq!(config.chunk_size, Some(512)); + assert_eq!(config.chunk_overlap, Some(64)); + assert_eq!(config.top_k, Some(7)); + assert_eq!(config.embedding_model.as_deref(), Some("some:model")); + } } diff --git a/src/config/paths.rs b/src/config/paths.rs index 49f2135..44589d9 100644 --- a/src/config/paths.rs +++ b/src/config/paths.rs @@ -16,7 +16,7 @@ use anyhow::{Context, Result, anyhow, bail}; use log::LevelFilter; use std::collections::HashSet; use std::env; -use std::fs::{read_dir, read_to_string}; +use std::fs::{read_dir, read_to_string, remove_file}; use std::path::{Path, PathBuf}; pub fn config_dir() -> PathBuf { @@ -414,6 +414,10 @@ pub fn list_rags() -> Vec { for entry in rd.flatten() { let name = entry.file_name(); if let Some(name) = name.to_string_lossy().strip_suffix(".yaml") { + if is_rag_sidecar_name(name) { + continue; + } + names.push(name.to_string()); } } @@ -424,6 +428,34 @@ pub fn list_rags() -> Vec { } } +pub(crate) fn is_rag_sidecar_name(name: &str) -> bool { + name.ends_with(".sbx-mixin") +} + +pub(crate) fn remove_rag_sidecars(dir: &Path, name: &str) -> Result<()> { + let duckdb_path = dir.join(format!("{name}.duckdb")); + if duckdb_path.exists() { + let _ = remove_file(&duckdb_path); + } + let wal_path = dir.join(format!("{name}.duckdb.wal")); + if wal_path.exists() { + let _ = remove_file(&wal_path); + } + let mixin_path = dir.join(format!("{name}.sbx-mixin.yaml")); + if mixin_path.exists() { + remove_file(&mixin_path).with_context(|| { + format!( + "Failed to remove the sandbox mixin for RAG '{name}' at '{}'. \ + The RAG was NOT deleted so you can retry; this host remains \ + whitelisted in the sandbox until the file is removed.", + mixin_path.display() + ) + })?; + } + + Ok(()) +} + pub fn list_macros() -> Vec { list_file_names(macros_dir(), ".yaml") } @@ -846,4 +878,72 @@ mod tests { } let _ = fs::remove_dir_all(&root); } + + fn sidecar_temp_dir(label: &str) -> PathBuf { + let unique = time::SystemTime::now() + .duration_since(time::UNIX_EPOCH) + .unwrap() + .as_nanos(); + let root = env::temp_dir().join(format!("coyote-{label}-test-{unique}")); + fs::create_dir_all(&root).unwrap(); + root + } + + #[test] + fn is_rag_sidecar_name_accepts_dotted_rag_names() { + assert!(!is_rag_sidecar_name("v2.docs")); + assert!(!is_rag_sidecar_name("myrag")); + assert!(is_rag_sidecar_name("myrag.sbx-mixin")); + assert!(is_rag_sidecar_name("v2.docs.sbx-mixin")); + } + + #[test] + fn remove_rag_sidecars_removes_duckdb_wal_and_mixin() { + let root = sidecar_temp_dir("rag-sidecars-both"); + let duckdb = root.join("docs.duckdb"); + let wal = root.join("docs.duckdb.wal"); + let mixin = root.join("docs.sbx-mixin.yaml"); + fs::write(&duckdb, "db").unwrap(); + fs::write(&wal, "wal").unwrap(); + fs::write(&mixin, "mixin").unwrap(); + + remove_rag_sidecars(&root, "docs").unwrap(); + + assert!(!duckdb.exists(), "the .duckdb sidecar must be removed"); + assert!(!wal.exists(), "the .duckdb.wal sidecar must be removed"); + assert!( + !mixin.exists(), + "the .sbx-mixin.yaml sidecar must be removed" + ); + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn remove_rag_sidecars_is_ok_when_absent() { + let root = sidecar_temp_dir("rag-sidecars-absent"); + assert!(remove_rag_sidecars(&root, "docs").is_ok()); + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn remove_rag_sidecars_runs_before_yaml_unlink() { + let root = sidecar_temp_dir("rag-sidecars-order"); + let yaml = root.join("docs.yaml"); + fs::write(&yaml, "rag").unwrap(); + let mixin = root.join("docs.sbx-mixin.yaml"); + fs::create_dir_all(&mixin).unwrap(); + fs::write(mixin.join("blocker"), "x").unwrap(); + + let err = remove_rag_sidecars(&root, "docs").unwrap_err(); + assert!( + err.to_string() + .contains("Failed to remove the sandbox mixin"), + "got: {err}" + ); + assert!( + yaml.exists(), + "the .yaml must survive a sidecar-removal failure so the delete is retryable" + ); + let _ = fs::remove_dir_all(&root); + } } diff --git a/src/config/request_context.rs b/src/config/request_context.rs index 7c07c4a..c0eea43 100644 --- a/src/config/request_context.rs +++ b/src/config/request_context.rs @@ -142,6 +142,7 @@ pub struct RequestContext { pub role: Option, pub session: Option, pub rag: Option>, + pub rag_key: Option, pub agent: Option, pub last_message: Option, @@ -176,6 +177,7 @@ impl RequestContext { role: None, session: None, rag: None, + rag_key: None, agent: None, last_message: None, tool_scope: ToolScope::default(), @@ -229,6 +231,7 @@ impl RequestContext { role: None, session: None, rag: None, + rag_key: None, agent: None, last_message: None, tool_scope: ToolScope { @@ -277,6 +280,7 @@ impl RequestContext { role: self.role.clone(), session: self.session.clone(), rag: self.rag.clone(), + rag_key: self.rag_key.clone(), agent: self.agent.clone(), last_message: self.last_message.clone(), tool_scope: self.tool_scope.clone(), @@ -315,6 +319,7 @@ impl RequestContext { role: None, session: None, rag: None, + rag_key: None, agent: None, last_message: None, tool_scope: ToolScope { @@ -2554,6 +2559,14 @@ impl RequestContext { match file_ext { Some(file_ext) => { if let Some(name) = name.to_string_lossy().strip_suffix(file_ext) { + // Sidecars are not independently deletable assets. + // Guarded on `kind == "rag"` because this scan is shared + // by all six kinds, and `session`/`macro` also use + // `.yaml`. The helper lives in paths.rs beside + // list_rags() so both filters cannot drift apart. + if kind == "rag" && paths::is_rag_sidecar_name(name) { + continue; + } names.push(name.to_string()); } } @@ -2590,6 +2603,13 @@ impl RequestContext { match file_ext { Some(ext) => { let path = dir.join(format!("{name}{ext}")); + // Sidecars FIRST. If this fails, the .yaml is still on disk, the + // RAG is still listed, and the user can retry. Unlinking the .yaml + // first would make the deletion unretryable while leaving an + // orphaned mixin whitelisting a host in every sandbox launch. + if kind == "rag" { + paths::remove_rag_sidecars(&dir, &name)?; + } remove_file(&path).with_context(|| { format!("Failed to delete {kind} at '{}'", path.display()) })?; @@ -2773,7 +2793,12 @@ impl RequestContext { } } "rag_top_k" => { - let value = value.parse().with_context(|| "Invalid value")?; + let value: usize = value.parse().with_context(|| "Invalid value")?; + if value == 0 { + bail!( + "rag_top_k must be >= 1; a top_k of 0 makes every query return no results." + ); + } if !self.set_rag_top_k(value)? { self.update_app_config(|app| app.rag_top_k = value); } @@ -3724,6 +3749,14 @@ impl RequestContext { .then(|| Arc::new(RwLock::new(Supervisor::new(max_concurrent, max_depth)))); self.rag = agent.rag(); + // Keep `rag_key` in lockstep with `rag`. Agent RAGs are cached under + // `RagKey::Agent()` (see `Agent::init`), so mirror that key exactly; + // leaving the previous key in place would let `.rebuild rag` invalidate an + // unrelated RAG's cache entry, and leaving it `None` would invalidate nothing. + self.rag_key = self + .rag + .is_some() + .then(|| RagKey::Agent(agent.name().to_string())); self.agent = Some(agent); self.supervisor = supervisor; self.inbox = None; @@ -3772,6 +3805,11 @@ impl RequestContext { self.pending_agents_guardrail_count = 0; self.todo_list = TodoList::default(); self.rag.take(); + // Cleared alongside `rag` so the pair never disagrees: an agent RAG is + // cached under `RagKey::Agent()`, and leaving that key behind + // would outlive the RAG it names. Latent rather than live today only + // because `rebuild_rag`/`edit_rag_docs` bail on `rag.is_none()` first. + self.rag_key = None; self.discontinuous_last_message(); } Ok(()) @@ -4079,10 +4117,11 @@ impl RequestContext { } let app = self.app.config.clone(); + let vault = self.app.vault.clone(); let rag_cache = self.rag_cache(); let working_mode = self.working_mode; - let rag: Arc = match rag { + let (rag, rag_key): (Arc, Option) = match rag { None => { let rag_path = self.rag_file(super::TEMP_RAG_NAME); if rag_path.exists() { @@ -4090,15 +4129,29 @@ impl RequestContext { format!("Failed to cleanup previous '{}' rag", super::TEMP_RAG_NAME) })?; } - Arc::new(Rag::init(&app, super::TEMP_RAG_NAME, &rag_path, &[], abort_signal).await?) + ( + Arc::new( + Rag::init( + &app, + super::TEMP_RAG_NAME, + &rag_path, + &[], + abort_signal, + false, + ) + .await?, + ), + None, + ) } Some(name) => { let rag_path = self.rag_file(name); let key = RagKey::Named(name.to_string()); - rag_cache - .load_with(key, || { + let loaded = rag_cache + .load_with(key.clone(), || { let app = app.clone(); + let vault = vault.clone(); let rag_path = rag_path.clone(); let abort_signal = abort_signal.clone(); async move { @@ -4106,16 +4159,39 @@ impl RequestContext { if working_mode.is_cmd() { bail!("Unknown RAG '{name}'"); } - Rag::init(&app, name, &rag_path, &[], abort_signal.clone()).await + Rag::init(&app, name, &rag_path, &[], abort_signal.clone(), true) + .await } else { - Rag::load(&app, name, &rag_path) + Rag::load_async(&app, &vault, name, &rag_path).await } } }) - .await? + .await?; + (loaded, Some(key)) } }; self.rag = Some(rag); + self.rag_key = rag_key; + Ok(()) + } + + pub async fn attach_rag(&mut self, name: &str) -> Result<()> { + let rag_path = self.rag_file(name); + if rag_path.exists() { + bail!( + "RAG '{name}' already exists at '{}'. \ + Use a different name, or delete the existing file first.", + rag_path.display() + ); + } + let app = self.app.config.as_ref(); + let vault = self.app.vault.clone(); + let rag = Rag::attach(app, &vault, name, &rag_path).await?; + let rag = Arc::new(rag); + let key = RagKey::Named(name.to_string()); + self.rag_cache().insert(key.clone(), &rag); + self.rag = Some(rag); + self.rag_key = Some(key); Ok(()) } @@ -4125,6 +4201,12 @@ impl RequestContext { None => bail!("No RAG"), }; + if rag.is_attached() { + bail!( + "Cannot edit documents on an attached RAG; Coyote does not own its source documents." + ); + } + let document_paths = rag.document_paths(); let temp_file = temp_file(&format!("-rag-{}", rag.name()), ".txt"); tokio::fs::write(&temp_file, &document_paths.join("\n")) @@ -4150,15 +4232,18 @@ impl RequestContext { bail!("No changes") } - let key = if self.agent.is_some() { - RagKey::Agent(rag.name().to_string()) - } else { - RagKey::Named(rag.name().to_string()) - }; - self.rag_cache().invalidate(&key); + if let Some(key) = self.rag_key.clone() { + self.rag_cache().invalidate(&key); + } - rag.refresh_document_paths(&new_document_paths, false, &self.app.config, abort_signal) - .await?; + rag.refresh_document_paths( + &new_document_paths, + false, + false, + &self.app.config, + abort_signal, + ) + .await?; self.rag = Some(Arc::new(rag)); Ok(()) } @@ -4169,15 +4254,25 @@ impl RequestContext { None => bail!("No RAG"), }; - let key = if self.agent.is_some() { - RagKey::Agent(rag.name().to_string()) - } else { - RagKey::Named(rag.name().to_string()) - }; - self.rag_cache().invalidate(&key); + if rag.is_attached() { + bail!( + "Cannot rebuild an attached RAG; Coyote does not own its source documents. \ + Re-index from the system that originally created '{}'.", + rag.name() + ); + } + + if let Some(key) = self.rag_key.clone() { + self.rag_cache().invalidate(&key); + } let document_paths = rag.document_paths().to_vec(); - rag.refresh_document_paths(&document_paths, true, &self.app.config, abort_signal) + println!( + "Rebuilding re-embeds every document ({} files). \ + This will call the embedding API and may take a while.", + rag.file_count() + ); + rag.refresh_document_paths(&document_paths, true, true, &self.app.config, abort_signal) .await?; self.rag = Some(Arc::new(rag)); Ok(()) @@ -4583,6 +4678,44 @@ mod tests { assert!(ctx.agent.is_none()); assert!(ctx.rag.is_none()); + assert_eq!(ctx.rag_key, None); + } + + #[test] + #[serial] + fn use_agent_does_not_carry_stale_rag_key() { + let _guard = TestConfigDirGuard::new(); + let mut ctx = create_test_ctx(); + let app = ctx.app.config.clone(); + let agent_name = format!( + "test_agent_{}", + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos() + ); + let agent_dir = paths::agent_data_dir(&agent_name); + create_dir_all(&agent_dir).unwrap(); + write( + agent_dir.join("config.yaml"), + format!("name: {agent_name}\ninstructions: hi\n"), + ) + .unwrap(); + + ctx.rag_key = Some(RagKey::Named("docs".to_string())); + + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { + ctx.use_agent(&app, &agent_name, None, utils::create_abort_signal()) + .await + .unwrap(); + }); + + assert!(ctx.rag.is_none()); + assert_eq!(ctx.rag_key, None); } #[test] @@ -5970,6 +6103,28 @@ mod tests { assert!(paths::list_rags().is_empty()); } + #[test] + #[serial] + fn list_rags_skips_sbx_mixin_sidecars() { + let _guard = TestConfigDirGuard::new(); + let rags_dir = paths::rags_dir(); + create_dir_all(&rags_dir).unwrap(); + write(rags_dir.join("docs.yaml"), "embedding_model: test").unwrap(); + write(rags_dir.join("docs.sbx-mixin.yaml"), "kind: mixin").unwrap(); + write(rags_dir.join("v2.docs.yaml"), "embedding_model: test").unwrap(); + + let names = paths::list_rags(); + assert!(names.contains(&"docs".to_string())); + assert!( + names.contains(&"v2.docs".to_string()), + "a dotted RAG name must still be listed: {names:?}" + ); + assert!( + !names.contains(&"docs.sbx-mixin".to_string()), + "the sandbox mixin sidecar must not appear as a RAG: {names:?}" + ); + } + #[test] #[serial] fn use_agent_errors_when_already_in_session() { diff --git a/src/graph/types.rs b/src/graph/types.rs index 020badf..fcd3292 100644 --- a/src/graph/types.rs +++ b/src/graph/types.rs @@ -367,6 +367,13 @@ pub struct RagNode { #[serde(default, skip_serializing_if = "Option::is_none")] pub graph_hops: Option, + /// Storage driver for this node's knowledge base ("yaml", "duckdb"). `None` + /// means "yaml". Only honored when the knowledge base is first built; + /// changing it afterwards has no effect until the RAG is deleted and + /// re-initialized. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub driver: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub state_updates: Option>, @@ -1152,4 +1159,100 @@ nodes: assert!(triage.next.as_ref().unwrap().is_fan_out()); assert_eq!(triage.next.as_ref().unwrap().as_slice().len(), 2); } + + fn rag_node_of(graph: &Graph, id: &str) -> RagNode { + match &graph.get_node(id).unwrap().node_type { + NodeType::Rag(r) => r.clone(), + other => panic!("expected a rag node, got {other:?}"), + } + } + + #[test] + fn rag_node_deserializes_an_explicit_driver() { + let yaml = r#" +name: kb +start: research +nodes: + research: + type: rag + documents: ["./docs"] + driver: duckdb + next: done + done: + type: end + output: ok +"#; + let graph: Graph = serde_yaml::from_str(yaml).unwrap(); + + assert_eq!( + rag_node_of(&graph, "research").driver.as_deref(), + Some("duckdb") + ); + } + + /// Workflows written before drivers existed must keep parsing, and must keep + /// asking for nothing, so `RagInitConfig` resolves them to the yaml default. + #[test] + fn rag_node_without_a_driver_stays_unset() { + let yaml = r#" +name: kb +start: research +nodes: + research: + type: rag + documents: ["./docs"] + next: done + done: + type: end + output: ok +"#; + let graph: Graph = serde_yaml::from_str(yaml).unwrap(); + + assert_eq!(rag_node_of(&graph, "research").driver, None); + } + + #[test] + fn rag_node_driver_survives_a_serialize_round_trip() { + let yaml = r#" +name: kb +start: research +nodes: + research: + type: rag + documents: ["./docs"] + driver: duckdb + next: done + done: + type: end + output: ok +"#; + let graph: Graph = serde_yaml::from_str(yaml).unwrap(); + let reparsed: Graph = + serde_yaml::from_str(&serde_yaml::to_string(&graph).unwrap()).unwrap(); + + assert_eq!( + rag_node_of(&reparsed, "research").driver.as_deref(), + Some("duckdb") + ); + } + + /// `skip_serializing_if` must keep `driver:` out of graphs that never set it. + #[test] + fn rag_node_without_a_driver_omits_the_key_when_serialized() { + let yaml = r#" +name: kb +start: research +nodes: + research: + type: rag + documents: ["./docs"] + next: done + done: + type: end + output: ok +"#; + let graph: Graph = serde_yaml::from_str(yaml).unwrap(); + + assert!(!serde_yaml::to_string(&graph).unwrap().contains("driver")); + } } diff --git a/src/graph/validator.rs b/src/graph/validator.rs index f655c8e..fe438f7 100644 --- a/src/graph/validator.rs +++ b/src/graph/validator.rs @@ -2,6 +2,7 @@ use super::state::template_root_keys; use super::types::{Graph, Node, NodeType}; use crate::client::{Model, ModelType}; use crate::config::{Agent, AppConfig, paths}; +use crate::rag::{GraphRagConfig, RagData}; use anyhow::{Result, bail}; use std::collections::{BTreeMap, HashSet, VecDeque}; use std::path::PathBuf; @@ -96,6 +97,51 @@ pub struct GraphValidator { skill_exists: fn(&str) -> bool, } +/// A minimal `RagData` whose only interesting field is `driver`. The numeric +/// arguments are the smallest values that satisfy `validate()`'s unrelated +/// floors (top_k >= 1, and chunk_size >= 1 with chunk_overlap < chunk_size for +/// a non-attached RAG). `RagData::new` sets `attached: false`, which is the +/// correct shape here: a graph rag node always builds its own local knowledge +/// base from `documents` and can never be attached. +fn rag_driver_probe(driver: &str) -> RagData { + let mut data = RagData::new( + String::new(), + 1, + 0, + None, + 1, + None, + GraphRagConfig::default(), + ); + data.driver = driver.to_string(); + data +} + +/// `Some(message)` when `driver` is one that `RagData::validate()` would reject. +/// +/// The set of valid drivers is defined in exactly one place, `RagData::validate()`, +/// so this asks that function rather than restating the list here. +/// +/// Fails open on purpose: the first probe below uses the default driver, which is +/// valid by definition. If even that one is rejected, `validate()` has grown a +/// precondition the probe fixture no longer satisfies, and every verdict from here +/// would be a false positive that rejects working graphs. In that case we decline +/// to judge and leave enforcement to RAG construction. The +/// `rag_driver_probe_fixture_is_accepted` test turns that silent degradation into a +/// loud failure. Both `validate()` calls are load-bearing; neither is redundant. +pub(crate) fn rag_driver_error(driver: &str) -> Option { + if rag_driver_probe(&RagData::default().driver) + .validate() + .is_err() + { + return None; + } + rag_driver_probe(driver) + .validate() + .err() + .map(|err| err.to_string()) +} + impl GraphValidator { pub fn new(base_dir: impl Into) -> Self { Self { @@ -154,6 +200,11 @@ impl GraphValidator { not be written to state", )); } + if let Some(driver) = &r.driver + && let Some(message) = rag_driver_error(driver) + { + result.error(ValidationError::with_node(node_id, message)); + } } } } @@ -1031,6 +1082,7 @@ mod tests { extractor_model: None, extractor_prompt: None, graph_hops: None, + driver: None, state_updates, timeout: None, }), @@ -1385,6 +1437,55 @@ mod tests { ); } + /// Guards the fail-open branch in `rag_driver_error`. If this fails, + /// `RagData::validate()` grew a precondition the probe fixture no longer + /// satisfies and rag-node driver validation has silently switched itself off. + /// Repair the fixture in `rag_driver_probe`; do not delete this test. + #[test] + fn rag_driver_probe_fixture_is_accepted() { + let default_driver = RagData::default().driver; + assert!( + rag_driver_probe(&default_driver).validate().is_ok(), + "probe fixture rejected for the default driver '{default_driver}'" + ); + } + + #[test] + fn rag_driver_error_defers_to_ragdata_validate() { + assert_eq!(rag_driver_error("yaml"), None); + assert_eq!(rag_driver_error("duckdb"), None); + + let message = rag_driver_error("duckdbb").expect("unknown driver must be rejected"); + assert!(message.contains("duckdbb"), "got: {message}"); + } + + #[test] + fn rag_node_with_unknown_driver_errors_naming_the_node() { + let mut node = rag_node("kb", &["./docs"], true); + if let NodeType::Rag(ref mut r) = node.node_type { + r.driver = Some("postgres".into()); + } + let graph = graph_with(vec![("kb", node), ("end", end_node("end"))], "kb"); + + let result = validator().validate(&graph); + + assert!(!result.is_valid()); + let err = result.into_result().unwrap_err().to_string(); + assert!(err.contains("[kb]"), "must name the node: {err}"); + assert!(err.contains("postgres"), "must name the driver: {err}"); + } + + #[test] + fn rag_node_with_duckdb_driver_produces_no_findings() { + let mut node = rag_node("kb", &["./docs"], true); + if let NodeType::Rag(ref mut r) = node.node_type { + r.driver = Some("duckdb".into()); + } + let graph = graph_with(vec![("kb", node), ("end", end_node("end"))], "kb"); + + assert!(validator().validate(&graph).is_valid()); + } + fn agent_node(id: &str, agent: &str, next: Option<&str>) -> Node { Node { id: id.into(), diff --git a/src/rag/mod.rs b/src/rag/mod.rs index b06928d..a71d7b7 100644 --- a/src/rag/mod.rs +++ b/src/rag/mod.rs @@ -5,13 +5,20 @@ use crate::config::*; use crate::utils::*; mod graph; +mod provider; +mod providers; mod serde_vectors; mod splitter; use self::graph::{KnowledgeGraph, extract_entities}; +use self::provider::RagProvider; +use self::providers::{DuckDbProvider, QdrantProvider, YamlProvider}; +use crate::sandbox::mcp_credentials; +use crate::vault::{Vault, interpolate_secrets}; use anyhow::{Context, Result, anyhow, bail}; use bm25::{Language, SearchEngine, SearchEngineBuilder}; +use gman::SecretError; use hnsw_rs::prelude::*; use indexmap::{IndexMap, IndexSet}; use inquire::{Confirm, Select, Text, required, validator::Validation}; @@ -56,8 +63,8 @@ pub struct Rag { name: String, path: String, embedding_model: Model, - hnsw: Hnsw<'static, f32, DistCosine>, bm25: SearchEngine, + provider: Box, data: RagData, last_sources: RwLock>, node_to_docs: IndexMap>, @@ -74,6 +81,19 @@ impl Debug for Rag { } } +// CLONING A `Rag` DOES NOT SNAPSHOT ITS BACKING STORE. +// +// `provider.duplicate(&self.data)` is a true snapshot for YamlProvider only. +// DuckDbProvider Arc-clones one shared `Mutex` over one file, and +// QdrantProvider addresses the same remote collection. So for those drivers the +// clone and the original are two views of ONE store. +// +// INVARIANT: after calling `rebuild_indexes` on a cloned `Rag`, the pre-clone +// instance MUST be discarded immediately and MUST NOT serve further queries. +// Cloning to READ is always fine; cloning to REBUILD makes the original a +// half-truth (pre-rebuild `data`, post-rebuild store). Note that reassigning +// `RequestContext.rag` drops only one holder of the old `Arc` — forked +// request contexts, agents, captured inputs and the RAG cache keep theirs. impl Clone for Rag { fn clone(&self) -> Self { Self { @@ -81,8 +101,8 @@ impl Clone for Rag { name: self.name.clone(), path: self.path.clone(), embedding_model: self.embedding_model.clone(), - hnsw: self.data.build_hnsw(), bm25: self.data.build_bm25(), + provider: self.provider.duplicate(&self.data), node_to_docs: self.data.knowledge_graph.build_node_to_docs(), data: self.data.clone(), last_sources: RwLock::new(None), @@ -101,6 +121,8 @@ pub struct RagInitConfig { pub extractor_model: Option, pub extractor_prompt: Option, pub graph_hops: Option, + /// `None` -> "yaml" + pub driver: Option, } #[derive(Debug, Clone, Default)] @@ -127,12 +149,13 @@ impl Rag { bail!("Cannot build RAG knowledge base '{name}' with no documents"); } println!("⚙ Initializing RAG..."); - let data = Self::resolve_init_data(app, config)?; + let mut data = Self::resolve_init_data(app, config)?; + data.driver = config.driver.clone().unwrap_or_else(|| "yaml".to_string()); let mut rag = Self::create(app, name, save_path, data)?; let loaders = app.document_loaders.clone(); let (spinner, spinner_rx) = Spinner::create(""); abortable_run_with_spinner_rx( - rag.sync_documents(doc_paths, true, loaders, Some(spinner)), + rag.sync_documents(doc_paths, true, false, loaders, Some(spinner)), spinner_rx, abort_signal, ) @@ -237,12 +260,18 @@ impl Rag { save_path: &Path, doc_paths: &[String], abort_signal: AbortSignal, + prompt_for_driver: bool, ) -> Result { if !*IS_STDOUT_TERMINAL { bail!("Failed to init rag in non-interactive mode"); } println!("⚙ Initializing RAG..."); let (embedding_model, chunk_size, chunk_overlap) = Self::create_config(app)?; + let driver = if prompt_for_driver { + select_rag_driver()? + } else { + "yaml".to_string() + }; let reranker_model = app.rag_reranker_model.clone(); let top_k = app.rag_top_k; let extractor_model = match app.rag_extractor_model.clone() { @@ -255,7 +284,7 @@ impl Rag { app.rag_graph_hops }; let extractor_prompt = app.rag_extractor_prompt.clone(); - let data = RagData::new( + let mut data = RagData::new( embedding_model.id(), chunk_size, chunk_overlap, @@ -268,6 +297,7 @@ impl Rag { graph_hops: Some(graph_hops), }, ); + data.driver = driver; let mut rag = Self::create(app, name, save_path, data)?; let mut paths = doc_paths.to_vec(); if paths.is_empty() { @@ -276,7 +306,7 @@ impl Rag { let loaders = app.document_loaders.clone(); let (spinner, spinner_rx) = Spinner::create(""); abortable_run_with_spinner_rx( - rag.sync_documents(&paths, true, loaders, Some(spinner)), + rag.sync_documents(&paths, true, false, loaders, Some(spinner)), spinner_rx, abort_signal, ) @@ -291,12 +321,285 @@ impl Rag { let err = || format!("Failed to load rag '{name}' at '{}'", path.display()); let content = fs::read_to_string(path).with_context(err)?; let data: RagData = serde_yaml::from_str(&content).with_context(err)?; + data.validate().with_context(err)?; Self::create(app, name, path, data) } - pub fn create(app: &AppConfig, name: &str, path: &Path, data: RagData) -> Result { - let hnsw = data.build_hnsw(); - let bm25 = data.build_bm25(); + /// Loads a RAG from a YAML file. External drivers need an async constructor + /// because building their provider performs a network preflight. + pub async fn load_async( + app: &AppConfig, + vault: &Vault, + name: &str, + path: &Path, + ) -> Result { + let err = || format!("Failed to load rag '{name}' at '{}'", path.display()); + let raw_content = fs::read_to_string(path).with_context(err)?; + + // Parsed WITHOUT secret interpolation, so `driver_config` keeps its + // `{{...}}` placeholders in `self.data`. Resolution happens below, into a + // function-local copy only — see `resolve_driver_config` for why the + // resolved values must never travel back into `data`. + let data: RagData = serde_yaml::from_str(&raw_content).with_context(err)?; + + data.validate().with_context(err)?; + + match data.driver.as_str() { + "qdrant" => { + let driver_config = resolve_driver_config(&data.driver_config, vault, name)?; + let host = driver_config + .get("host") + .context("qdrant driver requires 'host' in driver_config")? + .clone(); + let collection = driver_config + .get("collection") + .context("qdrant driver requires 'collection' in driver_config")? + .clone(); + let api_key = driver_config.get("api_key").map(String::as_str); + + let provider = QdrantProvider::new(&host, &collection, api_key).await?; + let embedding_model = + Model::retrieve_model(app, &data.embedding_model, ModelType::Embedding)?; + Ok(Rag { + app_config: Arc::new(app.clone()), + name: name.to_string(), + path: path.display().to_string(), + embedding_model, + bm25: data.build_bm25(), + provider: Box::new(provider), + node_to_docs: data.knowledge_graph.build_node_to_docs(), + data, + last_sources: RwLock::new(None), + }) + } + _ => Self::load(app, name, path), + } + } + + pub async fn attach( + app: &AppConfig, + vault: &Vault, + name: &str, + save_path: &Path, + ) -> Result { + if !*IS_STDOUT_TERMINAL { + bail!("Cannot run attach wizard in non-interactive mode"); + } + println!("⚙ Attaching to external RAG..."); + + let driver = Select::new("Select driver:", vec!["qdrant"]).prompt()?; + + let host = Text::new("Host (e.g. qdrant.company.com:6333):") + .with_validator(required!("This field is required")) + .with_validator(|input: &str| { + Ok(if input.contains('[') || input.contains(']') { + Validation::Invalid( + "Bracketed IPv6 literals are not supported; use a hostname.".into(), + ) + } else { + Validation::Valid + }) + }) + .prompt()?; + + let api_key_entry: Option<(String, String)> = { + let needs_key = Confirm::new("Does this instance require an API key?") + .with_default(true) + .prompt()?; + if needs_key { + let secret_name = Text::new("Vault secret name for API key:") + .with_default("QDRANT_API_KEY") + .with_validator(required!("This field is required")) + .prompt()?; + let resolved = resolve_or_create_api_key_secret(vault, &secret_name)?; + Some((secret_name, resolved)) + } else { + None + } + }; + + println!("⚙ Connecting to {host}..."); + let api_key = api_key_entry.as_ref().map(|(_, v)| v.as_str()); + let collections = QdrantProvider::list_collections(&host, api_key) + .await + .with_context(|| format!("Failed to connect to {host}. Check host and API key."))?; + + if collections.is_empty() { + bail!("No collections found in this Qdrant instance"); + } + println!( + "✓ Connected. {} collection(s) available.", + collections.len() + ); + + let collection = Select::new("Select collection:", collections).prompt()?; + + let sample_id = QdrantProvider::sample_point_id(&host, &collection, api_key).await?; + + // `None` means the scroll came back with no points at all: the collection + // is empty. Attaching is not necessarily wrong — another tool may be about + // to fill it — but accepting it silently yields a RAG that answers every + // query with nothing and never explains why, and none of the checks below + // can tell that apart from a misconfiguration. Ask, defaulting to no, so it + // cannot happen by accident. (`attach` already refuses to run + // non-interactively, so there is no unattended path through this prompt.) + if sample_id.is_none() { + println!( + "⚠️ Collection '{collection}' contains no points. Queries will return \ + nothing until something writes to it." + ); + let attach_anyway = Confirm::new("Attach to this empty collection anyway?") + .with_default(false) + .prompt()?; + if !attach_anyway { + bail!("Collection '{collection}' is empty; nothing to attach to."); + } + } + + println!("ℹ This collection must store document text in a 'page_content' payload field."); + + let dim = QdrantProvider::get_vector_dimension(&host, &collection, api_key) + .await + .unwrap_or(0); + // Queries send a single unnamed vector, which a named/multi-vector + // collection rejects with HTTP 400 every time. Checked separately from + // `dim` because `dim == 0` also means "the request failed". + if QdrantProvider::is_multi_vector(&host, &collection, api_key).await? { + bail!( + "Collection '{collection}' uses named (multi-vector) configuration. \ + Coyote queries with a single unnamed vector and would fail with HTTP 400 \ + on every request. Attach a single-vector collection instead." + ); + } + if dim > 0 { + println!("Collection uses {dim}-dim vectors."); + } + println!( + "⚠️ If the embedding model doesn't match what built this collection, \ + queries will return bad results." + ); + let models = list_models(app, ModelType::Embedding); + if models.is_empty() { + bail!("No available embedding model"); + } + let embedding_model_id = select_embedding_model(&models)?; + + let mut driver_config = IndexMap::new(); + driver_config.insert("host".to_string(), host.clone()); + driver_config.insert("collection".to_string(), collection.clone()); + if let Some((secret_name, _)) = &api_key_entry { + driver_config.insert("api_key".to_string(), format!("{{{{{secret_name}}}}}")); + } + + let data = RagData { + driver: driver.to_string(), + attached: true, + driver_config, + embedding_model: embedding_model_id, + chunk_size: app.rag_chunk_size.unwrap_or(1024), + chunk_overlap: app.rag_chunk_overlap.unwrap_or(50), + // A top_k of 0 makes every query return nothing. + top_k: app.rag_top_k.max(1), + ..RagData::default() + }; + data.validate()?; + + let embedding_model = + Model::retrieve_model(app, &data.embedding_model, ModelType::Embedding)?; + let provider = QdrantProvider::new(&host, &collection, api_key).await?; + let rag = Rag { + app_config: Arc::new(app.clone()), + name: name.to_string(), + path: save_path.display().to_string(), + embedding_model, + // Both empty: an attached RAG holds no local text and no local graph. + bm25: data.build_bm25(), + node_to_docs: IndexMap::new(), + provider: Box::new(provider), + data, + last_sources: RwLock::new(None), + }; + + rag.save()?; + println!("✓ Attached '{name}' → collection '{collection}' on {host}."); + + let env_var = api_key_entry.as_ref().map(|_| rag_env_var_name(name)); + let (header_name, value_format) = driver_auth_header(driver); + generate_rag_sbx_mixin( + save_path, + &host, + name, + env_var.as_deref(), + header_name, + value_format, + )?; + + Ok(rag) + } + + pub fn create(app: &AppConfig, name: &str, path: &Path, mut data: RagData) -> Result { + // Deliberately does NOT call rebuild_indexes: both callers construct the Rag + // before any documents are added, so rebuilding empty data would be a no-op. + // Actual population happens later via sync_documents. + let (provider, bm25): (Box, _) = match data.driver.as_str() { + "duckdb" => { + let db_path = providers::duckdb_path_from_yaml(path); + let dim = embedding_dim_for_model(&data.embedding_model); + let duck = DuckDbProvider::open(&db_path, dim)?; + // HYDRATE — mandatory, not an optimization. The YAML file for a duckdb + // RAG deliberately omits `vectors`, so `data.vectors` arrives empty from + // disk. Refilling it from the sidecar is what makes the NEXT incremental + // sync non-destructive: rebuild_indexes does CREATE OR REPLACE TABLE and + // writes exactly what data.vectors holds. Skip this and the first + // `.edit rag-docs` after a restart wipes every previously indexed vector. + // + // Guarded on is_empty() so a caller that already has vectors in memory + // is never overwritten by an empty table. + // + // WARNING: `?`, NOT `unwrap_or_default()`. A hydration failure must + // propagate. Degrading to an empty map here loads a RAG that looks healthy, + // answers every query with nothing, and then loses the store permanently + // on the first `.edit rag-docs`. The legitimate "nothing indexed yet" case + // is already Ok(empty) (open() runs CREATE TABLE IF NOT EXISTS) so `?` + // costs a new RAG nothing. + if data.vectors.is_empty() { + data.vectors = duck.read_all_vectors()?; + } + if data.vectors.is_empty() && !data.files.is_empty() { + println!( + "{} RAG '{name}' lists {} indexed file(s), but its vector store \ + '{}' holds no vectors, so every search will return nothing. A \ + duckdb RAG is two files: bring the .duckdb sidecar along with \ + the .yaml, or re-embed with `.rebuild rag`.", + warning_text("WARNING:"), + data.files.len(), + db_path.display() + ); + } + // data.files is always populated for duckdb, so build_bm25() is the only + // path; there is no from-DuckDB fallback. + let bm25 = data.build_bm25(); + (Box::new(duck), bm25) + } + "qdrant" => bail!( + "RAG '{name}' uses driver 'qdrant' without `attached: true`. \ + Coyote can currently only READ a pre-existing Qdrant \ + collection — attach one with `.rag attach`. Writing to a \ + Coyote-owned Qdrant collection is not supported yet." + ), + "yaml" => { + let bm25 = data.build_bm25(); + (Box::new(YamlProvider::from_data(&data)), bm25) + } + // Explicitly NOT a catch-all falling through to yaml. A typo'd driver + // used to build a yaml store, pay to embed the whole corpus, persist + // the bad driver, and only fail on the NEXT run, leaving the RAG + // unusable without hand-editing the YAML. + other => bail!( + "Unknown RAG driver '{other}' for RAG '{name}'. \ + Valid drivers: yaml, duckdb, qdrant." + ), + }; let node_to_docs = data.knowledge_graph.build_node_to_docs(); let embedding_model = Model::retrieve_model(app, &data.embedding_model, ModelType::Embedding)?; @@ -306,8 +609,8 @@ impl Rag { path: path.display().to_string(), data, embedding_model, - hnsw, bm25, + provider, node_to_docs, last_sources: RwLock::new(None), }; @@ -322,13 +625,20 @@ impl Rag { &mut self, document_paths: &[String], refresh: bool, + force_reingest: bool, app: &AppConfig, abort_signal: AbortSignal, ) -> Result<()> { let loaders = app.document_loaders.clone(); let (spinner, spinner_rx) = Spinner::create(""); abortable_run_with_spinner_rx( - self.sync_documents(document_paths, refresh, loaders, Some(spinner)), + self.sync_documents( + document_paths, + refresh, + force_reingest, + loaders, + Some(spinner), + ), spinner_rx, abort_signal, ) @@ -389,6 +699,13 @@ impl Rag { } pub fn set_last_sources(&self, ids: &[DocumentId]) { + if self.data.attached { + // `data.files` is empty for an attached RAG; the local index is not the + // source of truth. A static label is honest, an empty list is not. + *self.last_sources.write() = + Some("[Using attached RAG. Source list unavailable]".to_string()); + return; + } let mut sources: IndexMap> = IndexMap::new(); for id in ids { let (file_index, _) = id.split(); @@ -432,8 +749,18 @@ impl Rag { let path = Path::new(&self.path); ensure_parent_exists(path)?; - let content = serde_yaml::to_string(&self.data) - .with_context(|| format!("Failed to serde rag '{}'", self.name))?; + let content = if self.data.driver == "duckdb" { + // Embeddings live in the .duckdb sidecar; keep them out of the YAML file. + // Clone-and-empty rather than mutating self.data — the live map must stay + // complete for the next incremental sync, and save() takes &self, so any + // clear-then-restore would leave the object corrupted on an early return. + let mut on_disk = self.data.clone(); + on_disk.vectors.clear(); + serde_yaml::to_string(&on_disk) + } else { + serde_yaml::to_string(&self.data) + } + .with_context(|| format!("Failed to serde rag '{}'", self.name))?; fs::write(path, content).with_context(|| { format!("Failed to save rag '{}' to '{}'", self.name, path.display()) })?; @@ -455,6 +782,8 @@ impl Rag { .collect(); let data = json!({ "path": self.path, + "driver": self.driver(), + "attached": self.is_attached(), "embedding_model": self.embedding_model.id(), "chunk_size": self.data.chunk_size, "chunk_overlap": self.data.chunk_overlap, @@ -476,6 +805,18 @@ impl Rag { &self.name } + pub fn is_attached(&self) -> bool { + self.data.attached + } + + pub fn driver(&self) -> &str { + &self.data.driver + } + + pub fn file_count(&self) -> usize { + self.data.files.len() + } + pub fn is_temp(&self) -> bool { self.name == TEMP_RAG_NAME } @@ -539,6 +880,9 @@ impl Rag { } fn resolve_source(&self, id: &DocumentId) -> String { + if self.data.attached { + return self.data.attached_source_label(); + } let (file_index, _) = id.split(); self.data .files @@ -548,6 +892,10 @@ impl Rag { } fn format_sources(&self, ids: &[DocumentId]) -> String { + if self.data.attached { + return format!("- {}", self.data.attached_source_label()); + } + let mut seen = IndexSet::new(); for id in ids { let (file_index, _) = id.split(); @@ -565,9 +913,15 @@ impl Rag { &mut self, paths: &[String], refresh: bool, + force_reingest: bool, loaders: HashMap, spinner: Option, ) -> Result<()> { + debug_assert!( + !force_reingest || refresh, + "force_reingest requires refresh" + ); + let refresh = refresh || force_reingest; if let Some(spinner) = &spinner { let _ = spinner.set_message(String::new()); } @@ -685,11 +1039,9 @@ impl Rag { } in loaded_documents { let hash = sha256(&contents); - if let Some(file_ids) = to_deleted.get_mut(&hash) - && let Some((i, _)) = file_ids - .iter() - .enumerate() - .find(|(_, v)| self.data.files[*v].path == path) + if let Some((i, _)) = + find_hash_skip(force_reingest, &to_deleted, &self.data.files, &hash, &path) + && let Some(file_ids) = to_deleted.get_mut(&hash) { if file_ids.len() == 1 { to_deleted.swap_remove(&hash); @@ -795,9 +1147,18 @@ impl Rag { } progress(&spinner, "Building store".into()); - self.hnsw = self.data.build_hnsw(); + // Derived in-memory state is refreshed BEFORE the fallible provider rebuild. + // `self.data` has already been mutated at this point, so returning early on a + // provider error while `bm25`/`node_to_docs` still describe the previous corpus + // would leave this Rag internally inconsistent. Both are pure functions of + // `self.data` and cannot fail, so doing them first is always safe. self.bm25 = self.data.build_bm25(); self.node_to_docs = self.data.knowledge_graph.build_node_to_docs(); + // `refresh` is true for a full re-index (.rebuild rag / --rebuild-rag / + // initial build) and false for an incremental .edit rag-docs change. + // Passing it through is what stops a remote provider from wiping its + // collection on a one-file add. + self.provider.rebuild_indexes(&self.data, refresh).await?; Ok(()) } @@ -808,17 +1169,27 @@ impl Rag { top_k: usize, rerank_model: Option<&str>, ) -> Result> { - let (vector_search_results, keyword_search_results) = tokio::join!( - self.vector_search(query, top_k, 0.0), - self.keyword_search(query, top_k, 0.0), - ); + let keyword_leg = async { + if self.provider.has_native_keyword_search() { + self.provider + .keyword_search(query, top_k) + .await + .unwrap_or_else(|e| { + warn!("native keyword search failed, dropping the keyword ranker: {e}"); + Vec::new() + }) + } else { + self.keyword_search(query, top_k, 0.0) + } + }; + let (vector_search_results, keyword_search_results) = + tokio::join!(self.vector_search(query, top_k, 0.0), keyword_leg); let vector_search_results = vector_search_results?; debug!("vector_search_results: {vector_search_results:?}",); let vector_search_ids: Vec = vector_search_results.into_iter().map(|(v, _)| v).collect(); - let keyword_search_results = keyword_search_results?; debug!("keyword_search_results: {keyword_search_results:?}",); let keyword_search_ids: Vec = keyword_search_results.into_iter().map(|(v, _)| v).collect(); @@ -831,13 +1202,18 @@ impl Rag { .concat() .into_iter() .collect(); - let mut documents = vec![]; - let mut documents_ids = vec![]; - for id in ids { - if let Some(document) = self.data.get(id) { - documents_ids.push(id); - documents.push(document.page_content.to_string()); - } + let ids: Vec = ids.into_iter().collect(); + let fetched = self.provider.fetch_content(&ids).await?; + // Build both vectors from the SAME source in the SAME iteration — + // never zip two independently-built lists. The reranker returns + // positional indices into `documents`, so any drift between the two + // resolves reranked hits to the wrong document's text. A partial + // fetch simply yields a shorter pair, and both shrink together. + let mut documents_ids = Vec::with_capacity(fetched.len()); + let mut documents = Vec::with_capacity(fetched.len()); + for (id, text) in fetched { + documents_ids.push(id); + documents.push(text); } let data = RerankData::new(query.to_string(), documents, top_k); let list = client.rerank(&data).await.context("Failed to rerank")?; @@ -869,13 +1245,7 @@ impl Rag { ids } }; - let output = ids - .into_iter() - .filter_map(|id| { - let document = self.data.get(id)?; - Some((id, document.page_content.clone())) - }) - .collect(); + let output = self.provider.fetch_content(&ids).await?; Ok(output) } @@ -892,35 +1262,24 @@ impl Rag { ); let texts = splitter.split_text(query); let embeddings_data = EmbeddingsData::new(texts, true); - let embeddings = self.create_embeddings(embeddings_data, None).await?; - let output = self - .hnsw - .parallel_search(&embeddings, top_k, 30) - .into_iter() - .flat_map(|list| { - list.into_iter() - .filter_map(|v| { - let score = 1.0 - v.distance; - if score > min_score { - Some((DocumentId(v.d_id), score)) - } else { - None - } - }) - .collect::>() - }) - .collect(); - Ok(output) + let query_embeddings = self.create_embeddings(embeddings_data, None).await?; + + let mut results: Vec<(DocumentId, f32)> = vec![]; + for embedding in &query_embeddings { + let batch = self + .provider + .vector_search(embedding, top_k, min_score) + .await?; + results.extend(batch); + } + Ok(merge_vector_results(results)) } - async fn keyword_search( - &self, - query: &str, - top_k: usize, - min_score: f32, - ) -> Result> { + /// Local in-memory BM25 over `data.files`. This is empty for attached RAGs, which is + /// correct: they have no local text. + fn keyword_search(&self, query: &str, top_k: usize, min_score: f32) -> Vec<(DocumentId, f32)> { let results = self.bm25.search(query, top_k); - let output: Vec<(DocumentId, f32)> = results + results .into_iter() .filter_map(|v| { let score = v.score; @@ -930,8 +1289,7 @@ impl Rag { None } }) - .collect(); - Ok(output) + .collect() } fn graph_search(&self, query: &str, top_k: usize) -> Vec { @@ -1084,16 +1442,33 @@ impl Rag { #[derive(Clone, Serialize, Deserialize)] pub struct RagData { + #[serde(default = "RagData::default_driver")] + pub driver: String, + #[serde(default)] + pub attached: bool, + #[serde(default, skip_serializing_if = "IndexMap::is_empty")] + pub driver_config: IndexMap, + pub embedding_model: String, + #[serde(default)] pub chunk_size: usize, + #[serde(default)] pub chunk_overlap: usize, pub reranker_model: Option, + #[serde(default)] pub top_k: usize, pub batch_size: Option, + #[serde(default)] pub next_file_id: FileId, + #[serde(default)] pub document_paths: Vec, + #[serde(default)] pub files: IndexMap, - #[serde(with = "serde_vectors")] + #[serde( + default, + with = "serde_vectors", + skip_serializing_if = "IndexMap::is_empty" + )] pub vectors: IndexMap>, #[serde(default)] pub extractor_model: Option, @@ -1108,6 +1483,9 @@ pub struct RagData { impl Debug for RagData { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("RagData") + .field("driver", &self.driver) + .field("attached", &self.attached) + .field("driver_config", &self.driver_config) .field("embedding_model", &self.embedding_model) .field("chunk_size", &self.chunk_size) .field("chunk_overlap", &self.chunk_overlap) @@ -1135,6 +1513,9 @@ impl RagData { graph: GraphRagConfig, ) -> Self { Self { + driver: "yaml".to_string(), + attached: false, + driver_config: Default::default(), embedding_model, chunk_size, chunk_overlap, @@ -1152,11 +1533,79 @@ impl RagData { } } - pub fn get(&self, id: DocumentId) -> Option<&RagDocument> { - let (file_index, document_index) = id.split(); - let file = self.files.get(&file_index)?; - let document = file.documents.get(document_index)?; - Some(document) + fn default_driver() -> String { + "yaml".to_string() + } + + /// Citation label for an attached RAG. Its documents live in a remote + /// collection, so there is no local file path to cite. + fn attached_source_label(&self) -> String { + match self.driver_config.get("collection") { + Some(collection) => format!("[external collection: {collection}]"), + None => "[external collection]".to_string(), + } + } + + pub fn validate(&self) -> Result<()> { + if self.top_k == 0 { + bail!( + "top_k must be >= 1 (got 0). A top_k of 0 makes every query return \ + no results with no error. Set `top_k:` in the RAG YAML." + ); + } + + if !self.attached { + if self.chunk_size == 0 { + bail!( + "chunk_size must be >= 1 (got 0) for a non-attached RAG. A \ + chunk_size of 0 panics with a divide-by-zero while sizing \ + embedding batches. Set `chunk_size:` in the RAG YAML." + ); + } + + if self.chunk_overlap >= self.chunk_size { + bail!( + "chunk_overlap ({}) must be strictly less than chunk_size ({}).", + self.chunk_overlap, + self.chunk_size + ); + } + } + + match (self.driver.as_str(), self.attached) { + ("yaml", false) => Ok(()), + ("duckdb", false) => Ok(()), + ("qdrant", true) => Ok(()), + ("qdrant", false) => Ok(()), + ("yaml", true) => bail!( + "driver 'yaml' cannot be attached (attached: true). \ + Attached RAGs require an external driver (qdrant)." + ), + ("duckdb", true) => bail!( + "driver 'duckdb' cannot be attached (attached: true). \ + DuckDB is a local-only driver; use 'qdrant' for external collections." + ), + (other, _) => { + bail!("Unknown RAG driver '{other}'. Valid drivers: yaml, duckdb, qdrant.") + } + } + } + + /// Every (DocumentId, &RagDocument) in the corpus, in `files` order. + /// + /// This, NOT `vectors`, is the authoritative document id space. BM25, the + /// knowledge graph and content lookup all key off it; `vectors` is a subset, + /// since `add`'s zip truncates whenever fewer embeddings come back than + /// document ids were sent. + pub fn iter_documents(&self) -> impl Iterator { + self.files.iter().flat_map(|(file_index, file)| { + file.documents + .iter() + .enumerate() + .map(move |(document_index, document)| { + (DocumentId::new(*file_index, document_index), document) + }) + }) } pub fn del(&mut self, file_ids: Vec) { @@ -1194,13 +1643,12 @@ impl RagData { } pub fn build_bm25(&self) -> SearchEngine { - let mut documents = vec![]; - for (file_index, file) in self.files.iter() { - for (document_index, document) in file.documents.iter().enumerate() { - let id = DocumentId::new(*file_index, document_index); - documents.push(bm25::Document::new(id, &document.page_content)) - } - } + // Shares `iter_documents` with the providers' content maps so the BM25 key + // space and the content key space are identical by construction. + let documents: Vec<_> = self + .iter_documents() + .map(|(id, doc)| bm25::Document::new(id, &doc.page_content)) + .collect(); SearchEngineBuilder::::with_documents(Language::English, documents) .k1(1.5) .b(0.75) @@ -1208,6 +1656,20 @@ impl RagData { } } +impl Default for RagData { + fn default() -> Self { + RagData::new( + String::new(), + 0, + 0, + None, + 5, + None, + GraphRagConfig::default(), + ) + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RagFile { hash: String, @@ -1266,6 +1728,112 @@ impl DocumentId { } } +/// Writes the per-RAG sandbox sidecar that whitelists the external host and tells +/// the sbx proxy which header to rewrite with the stored credential. +/// +/// Two details are load-bearing and fail silently if guessed: +/// 1. The schema envelope is mandatory. `wrap_mixin_as_kit` copies this file +/// byte-for-byte to `spec.yaml` inside a kit dir handed to `sbx create --kit`, +/// with no `kind` rewrite — so an envelope-less file can break the launch +/// itself, not merely this RAG's traffic. +/// 2. The `service` declared here must be the id the host binds the value +/// under with `sbx secret set`; both derive from the RAG name through +/// `secret_service_id`, so they cannot spell it differently. +/// +/// `api_key_env` is `None` for a store that needs no credential: the host is +/// still allowed, but no binding is declared, because nothing binds a value. +fn generate_rag_sbx_mixin( + rag_yaml_path: &Path, + host: &str, + service_name: &str, + api_key_env: Option<&str>, + header_name: &str, + value_format: &str, +) -> Result<()> { + let base_url = QdrantProvider::normalize_base_url(host); + let Some(allow_entry) = mcp_credentials::allow_entry_for_url(&base_url) else { + eprintln!( + "Warning: host '{host}' has no representation in the sbx network allow \ + grammar, so no sandbox mixin was written for RAG '{service_name}'. \ + Queries to this RAG will be blocked inside the sandbox." + ); + + return Ok(()); + }; + + let credentials = api_key_env + .map(|env_var| mcp_credentials::CredentialEntry { + service: mcp_credentials::secret_service_id(service_name), + description: format!("API key for the attached RAG '{service_name}'"), + api_key: mcp_credentials::ApiKey { + name: env_var.to_string(), + proxy_managed: true, + inject: vec![rag_inject_rule(&allow_entry, header_name, value_format)], + }, + }) + .into_iter() + .collect(); + + let mixin_path = rag_yaml_path.with_extension("sbx-mixin.yaml"); + let content = mcp_credentials::render_mixin_document( + &format!("rag-{service_name}"), + &format!( + "Auto-generated by the Coyote attach wizard for RAG '{service_name}'. Allows \ + outbound traffic to its external vector store and declares the credential the \ + sbx proxy injects into each request. Do not edit manually." + ), + credentials, + &[allow_entry], + )?; + fs::write(&mixin_path, &content).with_context(|| { + format!( + "Failed to write sandbox mixin to '{}'", + mixin_path.display() + ) + })?; + + println!("✓ Sandbox mixin: '{}'.", mixin_path.display()); + Ok(()) +} + +fn rag_inject_rule( + domain: &str, + header_name: &str, + value_format: &str, +) -> mcp_credentials::InjectRule { + if header_name.eq_ignore_ascii_case("authorization") + && value_format.eq_ignore_ascii_case("Bearer %s") + { + mcp_credentials::InjectRule { + domain: domain.to_string(), + header: None, + format: None, + scheme: Some("bearer".to_string()), + } + } else { + mcp_credentials::InjectRule { + domain: domain.to_string(), + header: Some(header_name.to_string()), + format: Some(value_format.to_string()), + scheme: None, + } + } +} + +fn rag_env_var_name(rag_name: &str) -> String { + format!( + "{}_API_KEY", + rag_name.to_uppercase().replace(['-', ' '], "_") + ) +} + +fn driver_auth_header(driver: &str) -> (&'static str, &'static str) { + match driver { + "qdrant" => ("api-key", "%s"), + _ => ("Authorization", "Bearer %s"), + } +} + fn select_embedding_model(models: &[&Model]) -> Result { let max_width = models.iter().map(|v| v.id().len()).max().unwrap_or(0); let models: Vec<_> = models @@ -1278,6 +1846,27 @@ fn select_embedding_model(models: &[&Model]) -> Result { Ok(result.value) } +pub(crate) fn select_rag_driver() -> Result { + let options = vec![ + "yaml — portable, in-memory HNSW; usable from several Coyote processes at once (default)", + "duckdb — persistent on-disk store; vectors and content survive restarts; HNSW approximate search.", + ]; + let sel = Select::new("RAG storage driver:", options) + .with_starting_cursor(0) + .prompt()?; + if sel.starts_with("duckdb") { + println!( + "Note: several Coyote processes can query a duckdb RAG at the same time, \ + but while one process is ingesting or rebuilding it the others cannot \ + read it until that finishes. Changing its driver later means deleting \ + and recreating the RAG." + ); + Ok("duckdb".to_string()) + } else { + Ok("yaml".to_string()) + } +} + const EXTRACTOR_SKIP: &str = "Skip"; fn select_extractor_model(app: &AppConfig) -> Result> { @@ -1451,6 +2040,49 @@ fn progress(spinner: &Option, message: String) { } } +/// Decide whether a just-loaded document may skip re-chunking and re-embedding. +/// +/// Returns the position of the matching `FileId` within `to_deleted[hash]`, together with +/// that `FileId`. The caller needs the position to un-mark the file for deletion. `None` +/// means "ingest this document": either a full re-ingest was requested, or no +/// already-indexed file has both this content hash and this path. +fn find_hash_skip( + force_reingest: bool, + to_deleted: &IndexMap>, + files: &IndexMap, + hash: &str, + path: &str, +) -> Option<(usize, FileId)> { + if force_reingest { + return None; + } + let file_ids = to_deleted.get(hash)?; + file_ids + .iter() + .enumerate() + .find(|(_, v)| files[*v].path == path) + .map(|(i, v)| (i, *v)) +} + +/// Global score sort + dedup keeping the best score per document. +/// +/// NO overall cap: each `provider.vector_search` call already returns <= top_k, +/// so the pool is bounded by `top_k * query_chunks`, and `reciprocal_rank_fusion` +/// truncates to `top_k` itself. Capping here would let whichever query chunk has +/// the strongest absolute scores crowd out every other chunk's hits. +fn merge_vector_results(mut results: Vec<(DocumentId, f32)>) -> Vec<(DocumentId, f32)> { + debug_assert!( + results.iter().all(|(_, score)| score.is_finite()), + "provider returned a non-finite score; NaN silently degrades sort order" + ); + results.sort_by(|a, b| b.1.total_cmp(&a.1)); + let mut seen = IndexSet::new(); + results + .into_iter() + .filter(|(id, _)| seen.insert(*id)) + .collect() +} + fn reciprocal_rank_fusion( list_of_document_ids: Vec>, list_of_weights: Vec, @@ -1473,9 +2105,571 @@ fn reciprocal_rank_fusion( .collect() } +/// Map an embedding model id to its vector dimension. +/// +/// The DuckDB `FLOAT[N]` column type and its HNSW index are fixed at schema-creation +/// time, so this value must be decided before the first insert. An unrecognized model +/// falls back to 1536; if that is wrong, DuckDB raises a dimension-mismatch error on +/// the first insert rather than silently corrupting the schema, and the recovery is to +/// delete the sidecar and re-ingest from source. +fn embedding_dim_for_model(model_id: &str) -> usize { + match model_id { + m if m.contains("3-large") => 3072, + m if m.contains("3-small") || m.contains("ada-002") => 1536, + m if m.contains("nomic-embed-text") || m.contains("all-minilm") => 768, + m if m.contains("jina-embeddings-v2") => 1024, + _ => 1536, + } +} + +/// True only for "the vault does not hold this key". +/// +/// Everything else — an auth failure, a provider outage, or the vault being +/// disabled because Coyote is running inside a sandbox — must NOT be treated as +/// a missing secret. Offering to create one in those cases would prompt for a +/// value that cannot be stored and bury the real reason. +fn is_missing_secret(err: &anyhow::Error) -> bool { + matches!( + err.downcast_ref::(), + Some(SecretError::NotFound { .. }) + ) +} + +/// Reads `secret_name` out of the vault, offering to create it in place when the +/// vault simply does not hold it yet. +/// +/// Sending the user off to run `coyote --add-secret` mid-wizard discarded every +/// answer they had already given. `Vault::add_secret` does the masked prompt, +/// the provider write and the confirmation line, so this defers to it rather +/// than collecting or storing the value itself. +fn resolve_or_create_api_key_secret(vault: &Vault, secret_name: &str) -> Result { + let read_err = match vault.get_secret(secret_name, false) { + Ok(secret) => return Ok(secret), + Err(err) => err, + }; + if !is_missing_secret(&read_err) { + return Err(read_err) + .with_context(|| format!("Cannot read secret '{secret_name}' from the vault")); + } + + let create = Confirm::new(&format!( + "Secret '{secret_name}' is not in the vault. Create it now?" + )) + .with_default(true) + .prompt()?; + if !create { + bail!( + "This instance needs an API key, so '{secret_name}' has to exist before \ + attaching. Add it with `coyote --add-secret {secret_name}` and re-run, or \ + re-run and answer 'no' when asked whether the instance requires an API key." + ); + } + + vault + .add_secret(secret_name) + .with_context(|| format!("Failed to store secret '{secret_name}' in the vault"))?; + vault + .get_secret(secret_name, false) + .with_context(|| format!("Secret '{secret_name}' is unreadable after being stored")) +} + +/// Resolves `{{SECRET}}` placeholders in every `driver_config` value against the +/// vault, returning a DETACHED copy. +/// +/// Three properties this must preserve, each of which has already bitten: +/// +/// 1. The resolved values never go back into `RagData`. `Rag::save()` +/// serializes `self.data`, and `.set rag_top_k`, `.set rag_reranker_model` +/// and every post-sync save call it — so a resolved credential parked in +/// `data.driver_config` gets written to the RAG's YAML file in plaintext the +/// next time the user changes any setting. +/// 2. The literal `{{NAME}}` text survives in `data` and on disk. Sandbox +/// credential provisioning parses that placeholder back out of the file to +/// learn which vault secret to bind into the sandbox; resolve it away and +/// provisioning silently finds nothing to register. +/// 3. Only `driver_config` is interpolated, never the whole file. The rest of a +/// RAG file is ingested document text and vectors — where `{{...}}` is +/// ordinary content (Jinja, Mustache, Vue, Go templates) that would be read +/// as a secret reference, blanked to `""`, and persisted on the next save. +/// `driver_config` is small and is the only place credentials live. +fn resolve_driver_config( + driver_config: &IndexMap, + vault: &Vault, + rag_name: &str, +) -> Result> { + resolve_driver_config_with(driver_config, rag_name, |value| { + interpolate_secrets(value, vault) + }) +} + +/// Interpolation core, taking the resolver as an argument so it can be exercised +/// without a vault. Mirrors `interpolate_secrets` / `interpolate_secrets_with`. +fn resolve_driver_config_with( + driver_config: &IndexMap, + rag_name: &str, + mut interpolate: F, +) -> Result> +where + F: FnMut(&str) -> Result<(String, Vec)>, +{ + let mut resolved = IndexMap::with_capacity(driver_config.len()); + let mut missing: Vec = Vec::new(); + for (key, value) in driver_config { + let (value, value_missing) = interpolate(value).with_context(|| { + format!("Failed to resolve '{key}' in driver_config for RAG '{rag_name}'") + })?; + missing.extend(value_missing); + resolved.insert(key.clone(), value); + } + + // A secret the vault does not hold is NOT an error inside + // `interpolate_secrets`: it substitutes the empty string and only reports the + // name. Accepting that ships an empty credential, and the user sees an + // unexplained 401 from the server instead of the typo they made. + if !missing.is_empty() { + missing.sort(); + missing.dedup(); + bail!( + "RAG '{rag_name}' references secrets that are missing from the vault: {}. \ + Add them with `coyote --add-secret `, then try again.", + missing.join(", ") + ); + } + + Ok(resolved) +} + #[cfg(test)] mod tests { use super::*; + use std::path::PathBuf; + use std::time::{SystemTime, UNIX_EPOCH}; + + struct TempDir { + path: PathBuf, + } + + impl TempDir { + fn new(tag: &str) -> Self { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = env::temp_dir().join(format!("coyote-rag-{tag}-{unique}")); + fs::create_dir_all(&path).unwrap(); + Self { path } + } + } + + impl Drop for TempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.path); + } + } + + #[test] + fn attached_ragdata_serializes_driver_and_attached() { + let mut data = RagData { + driver: "qdrant".to_string(), + attached: true, + embedding_model: "text-embedding-3-small".to_string(), + top_k: 5, + ..Default::default() + }; + data.driver_config + .insert("host".into(), "localhost:6333".into()); + data.driver_config.insert("collection".into(), "c".into()); + data.driver_config + .insert("api_key".into(), "{{QDRANT_API_KEY}}".into()); + data.validate().unwrap(); + let yaml = serde_yaml::to_string(&data).unwrap(); + assert!(yaml.contains("attached: true")); + assert!(yaml.contains("driver: qdrant")); + // The placeholder is what reaches disk — never a resolved secret. + assert!(yaml.contains("{{QDRANT_API_KEY}}")); + } + + const FAKE_SECRET: &str = "sk-live-fake-value-for-tests"; + + fn attached_qdrant_data() -> RagData { + let mut data = RagData { + driver: "qdrant".to_string(), + attached: true, + embedding_model: "text-embedding-3-small".to_string(), + top_k: 5, + ..Default::default() + }; + data.driver_config + .insert("host".into(), "localhost:6333".into()); + data.driver_config.insert("collection".into(), "c".into()); + data.driver_config + .insert("api_key".into(), "{{QDRANT_API_KEY}}".into()); + data + } + + /// THE invariant behind `resolve_driver_config` returning a detached copy. + /// + /// `save()` serializes `self.data`, and `.set rag_top_k`, `.set + /// rag_reranker_model` and every post-sync save call it. If load ever bakes + /// the resolved credential into `data.driver_config`, the next trivial + /// setting change writes the user's plaintext API key into the RAG's YAML + /// file. The literal placeholder must also survive, because sandbox + /// credential provisioning parses it back off disk. + #[test] + fn a_save_after_load_writes_the_placeholder_not_the_resolved_secret() { + let dir = TempDir::new("driver-config-secret"); + let path = dir.path.join("kb.yaml"); + let data = attached_qdrant_data(); + + // Exactly what `load_async` does with the parsed data. + let resolved = resolve_driver_config_with(&data.driver_config, "kb", |value| { + Ok((value.replace("{{QDRANT_API_KEY}}", FAKE_SECRET), vec![])) + }) + .unwrap(); + assert_eq!( + resolved["api_key"], FAKE_SECRET, + "the live client still has to receive the real key" + ); + assert_eq!( + data.driver_config["api_key"], "{{QDRANT_API_KEY}}", + "resolution must not mutate the RagData that save() serializes" + ); + + let rag = Rag { + app_config: Arc::new(AppConfig::default()), + name: "kb".to_string(), + path: path.display().to_string(), + embedding_model: Model::new("openai", "text-embedding-3-small"), + bm25: data.build_bm25(), + provider: Box::new(YamlProvider::from_data(&data)), + node_to_docs: IndexMap::new(), + data, + last_sources: RwLock::new(None), + }; + assert!(rag.save().unwrap()); + + let on_disk = fs::read_to_string(&path).unwrap(); + assert!( + on_disk.contains("{{QDRANT_API_KEY}}"), + "sandbox provisioning parses this placeholder back off disk: {on_disk}" + ); + assert!( + !on_disk.contains(FAKE_SECRET), + "a save after load leaked the plaintext secret to {}", + path.display() + ); + } + + /// Every value is interpolated, not just `api_key` — a credential-bearing + /// field added later must not ship its raw placeholder to the server. + #[test] + fn resolution_covers_every_driver_config_value() { + let mut driver_config = IndexMap::new(); + driver_config.insert("host".to_string(), "{{QDRANT_HOST}}".to_string()); + driver_config.insert("collection".to_string(), "c".to_string()); + driver_config.insert("api_key".to_string(), "{{QDRANT_API_KEY}}".to_string()); + + let resolved = resolve_driver_config_with(&driver_config, "kb", |value| { + let out = value + .replace("{{QDRANT_HOST}}", "qdrant.internal:6333") + .replace("{{QDRANT_API_KEY}}", FAKE_SECRET); + Ok((out, vec![])) + }) + .unwrap(); + + assert_eq!(resolved["host"], "qdrant.internal:6333"); + assert_eq!(resolved["collection"], "c"); + assert_eq!(resolved["api_key"], FAKE_SECRET); + } + + /// Missing secrets are reported together, deduplicated, and name the RAG. + #[test] + fn missing_secrets_fail_the_load_instead_of_resolving_to_empty() { + let mut driver_config = IndexMap::new(); + driver_config.insert("host".to_string(), "{{QDRANT_HOST}}".to_string()); + driver_config.insert("api_key".to_string(), "{{QDRANT_API_KEY}}".to_string()); + + let err = resolve_driver_config_with(&driver_config, "kb", |value| { + // What `interpolate_secrets` really does for an absent secret: blank it + // out and report the name rather than returning Err. + Ok(( + String::new(), + vec![value.trim_matches(['{', '}']).to_string()], + )) + }) + .expect_err("an empty API key must not be accepted as a successful load"); + + let msg = err.to_string(); + assert!(msg.contains("kb"), "the RAG must be named: {msg}"); + assert!(msg.contains("QDRANT_HOST"), "got: {msg}"); + assert!(msg.contains("QDRANT_API_KEY"), "got: {msg}"); + } + + /// Only a genuine NotFound may trigger the attach wizard's "create it now?" + /// offer. The vault is disabled wholesale inside a sandbox, where creating a + /// secret is impossible — misreading that as "missing" would prompt for a + /// value that cannot be stored and hide why. + #[test] + fn only_a_not_found_error_counts_as_a_missing_secret() { + let not_found = anyhow::Error::new(SecretError::NotFound { + key: "QDRANT_API_KEY".to_string(), + provider: "local", + }); + assert!(is_missing_secret(¬_found)); + + let auth_failed = anyhow::Error::new(SecretError::AuthFailed { + provider: "local", + source: anyhow!("bad vault password"), + }); + assert!(!is_missing_secret(&auth_failed)); + + // What `Vault::get_secret` returns in sandbox mode: a plain anyhow error. + let sandboxed = anyhow!("Vault management is disabled in sandbox mode."); + assert!(!is_missing_secret(&sandboxed)); + } + + /// A qdrant RAG's vectors MUST survive serialization. + /// + /// `save()` omits vectors only for `driver == "duckdb"`. Qdrant must not join + /// that guard: Cosine collections L2-normalize on write, so the YAML copy is + /// the only place the unnormalized originals survive. This fails loudly the + /// day someone "tidies" the guard into `matches!(driver, "duckdb" | "qdrant")`. + #[test] + fn save_round_trips_qdrant_vectors_intact() { + let mut data = RagData { + driver: "qdrant".to_string(), + attached: true, + embedding_model: "text-embedding-3-small".to_string(), + top_k: 5, + ..Default::default() + }; + // Deliberately NOT unit-length: magnitude 5, so any normalization is visible. + data.vectors.insert(DocumentId(0), vec![3.0, 0.0, 0.0, 4.0]); + let yaml = serde_yaml::to_string(&data).unwrap(); + assert!( + yaml.contains("vectors:"), + "qdrant vectors must be serialized, not omitted" + ); + let back: RagData = serde_yaml::from_str(&yaml).unwrap(); + assert_eq!( + back.vectors.get(&DocumentId(0)), + Some(&vec![3.0, 0.0, 0.0, 4.0]), + "magnitudes must survive — Qdrant normalizes, the YAML copy must not" + ); + } + + /// Renders a sidecar into a scratch dir and returns its text plus the + /// parsed document. Parsing is not optional: a malformed document is + /// otherwise copied verbatim into `spec.yaml` and only rejected by sbx. + fn render_rag_mixin( + host: &str, + name: &str, + api_key_env: Option<&str>, + header_name: &str, + value_format: &str, + ) -> (String, serde_yaml::Value) { + let dir = TempDir::new("mixin"); + let yaml_path = dir.path.join(format!("{name}.yaml")); + generate_rag_sbx_mixin( + &yaml_path, + host, + name, + api_key_env, + header_name, + value_format, + ) + .unwrap(); + let text = fs::read_to_string(dir.path.join(format!("{name}.sbx-mixin.yaml"))).unwrap(); + let parsed = serde_yaml::from_str(&text).unwrap(); + (text, parsed) + } + + fn allow_list(parsed: &serde_yaml::Value) -> Vec { + parsed["permissions"]["network"]["allow"] + .as_sequence() + .expect("a mixin without an allow list whitelists nothing") + .iter() + .map(|v| v.as_str().unwrap().to_string()) + .collect() + } + + #[test] + fn generated_sbx_mixin_carries_the_schema_envelope() { + let (text, parsed) = render_rag_mixin( + "rag.example.com:6333", + "company-docs", + Some("COMPANY_DOCS_API_KEY"), + "api-key", + "%s", + ); + + assert!( + text.starts_with("schemaVersion:"), + "envelope must come first:\n{text}" + ); + assert_eq!(parsed["schemaVersion"].as_str(), Some("2")); + assert_eq!(parsed["kind"].as_str(), Some("mixin")); + assert_eq!(parsed["name"].as_str(), Some("rag-company-docs")); + assert!(parsed["description"].as_str().is_some()); + + assert_eq!(allow_list(&parsed), vec!["rag.example.com:6333"]); + + let credential = &parsed["credentials"][0]; + assert_eq!(credential["service"].as_str(), Some("company-docs")); + assert_eq!( + credential["apiKey"]["name"].as_str(), + Some("COMPANY_DOCS_API_KEY") + ); + assert_eq!(credential["apiKey"]["proxyManaged"].as_bool(), Some(true)); + + let inject = &credential["apiKey"]["inject"][0]; + assert_eq!(inject["domain"].as_str(), Some("rag.example.com:6333")); + assert_eq!(inject["header"].as_str(), Some("api-key")); + assert_eq!(inject["format"].as_str(), Some("%s")); + + assert!( + allow_list(&parsed).contains(&inject["domain"].as_str().unwrap().to_string()), + "every inject domain must also appear in allow:\n{text}" + ); + + for dead in ["allowedDomains", "serviceDomains", "serviceAuth"] { + assert!(!text.contains(dead), "v1 key '{dead}' survived:\n{text}"); + } + assert!( + parsed["network"].is_null(), + "v1 top-level `network` survived:\n{text}" + ); + assert!( + parsed["environment"].is_null(), + "v1 `environment.proxyManaged` survived:\n{text}" + ); + } + + #[test] + fn generated_sbx_mixin_allows_the_port_the_client_dials() { + let cases = [ + // No scheme means http, and http means port 80 — normalize_base_url + // does not silently append Qdrant's 6333. + ("rag.example.com", "rag.example.com:80"), + ("rag.example.com:7777", "rag.example.com:7777"), + ("http://localhost:6333", "localhost:6333"), + // https on the default port is the one bare-host case. + ("https://xyz.cloud.qdrant.io", "xyz.cloud.qdrant.io"), + ( + "https://xyz.cloud.qdrant.io:6333", + "xyz.cloud.qdrant.io:6333", + ), + ]; + for (host, expected) in cases { + let (_, parsed) = render_rag_mixin(host, "docs", Some("DOCS_API_KEY"), "api-key", "%s"); + assert_eq!( + allow_list(&parsed), + vec![expected.to_string()], + "host {host}" + ); + } + } + + #[test] + fn generated_sbx_mixin_spells_bearer_as_a_scheme() { + let (_, parsed) = render_rag_mixin( + "https://store.example.com", + "docs", + Some("DOCS_API_KEY"), + "Authorization", + "Bearer %s", + ); + + let inject = &parsed["credentials"][0]["apiKey"]["inject"][0]; + + assert_eq!(inject["scheme"].as_str(), Some("bearer")); + assert!(inject["header"].is_null()); + assert!(inject["format"].is_null()); + } + + #[test] + fn generated_sbx_mixin_service_id_matches_the_host_side_bind() { + let (_, parsed) = render_rag_mixin( + "https://store.example.com", + "My_Docs", + Some("MY_DOCS_API_KEY"), + "api-key", + "%s", + ); + + assert_eq!( + parsed["credentials"][0]["service"].as_str(), + Some(mcp_credentials::secret_service_id("My_Docs").as_str()) + ); + assert_eq!( + parsed["credentials"][0]["service"].as_str(), + Some("my-docs") + ); + } + + #[test] + fn generated_sbx_mixin_omits_credentials_when_there_is_no_api_key() { + let (text, parsed) = + render_rag_mixin("https://store.example.com", "docs", None, "api-key", "%s"); + + assert_eq!(allow_list(&parsed), vec!["store.example.com"]); + assert!( + parsed["credentials"].is_null(), + "no key means no credential declaration:\n{text}" + ); + } + + #[test] + fn rag_env_var_name_uppercases_and_underscores() { + assert_eq!(rag_env_var_name("company-docs"), "COMPANY_DOCS_API_KEY"); + assert_eq!(rag_env_var_name("my rag"), "MY_RAG_API_KEY"); + assert_eq!(rag_env_var_name("docs"), "DOCS_API_KEY"); + } + + #[test] + fn driver_auth_header_uses_a_bare_api_key_for_qdrant() { + assert_eq!(driver_auth_header("qdrant"), ("api-key", "%s")); + assert_eq!( + driver_auth_header("something-else"), + ("Authorization", "Bearer %s") + ); + } + + #[test] + fn attached_rag_citation_helpers_do_not_fall_back_to_the_empty_file_index() { + let mut data = RagData { + driver: "qdrant".to_string(), + attached: true, + embedding_model: "text-embedding-3-small".to_string(), + top_k: 5, + ..Default::default() + }; + data.driver_config + .insert("collection".into(), "company-kb".into()); + assert!(data.files.is_empty()); + + assert_eq!( + data.attached_source_label(), + "[external collection: company-kb]" + ); + + // Degrades to a generic label rather than "unknown" when the collection + // name is absent. + data.driver_config.shift_remove("collection"); + assert_eq!(data.attached_source_label(), "[external collection]"); + } + + #[test] + fn embedding_dim_for_model_maps_known_models() { + assert_eq!(embedding_dim_for_model("text-embedding-3-large"), 3072); + assert_eq!(embedding_dim_for_model("text-embedding-3-small"), 1536); + assert_eq!(embedding_dim_for_model("text-embedding-ada-002"), 1536); + assert_eq!(embedding_dim_for_model("nomic-embed-text"), 768); + assert_eq!(embedding_dim_for_model("all-minilm"), 768); + assert_eq!(embedding_dim_for_model("jina-embeddings-v2-base-en"), 1024); + // Unknown models fall back to the OpenAI-compatible default. + assert_eq!(embedding_dim_for_model("some-unknown-model"), 1536); + } #[test] fn document_id_round_trip() { @@ -1566,7 +2760,7 @@ mod tests { } #[test] - fn rag_data_get_returns_document() { + fn rag_data_iter_documents_yields_all_documents_in_file_order() { let mut data = RagData::new( "m".into(), 100, @@ -1583,15 +2777,22 @@ mod tests { }; data.files.insert(0, file); - let doc = data.get(DocumentId::new(0, 0)).unwrap(); - assert_eq!(doc.page_content, "first"); + let documents: Vec<_> = data + .iter_documents() + .map(|(id, doc)| (id, doc.page_content.as_str())) + .collect(); - let doc = data.get(DocumentId::new(0, 1)).unwrap(); - assert_eq!(doc.page_content, "second"); + assert_eq!( + documents, + vec![ + (DocumentId::new(0, 0), "first"), + (DocumentId::new(0, 1), "second"), + ] + ); } #[test] - fn rag_data_get_returns_none_for_missing_file() { + fn rag_data_iter_documents_is_empty_without_files() { let data = RagData::new( "m".into(), 100, @@ -1601,11 +2802,12 @@ mod tests { None, GraphRagConfig::default(), ); - assert!(data.get(DocumentId::new(99, 0)).is_none()); + + assert_eq!(data.iter_documents().count(), 0); } #[test] - fn rag_data_get_returns_none_for_missing_document() { + fn rag_data_iter_documents_ignores_vector_only_ids() { let mut data = RagData::new( "m".into(), 100, @@ -1621,7 +2823,10 @@ mod tests { documents: vec![RagDocument::new("only one")], }; data.files.insert(0, file); - assert!(data.get(DocumentId::new(0, 5)).is_none()); + data.vectors.insert(DocumentId::new(0, 5), vec![1.0]); + + let ids: Vec<_> = data.iter_documents().map(|(id, _)| id).collect(); + assert_eq!(ids, vec![DocumentId::new(0, 0)]); } #[test] @@ -1868,4 +3073,247 @@ mod tests { "higher-weight signal's top doc should rank first" ); } + + #[test] + fn merge_vector_results_empty_input() { + let result = merge_vector_results(vec![]); + + assert!(result.is_empty(), "empty input should produce empty output"); + } + + #[test] + fn merge_vector_results_keeps_best_score_per_document() { + let doc = DocumentId::new(0, 0); + + let result = merge_vector_results(vec![(doc, 0.2), (doc, 0.9)]); + + assert_eq!(result.len(), 1, "a document must not be double-counted"); + assert_eq!(result[0].0, doc); + assert_eq!( + result[0].1, 0.9, + "dedup must keep the highest score, not the first seen" + ); + } + + #[test] + fn merge_vector_results_sorts_globally_by_descending_score() { + let doc_a = DocumentId::new(0, 0); + let doc_b = DocumentId::new(1, 0); + let doc_c = DocumentId::new(2, 0); + + let result = merge_vector_results(vec![(doc_a, 0.5), (doc_c, 0.9), (doc_b, 0.1)]); + let ids: Vec = result.iter().map(|(id, _)| *id).collect(); + + assert_eq!(ids, vec![doc_c, doc_a, doc_b]); + } + + #[test] + fn merge_vector_results_does_not_truncate() { + let input: Vec<(DocumentId, f32)> = (0..10) + .map(|i| (DocumentId::new(i, 0), i as f32 / 10.0)) + .collect(); + + let result = merge_vector_results(input); + + assert_eq!( + result.len(), + 10, + "merging must not cap the pool; truncation belongs to reciprocal_rank_fusion" + ); + } + + fn hash_skip_fixture() -> (IndexMap, IndexMap>) { + let mut files: IndexMap = Default::default(); + files.insert( + 7, + RagFile { + hash: "abc".into(), + path: "test.txt".into(), + documents: vec![RagDocument::new("unchanged")], + }, + ); + let mut to_deleted: IndexMap> = Default::default(); + to_deleted.insert("abc".into(), vec![7]); + (files, to_deleted) + } + + #[test] + fn force_reingest_re_embeds_hash_identical_files() { + let (files, to_deleted) = hash_skip_fixture(); + + assert_eq!( + find_hash_skip(true, &to_deleted, &files, "abc", "test.txt"), + None, + "a forced re-ingest must not skip an unchanged file" + ); + } + + #[test] + fn refresh_without_force_still_hash_skips() { + let (files, to_deleted) = hash_skip_fixture(); + + assert_eq!( + find_hash_skip(false, &to_deleted, &files, "abc", "test.txt"), + Some((0, 7)), + "an unchanged file should be skipped and un-marked for deletion" + ); + } + + #[test] + fn find_hash_skip_returns_none_on_path_change() { + let (files, to_deleted) = hash_skip_fixture(); + + assert_eq!( + find_hash_skip(false, &to_deleted, &files, "abc", "moved.txt"), + None + ); + assert_eq!( + find_hash_skip(true, &to_deleted, &files, "abc", "moved.txt"), + None + ); + } + + #[test] + fn ragdata_new_has_yaml_driver_and_not_attached() { + let data = RagData::new( + "text-embedding-3-small".to_string(), + 1024, + 50, + None, + 5, + None, + GraphRagConfig::default(), + ); + + assert_eq!(data.driver, "yaml"); + assert!(!data.attached); + } + + #[test] + fn ragdata_deserializes_without_driver_field() { + let yaml = " +embedding_model: text-embedding-3-small +chunk_size: 1024 +chunk_overlap: 50 +top_k: 5 +next_file_id: 0 +document_paths: [] +files: {} +vectors: {} +"; + + let data: RagData = serde_yaml::from_str(yaml).unwrap(); + + assert_eq!(data.driver, "yaml"); + assert!(!data.attached); + } + + #[test] + fn ragdata_round_trips_driver_and_attached() { + let mut data = RagData::new( + "text-embedding-3-small".to_string(), + 1024, + 50, + None, + 5, + None, + GraphRagConfig::default(), + ); + data.driver = "qdrant".to_string(); + data.attached = true; + + let yaml = serde_yaml::to_string(&data).unwrap(); + let restored: RagData = serde_yaml::from_str(&yaml).unwrap(); + + assert_eq!(restored.driver, "qdrant"); + assert!(restored.attached); + } + + #[test] + fn ragdata_validate_rejects_yaml_attached() { + let mut data = RagData::new( + "m".into(), + 1024, + 50, + None, + 5, + None, + GraphRagConfig::default(), + ); + data.attached = true; + + let err = data.validate().unwrap_err().to_string(); + + assert!(err.contains("cannot be attached"), "got: {err}"); + } + + #[test] + fn ragdata_validate_accepts_qdrant_attached() { + let mut data = RagData::new( + "m".into(), + 1024, + 50, + None, + 5, + None, + GraphRagConfig::default(), + ); + data.driver = "qdrant".to_string(); + data.attached = true; + + assert!(data.validate().is_ok()); + } + + #[test] + fn ragdata_validate_rejects_zero_top_k_from_a_truncated_yaml() { + let yaml = " +embedding_model: text-embedding-3-small +chunk_size: 1024 +chunk_overlap: 50 +"; + let data: RagData = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(data.top_k, 0, "a missing top_k must default to 0"); + let err = data.validate().unwrap_err().to_string(); + assert!(err.contains("top_k must be >= 1"), "got: {err}"); + } + + #[test] + fn ragdata_validate_rejects_zero_chunk_size_when_not_attached() { + let yaml = " +embedding_model: text-embedding-3-small +top_k: 5 +"; + let data: RagData = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(data.chunk_size, 0); + let err = data.validate().unwrap_err().to_string(); + assert!(err.contains("chunk_size must be >= 1"), "got: {err}"); + } + + #[test] + fn ragdata_validate_allows_zero_chunk_size_when_attached() { + let yaml = " +driver: qdrant +attached: true +embedding_model: text-embedding-3-small +top_k: 5 +"; + let data: RagData = serde_yaml::from_str(yaml).unwrap(); + assert_eq!(data.chunk_size, 0); + assert!(data.validate().is_ok()); + } + + #[test] + fn ragdata_validate_rejects_overlap_not_less_than_chunk_size() { + let data = RagData::new( + "m".into(), + 100, + 100, + None, + 5, + None, + GraphRagConfig::default(), + ); + let err = data.validate().unwrap_err().to_string(); + assert!(err.contains("chunk_overlap"), "got: {err}"); + } } diff --git a/src/rag/provider.rs b/src/rag/provider.rs new file mode 100644 index 0000000..3c75fd2 --- /dev/null +++ b/src/rag/provider.rs @@ -0,0 +1,76 @@ +use super::{DocumentId, RagData}; +use anyhow::Result; +use async_trait::async_trait; + +/// Abstracts where RAG vector data is stored and queried. +/// +/// The Rag orchestrator owns: embeddings, chunking, BM25 keyword search, graph RAG, +/// entity extraction, RRF merging. Providers own: vector storage and content retrieval. +#[async_trait] +pub trait RagProvider: Send + Sync { + /// Vector similarity search. Returns (DocumentId, score) sorted by score desc. + /// `embedding` is a single query vector from Coyote's embedding model. + async fn vector_search( + &self, + embedding: &[f32], + top_k: usize, + min_score: f32, + ) -> Result>; + + /// Resolve document IDs to their page content. + /// + /// **Ordering contract:** implementations MUST return results in the same + /// relative order as the input `ids` slice. `hybrid_search` passes an + /// RRF-ranked list and feeds the result straight to the LLM. A provider + /// that returns rows in storage order (e.g. Qdrant `get_points`, DuckDB + /// `WHERE id IN (...)`) would silently discard the ranking. Implementations + /// that query an unordered backend must re-sort by input position before + /// returning. + /// + /// Returns only IDs that were found; callers must handle partial returns + /// (a missing ID is skipped, not an error). + /// YamlProvider: reads from an in-memory content map built from data.files. + /// DuckDbProvider: queries the documents table by id. + /// QdrantProvider: fetches payload from the remote collection. + async fn fetch_content(&self, ids: &[DocumentId]) -> Result>; + + /// Rebuild internal indexes from freshly updated RagData. + /// Called once at the end of every sync_documents pass. + /// + /// `full_rebuild` mirrors `sync_documents`' `refresh` parameter: + /// - `true`: a full re-index (`.rebuild rag`, `--rebuild-rag`, initial build). + /// Destructive strategies (wipe-then-reindex) are permitted. + /// - `false`: an incremental change (`.edit rag-docs` adding/removing a file). + /// Implementations MUST NOT wipe existing state; upsert only. + /// + /// The parameter is part of the signature from the outset so it is fixed + /// while there is exactly one implementor. Yaml/DuckDb ignore it, + /// rebuilding their local state wholesale is fast and always correct. + /// Only a remote provider is destructive enough to care. + async fn rebuild_indexes(&mut self, data: &RagData, full_rebuild: bool) -> Result<()>; + + /// Keyword / full-text search. Returns (DocumentId, BM25-style score) sorted desc. + /// + /// Default impl returns `Ok(vec![])`. Callers fall back to `Rag.bm25` (local in-memory + /// BM25 built from `data.files`). + /// + /// Callers check `has_native_keyword_search()` before deciding which path to take: + /// - true → call this method; skip `Rag.bm25` + /// - false → call `Rag.keyword_search()` which uses `Rag.bm25` (sync, infallible) + async fn keyword_search(&self, query: &str, top_k: usize) -> Result> { + let _ = (query, top_k); + + Ok(vec![]) + } + + /// Returns true if this provider implements a native keyword-search index. + /// When false, `Rag.hybrid_search` uses the local `Rag.bm25` field instead. + fn has_native_keyword_search(&self) -> bool { + false + } + + /// Deep-clone the provider with fresh indexes derived from `data`. + /// Required because Box is not Clone. + /// Called by Rag's Clone impl (which clones before mutating in rebuild_rag/edit_rag_docs). + fn duplicate(&self, data: &RagData) -> Box; +} diff --git a/src/rag/providers/duckdb.rs b/src/rag/providers/duckdb.rs new file mode 100644 index 0000000..aa9bfff --- /dev/null +++ b/src/rag/providers/duckdb.rs @@ -0,0 +1,1442 @@ +use crate::rag::provider::RagProvider; +use crate::rag::{DocumentId, RagData}; +use std::collections::HashMap; + +use anyhow::{Context, Result, anyhow, bail}; +use async_trait::async_trait; +use duckdb::types::Value; +use duckdb::{AccessMode, Config, Connection}; +use indexmap::IndexMap; +use log::warn; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, MutexGuard}; + +/// Serializes `INSTALL` across every thread in this process. DuckDB installs an +/// extension by downloading it to a temp file and then MOVING that file into +/// `~/.duckdb/extensions/...`. Two threads installing the same extension at once +/// both perform that move; on Windows the loser's move targets a file the winner +/// already holds open and fails with "Access is denied", where POSIX would let the +/// replacement through. Guards nothing but the install step. +/// +/// Lock order is `DuckDbProvider::conn` -> INSTALL_LOCK, never the reverse: +/// `ensure_writable` reopens the connection, and so may install, while holding the +/// `ConnHandle` guard, whereas nothing ever acquires a `ConnHandle` guard while +/// holding this lock. The cycle that would deadlock cannot form. +static INSTALL_LOCK: Mutex<()> = Mutex::new(()); + +/// Derive the DuckDB sidecar path from a RAG's YAML path: `docs.yaml` -> `docs.duckdb`. +pub(crate) fn duckdb_path_from_yaml(yaml_path: &Path) -> PathBuf { + yaml_path.with_extension("duckdb") +} + +/// The shared connection together with the access mode it was opened with. +/// +/// `conn` is an `Option` only so that an upgrade can DROP the read-only connection +/// before asking DuckDB for a read-write one. It is `Some` at every point an outside +/// caller can observe, and is never left `None` on a path that returns `Ok`. +struct ConnHandle { + conn: Option, + /// True when `conn` was opened READ_WRITE. This lives behind the same mutex as the + /// connection itself rather than next to it in `DuckDbProvider`, so that a + /// `duplicate()` clone sharing the `Arc` observes an upgrade performed through any + /// other handle instead of keeping its own stale copy of the mode. + writable: bool, +} + +impl ConnHandle { + fn conn(&self) -> Result<&Connection> { + self.conn.as_ref().ok_or_else(Self::lost) + } + + fn conn_mut(&mut self) -> Result<&mut Connection> { + self.conn.as_mut().ok_or_else(Self::lost) + } + + /// Only reachable when a read-write upgrade failed AND reopening read-only failed + /// too. Returning an error beats panicking inside a locked scope, which would + /// poison the mutex for the remaining life of the process. + fn lost() -> anyhow::Error { + anyhow!( + "The DuckDB connection was lost: upgrading it to read-write failed and the \ + store could not be reopened read-only afterwards. Another process is \ + holding the file; retry once it has released it." + ) + } +} + +pub struct DuckDbProvider { + path: PathBuf, + conn: Arc>, + /// Embedding dimension; fixed at open time because the `FLOAT[N]` column depends on it. + dim: usize, + /// True once an FTS index has been built on `documents`. Until then + /// `fts_main_documents.match_bm25` does not exist and any keyword query would + /// fail with a DuckDB catalog error. Backs `has_native_keyword_search`. + fts_ready: AtomicBool, +} + +impl DuckDbProvider { + /// Open (or create) the DuckDB file. `dim` is the embedding vector dimension, + /// supplied by the caller who knows the model. + /// + /// Opens READ-ONLY whenever the file already carries a complete schema, so that any + /// number of Coyote processes can query the same RAG at the same time. DuckDB allows + /// many concurrent readers XOR exactly one writer, so the exclusive read-write handle + /// is taken only when there is actually something to write: when the store has to be + /// created or initialized here, or lazily through `ensure_writable` on the rebuild + /// path. + pub fn open(db_path: &Path, dim: usize) -> Result { + let (conn, writable) = Self::open_for_workload(db_path, dim)?; + // A reopened file may already carry a live FTS index from a previous session, + // in which case keyword search works immediately. + let fts_exists = Self::probe_fts_index(&conn); + Ok(Self { + path: db_path.to_path_buf(), + conn: Arc::new(Mutex::new(ConnHandle { + conn: Some(conn), + writable, + })), + dim, + fts_ready: AtomicBool::new(fts_exists), + }) + } + + /// Pick the weakest access mode that can serve this store, returning the connection + /// and whether it came back writable. + fn open_for_workload(db_path: &Path, dim: usize) -> Result<(Connection, bool)> { + if db_path.exists() + && let Ok(conn) = Self::open_read_only(db_path) + && Self::store_is_initialized(&conn) + { + return Ok((conn, false)); + } + // Three cases land here: the file does not exist yet, it could not be opened + // read-only (another process holds it read-write), or it carries no usable + // schema. All of them need a read-write handle, and the read-write attempt is + // also what produces the actionable lock error for the middle case. + let conn = Self::open_read_write(db_path, dim)?; + Ok((conn, true)) + } + + /// Is this file already a fully initialized Coyote store? + /// + /// This gate decides whether a read-only open is viable, so it must be exact: every + /// statement in `init_schema` is rejected outright on a read-only handle, INCLUDING + /// `CREATE TABLE IF NOT EXISTS` against a table that already exists, which DuckDB + /// refuses rather than treating as a no-op. Anything missing therefore forces a + /// read-write open. The HNSW index is part of the check because a store whose tables + /// survived but whose index did not would otherwise be opened read-only and silently + /// serve every `vector_search` from a full scan. + fn store_is_initialized(conn: &Connection) -> bool { + let tables: i64 = conn + .query_row( + "SELECT count(*) FROM duckdb_tables() \ + WHERE table_name IN ('vectors', 'documents')", + [], + |r| r.get(0), + ) + .unwrap_or(0); + if tables < 2 { + return false; + } + conn.query_row( + "SELECT count(*) FROM duckdb_indexes() WHERE index_name = 'hnsw_idx'", + [], + |r| r.get::<_, i64>(0), + ) + .map(|n| n > 0) + .unwrap_or(false) + } + + /// Open the store read-only. Many processes may hold such a handle at once. + fn open_read_only(db_path: &Path) -> Result { + let config = Config::default() + .access_mode(AccessMode::ReadOnly) + .context("Failed to build a read-only DuckDB configuration")?; + let conn = Connection::open_with_flags(db_path, config).with_context(|| { + format!( + "Failed to open the DuckDB store at '{}' read-only", + db_path.display() + ) + })?; + Self::establish_session(&conn)?; + Ok(conn) + } + + /// Open the store read-write and make sure its schema exists. Exactly one process + /// may hold such a handle, and no reader from another process may hold it meanwhile. + fn open_read_write(db_path: &Path, dim: usize) -> Result { + let config = Config::default() + .access_mode(AccessMode::ReadWrite) + .context("Failed to build a read-write DuckDB configuration")?; + let conn = Connection::open_with_flags(db_path, config).with_context(|| { + format!( + "Failed to open the DuckDB store at '{}' for writing. Another Coyote \ + process (or another window) has this RAG open: a duckdb RAG supports MANY \ + concurrent READERS, but only ONE writer at a time, and a writer excludes \ + readers in other processes. Close that process, or wait for its sync to \ + finish, and retry.", + db_path.display() + ) + })?; + Self::establish_session(&conn)?; + Self::init_schema(&conn, dim)?; + Ok(conn) + } + + /// Install the per-connection session state that every connection needs, whatever + /// its access mode. + /// + /// Extension `LOAD`s and `SET` are per-CONNECTION, not per-database: a connection + /// opened later — an upgrade, in particular — starts with none of this and must run + /// it again. None of these statements write to the database, so they all succeed on + /// a read-only handle. + /// + /// Statement order is load-bearing. `hnsw_enable_experimental_persistence` is + /// registered BY the vss extension, so setting it before `LOAD vss` fails with + /// "Setting with name ... is not in the catalog, but it exists in the vss + /// extension". ensure vss (installing it if missing) -> ensure fts -> SET. + fn establish_session(conn: &Connection) -> Result<()> { + Self::ensure_extension(conn, "vss")?; + Self::ensure_extension(conn, "fts")?; + conn.execute_batch("SET hnsw_enable_experimental_persistence = true;") + .context("Failed to enable DuckDB HNSW index persistence") + } + + /// Create the tables and the vector index. Every statement here WRITES, so this only + /// ever runs on a read-write connection. + /// + /// Must be preceded by `establish_session`: without the `SET` it performs, a + /// CREATE INDEX ... USING HNSW on a file-backed database fails with "HNSW index + /// persistence is not yet supported by default". + fn init_schema(conn: &Connection, dim: usize) -> Result<()> { + conn.execute_batch(&format!( + "CREATE TABLE IF NOT EXISTS vectors ( + doc_id UBIGINT PRIMARY KEY, + embedding FLOAT[{dim}] + ); + CREATE INDEX IF NOT EXISTS hnsw_idx + ON vectors USING HNSW (embedding) + WITH (metric = 'cosine'); + CREATE TABLE IF NOT EXISTS documents ( + doc_id UBIGINT PRIMARY KEY, + page_content TEXT NOT NULL + );" + )) + .context("Failed to initialize DuckDB schema") + } + + /// Guarantee the shared connection is read-write, upgrading it in place if it is not. + /// EVERY write path must call this before touching the store. + /// + /// The upgrade replaces the `Connection` INSIDE the shared `Arc>`, so + /// `duplicate()` clones, which share that `Arc`, see it too. The read-only connection + /// is dropped before the read-write open because DuckDB tracks the file lock per + /// database instance and the old handle still holds one. + /// + /// On failure the store is reopened read-only so that queries keep working, and the + /// error is propagated so the caller aborts instead of writing. A failed upgrade must + /// leave the provider degraded, never bricked, and never silently read-only-with-a- + /// caller-that-thinks-it-wrote. + fn ensure_writable(&self) -> Result<()> { + let mut handle = self.lock_conn()?; + if handle.writable { + return Ok(()); + } + drop(handle.conn.take()); + match Self::open_read_write(&self.path, self.dim) { + Ok(conn) => { + handle.conn = Some(conn); + handle.writable = true; + Ok(()) + } + Err(e) => { + handle.conn = Self::open_read_only(&self.path).ok(); + Err(e.context(format!( + "Cannot write to the DuckDB RAG at '{}': it is open read-only and could \ + not be upgraded to read-write, because another Coyote process has this \ + RAG open. NOTHING WAS WRITTEN. Close the other process, or wait for it \ + to finish, and retry.", + self.path.display() + ))) + } + } + } + + /// Make a DuckDB extension available on `conn`, installing it if this machine does + /// not have it yet. `LOAD` is attempted first so an extension that is already + /// installed costs nothing and never touches the network; `INSTALL` is only reached + /// once, on a machine seeing the extension for the first time. + /// + /// `LOAD` is per-connection and so runs on every connection; only `INSTALL` is + /// serialized, and the second `LOAD` under the lock is what keeps it to one + /// install. Without that re-check, every thread that queued behind the winner + /// would still run a redundant `INSTALL` and re-trigger the same file move. + fn ensure_extension(conn: &Connection, name: &str) -> Result<()> { + if conn.execute_batch(&format!("LOAD {name};")).is_ok() { + return Ok(()); + } + // A poisoned lock means some other thread panicked mid-install; the lock owns + // no state to corrupt, so recover rather than failing every later open. + let _install_guard = INSTALL_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + // Re-check now that we hold the lock: whoever held it before us may already + // have installed the extension, in which case this `LOAD` finds it on disk. + if conn.execute_batch(&format!("LOAD {name};")).is_ok() { + return Ok(()); + } + conn.execute_batch(&format!("INSTALL {name};")) + .with_context(|| { + format!( + "Failed to install the DuckDB `{name}` extension. The duckdb RAG driver \ + needs it, and downloading it needs network access the first time. If this \ + machine is offline, connect once and retry, or run `INSTALL {name};` \ + yourself from a DuckDB shell." + ) + })?; + conn.execute_batch(&format!("LOAD {name};")) + .with_context(|| { + format!("Failed to load the DuckDB `{name}` extension after installing it.") + }) + } + + /// Read all `(doc_id, embedding)` pairs so `create()` can hydrate `data.vectors` + /// from disk. This is what makes the next incremental sync non-destructive, and is + /// mandatory rather than an optimization. + /// + /// 🔴 THIS PATTERN IS DUCKDB-ONLY. NEVER write the qdrant equivalent. Qdrant Cosine + /// collections L2-normalize stored vectors on write, so hydrating `data.vectors` + /// from a Qdrant read fills it with NORMALIZED vectors and the next save() destroys + /// the originals permanently. The symmetry with this function is exactly why that + /// bug is easy to introduce. + /// + /// ALL-OR-NOTHING. An empty `vectors` table is `Ok(empty)`; ANY decode failure or + /// non-finite embedding is an `Err`. A partially-hydrated map is worse than no map: + /// `rebuild_indexes` does `CREATE OR REPLACE TABLE` and writes exactly what it is + /// given, so a thinned map is committed as the new truth on the next sync. + pub(crate) fn read_all_vectors(&self) -> Result>> { + let handle = self.lock_conn()?; + let mut stmt = handle + .conn()? + .prepare("SELECT doc_id, embedding FROM vectors")?; + let raw: Vec<(u64, Vec)> = stmt + .query_map([], |row| { + let id: u64 = row.get(0)?; + // The duckdb crate does not implement `FromSql` for `Vec`, so the + // column is read as a `Value` and destructured. A FLOAT[N] column yields + // `Value::Array`, a FLOAT[] column yields `Value::List`; match both so + // the reader survives a file written under either schema. + let embedding: Vec = match row.get::<_, Value>(1)? { + Value::Array(vals) | Value::List(vals) => vals + .into_iter() + .map(|v| match v { + Value::Float(f) => f, + Value::Double(d) => d as f32, + _ => f32::NAN, + }) + .collect(), + _ => Vec::new(), + }; + Ok((id, embedding)) + })? + .collect::>>() + .with_context(|| { + format!( + "Failed to decode the `vectors` table in '{}'. The DuckDB sidecar is \ + unreadable; delete it and run `.rebuild rag` to re-ingest from source.", + self.path.display() + ) + })?; + + // Validate OUTSIDE the closure so the error can name the offending row. A + // non-finite embedding is NOT droppable: dropping it thins the map, and the + // thinned map is what the next CREATE OR REPLACE commits. + let mut out = IndexMap::with_capacity(raw.len()); + for (id, embedding) in raw { + if embedding.is_empty() || embedding.iter().any(|f| !f.is_finite()) { + bail!( + "Vector for doc_id {id} in '{}' is empty or contains a non-finite \ + value. Refusing to hydrate a partial vector map — that would erase \ + the remaining vectors on the next sync. Delete the sidecar and run \ + `.rebuild rag` to re-ingest from source.", + self.path.display() + ); + } + out.insert(DocumentId(id as usize), embedding); + } + + Ok(out) + } + + /// Does a LIVE FTS index exist on `documents`? Probed at open time so a reopened + /// database reports native keyword search accurately. + /// + /// A schema-existence check alone is NOT sufficient. After + /// `CREATE OR REPLACE TABLE documents`, the `fts_main_documents` schema still + /// exists and `match_bm25` still SUCCEEDS, but returns zero rows for every term. + /// The index is silently dead. The probe therefore asserts that a KNOWN row comes + /// back rather than merely that the call did not error. + fn probe_fts_index(conn: &Connection) -> bool { + let schema_exists = conn + .query_row( + "SELECT COUNT(*) FROM duckdb_schemas() WHERE schema_name = 'fts_main_documents'", + [], + |row| row.get::<_, i64>(0), + ) + .map(|n| n > 0) + .unwrap_or(false); + if !schema_exists { + return false; + } + conn.query_row( + "SELECT COUNT(*) FROM documents d + WHERE fts_main_documents.match_bm25( + d.doc_id, + (SELECT string_split(trim(page_content), ' ')[1] + FROM documents WHERE length(trim(page_content)) > 0 LIMIT 1) + ) IS NOT NULL", + [], + |row| row.get::<_, i64>(0), + ) + .map(|n| n > 0) + .unwrap_or(false) + } + + /// Lock the shared connection, converting mutex poisoning into an `anyhow` error. + /// + /// Never `.lock().unwrap()` here: a panic anywhere inside a locked scope poisons the + /// mutex permanently, and an unwrap would then turn every subsequent RAG query into + /// a panic for the remaining life of the process. + fn lock_conn(&self) -> Result> { + self.conn + .lock() + .map_err(|e| anyhow!("DuckDB connection mutex was poisoned: {e}")) + } +} + +#[async_trait] +impl RagProvider for DuckDbProvider { + // NOTE: the MutexGuard is held across the body of these `async fn`s. That is sound + // ONLY because no `.await` appears inside a locked scope. Adding one would make the + // generated future non-`Send` and break the `RagProvider: Send + Sync` bound. + async fn vector_search( + &self, + embedding: &[f32], + top_k: usize, + min_score: f32, + ) -> Result> { + if embedding.iter().any(|f| !f.is_finite()) { + bail!("Query embedding contains a non-finite value (NaN or infinity)"); + } + let vals: String = embedding + .iter() + .map(|f| f.to_string()) + .collect::>() + .join(", "); + let dim = self.dim; + let handle = self.lock_conn()?; + // array_cosine_distance requires a FLOAT[N] ARRAY, not the LIST type FLOAT[]. + // ORDER BY distance ASC is required for the planner to use hnsw_idx; the + // similarity form (DESC) does NOT trigger the ANN index. Distance is converted + // back to a similarity score on return. + let sql = format!( + "SELECT doc_id, \ + array_cosine_distance(embedding, [{vals}]::FLOAT[{dim}]) AS distance \ + FROM vectors ORDER BY distance ASC LIMIT {top_k}" + ); + let mut stmt = handle.conn()?.prepare(&sql)?; + let results = stmt + .query_map([], |row| { + let id: u64 = row.get(0)?; + // `array_cosine_distance` on a FLOAT[N] column returns FLOAT (f32), NOT + // DOUBLE. Reading it as f64 raises InvalidColumnType INSIDE the closure, + // which a bare `.filter_map(|r| r.ok())` would silently discard, + // yielding ZERO results with no error and no log line. + let distance: f32 = match row.get::<_, Value>(1)? { + Value::Float(f) => f, + Value::Double(d) => d as f32, + other => { + warn!("unexpected distance type from DuckDB: {other:?}"); + return Err(duckdb::Error::InvalidQuery); + } + }; + Ok((DocumentId(id as usize), 1.0_f32 - distance)) + })? + // Log-and-drop rather than a bare `.ok()`: a systematic decode failure here + // is otherwise indistinguishable from "no matches". + .filter_map(|r| match r { + Ok(v) => Some(v), + Err(e) => { + warn!("vector_search row decode failed: {e}"); + None + } + }) + .filter(|(_, score)| *score > min_score) + .collect(); + + Ok(results) + } + + async fn fetch_content(&self, ids: &[DocumentId]) -> Result> { + if ids.is_empty() { + return Ok(vec![]); + } + let handle = self.lock_conn()?; + let placeholders = ids.iter().map(|_| "?").collect::>().join(", "); + let sql = + format!("SELECT doc_id, page_content FROM documents WHERE doc_id IN ({placeholders})"); + let params: Vec = ids.iter().map(|id| Value::UBigInt(id.0 as u64)).collect(); + let mut stmt = handle.conn()?.prepare(&sql)?; + let mut rows: Vec<(DocumentId, String)> = stmt + .query_map(duckdb::params_from_iter(params.iter()), |row| { + let id: u64 = row.get(0)?; + let text: String = row.get(1)?; + Ok((DocumentId(id as usize), text)) + })? + .filter_map(|r| match r { + Ok(v) => Some(v), + Err(e) => { + warn!("fetch_content row decode failed: {e}"); + None + } + }) + .collect(); + let position: HashMap = + ids.iter().enumerate().map(|(i, id)| (*id, i)).collect(); + + rows.sort_by_key(|(id, _)| position.get(id).copied().unwrap_or(usize::MAX)); + + Ok(rows) + } + + async fn rebuild_indexes(&mut self, data: &RagData, _full_rebuild: bool) -> Result<()> { + // Local on-disk state. A wholesale table reset plus re-INSERT is always + // correct, so the incremental/full distinction is ignored. + // + // 🔴 ANTI-WIPE GUARD. The CREATE OR REPLACE TABLE below writes exactly what + // `data.vectors` holds. An empty map on a RAG that HAS indexed files, against a + // store that ALREADY holds vectors, is always a bug; a failed/skipped + // hydration, or a caller that emptied the live map. Refuse rather than commit + // the loss. Every legitimate empty-vector rebuild also has an empty `files` + // (a fresh RAG; the zero-document tests), so this cannot fire on a correct call. + // The `existing > 0` conjunct is load-bearing, not belt-and-braces: a populated + // fixture rebuilt against a fresh temp database has count 0 and must proceed. + if data.vectors.is_empty() && !data.files.is_empty() { + let existing: i64 = { + // Scoped: the guard MUST be dropped before `lock_conn()` is taken again + // below. `Mutex` is not reentrant; holding both self-deadlocks at + // runtime, with no compile error. + let handle = self.lock_conn()?; + handle + .conn()? + .query_row("SELECT count(*) FROM vectors", [], |r| r.get(0)) + .context("Failed to count existing vectors before rebuild")? + }; + if existing > 0 { + bail!( + "Refusing to rebuild the DuckDB store at '{}': the in-memory vector \ + map is empty, but this RAG has {} indexed file(s) and the store \ + already holds {existing} vector(s). Rebuilding would erase them. \ + Vector hydration failed, or `data.vectors` was cleared on a live \ + Rag. Nothing was written; the store is intact.", + self.path.display(), + data.files.len() + ); + } + } + // Validate BEFORE opening the transaction so a bad embedding aborts before any + // write, not mid-write with the guard held. + for (doc_id, embedding) in &data.vectors { + if embedding.iter().any(|f| !f.is_finite()) { + bail!( + "Embedding for document {} contains a non-finite value", + doc_id.0 + ); + } + } + let dim = self.dim; + // THE write path. Everything above this line only reads, so the upgrade happens + // here, after both guards have had their say: a rebuild that is going to be + // refused must not first take the exclusive lock away from other processes. + // + // This is also the point that makes a silently-dropped write impossible. If the + // upgrade fails, `?` aborts the rebuild before a single statement is issued and + // the caller gets the error. Nothing below can run on a read-only connection. + self.ensure_writable()?; + let mut handle = self.lock_conn()?; + let conn = handle.conn_mut()?; + let tx = conn + .transaction() + .context("Failed to begin DuckDB transaction")?; + + // Use `CREATE OR REPLACE TABLE`, not `DELETE FROM`. Deleting rows and + // re-inserting the SAME primary keys inside ONE transaction violates DuckDB's PK + // constraint: the index holds deleted keys until commit, so the second rebuild + // fails with a duplicate-key error. `CREATE OR REPLACE` drops the table and its + // indexes atomically, leaving no stale keys. It also drops the HNSW index, which + // is recreated after commit. + tx.execute_batch(&format!( + "CREATE OR REPLACE TABLE vectors ( + doc_id UBIGINT PRIMARY KEY, + embedding FLOAT[{dim}] + ); + CREATE OR REPLACE TABLE documents ( + doc_id UBIGINT PRIMARY KEY, + page_content TEXT NOT NULL + );" + )) + .context("Failed to reset DuckDB tables")?; + + { + // Plain INSERT: tables are empty after CREATE OR REPLACE, so no PK conflict. + // The embedding is bound as TEXT and cast in SQL. The duckdb crate's + // `bind_parameter` has no arm for List/Array, so binding a vector directly + // fails at runtime with "binding List parameters is not yet supported". + let mut vstmt = tx.prepare(&format!( + "INSERT INTO vectors (doc_id, embedding) VALUES (?, CAST(? AS FLOAT[{dim}]))" + ))?; + let mut dstmt = + tx.prepare("INSERT INTO documents (doc_id, page_content) VALUES (?, ?)")?; + + // 🔴 TWO INDEPENDENT LOOPS. `documents` is keyed on `files`, `vectors` on + // `vectors`. They are DIFFERENT key sets and neither is a subset of the + // other. Do not merge these into one loop over `&data.vectors` gated on a + // lookup into `files`: that produces keys(documents) ⊆ keys(vectors), and + // every id in `files \ vectors` (which `RagData::add`'s zip truncation + // really does produce) becomes a live, rankable id from BM25 and + // graph_search that resolves to nothing; no error, no log line. + for (id, doc) in data.iter_documents() { + dstmt.execute(duckdb::params![id.0 as u64, doc.page_content.as_str()])?; + } + + for (doc_id, embedding) in &data.vectors { + // Serialize as a DuckDB array literal: "[0.1,0.2,...]". Finiteness was + // validated above, so `to_string()` cannot emit NaN/inf here. + let embedding_text = { + let mut s = String::with_capacity(embedding.len() * 12 + 2); + s.push('['); + for (i, f) in embedding.iter().enumerate() { + if i > 0 { + s.push(','); + } + s.push_str(&f.to_string()); + } + s.push(']'); + s + }; + vstmt.execute(duckdb::params![doc_id.0 as u64, embedding_text.as_str()])?; + } + } // drop prepared statements before commit + + tx.commit().context("Failed to commit DuckDB transaction")?; + + // Recreate the HNSW index, dropped by CREATE OR REPLACE TABLE above. When + // CREATE INDEX USING HNSW fails, the preceding COMMIT still returns Ok, so the + // failure lands here; swallowing it would leave the database populated but + // UNINDEXED, with every vector_search silently falling back to a full scan. + conn.execute_batch( + "CREATE INDEX IF NOT EXISTS hnsw_idx \ + ON vectors USING HNSW (embedding) WITH (metric = 'cosine');", + ) + .context("Failed to recreate HNSW index")?; + + // Rebuild the FTS index (must be outside the transaction). This rebuild is + // MANDATORY on every pass, not an optimization: CREATE OR REPLACE TABLE above + // leaves the old fts_main_documents schema in place but DEAD; match_bm25 keeps + // succeeding while returning zero rows for every term. + // GUARDED on a non-empty table: FTS cannot index an empty table, and + // rebuild_indexes is legitimately called with zero documents (a fresh RAG, and + // three of the tests below). + let doc_count: i64 = conn + .query_row("SELECT count(*) FROM documents", [], |r| r.get(0)) + .context("Failed to count documents before FTS rebuild")?; + + if doc_count > 0 { + conn.execute_batch( + "DROP FUNCTION IF EXISTS fts_main_documents_match_bm25; + PRAGMA create_fts_index('documents', 'doc_id', 'page_content', overwrite=1);", + ) + .context("Failed to rebuild DuckDB FTS index")?; + } + + // Live only if an index was actually built. With zero documents there is no FTS + // index, so `has_native_keyword_search()` must stay false and hybrid_search must + // fall back to local BM25, which is also empty, and therefore correct. + self.fts_ready.store(doc_count > 0, Ordering::Relaxed); + + Ok(()) + } + + async fn keyword_search(&self, query: &str, top_k: usize) -> Result> { + let handle = self.lock_conn()?; + // match_bm25 returns NULL for non-matching rows; WHERE filters them out. + let mut stmt = handle.conn()?.prepare( + "SELECT doc_id, fts_main_documents.match_bm25(doc_id, ?) AS score + FROM documents + WHERE score IS NOT NULL + ORDER BY score DESC + LIMIT ?", + )?; + let results = stmt + .query_map(duckdb::params![query, top_k as u64], |row| { + let id: u64 = row.get(0)?; + // Same hazard as vector_search's distance column: guessing the width + // wrong raises InvalidColumnType INSIDE the closure, which a bare + // `.filter_map(|r| r.ok())` silently discards, yielding ZERO keyword + // hits, indistinguishable from "the query matched nothing". + let score: f32 = match row.get::<_, Value>(1)? { + Value::Double(d) => d as f32, + Value::Float(f) => f, + other => { + warn!("unexpected match_bm25 score type from DuckDB: {other:?}"); + return Err(duckdb::Error::InvalidQuery); + } + }; + Ok((DocumentId(id as usize), score)) + })? + .filter_map(|r| match r { + Ok(v) => Some(v), + Err(e) => { + warn!("keyword_search row decode failed: {e}"); + None + } + }) + .collect(); + Ok(results) + } + + fn has_native_keyword_search(&self) -> bool { + // NOT an unconditional `true`. The FTS schema only exists after + // rebuild_indexes has run the pragma (or after reopening a database where a + // previous session did). `create()` deliberately does NOT call rebuild_indexes, + // so a freshly created RAG reaches this point with no FTS index at all. + // Returning true there would route every query into keyword_search, whose `?` + // would propagate a DuckDB catalog error and fail the WHOLE search. + self.fts_ready.load(Ordering::Relaxed) + } + + fn duplicate(&self, _data: &RagData) -> Box { + // Do NOT call DuckDbProvider::open() here. `Connection::open()` instantiates a + // NEW DuckDB *database* handle on the same file; the first handle still holds + // the file lock, so the second open fails with a locking error. Cloning the Arc + // shares the already-open connection, serialized by the Mutex. + // + // This means DuckDbProvider clones are NOT independent snapshots: state lives on + // disk and a rebuild through one handle is immediately visible to the other. + // That is unavoidable for any on-disk store and is handled by the discipline + // documented on `Rag`'s Clone impl, the pre-clone instance must be discarded. + // + // Sharing the Arc also shares the ACCESS MODE, which lives inside the ConnHandle + // rather than beside it: when one handle upgrades itself to read-write, every + // clone is upgraded with it and none is left holding a stale "read-only" belief. + Box::new(DuckDbProvider { + path: self.path.clone(), + conn: Arc::clone(&self.conn), + dim: self.dim, + fts_ready: AtomicBool::new(self.fts_ready.load(Ordering::Relaxed)), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::rag::provider::RagProvider; + use crate::rag::{RagDocument, RagFile}; + use std::process::Command; + use std::time::{SystemTime, UNIX_EPOCH}; + use std::{env, fs}; + + struct TempDb { + path: PathBuf, + } + + impl TempDb { + fn new(tag: &str) -> Self { + let unique = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = env::temp_dir().join(format!("coyote-duckdb-{tag}-{unique}.duckdb")); + Self { path } + } + } + + impl Drop for TempDb { + fn drop(&mut self) { + let _ = fs::remove_file(&self.path); + let _ = fs::remove_file(self.path.with_extension("duckdb.wal")); + } + } + + /// ⚠️ NOTE THE EMPTY `files`. This fixture describes a RAG with ZERO documents. + /// Since `rebuild_indexes` populates the `documents` table from + /// `data.iter_documents()` (keyed on `files`), any test built on this helper alone + /// exercises the `vectors` table and NOTHING ELSE. No `documents` rows, no FTS + /// index. That is correct for the rebuild tests below, and is exactly why the + /// FTS-flag test and the `fetch_content` tests use `populated_rag_data()` instead. + /// Do not "simplify" them back onto this helper. + fn minimal_rag_data() -> RagData { + RagData { + embedding_model: "text-embedding-3-small".to_string(), + chunk_size: 1024, + chunk_overlap: 50, + top_k: 5, + driver: "duckdb".to_string(), + attached: false, + ..Default::default() + } + } + + /// Two files, one document each; the minimum fixture that produces `documents` + /// rows. + fn populated_rag_data() -> RagData { + let mut data = minimal_rag_data(); + debug_assert_eq!(DocumentId::new(0, 0), DocumentId(0)); + data.files.insert( + 0, + RagFile { + hash: "h0".to_string(), + path: "/tmp/a.md".to_string(), + documents: vec![RagDocument { + page_content: "alpha keyword".to_string(), + metadata: Default::default(), + }], + }, + ); + data.files.insert( + 1, + RagFile { + hash: "h1".to_string(), + path: "/tmp/b.md".to_string(), + documents: vec![RagDocument { + page_content: "beta keyword".to_string(), + metadata: Default::default(), + }], + }, + ); + data + } + + #[tokio::test] + async fn open_creates_schema() { + let db = TempDb::new("schema"); + let provider = DuckDbProvider::open(&db.path, 3).unwrap(); + let handle = provider.conn.lock().unwrap(); + let conn = handle.conn().unwrap(); + + let v: i64 = conn + .query_row("SELECT COUNT(*) FROM vectors", [], |r| r.get(0)) + .unwrap(); + let d: i64 = conn + .query_row("SELECT COUNT(*) FROM documents", [], |r| r.get(0)) + .unwrap(); + + assert_eq!(v, 0); + assert_eq!(d, 0); + } + + #[tokio::test] + async fn vector_search_returns_top_result() { + let db = TempDb::new("vsearch"); + let provider = DuckDbProvider::open(&db.path, 3).unwrap(); + { + let handle = provider.conn.lock().unwrap(); + let conn = handle.conn().unwrap(); + // The ::FLOAT[3] cast is REQUIRED: a bare [0.1, 0.2, 0.3] literal infers + // DOUBLE[], which does not match the FLOAT[N] ARRAY column type. + conn.execute( + "INSERT INTO vectors (doc_id, embedding) VALUES (0, [0.1, 0.2, 0.3]::FLOAT[3])", + [], + ) + .unwrap(); + } + + let results = provider + .vector_search(&[0.1, 0.2, 0.3], 5, 0.0) + .await + .unwrap(); + + assert_eq!(results.len(), 1); + assert_eq!(results[0].0.0, 0); + assert!(results[0].1 > 0.99); + } + + #[tokio::test] + async fn fetch_content_returns_stored_text() { + let db = TempDb::new("fetch"); + let provider = DuckDbProvider::open(&db.path, 3).unwrap(); + { + let handle = provider.conn.lock().unwrap(); + let conn = handle.conn().unwrap(); + conn.execute( + "INSERT INTO documents (doc_id, page_content) VALUES (42, 'hello world')", + [], + ) + .unwrap(); + } + + let results = provider.fetch_content(&[DocumentId(42)]).await.unwrap(); + + assert_eq!(results.len(), 1); + assert_eq!(results[0].1, "hello world"); + } + + #[tokio::test] + async fn rebuild_indexes_then_vector_search() { + let db = TempDb::new("rebuild"); + let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); + let data = minimal_rag_data(); + provider.rebuild_indexes(&data, true).await.unwrap(); + + let results = provider + .vector_search(&[0.1, 0.2, 0.3], 5, 0.0) + .await + .unwrap(); + + assert!(results.is_empty()); + } + + #[tokio::test] + async fn rebuild_indexes_is_idempotent_across_two_passes() { + let db = TempDb::new("idempotent"); + let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); + let mut data = minimal_rag_data(); + data.vectors.insert(DocumentId(0), vec![0.1, 0.2, 0.3]); + data.vectors.insert(DocumentId(1), vec![0.4, 0.5, 0.6]); + + provider.rebuild_indexes(&data, true).await.unwrap(); + + provider + .rebuild_indexes(&data, true) + .await + .expect("second rebuild must not violate the primary key constraint"); + + let handle = provider.conn.lock().unwrap(); + let conn = handle.conn().unwrap(); + let count: i64 = conn + .query_row("SELECT COUNT(*) FROM vectors", [], |r| r.get(0)) + .unwrap(); + assert_eq!(count, 2, "rebuild must replace rows, not duplicate them"); + } + + #[tokio::test] + async fn rebuild_indexes_preserves_prior_vectors_on_incremental_pass() { + let db = TempDb::new("incremental"); + let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); + + let mut data = minimal_rag_data(); + data.vectors.insert(DocumentId(0), vec![0.1, 0.2, 0.3]); + data.vectors.insert(DocumentId(1), vec![0.4, 0.5, 0.6]); + provider.rebuild_indexes(&data, true).await.unwrap(); + + let mut reloaded = minimal_rag_data(); + assert!( + reloaded.vectors.is_empty(), + "YAML-loaded data starts with no vectors" + ); + reloaded.vectors = provider.read_all_vectors().unwrap(); + assert_eq!( + reloaded.vectors.len(), + 2, + "hydration must restore both vectors" + ); + + reloaded.vectors.insert(DocumentId(2), vec![0.7, 0.8, 0.9]); + provider.rebuild_indexes(&reloaded, false).await.unwrap(); + + let handle = provider.conn.lock().unwrap(); + let conn = handle.conn().unwrap(); + let count: i64 = conn + .query_row("SELECT COUNT(*) FROM vectors", [], |r| r.get(0)) + .unwrap(); + // Without hydration this is 1 — the two originals silently vanish. + assert_eq!( + count, 3, + "an incremental pass must not destroy previously indexed vectors" + ); + } + + #[tokio::test] + async fn keyword_search_is_not_advertised_before_first_rebuild() { + let db = TempDb::new("ftsflag"); + let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); + assert!( + !provider.has_native_keyword_search(), + "a freshly opened DB has no FTS index yet" + ); + + let mut data = populated_rag_data(); + data.vectors.insert(DocumentId(0), vec![0.1, 0.2, 0.3]); + provider.rebuild_indexes(&data, true).await.unwrap(); + { + let handle = provider.conn.lock().unwrap(); + let conn = handle.conn().unwrap(); + let docs: i64 = conn + .query_row("SELECT COUNT(*) FROM documents", [], |r| r.get(0)) + .unwrap(); + assert_eq!(docs, 2, "the fixture must produce documents rows"); + } + assert!( + provider.has_native_keyword_search(), + "rebuild_indexes creates the FTS index and must flip the flag" + ); + } + + #[tokio::test] + async fn fetch_content_resolves_documents_without_vectors() { + let db = TempDb::new("docsnovec"); + let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); + + let mut data = populated_rag_data(); + let id_a = DocumentId::new(0, 0); + let id_b = DocumentId::new(1, 0); + data.vectors.insert(id_a, vec![0.1, 0.2, 0.3]); + assert!( + !data.vectors.contains_key(&id_b), + "precondition: b has no vector" + ); + + provider.rebuild_indexes(&data, true).await.unwrap(); + + let got = provider.fetch_content(&[id_a, id_b]).await.unwrap(); + assert_eq!( + got.len(), + 2, + "fetch_content must resolve BOTH documents; the one without a vector is \ + still reachable by keyword and graph search" + ); + assert_eq!(got[0].1, "alpha keyword"); + assert_eq!(got[1].1, "beta keyword"); + } + + #[tokio::test] + async fn duckdb_fetch_content_preserves_input_order() { + let db = TempDb::new("order"); + let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); + let data = populated_rag_data(); + provider.rebuild_indexes(&data, true).await.unwrap(); + + let id_a = DocumentId::new(0, 0); + let id_b = DocumentId::new(1, 0); + + let got = provider.fetch_content(&[id_b, id_a]).await.unwrap(); + assert_eq!(got.len(), 2); + assert_eq!(got[0].0, id_b, "input order must win over storage order"); + assert_eq!(got[0].1, "beta keyword"); + assert_eq!(got[1].0, id_a); + assert_eq!(got[1].1, "alpha keyword"); + } + + #[tokio::test] + async fn rebuild_indexes_refuses_to_wipe_when_vectors_are_empty_but_files_are_not() { + let db = TempDb::new("nowipe"); + let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); + + let mut data = populated_rag_data(); + data.vectors + .insert(DocumentId::new(0, 0), vec![0.1, 0.2, 0.3]); + data.vectors + .insert(DocumentId::new(1, 0), vec![0.4, 0.5, 0.6]); + provider.rebuild_indexes(&data, true).await.unwrap(); + + let broken = populated_rag_data(); + assert!( + broken.vectors.is_empty() && !broken.files.is_empty(), + "precondition" + ); + + let err = provider.rebuild_indexes(&broken, true).await.unwrap_err(); + assert!( + err.to_string().contains("Refusing to rebuild"), + "got: {err}" + ); + + let handle = provider.conn.lock().unwrap(); + let conn = handle.conn().unwrap(); + let count: i64 = conn + .query_row("SELECT COUNT(*) FROM vectors", [], |r| r.get(0)) + .unwrap(); + assert_eq!( + count, 2, + "the guard must abort BEFORE the CREATE OR REPLACE" + ); + } + + #[tokio::test] + async fn rebuild_indexes_allows_empty_vectors_when_files_are_also_empty() { + let db = TempDb::new("emptyok"); + let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); + let data = minimal_rag_data(); + provider + .rebuild_indexes(&data, true) + .await + .expect("a fresh RAG with nothing indexed must rebuild cleanly"); + } + + #[tokio::test] + async fn duplicate_shares_the_same_connection() { + let db = TempDb::new("dup"); + let provider = DuckDbProvider::open(&db.path, 3).unwrap(); + let dup = provider.duplicate(&minimal_rag_data()); + + { + let handle = provider.conn.lock().unwrap(); + let conn = handle.conn().unwrap(); + conn.execute( + "INSERT INTO documents (doc_id, page_content) VALUES (7, 'shared row')", + [], + ) + .unwrap(); + } + + let via_dup = dup.fetch_content(&[DocumentId(7)]).await.unwrap(); + + assert_eq!( + via_dup.len(), + 1, + "duplicate() must see writes made via the original" + ); + assert_eq!(via_dup[0].1, "shared row"); + } + + #[tokio::test] + async fn read_all_vectors_rejects_non_finite_embeddings() { + let db = TempDb::new("nonfinite"); + let provider = DuckDbProvider::open(&db.path, 3).unwrap(); + { + let handle = provider.conn.lock().unwrap(); + let conn = handle.conn().unwrap(); + conn.execute( + "INSERT INTO vectors (doc_id, embedding) VALUES (0, [0.1, 0.2, 0.3]::FLOAT[3])", + [], + ) + .unwrap(); + conn.execute( + // Each element is cast individually: a bare ['nan', 0.2, 0.3] literal + // mixes text and numerics, so DuckDB infers DECIMAL and the conversion + // fails before the row is ever stored. + "INSERT INTO vectors (doc_id, embedding) VALUES \ + (1, [CAST('nan' AS FLOAT), CAST(0.2 AS FLOAT), CAST(0.3 AS FLOAT)]::FLOAT[3])", + [], + ) + .unwrap(); + } + let err = provider.read_all_vectors().unwrap_err(); + assert!( + err.to_string().contains("non-finite"), + "hydration must abort rather than silently drop the row; got: {err}" + ); + } + + /// Was the shared connection opened read-write? Reads the flag that lives inside the + /// shared handle, which is the same one `ensure_writable` flips. + fn is_writable(provider: &DuckDbProvider) -> bool { + provider.conn.lock().unwrap().writable + } + + /// Build a fully initialized store, then let the read-write handle go so the file is + /// unlocked for the next opener. + async fn seed_store(path: &Path) { + let mut provider = DuckDbProvider::open(path, 3).unwrap(); + assert!(is_writable(&provider), "a fresh file must open read-write"); + let mut data = populated_rag_data(); + data.vectors + .insert(DocumentId::new(0, 0), vec![0.1, 0.2, 0.3]); + provider.rebuild_indexes(&data, true).await.unwrap(); + } + + #[tokio::test] + async fn a_fresh_file_is_opened_read_write() { + let db = TempDb::new("freshrw"); + let provider = DuckDbProvider::open(&db.path, 3).unwrap(); + + assert!( + is_writable(&provider), + "the schema has to be created, which writes, so a missing file must open \ + read-write" + ); + } + + #[tokio::test] + async fn an_initialized_store_is_reopened_read_only() { + let db = TempDb::new("reopenro"); + seed_store(&db.path).await; + + let provider = DuckDbProvider::open(&db.path, 3).unwrap(); + + assert!( + !is_writable(&provider), + "a store that needs no schema work must open read-only, so that other Coyote \ + processes can query it at the same time" + ); + } + + #[tokio::test] + async fn a_file_without_the_coyote_schema_is_opened_read_write() { + let db = TempDb::new("noschema"); + { + // A valid DuckDB file that is not one of ours. Opening it read-only would + // strand it forever: the init batch is refused on a read-only handle. + let conn = Connection::open(&db.path).unwrap(); + conn.execute_batch("CREATE TABLE unrelated (x INTEGER);") + .unwrap(); + } + + let provider = DuckDbProvider::open(&db.path, 3).unwrap(); + + assert!( + is_writable(&provider), + "a schema-less file must open read-write" + ); + let handle = provider.conn.lock().unwrap(); + let tables: i64 = handle + .conn() + .unwrap() + .query_row( + "SELECT count(*) FROM duckdb_tables() \ + WHERE table_name IN ('vectors', 'documents')", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(tables, 2, "the schema must have been created"); + } + + #[tokio::test] + async fn a_read_only_store_still_serves_vector_and_keyword_search() { + let db = TempDb::new("rosearch"); + seed_store(&db.path).await; + + let provider = DuckDbProvider::open(&db.path, 3).unwrap(); + assert!(!is_writable(&provider), "precondition: opened read-only"); + + let hits = provider + .vector_search(&[0.1, 0.2, 0.3], 5, 0.0) + .await + .unwrap(); + assert_eq!( + hits.len(), + 1, + "the persisted HNSW index must be queryable read-only" + ); + assert!(hits[0].1 > 0.99); + + assert!( + provider.has_native_keyword_search(), + "the FTS index built by the previous session must still be detected on a \ + read-only handle" + ); + let kw = provider.keyword_search("alpha", 5).await.unwrap(); + assert_eq!(kw.len(), 1, "keyword search must work read-only"); + + let docs = provider + .fetch_content(&[DocumentId::new(0, 0)]) + .await + .unwrap(); + assert_eq!(docs[0].1, "alpha keyword"); + } + + #[tokio::test] + async fn a_read_only_handle_refuses_a_direct_write() { + let db = TempDb::new("rorefuse"); + seed_store(&db.path).await; + let provider = DuckDbProvider::open(&db.path, 3).unwrap(); + assert!(!is_writable(&provider), "precondition: opened read-only"); + + let handle = provider.conn.lock().unwrap(); + let err = handle + .conn() + .unwrap() + .execute( + "INSERT INTO documents (doc_id, page_content) VALUES (99, 'nope')", + [], + ) + .unwrap_err(); + + // The backstop, not the primary defence: `rebuild_indexes` upgrades first and + // never reaches a write on a read-only handle. It matters anyway because DuckDB + // lets `transaction()` open and `commit()` return Ok on a read-only connection, + // so a write that slipped through would look like it had succeeded. + assert!( + err.to_string().contains("read-only mode"), + "a read-only handle must refuse writes loudly; got: {err}" + ); + } + + #[tokio::test] + async fn rebuild_indexes_upgrades_a_read_only_connection() { + let db = TempDb::new("upgrade"); + seed_store(&db.path).await; + + let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); + assert!(!is_writable(&provider), "precondition: opened read-only"); + + let mut data = populated_rag_data(); + data.vectors = provider.read_all_vectors().unwrap(); + data.vectors + .insert(DocumentId::new(1, 0), vec![0.4, 0.5, 0.6]); + provider.rebuild_indexes(&data, false).await.unwrap(); + + assert!( + is_writable(&provider), + "the write path must have upgraded the connection in place" + ); + + // The upgraded connection is a NEW connection, so the per-connection session + // state has to have been re-established on it. Without the re-run `SET`, the + // CREATE INDEX ... USING HNSW inside rebuild_indexes would already have failed. + let persisted: String = { + let handle = provider.conn.lock().unwrap(); + handle + .conn() + .unwrap() + .query_row( + "SELECT CAST(current_setting('hnsw_enable_experimental_persistence') \ + AS VARCHAR)", + [], + |r| r.get(0), + ) + .unwrap() + }; + assert_eq!( + persisted, "true", + "the upgraded connection must re-run the SET; session state does not carry \ + over from the dropped read-only connection" + ); + + drop(provider); + let reopened = DuckDbProvider::open(&db.path, 3).unwrap(); + let all = reopened.read_all_vectors().unwrap(); + assert_eq!(all.len(), 2, "the upgraded write must have reached disk"); + } + + #[tokio::test] + async fn an_upgrade_is_visible_through_duplicate_clones() { + let db = TempDb::new("upgradedup"); + seed_store(&db.path).await; + + let mut provider = DuckDbProvider::open(&db.path, 3).unwrap(); + assert!(!is_writable(&provider), "precondition: opened read-only"); + let dup = provider.duplicate(&minimal_rag_data()); + + let mut data = populated_rag_data(); + data.vectors = provider.read_all_vectors().unwrap(); + provider.rebuild_indexes(&data, false).await.unwrap(); + + // `duplicate()` shares the Arc, and the access mode lives inside it, so the clone + // must observe the upgrade rather than keep believing it is read-only. + let via_dup = dup.fetch_content(&[DocumentId::new(0, 0)]).await.unwrap(); + assert_eq!( + via_dup.len(), + 1, + "the clone must still read after an upgrade" + ); + assert!( + is_writable(&provider), + "the shared handle must report writable to every clone" + ); + } + + /// Two REAL OS processes reading one store at the same time. + /// + /// Ignored by default because it re-executes the test binary as a child process, + /// which is heavier and more environment-dependent than the rest of the suite. Run it + /// with: + /// cargo test --all -- --ignored duckdb_store_is_shared_across_processes + /// + /// It cannot be written as an ordinary in-process test: DuckDB keeps ONE database + /// instance per process, so a second open in the same process bypasses the file lock + /// entirely (a read-write open succeeds even while this process holds a read-only + /// one). Only separate processes exercise the lock this feature exists to avoid. + #[tokio::test] + #[ignore = "spawns a second OS process; run explicitly with --ignored"] + async fn duckdb_store_is_shared_across_processes() { + const CHILD_DB: &str = "COYOTE_DUCKDB_MULTIPROC_DB"; + const CHILD_EXPECT: &str = "COYOTE_DUCKDB_MULTIPROC_EXPECT"; + const TEST_NAME: &str = + "rag::providers::duckdb::tests::duckdb_store_is_shared_across_processes"; + + if let Ok(path) = env::var(CHILD_DB) { + let expect = env::var(CHILD_EXPECT).unwrap_or_default(); + let opened = DuckDbProvider::open(Path::new(&path), 3); + match expect.as_str() { + "readable" => { + let provider = opened.expect( + "a second process must be able to open a store that another \ + process holds READ-ONLY", + ); + assert!(!is_writable(&provider), "the child must land read-only"); + let hits = provider + .vector_search(&[0.1, 0.2, 0.3], 5, 0.0) + .await + .unwrap(); + assert_eq!(hits.len(), 1, "the child must read the seeded vector"); + } + "blocked" => { + let err = opened.err().expect( + "a second process must NOT be able to open a store that another \ + process holds READ-WRITE", + ); + let msg = format!("{err:#}"); + assert!( + msg.contains("concurrent READERS") && msg.contains("only ONE writer"), + "the lock error must explain the reader/writer rule; got: {msg}" + ); + } + other => panic!("unknown child expectation {other:?}"), + } + return; + } + + let db = TempDb::new("multiproc"); + seed_store(&db.path).await; + + let run_child = |expect: &str| { + Command::new(env::current_exe().unwrap()) + .args(["--exact", "--ignored", "--nocapture", TEST_NAME]) + .env(CHILD_DB, &db.path) + .env(CHILD_EXPECT, expect) + .output() + .expect("failed to spawn the child test process") + }; + + // Phase 1: this process holds a READ-ONLY handle. The child must get one too. + let provider = DuckDbProvider::open(&db.path, 3).unwrap(); + assert!(!is_writable(&provider), "precondition: parent is read-only"); + let out = run_child("readable"); + assert!( + out.status.success(), + "child could not share the read-only store:\n{}\n{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + let hits = provider + .vector_search(&[0.1, 0.2, 0.3], 5, 0.0) + .await + .unwrap(); + assert_eq!( + hits.len(), + 1, + "the parent must still read after the child ran" + ); + + // Phase 2: upgrade this process to READ-WRITE. The child must now be refused, + // with the message that explains why. + provider.ensure_writable().unwrap(); + let out = run_child("blocked"); + assert!( + out.status.success(), + "a writer must exclude other processes, with an actionable error:\n{}\n{}", + String::from_utf8_lossy(&out.stdout), + String::from_utf8_lossy(&out.stderr) + ); + } + + #[test] + fn duckdb_path_from_yaml_swaps_extension() { + let p = duckdb_path_from_yaml(Path::new("/tmp/rags/docs.yaml")); + + assert_eq!(p, PathBuf::from("/tmp/rags/docs.duckdb")); + } +} diff --git a/src/rag/providers/mod.rs b/src/rag/providers/mod.rs new file mode 100644 index 0000000..1a54ca8 --- /dev/null +++ b/src/rag/providers/mod.rs @@ -0,0 +1,9 @@ +mod yaml; +pub use self::yaml::YamlProvider; + +mod duckdb; +pub use self::duckdb::DuckDbProvider; +pub(crate) use self::duckdb::duckdb_path_from_yaml; + +mod qdrant; +pub use self::qdrant::QdrantProvider; diff --git a/src/rag/providers/qdrant.rs b/src/rag/providers/qdrant.rs new file mode 100644 index 0000000..da3b84b --- /dev/null +++ b/src/rag/providers/qdrant.rs @@ -0,0 +1,828 @@ +use crate::rag::provider::RagProvider; +use crate::rag::{DocumentId, RagData}; + +use anyhow::{Context, Result, bail}; +use async_trait::async_trait; +use parking_lot::RwLock; +use reqwest::header::{HeaderMap, HeaderValue}; +use reqwest::{Client, Response, StatusCode}; +use serde_json::Value; +use std::collections::HashMap; +use std::sync::Arc; +use url::{Host, Url}; + +/// Marks a `DocumentId` that stands in for a point id Coyote cannot carry +/// directly. Qdrant accepts UUID strings as point ids, and that is what +/// LangChain writes by default. +/// +/// `DocumentId` packs `(file_index, document_index)` into one `usize` with the +/// file index in the high half, so this bit is only reachable at a file index of +/// 2^31. Nothing local gets near that, and an attached RAG builds no local index +/// at all — `data.files` and `data.vectors` stay empty and every +/// `DocumentId::split` caller early-returns on `data.attached`. Along the +/// attached path the id is an opaque key carried through RRF, which is what +/// makes a synthetic one safe here and nowhere else. +const SYNTHETIC_ID_TAG: usize = 1 << (usize::BITS - 1); + +/// Two-way map between a raw Qdrant point id and the `DocumentId` the retrieval +/// pipeline sees. +/// +/// Only ids that cannot survive the round trip are interned. A plain `u64` that +/// fits below the tag keeps mapping to itself, so integer-keyed collections +/// behave exactly as they did before this map existed. +#[derive(Default)] +struct PointIdInterner { + handles: HashMap, + raw: HashMap, + next: usize, +} + +impl PointIdInterner { + /// The `DocumentId` for a raw point id, minting a handle if one is needed. + /// + /// `None` only for a missing id, which is a malformed response. + fn document_id(&mut self, raw: &Value) -> Option { + if raw.is_null() { + return None; + } + // The pre-existing integer path, unchanged. `try_from` rather than `as` + // so a value too wide for the target's `usize` is interned instead of + // silently truncated into a different point. + if let Some(n) = raw.as_u64() + && let Ok(n) = usize::try_from(n) + && n & SYNTHETIC_ID_TAG == 0 + { + return Some(DocumentId(n)); + } + Some(self.intern(raw)) + } + + fn intern(&mut self, raw: &Value) -> DocumentId { + // Keyed on the JSON rendering, so the string "1" and the integer 1 are + // not conflated into one point. + let key = raw.to_string(); + if let Some(handle) = self.handles.get(&key) { + return *handle; + } + let handle = DocumentId(SYNTHETIC_ID_TAG | self.next); + self.next += 1; + self.handles.insert(key, handle); + self.raw.insert(handle, raw.clone()); + handle + } + + /// The original id for a handle, or `None` when the id was never interned — + /// i.e. it is a plain integer that is already its own id. + fn raw_id(&self, handle: DocumentId) -> Option<&Value> { + self.raw.get(&handle) + } + + /// Builds the `ids` array for an outbound `/points` fetch. Every entry is the + /// id Qdrant issued, integer or string; a synthetic handle must never leave + /// this process. + fn outbound_ids(&self, ids: &[DocumentId]) -> Vec { + ids.iter() + .map(|id| match self.raw_id(*id) { + Some(raw) => raw.clone(), + None => Value::from(id.0 as u64), + }) + .collect() + } +} + +fn parse_search_hits( + interner: &mut PointIdInterner, + body: &Value, + min_score: f32, +) -> Result> { + let hits = body["result"] + .as_array() + .context("Unexpected /points/search response shape")?; + + Ok(hits + .iter() + .filter_map(|pt| { + let score = pt["score"].as_f64()? as f32; + Some((interner.document_id(&pt["id"])?, score)) + }) + .filter(|(_, score)| min_score <= 0.0 || *score > min_score) + .collect()) +} + +fn parse_points(interner: &mut PointIdInterner, body: &Value) -> Result> { + let points = body["result"] + .as_array() + .context("Unexpected /points response shape")?; + + Ok(points + .iter() + .filter_map(|pt| { + let text = pt["payload"]["page_content"].as_str()?.to_string(); + Some((interner.document_id(&pt["id"])?, text)) + }) + .collect()) +} + +/// Render Qdrant's error envelope into a human-readable message. +/// +/// `body` is the raw response text. Two shapes have to be tolerated: +/// * application-level errors carry `{"status": {"error": "..."}, "time": 0.0}`, +/// while successful responses carry a bare string `{"status": "ok", ...}` — so +/// `status` is string-or-object and a struct with `status: String` fails to +/// parse every error body; +/// * routing-level 404s (a wrong HTTP verb) return an EMPTY body with no JSON at +/// all, which without the length check surfaces as "EOF while parsing a value" +/// instead of the actual 404. +fn format_error_body(status: StatusCode, body: &str) -> String { + if body.is_empty() { + return format!("HTTP {status} (empty body — check the HTTP verb and path)"); + } + serde_json::from_str::(body) + .ok() + .and_then(|v| v["status"]["error"].as_str().map(str::to_string)) + .unwrap_or_else(|| format!("HTTP {status}: {body}")) +} + +/// Read the vector dimension out of a parsed `GET /collections/{name}` response. +fn vector_dimension_from_collection(body: &Value) -> Result { + let params = &body["result"]["config"]["params"]; + params["vectors"]["size"] + .as_u64() + .or_else(|| { + params["vectors"] + .as_object() + .and_then(|m| m.values().next()) + .and_then(|v| v["size"].as_u64()) + }) + .context("Could not determine vector dimension from collection config") +} + +/// True if a parsed `GET /collections/{name}` response describes a NAMED +/// (multi-vector) collection. +/// +/// `vector_search` posts an unnamed vector, which a named-vector collection +/// rejects with HTTP 400 on every query, so attaching one yields a RAG that is +/// silently 100% broken. A named collection holding a SINGLE vector is +/// structurally a map, identical in kind to the multi-named case, and rejects +/// the same way; testing for a numeric `size` directly under `vectors` catches +/// it, whereas counting keys (`len() > 1`) would wrongly accept it. +fn is_multi_vector_config(body: &Value) -> bool { + body["result"]["config"]["params"]["vectors"]["size"] + .as_u64() + .is_none() +} + +/// Query-only client for an external Qdrant collection. +/// +/// Attach-only: this provider never writes to the remote collection. Coyote does +/// not own the data, and `rebuild_indexes` refuses rather than pretending to. +pub struct QdrantProvider { + client: Client, + base_url: String, + collection: String, + point_ids: Arc>, +} + +impl QdrantProvider { + fn skips_proxy(base_url: &str) -> bool { + let Ok(url) = Url::parse(base_url) else { + return false; + }; + match url.host() { + Some(Host::Domain(name)) => { + name == "localhost" || name.ends_with(".localhost") || name.ends_with(".local") + } + Some(Host::Ipv4(ip)) => ip.is_loopback() || ip.is_private() || ip.is_link_local(), + // No stable is_unique_local, so fc00::/7 is matched directly. + Some(Host::Ipv6(ip)) => ip.is_loopback() || ip.segments()[0] & 0xfe00 == 0xfc00, + None => false, + } + } + + fn make_client(base_url: &str, api_key: Option<&str>) -> Result { + let mut headers = HeaderMap::new(); + if let Some(key) = api_key { + let mut value = + HeaderValue::from_str(key).context("api-key header value is not valid ASCII")?; + value.set_sensitive(true); + headers.insert("api-key", value); + } + let mut builder = Client::builder().default_headers(headers); + if Self::skips_proxy(base_url) { + builder = builder.no_proxy(); + } + builder.build().context("Failed to build reqwest client") + } + + pub(crate) fn normalize_base_url(host: &str) -> String { + if host.starts_with("http://") || host.starts_with("https://") { + host.to_string() + } else { + format!("http://{host}") + } + } + + async fn error_message(resp: Response) -> String { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + format_error_body(status, &body) + } + + /// Shared `GET /collections/{name}` fetch. Both the dimension and the + /// multi-vector probe discriminate on this same response. + async fn fetch_collection( + host: &str, + collection: &str, + api_key: Option<&str>, + ) -> Result { + let base_url = Self::normalize_base_url(host); + let client = Self::make_client(&base_url, api_key)?; + let resp = client + .get(format!("{base_url}/collections/{collection}")) + .send() + .await + .with_context(|| format!("Failed to connect to {host}"))?; + if !resp.status().is_success() { + bail!( + "Failed to read collection '{collection}': {}", + Self::error_message(resp).await + ); + } + + Ok(resp.json().await?) + } + + pub async fn new(host: &str, collection: &str, api_key: Option<&str>) -> Result { + let base_url = Self::normalize_base_url(host); + let client = Self::make_client(&base_url, api_key)?; + let resp = client + .get(format!("{base_url}/collections/{collection}")) + .send() + .await + .with_context(|| format!("Failed to connect to {host}"))?; + if !resp.status().is_success() { + bail!( + "Collection '{collection}' not accessible at {host}: {}", + Self::error_message(resp).await + ); + } + + Ok(Self { + client, + base_url, + collection: collection.to_string(), + point_ids: Arc::default(), + }) + } + + pub async fn list_collections(host: &str, api_key: Option<&str>) -> Result> { + let base_url = Self::normalize_base_url(host); + let client = Self::make_client(&base_url, api_key)?; + let resp = client + .get(format!("{base_url}/collections")) + .send() + .await + .with_context(|| format!("Failed to connect to {host}"))?; + if !resp.status().is_success() { + bail!( + "Failed to list collections: {}", + Self::error_message(resp).await + ); + } + + let body: Value = resp.json().await?; + let names = body["result"]["collections"] + .as_array() + .context("Unexpected /collections response shape")? + .iter() + .filter_map(|v| v["name"].as_str().map(str::to_string)) + .collect(); + + Ok(names) + } + + pub async fn get_vector_dimension( + host: &str, + collection: &str, + api_key: Option<&str>, + ) -> Result { + let body = Self::fetch_collection(host, collection, api_key).await?; + + vector_dimension_from_collection(&body) + } + + pub async fn is_multi_vector( + host: &str, + collection: &str, + api_key: Option<&str>, + ) -> Result { + let body = Self::fetch_collection(host, collection, api_key).await?; + + Ok(is_multi_vector_config(&body)) + } + + pub async fn sample_point_id( + host: &str, + collection: &str, + api_key: Option<&str>, + ) -> Result> { + let base_url = Self::normalize_base_url(host); + let client = Self::make_client(&base_url, api_key)?; + let url = format!("{base_url}/collections/{collection}/points/scroll"); + let body = serde_json::json!({ "limit": 1, "with_payload": false }); + + let resp = client + .post(&url) + .json(&body) + .send() + .await + .with_context(|| format!("Failed to connect to {host}"))?; + + if !resp.status().is_success() { + bail!( + "Failed to sample a point from '{collection}': {}", + Self::error_message(resp).await + ); + } + + let data: Value = resp.json().await?; + let id_val = data["result"]["points"] + .as_array() + .and_then(|pts| pts.first()) + .map(|pt| pt["id"].to_string()); + + Ok(id_val) + } +} + +#[async_trait] +impl RagProvider for QdrantProvider { + async fn vector_search( + &self, + embedding: &[f32], + top_k: usize, + min_score: f32, + ) -> Result> { + let url = format!( + "{}/collections/{}/points/search", + self.base_url, self.collection + ); + // `score_threshold` is deliberately NOT sent. It is metric-aware: on Cosine + // collections 0.0 means "no floor" as expected, but Euclid collections score + // by negative distance, where 0.0 filters everything out. The attach wizard + // does not pin the distance metric, so filter locally instead; i.e. where a + // 0.0 floor is correctly treated as "no floor" (see `parse_search_hits`). + let body = serde_json::json!({ + "vector": embedding, + "limit": top_k, + "with_payload": false, + }); + let resp = self.client.post(&url).json(&body).send().await?; + if !resp.status().is_success() { + bail!( + "Qdrant search on '{}' failed: {}", + self.collection, + Self::error_message(resp).await + ); + } + let data: Value = resp.json().await?; + // The interner is what lets a UUID-keyed collection work: a string id gets + // a synthetic handle here and the original is replayed by `fetch_content`. + let mut interner = self.point_ids.write(); + + parse_search_hits(&mut interner, &data, min_score) + } + + async fn fetch_content(&self, ids: &[DocumentId]) -> Result> { + if ids.is_empty() { + return Ok(vec![]); + } + let url = format!("{}/collections/{}/points", self.base_url, self.collection); + // Qdrant is asked for the ids it issued, never for a synthetic handle. + let id_list = self.point_ids.read().outbound_ids(ids); + let body = serde_json::json!({ + "ids": id_list, + "with_payload": true, + }); + + let resp = self.client.post(&url).json(&body).send().await?; + + if !resp.status().is_success() { + bail!( + "Qdrant point fetch on '{}' failed: {}", + self.collection, + Self::error_message(resp).await + ); + } + let data: Value = resp.json().await?; + let mut rows = { + let mut interner = self.point_ids.write(); + parse_points(&mut interner, &data)? + }; + // `/points` does not guarantee response order matches request order, and the + // caller's RRF ranking is carried by that order. Restore it. + let position: HashMap = + ids.iter().enumerate().map(|(i, id)| (*id, i)).collect(); + rows.sort_by_key(|(id, _)| position.get(id).copied().unwrap_or(usize::MAX)); + + Ok(rows) + } + + async fn rebuild_indexes(&mut self, data: &RagData, _full_rebuild: bool) -> Result<()> { + // Both arms refuse. A silent `Ok(())` would make `.rebuild rag` and + // `.edit rag-docs` look like they worked while writing nothing to the + // remote, leaving the user believing the collection was updated. + if data.attached { + bail!( + "This RAG is attached to an external Qdrant collection. Coyote does not own \ + its documents and cannot rebuild it. Manage the collection directly, or \ + create a Coyote-owned RAG with `.rag `." + ); + } + bail!("Writing to Qdrant is not supported yet (attach-only)."); + } + + fn duplicate(&self, _data: &RagData) -> Box { + // Cloning the client shares the connection pool and the injected api-key + // header. Sharing is correct: both handles address the same remote + // collection, and neither of them writes to it. + // + // The point-id map is shared for the same reason, and because it MUST be: + // `Rag::clone()` hands the clone `DocumentId`s that the original minted, + // so a fresh map would resolve them to nothing and `fetch_content` would + // ask Qdrant for a synthetic handle — zero results, no error. Resetting it + // would also re-mint handles for ids the original still holds. + Box::new(Self { + client: self.client.clone(), + base_url: self.base_url.clone(), + collection: self.collection.clone(), + point_ids: Arc::clone(&self.point_ids), + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn error_message_reads_the_object_status_envelope() { + let body = + r#"{"status": {"error": "Wrong input: Not existing vector name error:"}, "time": 0.0}"#; + let msg = format_error_body(StatusCode::BAD_REQUEST, body); + assert!(msg.contains("Not existing vector name"), "got: {msg}"); + assert!( + !msg.contains("EOF"), + "must not fall through to a parse error" + ); + } + + #[test] + fn error_message_survives_the_string_status_and_the_empty_body() { + let ok = format_error_body(StatusCode::OK, r#"{"status": "ok", "time": 0.0}"#); + assert!( + ok.contains("200"), + "no `status.error` present → fall back to status+body: {ok}" + ); + + let empty = format_error_body(StatusCode::NOT_FOUND, ""); + + assert!(empty.contains("empty body"), "got: {empty}"); + assert!( + empty.contains("verb"), + "the message must point at the likely cause: {empty}" + ); + } + + #[test] + fn vector_dimension_handles_both_collection_shapes() { + let unnamed = serde_json::json!({ + "result": {"config": {"params": {"vectors": {"size": 1536, "distance": "Cosine"}}}} + }); + assert_eq!(vector_dimension_from_collection(&unnamed).unwrap(), 1536); + + let named = serde_json::json!({ + "result": {"config": {"params": {"vectors": {"text": {"size": 768, "distance": "Cosine"}}}}} + }); + assert_eq!(vector_dimension_from_collection(&named).unwrap(), 768); + + let junk = serde_json::json!({"result": {"config": {"params": {}}}}); + assert!(vector_dimension_from_collection(&junk).is_err()); + } + + #[test] + fn is_multi_vector_rejects_the_named_single_collection() { + let unnamed = serde_json::json!({ + "result": {"config": {"params": {"vectors": {"size": 1536, "distance": "Cosine"}}}} + }); + assert!(!is_multi_vector_config(&unnamed)); + + let named_single = serde_json::json!({ + "result": {"config": {"params": {"vectors": {"text": {"size": 1536}}}}} + }); + assert!( + is_multi_vector_config(&named_single), + "named-single must be rejected too" + ); + + let named_multi = serde_json::json!({ + "result": {"config": {"params": {"vectors": {"text": {"size": 1536}, "image": {"size": 512}}}}} + }); + assert!(is_multi_vector_config(&named_multi)); + } + + #[test] + fn normalize_base_url_only_adds_a_scheme_when_missing() { + assert_eq!( + QdrantProvider::normalize_base_url("qdrant.example.com:6333"), + "http://qdrant.example.com:6333" + ); + assert_eq!( + QdrantProvider::normalize_base_url("https://xyz.cloud.qdrant.io"), + "https://xyz.cloud.qdrant.io" + ); + assert_eq!( + QdrantProvider::normalize_base_url("http://localhost:6333"), + "http://localhost:6333" + ); + } + + #[tokio::test] + async fn rebuild_indexes_refuses_for_attached_and_unattached_alike() { + let mut provider = QdrantProvider { + client: Client::new(), + base_url: "http://localhost:6333".to_string(), + collection: "c".to_string(), + point_ids: Arc::default(), + }; + + let attached = RagData { + driver: "qdrant".to_string(), + attached: true, + ..Default::default() + }; + let err = provider + .rebuild_indexes(&attached, true) + .await + .expect_err("an attached qdrant RAG must never report a successful rebuild"); + assert!(err.to_string().contains("cannot rebuild"), "got: {err}"); + + // `attached: false` is reserved for the (unimplemented) write path. It must + // also refuse: silently succeeding would run a full paid embedding pass and + // then discard every vector. + let owned = RagData { + driver: "qdrant".to_string(), + attached: false, + ..Default::default() + }; + let err = provider + .rebuild_indexes(&owned, true) + .await + .expect_err("writing to qdrant is unimplemented and must fail loudly"); + assert!(err.to_string().contains("not supported yet"), "got: {err}"); + } + + #[tokio::test] + async fn fetch_content_short_circuits_on_an_empty_id_list() { + let provider = QdrantProvider { + client: Client::new(), + base_url: "http://127.0.0.1:1".to_string(), + collection: "c".to_string(), + point_ids: Arc::default(), + }; + + assert!(provider.fetch_content(&[]).await.unwrap().is_empty()); + } + + #[test] + fn local_and_private_hosts_skip_the_proxy() { + for host in [ + "http://localhost:6333", + "http://127.0.0.1:6333", + "http://192.168.0.56:6333", + "http://10.1.2.3:6333", + "http://172.16.4.5:6333", + "http://qdrant.local:6333", + "http://[::1]:6333", + ] { + assert!( + QdrantProvider::skips_proxy(host), + "{host} should not be proxied" + ); + } + } + + #[test] + fn public_hosts_still_honour_the_environment() { + for host in [ + "https://qdrant.example.com", + "http://8.8.8.8:6333", + "https://xyz.eu-central.aws.cloud.qdrant.io:6333", + "http://172.32.0.1:6333", + ] { + assert!( + !QdrantProvider::skips_proxy(host), + "{host} must keep the environment's proxy" + ); + } + } + + /// Euclid collections score by NEGATIVE distance, so the 0.0 the caller + /// passes must mean "no floor". Filtering on it drops every hit — the exact + /// bug that keeps Qdrant's own `score_threshold` off the wire. + #[test] + fn a_zero_floor_keeps_negative_euclid_scores() { + let mut interner = PointIdInterner::default(); + let search = serde_json::json!({ + "result": [ + {"id": 1, "score": -0.12}, + {"id": 2, "score": -8.5}, + ] + }); + + let hits = parse_search_hits(&mut interner, &search, 0.0).unwrap(); + + assert_eq!(hits.len(), 2, "a 0.0 floor must not drop negative scores"); + } + + #[test] + fn a_positive_floor_still_filters() { + let mut interner = PointIdInterner::default(); + let search = serde_json::json!({ + "result": [ + {"id": 1, "score": 0.9}, + {"id": 2, "score": 0.2}, + ] + }); + + let hits = parse_search_hits(&mut interner, &search, 0.5).unwrap(); + + assert_eq!(hits.len(), 1); + assert_eq!(hits[0].0, DocumentId(1)); + } + + /// A UUID-keyed collection has to survive the whole `vector_search` → + /// `fetch_content` round trip, and the fetch must ask Qdrant for the ORIGINAL + /// string id. Parsing ids with `as_u64()` used to drop these hits inside a + /// `filter_map`, i.e. zero results and no error. + #[test] + fn uuid_point_ids_round_trip_and_are_requested_verbatim() { + let mut interner = PointIdInterner::default(); + let first_uuid = "3f1b0c2e-1111-4000-8000-000000000001"; + let second_uuid = "3f1b0c2e-2222-4000-8000-000000000002"; + + let search = serde_json::json!({ + "result": [ + {"id": first_uuid, "score": 0.91}, + {"id": second_uuid, "score": 0.42}, + ] + }); + let hits = parse_search_hits(&mut interner, &search, 0.0).unwrap(); + assert_eq!(hits.len(), 2, "string ids must not be silently dropped"); + + let ids: Vec = hits.iter().map(|(id, _)| *id).collect(); + assert_eq!( + interner.outbound_ids(&ids), + vec![Value::from(first_uuid), Value::from(second_uuid)], + "the fetch must send the ids Qdrant issued, not the handles" + ); + + // Qdrant may answer /points in any order; the handles still map back and + // the caller's RRF ranking is recoverable. + let points = serde_json::json!({ + "result": [ + {"id": second_uuid, "payload": {"page_content": "second"}}, + {"id": first_uuid, "payload": {"page_content": "first"}}, + ] + }); + let mut rows = parse_points(&mut interner, &points).unwrap(); + let position: HashMap = + ids.iter().enumerate().map(|(i, id)| (*id, i)).collect(); + rows.sort_by_key(|(id, _)| position.get(id).copied().unwrap_or(usize::MAX)); + assert_eq!( + rows, + vec![ + (ids[0], "first".to_string()), + (ids[1], "second".to_string()) + ] + ); + } + + /// Integer-keyed collections must be untouched by the interner: the id maps to + /// itself on the way in and goes back out as the same integer. + #[test] + fn integer_point_ids_are_passed_through_untouched() { + let mut interner = PointIdInterner::default(); + let search = serde_json::json!({ + "result": [{"id": 7, "score": 0.9}, {"id": 0, "score": 0.5}] + }); + + let hits = parse_search_hits(&mut interner, &search, 0.0).unwrap(); + assert_eq!( + hits, + vec![(DocumentId(7), 0.9_f32), (DocumentId(0), 0.5_f32)] + ); + + let ids: Vec = hits.iter().map(|(id, _)| *id).collect(); + assert_eq!( + interner.outbound_ids(&ids), + vec![Value::from(7_u64), Value::from(0_u64)], + "integer ids must not be regressed into synthetic handles" + ); + assert!( + interner.raw_id(DocumentId(7)).is_none(), + "a plain integer id is its own id and needs no map entry" + ); + } + + /// Synthetic handles are stable per point id and live in a range no packed + /// `DocumentId` can reach. + #[test] + fn synthetic_handles_are_stable_and_never_collide_with_packed_ids() { + let mut interner = PointIdInterner::default(); + let uuid = Value::from("9d2f0a11-3333-4000-8000-00000000000a"); + + let handle = interner.document_id(&uuid).unwrap(); + assert_eq!( + interner.document_id(&uuid).unwrap(), + handle, + "the same point id must keep the same handle across queries" + ); + assert_ne!( + interner.document_id(&Value::from("other")).unwrap(), + handle, + "distinct point ids must not share a handle" + ); + assert_ne!(handle.0 & SYNTHETIC_ID_TAG, 0, "a handle carries the tag"); + + // A packed (file_index, document_index) never sets the tag bit: it is the + // top bit of the file index, which would take 2^31 indexed files. + for (file_index, document_index) in [(0, 0), (1, 0), (0, 4242), (1_000_000, 999)] { + assert_eq!( + DocumentId::new(file_index, document_index).0 & SYNTHETIC_ID_TAG, + 0, + "packed ({file_index}, {document_index}) must stay out of the handle range" + ); + } + + // The one integer id that WOULD land on the tag is interned instead of + // being handed back as itself, so it cannot alias a handle. + let collides = Value::from(SYNTHETIC_ID_TAG as u64); + let interned = interner.document_id(&collides).unwrap(); + assert_eq!(interner.raw_id(interned), Some(&collides)); + assert_eq!( + interner.outbound_ids(&[interned]), + vec![collides], + "the original integer must still be what Qdrant is asked for" + ); + } + + /// `duplicate()` shares the map rather than resetting it: `Rag::clone()` hands + /// the clone `DocumentId`s the original minted, and a fresh map would turn + /// those into requests for a synthetic handle — zero results, no error. + #[test] + fn duplicate_shares_the_point_id_map() { + let provider = QdrantProvider { + client: Client::new(), + base_url: "http://127.0.0.1:1".to_string(), + collection: "c".to_string(), + point_ids: Arc::default(), + }; + let uuid = Value::from("c0ffee00-4444-4000-8000-000000000007"); + let handle = provider.point_ids.write().document_id(&uuid).unwrap(); + + let dup = provider.duplicate(&RagData { + driver: "qdrant".to_string(), + attached: true, + ..Default::default() + }); + // Downcasting is not available through `dyn RagProvider`, so go via the + // shared Arc: the clone must observe the original's interning. + assert_eq!(Arc::strong_count(&provider.point_ids), 2); + assert_eq!(provider.point_ids.read().raw_id(handle), Some(&uuid)); + drop(dup); + } + + #[tokio::test] + #[ignore] + async fn qdrant_list_collections_requires_running_instance() { + let collections = QdrantProvider::list_collections("http://localhost:6333", None) + .await + .unwrap(); + + assert!(!collections.is_empty()); + } + + #[tokio::test] + #[ignore] + async fn qdrant_vector_search_returns_results() { + let provider = QdrantProvider::new("http://localhost:6333", "test-collection", None) + .await + .unwrap(); + let embedding = vec![0.0f32; 1536]; + + let results = provider.vector_search(&embedding, 5, 0.0).await.unwrap(); + + assert!(results.len() <= 5); + } +} diff --git a/src/rag/providers/yaml.rs b/src/rag/providers/yaml.rs new file mode 100644 index 0000000..e56b396 --- /dev/null +++ b/src/rag/providers/yaml.rs @@ -0,0 +1,219 @@ +use crate::rag::provider::RagProvider; +use crate::rag::{DocumentId, RagData}; + +use anyhow::Result; +use async_trait::async_trait; +use hnsw_rs::prelude::*; +use indexmap::IndexMap; + +pub struct YamlProvider { + hnsw: Hnsw<'static, f32, DistCosine>, + content_map: IndexMap, +} + +impl YamlProvider { + pub fn from_data(data: &RagData) -> Self { + Self { + hnsw: data.build_hnsw(), + content_map: Self::build_content_map(data), + } + } + + fn build_content_map(data: &RagData) -> IndexMap { + data.iter_documents() + .map(|(id, doc)| (id, doc.page_content.clone())) + .collect() + } +} + +#[async_trait] +impl RagProvider for YamlProvider { + async fn vector_search( + &self, + embedding: &[f32], + top_k: usize, + min_score: f32, + ) -> Result> { + let results = self + .hnsw + .parallel_search(&[embedding.to_vec()], top_k, 30) + .into_iter() + .flat_map(|list| { + list.into_iter().filter_map(|v| { + let score = 1.0 - v.distance; + if score > min_score { + Some((DocumentId(v.d_id), score)) + } else { + None + } + }) + }) + .collect(); + + Ok(results) + } + + async fn fetch_content(&self, ids: &[DocumentId]) -> Result> { + Ok(ids + .iter() + .filter_map(|id| self.content_map.get(id).map(|text| (*id, text.clone()))) + .collect()) + } + + async fn rebuild_indexes(&mut self, data: &RagData, _full_rebuild: bool) -> Result<()> { + self.hnsw = data.build_hnsw(); + + self.content_map = Self::build_content_map(data); + + Ok(()) + } + + fn duplicate(&self, data: &RagData) -> Box { + Box::new(YamlProvider::from_data(data)) + } +} + +#[cfg(test)] +mod provider_tests { + use super::*; + use crate::rag::{RagDocument, RagFile}; + + fn minimal_rag_data() -> RagData { + RagData { + embedding_model: "text-embedding-3-small".to_string(), + chunk_size: 1024, + chunk_overlap: 50, + top_k: 5, + driver: "yaml".to_string(), + attached: false, + ..Default::default() + } + } + + /// Two files, one chunk each, with vectors, the minimum needed to exercise + /// `build_content_map` and the `fetch_content` ordering contract. + /// `DocumentId::new(f, d)` packs (file_index, document_index); `RagData::add` + /// is the real insertion path but a direct literal is sufficient and avoids + /// the embedding pipeline. + fn populated_rag_data() -> RagData { + let mut data = minimal_rag_data(); + // `files` must be populated: build_content_map iterates data.iter_documents(), + // which enumerates `files`. Populating `vectors` alone would produce an EMPTY + // content map, and every assertion below would vacuously pass on a broken impl. + // The vectors inserted at the end are for the HNSW side only. + data.files.insert( + 0, + RagFile { + hash: "h0".to_string(), + path: "/tmp/a.md".to_string(), + documents: vec![RagDocument { + page_content: "alpha".to_string(), + metadata: Default::default(), + }], + }, + ); + data.files.insert( + 1, + RagFile { + hash: "h1".to_string(), + path: "/tmp/b.md".to_string(), + documents: vec![RagDocument { + page_content: "beta".to_string(), + metadata: Default::default(), + }], + }, + ); + data.vectors + .insert(DocumentId::new(0, 0), vec![1.0, 0.0, 0.0]); + data.vectors + .insert(DocumentId::new(1, 0), vec![0.0, 1.0, 0.0]); + data + } + + #[tokio::test] + async fn yaml_provider_empty_data_returns_nothing() { + let data = minimal_rag_data(); + let provider = YamlProvider::from_data(&data); + + let results = provider.fetch_content(&[]).await.unwrap(); + + assert!(results.is_empty()); + } + + #[tokio::test] + async fn yaml_provider_fetch_content_preserves_input_order() { + let data = populated_rag_data(); + let provider = YamlProvider::from_data(&data); + + let a = DocumentId::new(0, 0); + let b = DocumentId::new(1, 0); + + let forward = provider.fetch_content(&[a, b]).await.unwrap(); + assert_eq!(forward.len(), 2, "both documents must resolve"); + assert_eq!(forward[0].1, "alpha"); + assert_eq!(forward[1].1, "beta"); + + let reversed = provider.fetch_content(&[b, a]).await.unwrap(); + + assert_eq!( + reversed[0].1, "beta", + "fetch_content must honor input order" + ); + assert_eq!(reversed[1].1, "alpha"); + } + + #[tokio::test] + async fn yaml_provider_fetch_content_skips_missing_ids() { + let data = populated_rag_data(); + let provider = YamlProvider::from_data(&data); + + let a = DocumentId::new(0, 0); + let missing = DocumentId::new(99, 0); + let b = DocumentId::new(1, 0); + + let out = provider.fetch_content(&[a, missing, b]).await.unwrap(); + assert_eq!(out.len(), 2, "missing id is skipped, not an error"); + assert_eq!(out[0].1, "alpha"); + assert_eq!(out[1].1, "beta"); + } + + #[tokio::test] + async fn yaml_provider_duplicate_returns_equivalent_content() { + let data = populated_rag_data(); + let provider = YamlProvider::from_data(&data); + let dup = provider.duplicate(&data); + let ids = [DocumentId::new(0, 0), DocumentId::new(1, 0)]; + + let r1 = provider.fetch_content(&ids).await.unwrap(); + let r2 = dup.fetch_content(&ids).await.unwrap(); + + assert_eq!(r1.len(), 2, "fixture must resolve both documents"); + assert_eq!( + r1, r2, + "duplicate must resolve the same content as the original" + ); + } + + #[tokio::test] + async fn yaml_provider_content_is_keyed_on_files_not_vectors() { + let mut data = populated_rag_data(); + let orphan = DocumentId::new(9, 0); + data.vectors.insert(orphan, vec![0.0, 0.0, 1.0]); + + let provider = YamlProvider::from_data(&data); + + let out = provider.fetch_content(&[orphan]).await.unwrap(); + assert!( + out.is_empty(), + "an id present only in `vectors` must not resolve to content" + ); + + let real = provider + .fetch_content(&[DocumentId::new(0, 0), DocumentId::new(1, 0)]) + .await + .unwrap(); + assert_eq!(real.len(), 2, "file-backed documents must still resolve"); + assert_eq!(real[0].1, "alpha"); + assert_eq!(real[1].1, "beta"); + } +} diff --git a/src/render/markdown.rs b/src/render/markdown.rs index 5a1a975..be54fa0 100644 --- a/src/render/markdown.rs +++ b/src/render/markdown.rs @@ -1749,6 +1749,86 @@ std::error::Error>> { ); } + fn strip_ansi(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut chars = text.chars(); + while let Some(c) = chars.next() { + if c != '\u{1b}' { + out.push(c); + continue; + } + if chars.next() == Some('[') { + for c in chars.by_ref() { + if ('\u{40}'..='\u{7e}').contains(&c) { + break; + } + } + } + } + out + } + + #[test] + fn strip_ansi_removes_sgr_and_keeps_text() { + assert_eq!(strip_ansi("\x1b[1mbold\x1b[0m"), "bold"); + assert_eq!(strip_ansi("\x1b[38;5;120mx\x1b[39m"), "x"); + assert_eq!(strip_ansi("plain"), "plain"); + } + + #[test] + fn render_table_pads_columns_by_display_width() { + use unicode_width::UnicodeWidthStr; + + const WRAP_WIDTH: u16 = 80; + + let options = RenderOptions::default(); + let mut render = MarkdownRender::init(options).unwrap(); + render.wrap_width = Some(WRAP_WIDTH); + + let header = vec![ + "**Setting**".into(), + "*Default*".into(), + "`Description`".into(), + ]; + let alignments = vec![ + CellAlignment::Left, + CellAlignment::Right, + CellAlignment::Center, + ]; + let rows = vec![ + vec![ + "**temperature**".into(), + "`0.7`".into(), + "Controls how *random* the sampled reply is allowed to be".into(), + ], + vec![ + "**top_p**".into(), + "`1.0`".into(), + "Nucleus sampling cutoff, applied **after** temperature".into(), + ], + ]; + + let output = render.render_table(header, alignments, rows); + + assert!( + output.contains('\u{1b}'), + "fixture must actually contain ANSI escapes: {output:?}", + ); + + let widths: Vec = output + .lines() + .map(|line| strip_ansi(line).width()) + .collect(); + assert!(!widths.is_empty(), "table rendered no lines"); + + for (index, width) in widths.iter().enumerate() { + assert_eq!( + *width, WRAP_WIDTH as usize, + "line {index} display width; all widths were {widths:?} in output:\n{output}", + ); + } + } + #[test] fn state_machine_renders_full_table_and_flushes_on_paragraph() { let options = RenderOptions::default(); diff --git a/src/repl/mod.rs b/src/repl/mod.rs index c33d067..7ade0a6 100644 --- a/src/repl/mod.rs +++ b/src/repl/mod.rs @@ -53,7 +53,7 @@ pub const DEFAULT_CONTINUATION_PROMPT: &str = indoc! {" 4. Continue with the next pending item now. Call tools immediately." }; -static REPL_COMMANDS: LazyLock<[ReplCommand; 59]> = LazyLock::new(|| { +static REPL_COMMANDS: LazyLock<[ReplCommand; 60]> = LazyLock::new(|| { [ ReplCommand::new(".help", "Show this help guide", AssertState::pass()), ReplCommand::new(".info", "Show system info", AssertState::pass()), @@ -217,6 +217,11 @@ static REPL_COMMANDS: LazyLock<[ReplCommand; 59]> = LazyLock::new(|| { "Initialize or access RAG", AssertState::False(StateFlags::AGENT), ), + ReplCommand::new( + ".rag attach", + "Attach to a pre-existing external RAG", + AssertState::False(StateFlags::AGENT), + ), ReplCommand::new( ".edit rag-docs", "Add or remove documents from an existing RAG", @@ -884,9 +889,17 @@ pub async fn run_repl_command( let version = args.map(|s| s.trim().to_string()); task::spawn_blocking(move || config::run_self_update(version, false)).await??; } - ".rag" => { - ctx.use_rag(args, abort_signal.clone()).await?; - } + ".rag" => match split_first_arg(args) { + Some(("attach", rest)) => match rest { + Some(name) if !name.trim().is_empty() => { + ctx.attach_rag(name.trim()).await?; + } + _ => println!("Usage: .rag attach "), + }, + _ => { + ctx.use_rag(args, abort_signal.clone()).await?; + } + }, ".agent" => match split_first_arg(args) { Some((agent_name, args)) => { let (new_args, _) = split_args_text(args.unwrap_or_default(), cfg!(windows)); @@ -1711,8 +1724,8 @@ mod tests { } #[test] - fn repl_commands_has_59_entries() { - assert_eq!(REPL_COMMANDS.len(), 59); + fn repl_commands_has_60_entries() { + assert_eq!(REPL_COMMANDS.len(), 60); } #[test] diff --git a/src/sandbox/mcp_credentials.rs b/src/sandbox/mcp_credentials.rs index 0132e21..8e73be3 100644 --- a/src/sandbox/mcp_credentials.rs +++ b/src/sandbox/mcp_credentials.rs @@ -387,7 +387,7 @@ pub(crate) fn collect_server_allow_entries( out.into_iter().collect() } -fn allow_entry_for_url(raw: &str) -> Option { +pub(crate) fn allow_entry_for_url(raw: &str) -> Option { let url = Url::parse(raw).ok()?; let scheme = url.scheme(); if scheme != "https" && scheme != "http" { @@ -426,8 +426,8 @@ fn placeholders(text: &str) -> Result> { struct CredentialsMixin { schema_version: &'static str, kind: &'static str, - name: &'static str, - description: &'static str, + name: String, + description: String, #[serde(skip_serializing_if = "Vec::is_empty")] credentials: Vec, #[serde(skip_serializing_if = "Option::is_none")] @@ -435,20 +435,20 @@ struct CredentialsMixin { } #[derive(Serialize)] -struct CredentialEntry { - service: String, - description: String, +pub(crate) struct CredentialEntry { + pub service: String, + pub description: String, #[serde(rename = "apiKey")] - api_key: ApiKey, + pub api_key: ApiKey, } #[derive(Serialize)] #[serde(rename_all = "camelCase")] -struct ApiKey { - name: String, - proxy_managed: bool, +pub(crate) struct ApiKey { + pub name: String, + pub proxy_managed: bool, #[serde(skip_serializing_if = "Vec::is_empty")] - inject: Vec, + pub inject: Vec, } #[derive(Serialize)] @@ -461,40 +461,24 @@ struct Network { allow: Vec, } -pub(crate) fn render_mixin_yaml( - credentials: &[CredentialSpec], - server_allow_entries: &[String], +pub(crate) fn render_mixin_document( + name: &str, + description: &str, + credentials: Vec, + extra_allow_entries: &[String], ) -> Result { let mut allow: BTreeSet = credentials .iter() - .flat_map(|c| c.inject.iter().map(|r| r.domain.clone())) + .flat_map(|c| c.api_key.inject.iter().map(|r| r.domain.clone())) .collect(); - allow.extend(server_allow_entries.iter().cloned()); + allow.extend(extra_allow_entries.iter().cloned()); let mixin = CredentialsMixin { schema_version: "2", kind: "mixin", - name: MCP_MIXIN_NAME, - description: "Auto-generated by Coyote at launch: allows network egress to the user's \ - remote MCP servers and declares their credentials so Docker Sandboxes \ - binds them (bindings are approved on first interactive run). Values are \ - pre-seeded from Coyote's vault via `sbx secret set`.", - credentials: credentials - .iter() - .map(|c| CredentialEntry { - service: c.service_id.clone(), - description: format!( - "Coyote vault secret '{}', used by MCP server(s) {}", - c.secret_name, - quoted_list(&c.servers) - ), - api_key: ApiKey { - name: c.env_var.clone(), - proxy_managed: c.proxy_managed, - inject: c.inject.clone(), - }, - }) - .collect(), + name: name.to_string(), + description: description.to_string(), + credentials, permissions: (!allow.is_empty()).then(|| Permissions { network: Network { allow: allow.into_iter().collect(), @@ -502,7 +486,39 @@ pub(crate) fn render_mixin_yaml( }), }; - serde_yaml::to_string(&mixin).context("Failed to serialize generated MCP credentials mixin") + serde_yaml::to_string(&mixin).context("Failed to serialize generated sandbox mixin") +} + +pub(crate) fn render_mixin_yaml( + credentials: &[CredentialSpec], + server_allow_entries: &[String], +) -> Result { + let entries = credentials + .iter() + .map(|c| CredentialEntry { + service: c.service_id.clone(), + description: format!( + "Coyote vault secret '{}', used by MCP server(s) {}", + c.secret_name, + quoted_list(&c.servers) + ), + api_key: ApiKey { + name: c.env_var.clone(), + proxy_managed: c.proxy_managed, + inject: c.inject.clone(), + }, + }) + .collect(); + + render_mixin_document( + MCP_MIXIN_NAME, + "Auto-generated by Coyote at launch: allows network egress to the user's remote MCP \ + servers and declares their credentials so Docker Sandboxes binds them (bindings are \ + approved on first interactive run). Values are pre-seeded from Coyote's vault via \ + `sbx secret set`.", + entries, + server_allow_entries, + ) } #[cfg(test)] diff --git a/src/sandbox/mixins.rs b/src/sandbox/mixins.rs index e25c664..3123062 100644 --- a/src/sandbox/mixins.rs +++ b/src/sandbox/mixins.rs @@ -10,6 +10,7 @@ use sha2::{Digest, Sha256}; use crate::config::paths; const SBX_MIXIN_FILE_NAME: &str = "sbx-mixin.yaml"; +const SBX_MIXIN_FILE_SUFFIX: &str = ".sbx-mixin.yaml"; const KIT_SPEC_FILE_NAME: &str = "spec.yaml"; #[derive(Debug, Clone)] @@ -66,10 +67,16 @@ pub fn discover() -> Result> { push_if_exists(&mut out, paths::sbx_mixin_file())?; push_if_exists(&mut out, paths::global_tools_sbx_mixin_file())?; - for path in collect_subdir_mixins(&paths::functions_dir()) { + for path in collect_mixins(&paths::functions_dir(), &[ScanMode::SubdirNamed]) { out.push(read_mixin(path)?); } - for path in collect_subdir_mixins(&paths::agents_data_dir()) { + for path in collect_mixins( + &paths::agents_data_dir(), + &[ScanMode::SubdirNamed, ScanMode::SubdirFlat], + ) { + out.push(read_mixin(path)?); + } + for path in collect_mixins(&paths::rags_dir(), &[ScanMode::Flat]) { out.push(read_mixin(path)?); } @@ -156,7 +163,73 @@ fn read_mixin(path: PathBuf) -> Result { }) } -fn collect_subdir_mixins(dir: &Path) -> Vec { +/// One on-disk layout a mixin scan can look for. A scan takes a set of these, +/// and each mode contributes only the shape it names. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ScanMode { + /// `/*.sbx-mixin.yaml` + Flat, + /// `/*/sbx-mixin.yaml` + SubdirNamed, + /// `/*/*.sbx-mixin.yaml` + SubdirFlat, +} + +/// Collects mixin paths under `dir` for every requested layout. Missing or +/// unreadable directories yield nothing rather than an error — these paths are +/// all optional on disk. +/// +/// Order is deterministic: flat matches first (sorted by file name), then each +/// subdirectory in sorted order, contributing its named mixin before its +/// suffixed ones. +fn collect_mixins(dir: &Path, modes: &[ScanMode]) -> Vec { + let mut result = Vec::new(); + + if modes.contains(&ScanMode::Flat) { + result.extend(suffixed_mixins_in(dir)); + } + + let named = modes.contains(&ScanMode::SubdirNamed); + let subdir_flat = modes.contains(&ScanMode::SubdirFlat); + if !named && !subdir_flat { + return result; + } + + for subdir in subdirs_of(dir) { + if named { + let candidate = subdir.join(SBX_MIXIN_FILE_NAME); + if candidate.exists() { + result.push(candidate); + } + } + if subdir_flat { + result.extend(suffixed_mixins_in(&subdir)); + } + } + + result +} + +fn suffixed_mixins_in(dir: &Path) -> Vec { + let mut result = Vec::new(); + let Ok(rd) = read_dir(dir) else { return result }; + + let mut entries: Vec<_> = rd + .flatten() + .filter(|e| e.file_type().map(|t| t.is_file()).unwrap_or(false)) + .filter(|e| { + e.file_name() + .to_str() + .is_some_and(|n| n.ends_with(SBX_MIXIN_FILE_SUFFIX)) + }) + .collect(); + entries.sort_by_key(|e| e.file_name()); + + result.extend(entries.into_iter().map(|e| e.path())); + result +} + +fn subdirs_of(dir: &Path) -> Vec { let mut result = Vec::new(); let Ok(rd) = read_dir(dir) else { return result }; @@ -166,13 +239,7 @@ fn collect_subdir_mixins(dir: &Path) -> Vec { .collect(); entries.sort_by_key(|e| e.file_name()); - for entry in entries { - let candidate = entry.path().join(SBX_MIXIN_FILE_NAME); - if candidate.exists() { - result.push(candidate); - } - } - + result.extend(entries.into_iter().map(|e| e.path())); result } @@ -192,6 +259,13 @@ mod tests { root } + fn file_names(paths: &[PathBuf]) -> Vec<&str> { + paths + .iter() + .map(|p| p.file_name().unwrap().to_str().unwrap()) + .collect() + } + #[test] fn summarize_counts_installs_and_domains() { let root = unique_root("sbx-mixin-counts"); @@ -275,7 +349,7 @@ network: } #[test] - fn collect_subdir_mixins_sorts_and_skips_missing() { + fn subdir_named_scan_sorts_and_skips_missing() { let root = unique_root("sbx-mixin-subdirs"); for name in ["zebra", "apple", "no-mixin", "mango"] { let dir = root.join(name); @@ -285,7 +359,7 @@ network: } } - let found = collect_subdir_mixins(&root); + let found = collect_mixins(&root, &[ScanMode::SubdirNamed]); let names: Vec = found .iter() .map(|p| { @@ -303,9 +377,9 @@ network: } #[test] - fn collect_subdir_mixins_returns_empty_for_missing_dir() { + fn subdir_named_scan_returns_empty_for_missing_dir() { let absent = env::temp_dir().join("coyote-definitely-not-here-xyz"); - let found = collect_subdir_mixins(&absent); + let found = collect_mixins(&absent, &[ScanMode::SubdirNamed]); assert!(found.is_empty()); } @@ -483,4 +557,126 @@ network: ); } } + + #[test] + fn flat_scan_matches_rag_sidecars_by_suffix() { + let root = unique_root("flat-mixins"); + fs::write(root.join("company-docs.sbx-mixin.yaml"), "kind: mixin\n").unwrap(); + fs::write(root.join("alpha.sbx-mixin.yaml"), "kind: mixin\n").unwrap(); + fs::write(root.join("company-docs.yaml"), "driver: qdrant\n").unwrap(); + fs::write(root.join("notes.yaml"), "driver: yaml\n").unwrap(); + fs::create_dir_all(root.join("decoy.sbx-mixin.yaml")).unwrap(); + + let found = collect_mixins(&root, &[ScanMode::Flat]); + assert_eq!( + file_names(&found), + vec!["alpha.sbx-mixin.yaml", "company-docs.sbx-mixin.yaml"] + ); + + let _ = fs::remove_dir_all(&root); + } + + /// Every scan site in `discover()` picks its modes assuming each mode owns + /// exactly one layout and nothing else. `agents_data_dir()` requests two + /// modes at once, so an overlap would collect the same file twice and + /// `create_sandbox` would pass it as two `--kit` flags. + #[test] + fn each_scan_mode_owns_exactly_one_layout() { + let root = unique_root("scan-mode-ownership"); + let agent = root.join("researcher"); + fs::create_dir_all(&agent).unwrap(); + let flat = root.join("company-docs.sbx-mixin.yaml"); + let subdir_named = agent.join("sbx-mixin.yaml"); + let subdir_flat = agent.join("handbook.sbx-mixin.yaml"); + for path in [&flat, &subdir_named, &subdir_flat] { + fs::write(path, "kind: mixin\n").unwrap(); + } + + assert_eq!(collect_mixins(&root, &[ScanMode::Flat]), vec![flat.clone()]); + assert_eq!( + collect_mixins(&root, &[ScanMode::SubdirNamed]), + vec![subdir_named.clone()] + ); + assert_eq!( + collect_mixins(&root, &[ScanMode::SubdirFlat]), + vec![subdir_flat.clone()] + ); + + let all = collect_mixins( + &root, + &[ScanMode::Flat, ScanMode::SubdirNamed, ScanMode::SubdirFlat], + ); + assert_eq!(all, vec![flat, subdir_named, subdir_flat]); + + let mut deduped = all.clone(); + deduped.sort(); + deduped.dedup(); + assert_eq!( + deduped.len(), + all.len(), + "no mixin may be collected twice: {all:?}" + ); + + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn flat_scan_tolerates_a_missing_directory() { + let root = unique_root("flat-missing"); + let absent = root.join("nope"); + assert!(collect_mixins(&absent, &[ScanMode::Flat]).is_empty()); + + let _ = fs::remove_dir_all(&root); + } + + /// `generate_rag_sbx_mixin` writes an agent-scoped RAG sidecar next to the + /// rag yaml, at `//.sbx-mixin.yaml`. Before `SubdirFlat` + /// existed, nothing scanned that shape and attaching a Qdrant RAG from + /// inside an agent produced no network allow rule and no credential. + #[test] + fn agent_scoped_rag_sidecar_is_discovered() { + let root = unique_root("agent-scoped-rag"); + let agent = root.join("researcher"); + fs::create_dir_all(&agent).unwrap(); + fs::write(agent.join("company-docs.sbx-mixin.yaml"), "kind: mixin\n").unwrap(); + fs::write(agent.join("company-docs.yaml"), "driver: qdrant\n").unwrap(); + + let found = collect_mixins(&root, &[ScanMode::SubdirNamed, ScanMode::SubdirFlat]); + assert_eq!(found, vec![agent.join("company-docs.sbx-mixin.yaml")]); + + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn agent_level_mixin_and_rag_sidecars_are_both_discovered() { + let root = unique_root("agent-both-shapes"); + let agent = root.join("researcher"); + fs::create_dir_all(&agent).unwrap(); + fs::write(agent.join("sbx-mixin.yaml"), "kind: mixin\n").unwrap(); + fs::write(agent.join("zebra.sbx-mixin.yaml"), "kind: mixin\n").unwrap(); + fs::write(agent.join("alpha.sbx-mixin.yaml"), "kind: mixin\n").unwrap(); + + let found = collect_mixins(&root, &[ScanMode::SubdirNamed, ScanMode::SubdirFlat]); + assert_eq!( + file_names(&found), + vec![ + "sbx-mixin.yaml", + "alpha.sbx-mixin.yaml", + "zebra.sbx-mixin.yaml" + ] + ); + + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn subdir_flat_scan_ignores_a_directory_named_like_a_mixin() { + let root = unique_root("subdir-flat-decoy"); + let agent = root.join("researcher"); + fs::create_dir_all(agent.join("decoy.sbx-mixin.yaml")).unwrap(); + + assert!(collect_mixins(&root, &[ScanMode::SubdirFlat]).is_empty()); + + let _ = fs::remove_dir_all(&root); + } } diff --git a/src/sandbox/mod.rs b/src/sandbox/mod.rs index cf7ad85..e7b0b67 100644 --- a/src/sandbox/mod.rs +++ b/src/sandbox/mod.rs @@ -10,7 +10,7 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use which::which; -mod mcp_credentials; +pub(crate) mod mcp_credentials; mod mixins; pub(crate) use mcp_credentials::sandbox_secret_env_var; @@ -19,6 +19,7 @@ use crate::config::AppConfig; use crate::config::Config; use crate::config::VAULT_DATA_FILE_NAME; use crate::config::paths; +use crate::rag::RagData; use crate::sandbox::mcp_credentials::MCP_MIXIN_NAME; use crate::sandbox::mixins::DiscoveredMixin; use crate::utils::run_command_with_output; @@ -53,6 +54,10 @@ pub fn launch(name: Option, fresh: bool) -> Result<()> { let vault = Vault::init(&bootstrap)?; let registered = sbx_registered_services()?; inject_llm_secret(&config_content, &vault, ®istered)?; + if !fresh { + inject_rag_secrets(&vault, ®istered)?; + } + let credentials_mixin = if fresh { None } else { @@ -309,6 +314,87 @@ fn inject_mcp_secrets(vault: &Vault, registered: &HashSet) -> Result) -> Result<()> { + let rags_dir = paths::rags_dir(); + if !rags_dir.exists() { + return Ok(()); + } + for entry in fs::read_dir(&rags_dir)?.flatten() { + let path = entry.path(); + if path.extension().and_then(|e| e.to_str()) != Some("yaml") { + continue; + } + let stem = match path.file_stem().and_then(|s| s.to_str()) { + Some(s) if !paths::is_rag_sidecar_name(s) => s.to_string(), + _ => continue, + }; + let Ok(raw) = fs::read_to_string(&path) else { + continue; + }; + let Ok(data) = serde_yaml::from_str::(&raw) else { + continue; + }; + if !data.attached { + continue; + } + let secret_names = driver_config_secret_names(&data); + let Some((primary, extra)) = secret_names.split_first() else { + continue; + }; + + let service_id = mcp_credentials::secret_service_id(&stem); + if !service_id.is_empty() && !registered.contains(&service_id) { + bind_rag_secret(vault, &service_id, primary, &stem)?; + } + + for name in extra { + let id = mcp_credentials::secret_service_id(name); + if !id.is_empty() && !registered.contains(&id) { + bind_rag_secret(vault, &id, name, &stem)?; + } + } + } + + Ok(()) +} + +fn driver_config_secret_names(data: &RagData) -> Vec { + let mut names: Vec = Vec::new(); + for value in data.driver_config.values() { + let trimmed = value.trim(); + let Ok(Some(caps)) = SECRET_RE.captures(trimmed) else { + continue; + }; + if caps.get(0).map(|m| m.as_str()) != Some(trimmed) { + continue; + } + let Some(name) = caps.get(1).map(|m| m.as_str().trim()) else { + continue; + }; + if !name.is_empty() && !names.iter().any(|n| n == name) { + names.push(name.to_string()); + } + } + names +} + +fn bind_rag_secret(vault: &Vault, service_id: &str, secret_name: &str, stem: &str) -> Result<()> { + match vault.get_secret(secret_name, false) { + Ok(secret_value) => { + sbx_secret_set(service_id, &secret_value) + .context("Failed to register RAG secret with sbx")?; + } + Err(e) => { + eprintln!( + "Warning: could not load secret '{secret_name}' for RAG '{stem}': {e}. \ + Queries to this RAG will fail inside the sandbox. \ + Run `coyote --add-secret {secret_name}` to fix." + ); + } + } + Ok(()) +} + fn provider_to_sbx_service(provider_type: &str, client_name: Option<&str>) -> String { match provider_type { "claude" => "anthropic".to_string(), @@ -588,6 +674,64 @@ fn chown_agent_recursive(sandbox: &str, path: &str) -> Result<()> { #[cfg(test)] mod tests { use super::*; + + fn rag_with(driver_config: &[(&str, &str)]) -> RagData { + let mut data = RagData::new("m".into(), 1024, 50, None, 5, None, Default::default()); + data.driver = "qdrant".to_string(); + data.attached = true; + for (k, v) in driver_config { + data.driver_config.insert(k.to_string(), v.to_string()); + } + data + } + + #[test] + fn secret_names_are_found_whatever_the_field_is_called() { + let data = rag_with(&[ + ("host", "qdrant.example.com:6333"), + ("collection", "docs"), + ("token", "{{SOME_TOKEN}}"), + ]); + + assert_eq!(driver_config_secret_names(&data), vec!["SOME_TOKEN"]); + } + + #[test] + fn a_literal_credential_is_not_treated_as_a_secret_name() { + let data = rag_with(&[("api_key", "sk-a-real-looking-key")]); + + assert!(driver_config_secret_names(&data).is_empty()); + } + + #[test] + fn plain_values_are_never_mistaken_for_secrets() { + let data = rag_with(&[("host", "localhost:6333"), ("collection", "docs")]); + + assert!(driver_config_secret_names(&data).is_empty()); + } + + #[test] + fn a_partial_placeholder_is_not_a_credential() { + let data = rag_with(&[("api_key", "Bearer {{KEY}}")]); + + assert!(driver_config_secret_names(&data).is_empty()); + } + + #[test] + fn several_secrets_are_all_found_and_deduped() { + let data = rag_with(&[ + ("api_key", "{{QDRANT_KEY}}"), + ("host", "localhost:6333"), + ("token", "{{ OTHER_TOKEN }}"), + ("fallback_key", "{{QDRANT_KEY}}"), + ]); + + assert_eq!( + driver_config_secret_names(&data), + vec!["QDRANT_KEY", "OTHER_TOKEN"], + "order follows driver_config, and a repeat is not registered twice" + ); + } use std::time::{SystemTime, UNIX_EPOCH}; #[test]