mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-05 07:29:40 +00:00
Upgrade Qdrant Edge to 0.8.0 and make unreadable vector stores repairable (#982)
Build and Release / Determine run mode (push) Waiting to run
Build and Release / Read metadata (push) Blocked by required conditions
Build and Release / Sync Flatpak repo (push) Blocked by required conditions
Build and Release / Collect Flatpak artifacts (push) Blocked by required conditions
Build and Release / Verify (push) Waiting to run
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Determine run mode (push) Waiting to run
Build and Release / Read metadata (push) Blocked by required conditions
Build and Release / Sync Flatpak repo (push) Blocked by required conditions
Build and Release / Collect Flatpak artifacts (push) Blocked by required conditions
Build and Release / Verify (push) Waiting to run
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
This commit is contained in:
Thorsten Sommer authored and 
GitHub committed
1 parent
557d0b1409
commit
e9aaff5774
26 files changed
+750
-571
No files matched your search
Generated
+137
-410
@@ -113,6 +113,12 @@ version = "0.2.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
|
||||
|
||||
[[package]]
|
||||
name = "allocator-api2"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c880a97d28a3681c0267bd29cff89621202715b065127cd445fa0f0fe0aa2880"
|
||||
|
||||
[[package]]
|
||||
name = "android_system_properties"
|
||||
version = "0.1.5"
|
||||
@@ -884,6 +890,15 @@ dependencies = [
|
||||
"constant_time_eq 0.1.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "blink-alloc"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ce4c15bad517bc0fb4a44523adf470e2c3eb3a365769327acdba849948ea3705"
|
||||
dependencies = [
|
||||
"allocator-api2 0.4.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.10.4"
|
||||
@@ -952,14 +967,6 @@ dependencies = [
|
||||
"piper",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bm25"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
|
||||
dependencies = [
|
||||
"murmur3_32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "brotli"
|
||||
version = "8.0.2"
|
||||
@@ -1366,53 +1373,6 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "common"
|
||||
version = "0.0.0"
|
||||
source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"aligned-vec",
|
||||
"atomicwrites",
|
||||
"bincode 1.3.3",
|
||||
"bitvec",
|
||||
"bytemuck",
|
||||
"chrono",
|
||||
"fs-err",
|
||||
"fs4",
|
||||
"fs_extra",
|
||||
"io-uring",
|
||||
"itertools",
|
||||
"log",
|
||||
"memmap2",
|
||||
"nix 0.31.3",
|
||||
"num-traits",
|
||||
"num_cpus",
|
||||
"ordered-float 5.3.0",
|
||||
"parking_lot",
|
||||
"ph",
|
||||
"procfs",
|
||||
"quick_cache",
|
||||
"rand 0.10.2",
|
||||
"roaring",
|
||||
"schemars 0.8.22",
|
||||
"self_cell",
|
||||
"semver",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"slab",
|
||||
"strum",
|
||||
"tap",
|
||||
"tar",
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
"thread-priority",
|
||||
"tokio",
|
||||
"validator",
|
||||
"walkdir",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "compact_str"
|
||||
version = "0.9.1"
|
||||
@@ -1550,6 +1510,17 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core_affinity"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a034b3a7b624016c6e13f5df875747cc25f884156aad2abd12b6c46797971342"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"num_cpus",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpubits"
|
||||
version = "0.1.1"
|
||||
@@ -1810,20 +1781,6 @@ version = "2.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8"
|
||||
|
||||
[[package]]
|
||||
name = "dataset"
|
||||
version = "0.0.0"
|
||||
source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"flate2",
|
||||
"fs-err",
|
||||
"indicatif",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dbus"
|
||||
version = "0.9.7"
|
||||
@@ -2882,10 +2839,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"js-sys",
|
||||
"libc",
|
||||
"wasi 0.11.1+wasi-snapshot-preview1",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3046,42 +3001,6 @@ dependencies = [
|
||||
"system-deps",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gpu"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
|
||||
dependencies = [
|
||||
"log",
|
||||
"parking_lot",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gridstore"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bitvec",
|
||||
"bytemuck",
|
||||
"common",
|
||||
"dataset",
|
||||
"ecow",
|
||||
"fs-err",
|
||||
"itertools",
|
||||
"log",
|
||||
"lz4_flex",
|
||||
"parking_lot",
|
||||
"rand 0.10.2",
|
||||
"serde",
|
||||
"serde_cbor",
|
||||
"serde_json",
|
||||
"smallvec",
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gtk"
|
||||
version = "0.18.2"
|
||||
@@ -3195,7 +3114,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"allocator-api2",
|
||||
"allocator-api2 0.2.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3204,7 +3123,7 @@ version = "0.15.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
|
||||
dependencies = [
|
||||
"allocator-api2",
|
||||
"allocator-api2 0.2.21",
|
||||
"equivalent",
|
||||
"foldhash 0.1.5",
|
||||
]
|
||||
@@ -3215,7 +3134,7 @@ version = "0.16.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
|
||||
dependencies = [
|
||||
"allocator-api2",
|
||||
"allocator-api2 0.2.21",
|
||||
"equivalent",
|
||||
"foldhash 0.2.0",
|
||||
]
|
||||
@@ -3343,6 +3262,12 @@ version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
|
||||
|
||||
[[package]]
|
||||
name = "humantime"
|
||||
version = "2.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15"
|
||||
|
||||
[[package]]
|
||||
name = "hybrid-array"
|
||||
version = "0.4.12"
|
||||
@@ -3761,7 +3686,6 @@ checksum = "25470f23803092da7d239834776d653104d551bc4d7eacaf31e6837854b8e9eb"
|
||||
dependencies = [
|
||||
"console",
|
||||
"portable-atomic",
|
||||
"rayon",
|
||||
"unicode-width",
|
||||
"unit-prefix",
|
||||
"web-time",
|
||||
@@ -3885,6 +3809,15 @@ dependencies = [
|
||||
"either",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.15.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc"
|
||||
dependencies = [
|
||||
"either",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.11"
|
||||
@@ -4221,17 +4154,11 @@ dependencies = [
|
||||
"imgref",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lru-slab"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
|
||||
|
||||
[[package]]
|
||||
name = "lz4_flex"
|
||||
version = "0.13.1"
|
||||
version = "0.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e"
|
||||
checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226"
|
||||
|
||||
[[package]]
|
||||
name = "lzma-rust2"
|
||||
@@ -4258,16 +4185,6 @@ version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30"
|
||||
|
||||
[[package]]
|
||||
name = "macros"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "markup5ever"
|
||||
version = "0.38.0"
|
||||
@@ -4382,7 +4299,7 @@ dependencies = [
|
||||
"strum_macros",
|
||||
"symphonia",
|
||||
"sys-locale",
|
||||
"sysinfo 0.39.6",
|
||||
"sysinfo",
|
||||
"tauri",
|
||||
"tauri-build",
|
||||
"tauri-plugin-dialog",
|
||||
@@ -5291,7 +5208,7 @@ dependencies = [
|
||||
"console_error_panic_hook",
|
||||
"console_log",
|
||||
"image",
|
||||
"itertools",
|
||||
"itertools 0.14.0",
|
||||
"js-sys",
|
||||
"libloading 0.8.6",
|
||||
"log",
|
||||
@@ -5523,16 +5440,6 @@ dependencies = [
|
||||
"portable-atomic",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "posting_list"
|
||||
version = "0.0.0"
|
||||
source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
|
||||
dependencies = [
|
||||
"bitpacking",
|
||||
"common",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "powerfmt"
|
||||
version = "0.2.0"
|
||||
@@ -5757,25 +5664,97 @@ checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea"
|
||||
|
||||
[[package]]
|
||||
name = "qdrant-edge"
|
||||
version = "0.7.2"
|
||||
source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b8072302c87506a34bffec9bc16dbdcd36df8ab1321406b6e141530348c7e54"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"bm25",
|
||||
"common",
|
||||
"aligned-vec",
|
||||
"arrayvec 0.7.6",
|
||||
"atomic_refcell",
|
||||
"atomicwrites",
|
||||
"bincode 1.3.3",
|
||||
"bitpacking",
|
||||
"bitvec",
|
||||
"blink-alloc",
|
||||
"bytemuck",
|
||||
"byteorder",
|
||||
"cc",
|
||||
"cgroups-rs",
|
||||
"charabia",
|
||||
"chrono",
|
||||
"core_affinity",
|
||||
"crc32c",
|
||||
"data-encoding",
|
||||
"docopt",
|
||||
"duplicate",
|
||||
"ecow",
|
||||
"env_logger",
|
||||
"fnv",
|
||||
"fs-err",
|
||||
"itertools",
|
||||
"fs4",
|
||||
"fs_extra",
|
||||
"geo",
|
||||
"geohash",
|
||||
"half 2.7.1",
|
||||
"humantime",
|
||||
"indexmap 2.14.0",
|
||||
"integer-encoding",
|
||||
"io-uring",
|
||||
"itertools 0.15.0",
|
||||
"log",
|
||||
"lz4_flex",
|
||||
"macro_rules_attribute",
|
||||
"memmap2",
|
||||
"murmur3_32",
|
||||
"nix 0.31.3",
|
||||
"nom 8.0.0",
|
||||
"num-cmp",
|
||||
"num-derive",
|
||||
"num-traits",
|
||||
"num_cpus",
|
||||
"once_cell",
|
||||
"ordered-float 5.3.0",
|
||||
"parking_lot",
|
||||
"permutation_iterator",
|
||||
"ph",
|
||||
"procfs",
|
||||
"qdrant-rust-stemmers",
|
||||
"quick_cache",
|
||||
"rand 0.10.2",
|
||||
"segment",
|
||||
"rand_distr",
|
||||
"rayon",
|
||||
"rmp-serde",
|
||||
"roaring",
|
||||
"rustix 1.1.4",
|
||||
"schemars 0.8.22",
|
||||
"self_cell",
|
||||
"semver",
|
||||
"serde",
|
||||
"serde-untagged",
|
||||
"serde-value",
|
||||
"serde_cbor",
|
||||
"serde_json",
|
||||
"shard",
|
||||
"sparse",
|
||||
"serde_variant",
|
||||
"sha2 0.11.0",
|
||||
"siphasher",
|
||||
"slab",
|
||||
"smallvec",
|
||||
"strum",
|
||||
"sysinfo",
|
||||
"tap",
|
||||
"tar",
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
"thread-priority",
|
||||
"tinyvec",
|
||||
"tokio",
|
||||
"tonic",
|
||||
"uuid",
|
||||
"wal",
|
||||
"validator",
|
||||
"vaporetto",
|
||||
"walkdir",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5797,27 +5776,6 @@ dependencies = [
|
||||
"bytemuck",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quantization"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
|
||||
dependencies = [
|
||||
"arrayvec 0.7.6",
|
||||
"bytemuck",
|
||||
"cc",
|
||||
"common",
|
||||
"fs-err",
|
||||
"num-traits",
|
||||
"ordered-float 5.3.0",
|
||||
"parking_lot",
|
||||
"permutation_iterator",
|
||||
"rand 0.10.2",
|
||||
"rayon",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"strum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-error"
|
||||
version = "2.0.1"
|
||||
@@ -5854,73 +5812,16 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "quick_cache"
|
||||
version = "0.6.22"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d1c821816e9b928e20e92ed59bb3ac4aab321d16ca2316871c9fe7ca739cd477"
|
||||
checksum = "403c1a912fec895cafb223201e368234842acb9220aaf08ab042ae89ba5f135c"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"equivalent",
|
||||
"hashbrown 0.16.1",
|
||||
"foldhash 0.2.0",
|
||||
"hashbrown 0.17.0",
|
||||
"parking_lot",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quinn"
|
||||
version = "0.11.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"cfg_aliases",
|
||||
"pin-project-lite",
|
||||
"quinn-proto",
|
||||
"quinn-udp",
|
||||
"rustc-hash",
|
||||
"rustls",
|
||||
"socket2",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"web-time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quinn-proto"
|
||||
version = "0.11.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"bytes",
|
||||
"getrandom 0.4.2",
|
||||
"lru-slab",
|
||||
"rand 0.10.2",
|
||||
"rand_pcg",
|
||||
"ring",
|
||||
"rustc-hash",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"slab",
|
||||
"thiserror 2.0.18",
|
||||
"tinyvec",
|
||||
"tracing",
|
||||
"web-time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quinn-udp"
|
||||
version = "0.5.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd"
|
||||
dependencies = [
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"socket2",
|
||||
"tracing",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.45"
|
||||
@@ -6065,7 +5966,7 @@ dependencies = [
|
||||
"built",
|
||||
"cfg-if",
|
||||
"interpolate_name",
|
||||
"itertools",
|
||||
"itertools 0.14.0",
|
||||
"libc",
|
||||
"libfuzzer-sys",
|
||||
"log",
|
||||
@@ -6131,7 +6032,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f"
|
||||
dependencies = [
|
||||
"either",
|
||||
"itertools",
|
||||
"itertools 0.14.0",
|
||||
"rayon",
|
||||
]
|
||||
|
||||
@@ -6266,10 +6167,8 @@ checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2",
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
@@ -6280,7 +6179,6 @@ dependencies = [
|
||||
"log",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"quinn",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"rustls-platform-verifier",
|
||||
@@ -6526,7 +6424,6 @@ version = "1.14.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd"
|
||||
dependencies = [
|
||||
"web-time",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
@@ -6700,78 +6597,6 @@ dependencies = [
|
||||
"xxhash-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "segment"
|
||||
version = "0.6.0"
|
||||
source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"atomic_refcell",
|
||||
"atomicwrites",
|
||||
"bincode 1.3.3",
|
||||
"bitvec",
|
||||
"bytemuck",
|
||||
"byteorder",
|
||||
"cc",
|
||||
"cgroups-rs",
|
||||
"charabia",
|
||||
"chrono",
|
||||
"common",
|
||||
"data-encoding",
|
||||
"duplicate",
|
||||
"ecow",
|
||||
"fnv",
|
||||
"fs-err",
|
||||
"fs_extra",
|
||||
"geo",
|
||||
"geohash",
|
||||
"gpu",
|
||||
"gridstore",
|
||||
"half 2.7.1",
|
||||
"indexmap 2.14.0",
|
||||
"integer-encoding",
|
||||
"io-uring",
|
||||
"itertools",
|
||||
"log",
|
||||
"macro_rules_attribute",
|
||||
"macros",
|
||||
"memmap2",
|
||||
"nom 8.0.0",
|
||||
"num-cmp",
|
||||
"num-derive",
|
||||
"num-traits",
|
||||
"ordered-float 5.3.0",
|
||||
"parking_lot",
|
||||
"posting_list",
|
||||
"procfs",
|
||||
"qdrant-rust-stemmers",
|
||||
"quantization",
|
||||
"rand 0.10.2",
|
||||
"rayon",
|
||||
"roaring",
|
||||
"schemars 0.8.22",
|
||||
"self_cell",
|
||||
"serde",
|
||||
"serde-untagged",
|
||||
"serde-value",
|
||||
"serde_cbor",
|
||||
"serde_json",
|
||||
"serde_variant",
|
||||
"sha2 0.11.0",
|
||||
"smallvec",
|
||||
"sparse",
|
||||
"strum",
|
||||
"sysinfo 0.38.4",
|
||||
"tap",
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
"tinyvec",
|
||||
"uuid",
|
||||
"validator",
|
||||
"vaporetto",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "selectors"
|
||||
version = "0.36.1"
|
||||
@@ -7050,39 +6875,6 @@ dependencies = [
|
||||
"digest 0.11.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shard"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"chrono",
|
||||
"common",
|
||||
"fs-err",
|
||||
"fs4",
|
||||
"indexmap 2.14.0",
|
||||
"itertools",
|
||||
"log",
|
||||
"ordered-float 5.3.0",
|
||||
"parking_lot",
|
||||
"rand 0.10.2",
|
||||
"rmp-serde",
|
||||
"schemars 0.8.22",
|
||||
"segment",
|
||||
"serde",
|
||||
"serde_cbor",
|
||||
"serde_json",
|
||||
"smallvec",
|
||||
"sparse",
|
||||
"strum",
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
"tonic",
|
||||
"uuid",
|
||||
"validator",
|
||||
"wal",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shared_child"
|
||||
version = "1.0.0"
|
||||
@@ -7223,32 +7015,6 @@ dependencies = [
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sparse"
|
||||
version = "0.1.0"
|
||||
source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
|
||||
dependencies = [
|
||||
"bincode 1.3.3",
|
||||
"bitpacking",
|
||||
"common",
|
||||
"fs-err",
|
||||
"gridstore",
|
||||
"half 2.7.1",
|
||||
"itertools",
|
||||
"log",
|
||||
"memmap2",
|
||||
"ordered-float 5.3.0",
|
||||
"parking_lot",
|
||||
"rand 0.10.2",
|
||||
"schemars 0.8.22",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"typed-arena",
|
||||
"validator",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "spm_precompiled"
|
||||
version = "0.1.4"
|
||||
@@ -7590,20 +7356,6 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sysinfo"
|
||||
version = "0.38.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"memchr",
|
||||
"ntapi",
|
||||
"objc2-core-foundation",
|
||||
"objc2-io-kit",
|
||||
"windows 0.62.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sysinfo"
|
||||
version = "0.39.6"
|
||||
@@ -8080,7 +7832,7 @@ dependencies = [
|
||||
"serde_with",
|
||||
"swift-rs",
|
||||
"thiserror 2.0.18",
|
||||
"toml 0.9.12+spec-1.1.0",
|
||||
"toml 1.1.4+spec-1.1.0",
|
||||
"url",
|
||||
"urlpattern",
|
||||
"uuid",
|
||||
@@ -8260,7 +8012,7 @@ dependencies = [
|
||||
"esaxx-rs",
|
||||
"getrandom 0.3.1",
|
||||
"indicatif",
|
||||
"itertools",
|
||||
"itertools 0.14.0",
|
||||
"log",
|
||||
"macro_rules_attribute",
|
||||
"monostate",
|
||||
@@ -8626,12 +8378,6 @@ version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
|
||||
|
||||
[[package]]
|
||||
name = "typed-arena"
|
||||
version = "2.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a"
|
||||
|
||||
[[package]]
|
||||
name = "typed-path"
|
||||
version = "0.12.2"
|
||||
@@ -8948,25 +8694,6 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wal"
|
||||
version = "0.1.4"
|
||||
source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"crc32c",
|
||||
"docopt",
|
||||
"env_logger",
|
||||
"fs-err",
|
||||
"fs4",
|
||||
"log",
|
||||
"memmap2",
|
||||
"rand 0.10.2",
|
||||
"rand_distr",
|
||||
"rustix 1.1.4",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "walkdir"
|
||||
version = "2.5.0"
|
||||
|
||||
+3
-6
@@ -67,7 +67,7 @@ tempfile = "3.27.0"
|
||||
strum_macros = "0.28.0"
|
||||
sysinfo = "0.39.6"
|
||||
bytes = "1.12.1"
|
||||
qdrant-edge = "0.7.2"
|
||||
qdrant-edge = "0.8.0"
|
||||
|
||||
# Prompt-injection detection. `regex` gives us linear-time matching without backtracking, so
|
||||
# a hostile document cannot make a scan blow up, and `aho-corasick` matches the ~1600 fixed
|
||||
@@ -79,12 +79,9 @@ image = { version = "0.25.10", default-features = false, features = ["jpeg", "pn
|
||||
tokenizers = "0.23.1"
|
||||
|
||||
[patch.crates-io]
|
||||
# Issue: It was not possible to build qdrant-edge for macOS. See PR 9312: https://github.com/qdrant/qdrant/pull/9312
|
||||
# State: The PR was merged, but not yet released. We use the git version for now.
|
||||
qdrant-edge = { git = "https://github.com/SommerEngineering/qdrant.git", rev = "462c84d82ced126e4a2b7914544bfde16a509eb1" }
|
||||
|
||||
# Issue: This repo was not updated since 2020. The rand crate was outdated. We patched it to use a newer version of rand.
|
||||
# State: There is a PR for a long time, but it was not merged. We use the git version for now.
|
||||
# State: There is a PR for a long time, but it was not merged. We use the git version for now. Qdrant Edge still depends
|
||||
# on this crate in version 0.1.2, so the patch stays relevant: https://github.com/asimihsan/permutation-iterator-rs/pull/14
|
||||
permutation_iterator = { git = "https://github.com/SommerEngineering/permutation-iterator-rs.git", rev = "76836ed316d18dfef530ba908f58481c343e80d7" }
|
||||
|
||||
[target.'cfg(target_os = "windows")'.dependencies]
|
||||
|
||||
+23
-69
@@ -2,81 +2,35 @@
|
||||
|
||||
This directory documents temporary patches for third-party Rust dependencies.
|
||||
|
||||
## Qdrant Edge
|
||||
## permutation_iterator
|
||||
|
||||
AI Studio temporarily uses a pinned commit from `SommerEngineering/qdrant` for `qdrant-edge`.
|
||||
The fork commit exposes Qdrant's internal `lib/edge` crate as `qdrant-edge` and applies the
|
||||
trait-solver fix from Qdrant PR #9312.
|
||||
`qdrant-edge` depends on `permutation_iterator 0.1.2`, and that crate has seen no release since
|
||||
2019. Its published version pulls in an outdated `rand` line, which drags a second copy of the
|
||||
whole `rand` family into our dependency tree: `rand 0.7.3`, `rand_core 0.5.1`, `rand_chacha 0.2`,
|
||||
`rand_hc`, `getrandom 0.1.16`, `wasi 0.9` and `cfg-if 0.1`, next to the current ones everything
|
||||
else uses.
|
||||
|
||||
When updating to a newer Qdrant Edge version, replace the placeholder values first:
|
||||
The fork `SommerEngineering/permutation-iterator-rs` is the published 0.1.2 with `rand` raised to
|
||||
0.8, so it still satisfies what `qdrant-edge` asks for. AI Studio pins it in `runtime/Cargo.toml`:
|
||||
|
||||
```bash
|
||||
export QDRANT_EDGE_VERSION="0.7.2"
|
||||
export QDRANT_BRANCH="ai-studio-qdrant-edge-${QDRANT_EDGE_VERSION}"
|
||||
export AISTUDIO_REPO="xxx/mindwork-ai-studio"
|
||||
export QDRANT_REPO="xxx/qdrant"
|
||||
```toml
|
||||
[patch.crates-io]
|
||||
permutation_iterator = { git = "https://github.com/SommerEngineering/permutation-iterator-rs.git", rev = "..." }
|
||||
```
|
||||
|
||||
1. Sync the Qdrant fork with upstream:
|
||||
The same change was offered upstream in
|
||||
[asimihsan/permutation-iterator-rs#14](https://github.com/asimihsan/permutation-iterator-rs/pull/14),
|
||||
where it has been waiting since 2021. This is tree hygiene, not a build failure: without the patch
|
||||
the runtime still builds, it just carries the old `rand` family along.
|
||||
|
||||
```bash
|
||||
cd "$QDRANT_REPO"
|
||||
git remote add upstream https://github.com/qdrant/qdrant.git 2>/dev/null || true
|
||||
git fetch upstream
|
||||
git fetch origin
|
||||
git switch master
|
||||
git merge --ff-only upstream/master
|
||||
git push origin master
|
||||
```
|
||||
### When this patch can go
|
||||
|
||||
2. Create a fresh AI Studio branch in the Qdrant fork:
|
||||
Either of these is enough, and both are worth a look whenever `qdrant-edge` is updated:
|
||||
|
||||
```bash
|
||||
cd "$QDRANT_REPO"
|
||||
git switch -c "$QDRANT_BRANCH" master
|
||||
```
|
||||
- crates.io carries a `permutation_iterator` newer than 0.1.2 which uses a current `rand`. Then the
|
||||
`[patch.crates-io]` entry goes, and `qdrant-edge`'s own requirement decides the version.
|
||||
- `qdrant-edge` stops depending on `permutation_iterator` at all. Check with
|
||||
`cargo tree -i permutation_iterator` after the update.
|
||||
|
||||
3. Apply the AI Studio patch if upstream has not released the fix yet:
|
||||
|
||||
```bash
|
||||
cd "$QDRANT_REPO"
|
||||
git apply "$AISTUDIO_REPO/runtime/patches/qdrant-edge-ai-studio.patch"
|
||||
```
|
||||
|
||||
4. Update the exposed `qdrant-edge` version in the fork:
|
||||
|
||||
```bash
|
||||
cd "$QDRANT_REPO"
|
||||
perl -0pi -e "s/name = \"qdrant-edge\"\\nversion = \"[^\"]+\"/name = \"qdrant-edge\"\\nversion = \"$ENV{QDRANT_EDGE_VERSION}\"/" lib/edge/Cargo.toml
|
||||
```
|
||||
|
||||
5. Commit and push the fork branch:
|
||||
|
||||
```bash
|
||||
cd "$QDRANT_REPO"
|
||||
git diff
|
||||
git add lib/edge/Cargo.toml lib/segment/src/common/anonymize.rs
|
||||
git commit -m "Expose qdrant-edge ${QDRANT_EDGE_VERSION} package for AI Studio"
|
||||
git push origin "$QDRANT_BRANCH"
|
||||
export QDRANT_EDGE_COMMIT="$(git rev-parse HEAD)"
|
||||
echo "$QDRANT_EDGE_COMMIT"
|
||||
```
|
||||
|
||||
6. Update AI Studio to use the new Qdrant Edge version and fork commit:
|
||||
|
||||
```bash
|
||||
cd "$AISTUDIO_REPO"
|
||||
perl -0pi -e "s/qdrant-edge = \"[^\"]+\"/qdrant-edge = \"$ENV{QDRANT_EDGE_VERSION}\"/" runtime/Cargo.toml
|
||||
perl -0pi -e "s/rev = \"[0-9a-f]+\"/rev = \"$ENV{QDRANT_EDGE_COMMIT}\"/" runtime/Cargo.toml
|
||||
```
|
||||
|
||||
7. Refresh the AI Studio lock file and verify the Rust runtime:
|
||||
|
||||
```bash
|
||||
cd "$AISTUDIO_REPO/runtime"
|
||||
cargo update -p qdrant-edge
|
||||
cargo check
|
||||
```
|
||||
|
||||
Remove the patch and the `[patch.crates-io]` override once Qdrant publishes a fixed `qdrant-edge`
|
||||
release on crates.io.
|
||||
Afterward, `grep 'name = "rand"' -A 2 runtime/Cargo.lock` must not show a 0.7 version anymore.
|
||||
`SommerEngineering/permutation-iterator-rs` can then be deleted.
|
||||
@@ -1,26 +0,0 @@
|
||||
diff --git a/lib/edge/Cargo.toml b/lib/edge/Cargo.toml
|
||||
index 7c2cf6037..d21e3c053 100644
|
||||
--- a/lib/edge/Cargo.toml
|
||||
+++ b/lib/edge/Cargo.toml
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
-name = "edge"
|
||||
-version = "0.1.0"
|
||||
+name = "qdrant-edge"
|
||||
+version = "0.7.2"
|
||||
authors = ["Qdrant Team <info@qdrant.tech>"]
|
||||
license = "Apache-2.0"
|
||||
edition = "2024"
|
||||
diff --git a/lib/segment/src/common/anonymize.rs b/lib/segment/src/common/anonymize.rs
|
||||
index 6b5d19b12..c73d24433 100644
|
||||
--- a/lib/segment/src/common/anonymize.rs
|
||||
+++ b/lib/segment/src/common/anonymize.rs
|
||||
@@ -105,7 +105,7 @@ where
|
||||
{
|
||||
collection_opt
|
||||
.as_ref()
|
||||
- .map(|c| anonymize_collection_values(c))
|
||||
+ .map(|c| anonymize_collection_values::<C, K, V>(c))
|
||||
}
|
||||
|
||||
impl Anonymize for String {
|
||||
@@ -1,4 +1,4 @@
|
||||
use std::collections::HashMap;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
@@ -11,8 +11,9 @@ use qdrant_edge::external::uuid::Uuid;
|
||||
use qdrant_edge::{
|
||||
Condition, Distance, EdgeConfig, EdgeOptimizersConfig, EdgeShard, EdgeVectorParams,
|
||||
FieldCondition, Filter, HnswIndexConfig, Match, MatchValue, NamedQuery, Payload, PointId,
|
||||
PointInsertOperations, PointOperations, PointStruct, QueryEnum, ScoredPoint, SearchRequest,
|
||||
UpdateOperation, ValueVariants, VectorInternal, Vectors, WithPayloadInterface, WithVector,
|
||||
PointInsertOperations, PointOperations, PointStruct, QueryEnum, QueryRequest, ScoredPoint,
|
||||
ScoringQuery, UpdateOperation, ValueVariants, VectorInternal, Vectors, WithPayloadInterface,
|
||||
WithVector,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tauri::Manager;
|
||||
@@ -32,6 +33,10 @@ const STORE_INITIALIZATION_MARKER_TEMP: &str = "store_name.tmp";
|
||||
const STORE_DISPLAY_NAME_MARKER: &str = "data_source_name.txt";
|
||||
const STORE_DISPLAY_NAME_MARKER_TEMP: &str = "data_source_name.tmp";
|
||||
|
||||
/// Marks a response whose store exists on disk but cannot be opened. The .NET side keys its repair
|
||||
/// offer off this value instead of parsing `issue`, so rewording the message stays harmless.
|
||||
const ISSUE_CODE_STORE_UNREADABLE: &str = "store-unreadable";
|
||||
|
||||
type QdrantEdgeResult<T> = Result<T, Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
static QDRANT_EDGE_DATABASE: Lazy<Mutex<Option<QdrantEdgeDatabase>>> =
|
||||
@@ -127,9 +132,39 @@ pub struct DeleteQdrantEdgeStoreRequest {
|
||||
pub struct QdrantEdgeResponse<T> {
|
||||
pub success: bool,
|
||||
pub issue: String,
|
||||
pub issue_code: &'static str,
|
||||
pub data: Option<T>,
|
||||
}
|
||||
|
||||
/// A vector store which is initialized on disk but which Qdrant Edge refuses to open.
|
||||
///
|
||||
/// This is deliberately its own error type rather than one more formatted string: a broken store
|
||||
/// is the one failure the user can act on, and the request layer has to recognize it to label the
|
||||
/// response. Nothing here deletes the store -- rebuilding the embeddings costs the user time and,
|
||||
/// with a cloud embedding provider, money, so that stays their decision.
|
||||
#[derive(Debug)]
|
||||
struct StoreUnreadableError {
|
||||
store_name: String,
|
||||
message: String,
|
||||
}
|
||||
|
||||
impl StoreUnreadableError {
|
||||
fn new(store_name: &str, path: &Path, source: impl std::fmt::Display) -> Self {
|
||||
Self {
|
||||
store_name: store_name.to_string(),
|
||||
message: format!("Failed to load vector store '{store_name}' from '{}': {source}", path.display()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for StoreUnreadableError {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter.write_str(&self.message)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for StoreUnreadableError {}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct QdrantEdgeEnsureStoreResult {
|
||||
pub created: bool,
|
||||
@@ -168,6 +203,10 @@ pub struct QdrantEdgeInfo {
|
||||
pub struct QdrantEdgeDatabase {
|
||||
base_path: PathBuf,
|
||||
shards: HashMap<String, EdgeShard>,
|
||||
|
||||
/// Stores whose unreadability has already been logged. A broken store is hit by every single
|
||||
/// request against it, and one log line per request would bury everything else.
|
||||
reported_unreadable_stores: HashSet<String>,
|
||||
}
|
||||
|
||||
impl QdrantEdgeDatabase {
|
||||
@@ -175,9 +214,16 @@ impl QdrantEdgeDatabase {
|
||||
Self {
|
||||
base_path,
|
||||
shards: HashMap::new(),
|
||||
reported_unreadable_stores: HashSet::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether this store's defect still has to be written to the log. True exactly once per store,
|
||||
/// until the store loads again.
|
||||
fn report_unreadable_store(&mut self, store_name: &str) -> bool {
|
||||
self.reported_unreadable_stores.insert(store_name.to_string())
|
||||
}
|
||||
|
||||
fn store_path(&self, store_name: &str) -> QdrantEdgeResult<PathBuf> {
|
||||
validate_store_name(store_name)?;
|
||||
Ok(self.base_path.join("stores").join(store_directory_name(store_name)))
|
||||
@@ -191,9 +237,10 @@ impl QdrantEdgeDatabase {
|
||||
}
|
||||
|
||||
let shard = if is_initialized {
|
||||
EdgeShard::load(&path, None).map_err(|error| {
|
||||
format!("Failed to load vector store '{store_name}' from '{}': {error}", path.display())
|
||||
})?
|
||||
match EdgeShard::load(&path, None) {
|
||||
Ok(shard) => shard,
|
||||
Err(error) => return Err(StoreUnreadableError::new(store_name, &path, error).into()),
|
||||
}
|
||||
} else {
|
||||
fs::create_dir_all(&path).map_err(|error| {
|
||||
format!("Failed to create directory for vector store '{store_name}' at '{}': {error}", path.display())
|
||||
@@ -215,6 +262,7 @@ impl QdrantEdgeDatabase {
|
||||
shard
|
||||
};
|
||||
|
||||
self.reported_unreadable_stores.remove(store_name);
|
||||
self.shards.insert(store_name.to_string(), shard);
|
||||
Ok((self.shards.get(store_name).unwrap(), !is_initialized))
|
||||
}
|
||||
@@ -230,9 +278,12 @@ impl QdrantEdgeDatabase {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let shard = EdgeShard::load(&path, None).map_err(|error| {
|
||||
format!("Failed to load vector store '{store_name}' from '{}': {error}", path.display())
|
||||
})?;
|
||||
let shard = match EdgeShard::load(&path, None) {
|
||||
Ok(shard) => shard,
|
||||
Err(error) => return Err(StoreUnreadableError::new(store_name, &path, error).into()),
|
||||
};
|
||||
|
||||
self.reported_unreadable_stores.remove(store_name);
|
||||
self.shards.insert(store_name.to_string(), shard);
|
||||
Ok(self.shards.get(store_name))
|
||||
}
|
||||
@@ -306,7 +357,7 @@ impl QdrantEdgeDatabase {
|
||||
shard.update(UpdateOperation::PointOperation(
|
||||
PointOperations::UpsertPoints(PointInsertOperations::PointsList(points)),
|
||||
))?;
|
||||
shard.flush();
|
||||
shard.flush()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -320,18 +371,19 @@ impl QdrantEdgeDatabase {
|
||||
return Ok(vec![]);
|
||||
};
|
||||
|
||||
let search_results = shard.search(SearchRequest {
|
||||
query: QueryEnum::Nearest(NamedQuery::new(
|
||||
let search_results = shard.query(QueryRequest {
|
||||
prefetches: Vec::new(),
|
||||
query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery::new(
|
||||
VectorInternal::Dense(vector),
|
||||
VECTOR_NAME,
|
||||
)),
|
||||
)))),
|
||||
filter: None,
|
||||
params: None,
|
||||
score_threshold: None,
|
||||
limit: max_matches,
|
||||
offset: 0,
|
||||
with_payload: Some(WithPayloadInterface::Bool(true)),
|
||||
with_vector: Some(WithVector::Bool(false)),
|
||||
score_threshold: None,
|
||||
params: None,
|
||||
with_vector: WithVector::Bool(false),
|
||||
with_payload: WithPayloadInterface::Bool(true),
|
||||
})?;
|
||||
|
||||
Ok(search_results
|
||||
@@ -348,7 +400,7 @@ impl QdrantEdgeDatabase {
|
||||
shard.update(UpdateOperation::PointOperation(
|
||||
PointOperations::DeletePointsByFilter(match_keyword_filter("file_path", file_path)?),
|
||||
))?;
|
||||
shard.flush();
|
||||
shard.flush()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -361,7 +413,7 @@ impl QdrantEdgeDatabase {
|
||||
if optimized {
|
||||
info!(Source = "Qdrant Edge"; "Optimized vector store '{}'.", store_name);
|
||||
}
|
||||
shard.flush();
|
||||
shard.flush()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -503,6 +555,7 @@ where
|
||||
return Json(QdrantEdgeResponse {
|
||||
success: false,
|
||||
issue: "Qdrant Edge is not available.".to_string(),
|
||||
issue_code: "",
|
||||
data: None,
|
||||
});
|
||||
};
|
||||
@@ -511,14 +564,36 @@ where
|
||||
Ok(data) => Json(QdrantEdgeResponse {
|
||||
success: true,
|
||||
issue: String::new(),
|
||||
issue_code: "",
|
||||
data: Some(data),
|
||||
}),
|
||||
Err(e) => {
|
||||
let issue = e.to_string();
|
||||
error!(Source = "Qdrant Edge"; "Qdrant Edge request failed: {issue}");
|
||||
|
||||
//
|
||||
// An unreadable store keeps failing for as long as the user leaves it alone, so it is
|
||||
// logged once and then only answered. Every other failure is logged as it happens,
|
||||
// because those are one-offs worth seeing each time.
|
||||
//
|
||||
let issue_code = match e.downcast_ref::<StoreUnreadableError>() {
|
||||
Some(unreadable) => {
|
||||
if database.report_unreadable_store(&unreadable.store_name) {
|
||||
error!(Source = "Qdrant Edge"; "Qdrant Edge request failed: {issue}");
|
||||
}
|
||||
|
||||
ISSUE_CODE_STORE_UNREADABLE
|
||||
},
|
||||
|
||||
None => {
|
||||
error!(Source = "Qdrant Edge"; "Qdrant Edge request failed: {issue}");
|
||||
""
|
||||
},
|
||||
};
|
||||
|
||||
Json(QdrantEdgeResponse {
|
||||
success: false,
|
||||
issue,
|
||||
issue_code,
|
||||
data: None,
|
||||
})
|
||||
},
|
||||
@@ -595,7 +670,7 @@ fn remove_obsolete_qdrant_path(path: &Path) {
|
||||
|
||||
fn edge_config(vector_size: usize) -> EdgeConfig {
|
||||
EdgeConfig {
|
||||
on_disk_payload: true,
|
||||
on_disk_payload: Some(true),
|
||||
vectors: HashMap::from([(
|
||||
VECTOR_NAME.to_string(),
|
||||
EdgeVectorParams {
|
||||
@@ -609,13 +684,20 @@ fn edge_config(vector_size: usize) -> EdgeConfig {
|
||||
},
|
||||
)]),
|
||||
sparse_vectors: HashMap::new(),
|
||||
hnsw_config: hnsw_config(),
|
||||
hnsw_config: Some(hnsw_config()),
|
||||
quantization_config: None,
|
||||
optimizers: edge_optimizers_config(),
|
||||
optimizers: Some(edge_optimizers_config()),
|
||||
wal_options: None,
|
||||
max_search_threads: None,
|
||||
search_pool_core: None,
|
||||
}
|
||||
}
|
||||
|
||||
// `on_disk` is deprecated in favor of `memory`, but Qdrant Edge does not re-export the `Memory`
|
||||
// type, so the new field cannot be named from here. Leaving both unset is not an option either:
|
||||
// the effective placement would fall back to cached instead of on-disk, which is a real change
|
||||
// and would have the optimizers rebuild the HNSW graph.
|
||||
#[allow(deprecated)]
|
||||
fn hnsw_config() -> HnswIndexConfig {
|
||||
HnswIndexConfig {
|
||||
m: HNSW_M,
|
||||
@@ -623,6 +705,7 @@ fn hnsw_config() -> HnswIndexConfig {
|
||||
full_scan_threshold: HNSW_FULL_SCAN_THRESHOLD_KB,
|
||||
max_indexing_threads: HNSW_MAX_INDEXING_THREADS,
|
||||
on_disk: Some(true),
|
||||
memory: None,
|
||||
payload_m: None,
|
||||
inline_storage: None,
|
||||
}
|
||||
@@ -902,6 +985,44 @@ mod tests {
|
||||
fs::remove_dir_all(test_directory).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unreadable_store_is_reported_but_never_deleted() {
|
||||
let test_directory = std::env::temp_dir().join(format!(
|
||||
"ai-studio-qdrant-unreadable-{}-{}",
|
||||
std::process::id(),
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos()
|
||||
));
|
||||
let store_name = "rag_6cc665a82b1e4d42bc748015b7b391ec";
|
||||
|
||||
let mut database = QdrantEdgeDatabase::new(test_directory.clone());
|
||||
assert!(database.ensure_store_exists(store_name, "Some source", 3).unwrap().created);
|
||||
let store_path = database.store_path(store_name).unwrap();
|
||||
|
||||
// Release the shard before breaking it, so the files are not held open any more.
|
||||
drop(database);
|
||||
fs::write(store_path.join("edge_config.json"), "this is not a config").unwrap();
|
||||
|
||||
let mut database = QdrantEdgeDatabase::new(test_directory.clone());
|
||||
let error = database.get_existing_store(store_name).unwrap_err();
|
||||
assert!(
|
||||
error.downcast_ref::<StoreUnreadableError>().is_some(),
|
||||
"a store which cannot be opened has to be recognizable as such, not just a message"
|
||||
);
|
||||
|
||||
// The whole point: the user's embeddings survive a defect until they ask for a rebuild.
|
||||
assert!(store_path.join("segments").is_dir());
|
||||
assert!(store_path.join(STORE_INITIALIZATION_MARKER).is_file());
|
||||
|
||||
// And the defect is logged once, not once per request.
|
||||
assert!(database.report_unreadable_store(store_name));
|
||||
assert!(!database.report_unreadable_store(store_name));
|
||||
|
||||
fs::remove_dir_all(test_directory).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn point_ids_must_be_valid_uuids() {
|
||||
assert!(to_point_id("6cc665a8-2b1e-4d42-bc74-8015b7b391ec").is_ok());
|
||||
|
||||
Reference in new issue
Block a user