mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-08-24 20:52:11 +00:00
clean up of code
This commit is contained in:
parent
2e3a774124
commit
7450473a8d
16
.github/workflows/build-and-release.yml
vendored
16
.github/workflows/build-and-release.yml
vendored
@ -329,8 +329,8 @@ jobs:
|
|||||||
pdfium_version=$(sed -n '11p' metadata.txt)
|
pdfium_version=$(sed -n '11p' metadata.txt)
|
||||||
pdfium_version=$(echo $pdfium_version | cut -d'.' -f3)
|
pdfium_version=$(echo $pdfium_version | cut -d'.' -f3)
|
||||||
|
|
||||||
# Next line is the Qdrant version:
|
# Next line is the vector store version:
|
||||||
qdrant_version="v$(sed -n '12p' metadata.txt)"
|
vector_store_version="$(sed -n '12p' metadata.txt)"
|
||||||
|
|
||||||
# Write the metadata to the environment:
|
# Write the metadata to the environment:
|
||||||
echo "APP_VERSION=${app_version}" >> $GITHUB_ENV
|
echo "APP_VERSION=${app_version}" >> $GITHUB_ENV
|
||||||
@ -344,7 +344,7 @@ jobs:
|
|||||||
echo "TAURI_VERSION=${tauri_version}" >> $GITHUB_ENV
|
echo "TAURI_VERSION=${tauri_version}" >> $GITHUB_ENV
|
||||||
echo "ARCHITECTURE=${{ matrix.dotnet_runtime }}" >> $GITHUB_ENV
|
echo "ARCHITECTURE=${{ matrix.dotnet_runtime }}" >> $GITHUB_ENV
|
||||||
echo "PDFIUM_VERSION=${pdfium_version}" >> $GITHUB_ENV
|
echo "PDFIUM_VERSION=${pdfium_version}" >> $GITHUB_ENV
|
||||||
echo "QDRANT_VERSION=${qdrant_version}" >> $GITHUB_ENV
|
echo "VECTOR_STORE_VERSION=${vector_store_version}" >> $GITHUB_ENV
|
||||||
|
|
||||||
# Log the metadata:
|
# Log the metadata:
|
||||||
echo "App version: '${formatted_app_version}'"
|
echo "App version: '${formatted_app_version}'"
|
||||||
@ -357,7 +357,7 @@ jobs:
|
|||||||
echo "Tauri version: '${tauri_version}'"
|
echo "Tauri version: '${tauri_version}'"
|
||||||
echo "Architecture: '${{ matrix.dotnet_runtime }}'"
|
echo "Architecture: '${{ matrix.dotnet_runtime }}'"
|
||||||
echo "PDFium version: '${pdfium_version}'"
|
echo "PDFium version: '${pdfium_version}'"
|
||||||
echo "Qdrant version: '${qdrant_version}'"
|
echo "Vector store version: '${vector_store_version}'"
|
||||||
|
|
||||||
- name: Read and format metadata (Windows)
|
- name: Read and format metadata (Windows)
|
||||||
if: matrix.platform == 'windows-latest'
|
if: matrix.platform == 'windows-latest'
|
||||||
@ -402,8 +402,8 @@ jobs:
|
|||||||
$pdfium_version = $metadata[10]
|
$pdfium_version = $metadata[10]
|
||||||
$pdfium_version = $pdfium_version.Split('.')[2]
|
$pdfium_version = $pdfium_version.Split('.')[2]
|
||||||
|
|
||||||
# Next line is the necessary Qdrant version:
|
# Next line is the vector store version:
|
||||||
$qdrant_version = "v$($metadata[11])"
|
$vector_store_version = $metadata[11]
|
||||||
|
|
||||||
# Write the metadata to the environment:
|
# Write the metadata to the environment:
|
||||||
Write-Output "APP_VERSION=${app_version}" >> $env:GITHUB_ENV
|
Write-Output "APP_VERSION=${app_version}" >> $env:GITHUB_ENV
|
||||||
@ -416,7 +416,7 @@ jobs:
|
|||||||
Write-Output "MUD_BLAZOR_VERSION=${mud_blazor_version}" >> $env:GITHUB_ENV
|
Write-Output "MUD_BLAZOR_VERSION=${mud_blazor_version}" >> $env:GITHUB_ENV
|
||||||
Write-Output "ARCHITECTURE=${{ matrix.dotnet_runtime }}" >> $env:GITHUB_ENV
|
Write-Output "ARCHITECTURE=${{ matrix.dotnet_runtime }}" >> $env:GITHUB_ENV
|
||||||
Write-Output "PDFIUM_VERSION=${pdfium_version}" >> $env:GITHUB_ENV
|
Write-Output "PDFIUM_VERSION=${pdfium_version}" >> $env:GITHUB_ENV
|
||||||
Write-Output "QDRANT_VERSION=${qdrant_version}" >> $env:GITHUB_ENV
|
Write-Output "VECTOR_STORE_VERSION=${vector_store_version}" >> $env:GITHUB_ENV
|
||||||
|
|
||||||
# Log the metadata:
|
# Log the metadata:
|
||||||
Write-Output "App version: '${formatted_app_version}'"
|
Write-Output "App version: '${formatted_app_version}'"
|
||||||
@ -429,7 +429,7 @@ jobs:
|
|||||||
Write-Output "Tauri version: '${tauri_version}'"
|
Write-Output "Tauri version: '${tauri_version}'"
|
||||||
Write-Output "Architecture: '${{ matrix.dotnet_runtime }}'"
|
Write-Output "Architecture: '${{ matrix.dotnet_runtime }}'"
|
||||||
Write-Output "PDFium version: '${pdfium_version}'"
|
Write-Output "PDFium version: '${pdfium_version}'"
|
||||||
Write-Output "Qdrant version: '${qdrant_version}'"
|
Write-Output "Vector store version: '${vector_store_version}'"
|
||||||
|
|
||||||
- name: Setup .NET
|
- name: Setup .NET
|
||||||
uses: actions/setup-dotnet@v4
|
uses: actions/setup-dotnet@v4
|
||||||
|
|||||||
@ -41,7 +41,7 @@ public sealed partial class RustService
|
|||||||
using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
|
||||||
cts.CancelAfter(TimeSpan.FromMinutes(5));
|
cts.CancelAfter(TimeSpan.FromMinutes(5));
|
||||||
|
|
||||||
var response = await this.http.PostAsJsonAsync(path, request, this.jsonRustSerializerOptions, cts.Token);
|
using var response = await this.http.PostAsJsonAsync(path, request, this.jsonRustSerializerOptions, cts.Token);
|
||||||
response.EnsureSuccessStatusCode();
|
response.EnsureSuccessStatusCode();
|
||||||
|
|
||||||
var operation = await response.Content.ReadFromJsonAsync<DatabaseOperationResponse>(this.jsonRustSerializerOptions, cts.Token);
|
var operation = await response.Content.ReadFromJsonAsync<DatabaseOperationResponse>(this.jsonRustSerializerOptions, cts.Token);
|
||||||
|
|||||||
@ -66,16 +66,6 @@
|
|||||||
"MudBlazor": "8.11.0"
|
"MudBlazor": "8.11.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"Qdrant.Client": {
|
|
||||||
"type": "Direct",
|
|
||||||
"requested": "[1.18.1, )",
|
|
||||||
"resolved": "1.18.1",
|
|
||||||
"contentHash": "eBwFLihGMvN02/jr/BNdcop2XmtA10y8VMOclVZ7K2H8yheAhl7jbkf7I8e4X3RYpT+cAxgcalP4xmOhgs4KJg==",
|
|
||||||
"dependencies": {
|
|
||||||
"Google.Protobuf": "3.31.0",
|
|
||||||
"Grpc.Net.Client": "2.71.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"ReverseMarkdown": {
|
"ReverseMarkdown": {
|
||||||
"type": "Direct",
|
"type": "Direct",
|
||||||
"requested": "[5.0.0, )",
|
"requested": "[5.0.0, )",
|
||||||
@ -90,33 +80,6 @@
|
|||||||
"resolved": "3.2.449",
|
"resolved": "3.2.449",
|
||||||
"contentHash": "uA9sYDy4VepL3xwzBTLcP2LyuVYMt0ZIT3gaSiXvGoX15Ob+rOP+hGydhevlSVd+rFo+Y+VQFEHDuWU8HBW+XA=="
|
"contentHash": "uA9sYDy4VepL3xwzBTLcP2LyuVYMt0ZIT3gaSiXvGoX15Ob+rOP+hGydhevlSVd+rFo+Y+VQFEHDuWU8HBW+XA=="
|
||||||
},
|
},
|
||||||
"Google.Protobuf": {
|
|
||||||
"type": "Transitive",
|
|
||||||
"resolved": "3.31.0",
|
|
||||||
"contentHash": "OZXSf6igaJBeo+kAzMhYF0R5zp0nRgf4G0Uis/IsGKACc4RGP9bQPLpHLengIFuASl0lY92utMB8rRpTx4TaOg=="
|
|
||||||
},
|
|
||||||
"Grpc.Core.Api": {
|
|
||||||
"type": "Transitive",
|
|
||||||
"resolved": "2.71.0",
|
|
||||||
"contentHash": "QquqUC37yxsDzd1QaDRsH2+uuznWPTS8CVE2Yzwl3CvU4geTNkolQXoVN812M2IwT6zpv3jsZRc9ExJFNFslTg=="
|
|
||||||
},
|
|
||||||
"Grpc.Net.Client": {
|
|
||||||
"type": "Transitive",
|
|
||||||
"resolved": "2.71.0",
|
|
||||||
"contentHash": "U1vr20r5ngoT9nlb7wejF28EKN+taMhJsV9XtK9MkiepTZwnKxxiarriiMfCHuDAfPUm9XUjFMn/RIuJ4YY61w==",
|
|
||||||
"dependencies": {
|
|
||||||
"Grpc.Net.Common": "2.71.0",
|
|
||||||
"Microsoft.Extensions.Logging.Abstractions": "6.0.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"Grpc.Net.Common": {
|
|
||||||
"type": "Transitive",
|
|
||||||
"resolved": "2.71.0",
|
|
||||||
"contentHash": "v0c8R97TwRYwNXlC8GyRXwYTCNufpDfUtj9la+wUrZFzVWkFJuNAltU+c0yI3zu0jl54k7en6u2WKgZgd57r2Q==",
|
|
||||||
"dependencies": {
|
|
||||||
"Grpc.Core.Api": "2.71.0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"LuaCSharp.Annotations": {
|
"LuaCSharp.Annotations": {
|
||||||
"type": "Transitive",
|
"type": "Transitive",
|
||||||
"resolved": "0.5.5",
|
"resolved": "0.5.5",
|
||||||
|
|||||||
@ -1,7 +1,7 @@
|
|||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::Mutex;
|
||||||
|
|
||||||
use axum::Json;
|
use axum::Json;
|
||||||
use log::{error, info, warn};
|
use log::{error, info, warn};
|
||||||
@ -29,8 +29,8 @@ const VECTOR_INDEXING_THRESHOLD_KB: usize = 10_000;
|
|||||||
|
|
||||||
type QdrantEdgeResult<T> = Result<T, Box<dyn std::error::Error + Send + Sync>>;
|
type QdrantEdgeResult<T> = Result<T, Box<dyn std::error::Error + Send + Sync>>;
|
||||||
|
|
||||||
static QDRANT_EDGE_DATABASE: Lazy<Arc<Mutex<Option<QdrantEdgeDatabase>>>> =
|
static QDRANT_EDGE_DATABASE: Lazy<Mutex<Option<QdrantEdgeDatabase>>> =
|
||||||
Lazy::new(|| Arc::new(Mutex::new(None)));
|
Lazy::new(|| Mutex::new(None));
|
||||||
|
|
||||||
static QDRANT_EDGE_STATUS: Lazy<Mutex<QdrantEdgeStatusInfo>> =
|
static QDRANT_EDGE_STATUS: Lazy<Mutex<QdrantEdgeStatusInfo>> =
|
||||||
Lazy::new(|| Mutex::new(QdrantEdgeStatusInfo::default()));
|
Lazy::new(|| Mutex::new(QdrantEdgeStatusInfo::default()));
|
||||||
@ -186,6 +186,7 @@ impl QdrantEdgeDatabase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn ensure_store_exists(&mut self, store_name: &str, vector_size: usize) -> QdrantEdgeResult<()> {
|
fn ensure_store_exists(&mut self, store_name: &str, vector_size: usize) -> QdrantEdgeResult<()> {
|
||||||
|
validate_vector_size(vector_size)?;
|
||||||
self.get_or_create_store(store_name, vector_size)?;
|
self.get_or_create_store(store_name, vector_size)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
@ -196,6 +197,7 @@ impl QdrantEdgeDatabase {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let vector_size = first_point.vector.len();
|
let vector_size = first_point.vector.len();
|
||||||
|
validate_vector_size(vector_size)?;
|
||||||
if points.iter().any(|point| point.vector.len() != vector_size) {
|
if points.iter().any(|point| point.vector.len() != vector_size) {
|
||||||
return Err("All vectors in one insert request must have the same size.".into());
|
return Err("All vectors in one insert request must have the same size.".into());
|
||||||
}
|
}
|
||||||
@ -241,10 +243,14 @@ impl QdrantEdgeDatabase {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn qdrant_edge_base_path() -> PathBuf {
|
fn qdrant_edge_base_path() -> QdrantEdgeResult<PathBuf> {
|
||||||
Path::new(DATA_DIRECTORY.get().unwrap())
|
let data_directory = DATA_DIRECTORY
|
||||||
|
.get()
|
||||||
|
.ok_or("The data directory has not been initialized.")?;
|
||||||
|
|
||||||
|
Ok(Path::new(data_directory)
|
||||||
.join("databases")
|
.join("databases")
|
||||||
.join("vector_database")
|
.join("vector_database"))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn qdrant_edge_info(_token: APIToken) -> Json<QdrantEdgeServiceInfo> {
|
pub async fn qdrant_edge_info(_token: APIToken) -> Json<QdrantEdgeServiceInfo> {
|
||||||
@ -298,7 +304,16 @@ pub fn start_qdrant_edge_database<R: tauri::Runtime>(app_handle: tauri::AppHandl
|
|||||||
set_qdrant_edge_starting();
|
set_qdrant_edge_starting();
|
||||||
remove_obsolete_qdrant_sidecar_files(&app_handle);
|
remove_obsolete_qdrant_sidecar_files(&app_handle);
|
||||||
|
|
||||||
let path = qdrant_edge_base_path();
|
let path = match qdrant_edge_base_path() {
|
||||||
|
Ok(path) => path,
|
||||||
|
Err(e) => {
|
||||||
|
let reason = format!("Qdrant Edge cannot be started: {e}");
|
||||||
|
error!(Source = "Qdrant Edge"; "{reason}");
|
||||||
|
set_qdrant_edge_unavailable(reason);
|
||||||
|
return;
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
match fs::create_dir_all(&path) {
|
match fs::create_dir_all(&path) {
|
||||||
Ok(_) => {
|
Ok(_) => {
|
||||||
let database = QdrantEdgeDatabase::new(path.clone());
|
let database = QdrantEdgeDatabase::new(path.clone());
|
||||||
@ -457,6 +472,14 @@ fn has_existing_store(path: &Path) -> bool {
|
|||||||
path.join("edge_config.json").exists() || path.join("segments").exists()
|
path.join("edge_config.json").exists() || path.join("segments").exists()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn validate_vector_size(vector_size: usize) -> QdrantEdgeResult<()> {
|
||||||
|
if vector_size == 0 {
|
||||||
|
return Err("Vector size must be greater than zero.".into());
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn vector_store_version() -> QdrantEdgeResult<String> {
|
fn vector_store_version() -> QdrantEdgeResult<String> {
|
||||||
let metadata = META_DATA
|
let metadata = META_DATA
|
||||||
.lock()
|
.lock()
|
||||||
@ -526,6 +549,10 @@ fn validate_store_name(store_name: &str) -> QdrantEdgeResult<()> {
|
|||||||
return Err("Vector store name cannot be empty.".into());
|
return Err("Vector store name cannot be empty.".into());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if matches!(store_name, "." | "..") {
|
||||||
|
return Err(format!("Vector store name '{store_name}' is not supported.").into());
|
||||||
|
}
|
||||||
|
|
||||||
if store_name
|
if store_name
|
||||||
.chars()
|
.chars()
|
||||||
.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-' || c == '.')
|
.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-' || c == '.')
|
||||||
@ -535,3 +562,19 @@ fn validate_store_name(store_name: &str) -> QdrantEdgeResult<()> {
|
|||||||
|
|
||||||
Err(format!("Vector store name '{store_name}' contains unsupported characters.").into())
|
Err(format!("Vector store name '{store_name}' contains unsupported characters.").into())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn validate_store_name_allows_safe_store_names() {
|
||||||
|
assert!(validate_store_name("rag_1234-abcd.ef").is_ok());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn validate_store_name_rejects_path_traversal_names() {
|
||||||
|
assert!(validate_store_name(".").is_err());
|
||||||
|
assert!(validate_store_name("..").is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user