diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index 5a29415a..40df7acf 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -9598,12 +9598,18 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1041509726"] = "Text" -- Office Files UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1063218378"] = "Office Files" +-- Spreadsheet +UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1313839225"] = "Spreadsheet" + -- Executable UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1364437037"] = "Executable" -- Mail UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1399880782"] = "Mail" +-- Delimited table +UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1405737676"] = "Delimited table" + -- Source like UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1487238587"] = "Source like" @@ -9631,6 +9637,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2502277006"] = "Custom" -- Visual briefing image UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2505088878"] = "Visual briefing image" +-- Shortcut +UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2547828883"] = "Shortcut" + -- Media UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T3507473059"] = "Media" diff --git a/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor index c955c1fc..cc1af8df 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor @@ -1,4 +1,5 @@ @using AIStudio.Settings.DataModel +@using AIStudio.Tools.Validation @using AIStudio.Provider @using AIStudio.Tools.ERIClient.DataModel @inherits MSGComponentBase @@ -12,8 +13,8 @@ @bind-Text="@this.dataName" Label="@T("Data Source Name")" Class="mb-6" - MaxLength="40" - Counter="40" + MaxLength="@DataSourceValidation.MAX_NAME_LENGTH" + Counter="@DataSourceValidation.MAX_NAME_LENGTH" Immediate="@true" Validation="@this.dataSourceValidation.ValidatingName" Adornment="Adornment.Start" diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor index 5f384525..0ffa4a08 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor @@ -1,4 +1,5 @@ @using AIStudio.Settings.DataModel +@using AIStudio.Tools.Validation @using AIStudio.Provider @inherits MSGComponentBase @@ -11,8 +12,8 @@ @bind-Text="@this.dataName" Label="@T("Data Source Name")" Class="mb-6" - MaxLength="40" - Counter="40" + MaxLength="@DataSourceValidation.MAX_NAME_LENGTH" + Counter="@DataSourceValidation.MAX_NAME_LENGTH" Immediate="@true" Validation="@this.dataSourceValidation.ValidatingName" Adornment="Adornment.Start" diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor index 09b87154..8248e8be 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor @@ -1,4 +1,5 @@ @using AIStudio.Settings.DataModel +@using AIStudio.Tools.Validation @using AIStudio.Provider @inherits MSGComponentBase @@ -11,8 +12,8 @@ @bind-Text="@this.dataName" Label="@T("Data Source Name")" Class="mb-6" - MaxLength="40" - Counter="40" + MaxLength="@DataSourceValidation.MAX_NAME_LENGTH" + Counter="@DataSourceValidation.MAX_NAME_LENGTH" Immediate="@true" Validation="@this.dataSourceValidation.ValidatingName" Adornment="Adornment.Start" diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs index 55bd9bbb..ca35d491 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs @@ -8,6 +8,7 @@ using AIStudio.Tools.ERIClient.DataModel; using AIStudio.Tools.PluginSystem; using AIStudio.Tools.RAG; using AIStudio.Tools.Services; +using AIStudio.Tools.Validation; using SharedTools; @@ -168,9 +169,9 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource return false; } - if (!table.TryGetValue("Name", out var nameValue) || !nameValue.TryRead(out var name) || string.IsNullOrWhiteSpace(name) || name.Length > 40 || name.Any(char.IsControl)) + if (!table.TryGetValue("Name", out var nameValue) || !nameValue.TryRead(out var name) || !DataSourceValidation.IsNameValid(name)) { - LOGGER.LogWarning($"The configured data source {idx} does not contain a valid name of at most 40 characters without control characters. (Plugin ID: {configPluginId})"); + LOGGER.LogWarning($"The configured data source {idx} does not contain a valid name of at most {DataSourceValidation.MAX_NAME_LENGTH} characters without control characters. (Plugin ID: {configPluginId})"); return false; } diff --git a/app/MindWork AI Studio/Tools/Databases/VectorStore/QdrantEdgeClientImplementation.cs b/app/MindWork AI Studio/Tools/Databases/VectorStore/QdrantEdgeClientImplementation.cs index a72ef8cd..c00d8348 100644 --- a/app/MindWork AI Studio/Tools/Databases/VectorStore/QdrantEdgeClientImplementation.cs +++ b/app/MindWork AI Studio/Tools/Databases/VectorStore/QdrantEdgeClientImplementation.cs @@ -83,7 +83,7 @@ public sealed class QdrantEdgeClientImplementation( } public override async Task EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token) => - await rustService.ExecuteDatabaseQuery( DATABASE_NAME, ENSURE_PATH, + await rustService.ExecuteDatabaseQuery(DATABASE_NAME, ENSURE_PATH, new EnsureVectorStoreRequest(storeName, dataSourceName, vectorSize), token) ?? throw new InvalidOperationException("The vector store ensure response was empty."); public override Task InsertEmbedding(string storeName, IReadOnlyList points, CancellationToken token) => diff --git a/app/MindWork AI Studio/Tools/Rust/FileTypes.cs b/app/MindWork AI Studio/Tools/Rust/FileTypes.cs index e8547a8b..68723b8c 100644 --- a/app/MindWork AI Studio/Tools/Rust/FileTypes.cs +++ b/app/MindWork AI Studio/Tools/Rust/FileTypes.cs @@ -30,7 +30,9 @@ public static class FileTypes public static readonly FileTypeFilter RUST = FileTypeFilter.Leaf("Rust", "rs"); public static readonly FileTypeFilter LUA = FileTypeFilter.Leaf("Lua", "lua"); public static readonly FileTypeFilter PHP = FileTypeFilter.Leaf("PHP", "php"); - public static readonly FileTypeFilter WEB = FileTypeFilter.Leaf("HTML/CSS", "html", "css"); + public static readonly FileTypeFilter HTML = FileTypeFilter.Leaf("HTML", "html", "htm"); + public static readonly FileTypeFilter CSS = FileTypeFilter.Leaf("CSS", "css"); + public static readonly FileTypeFilter WEB = FileTypeFilter.Parent("HTML/CSS", HTML, CSS); /// /// Gets the standalone HTML filter used for visual briefing import and export. @@ -52,15 +54,18 @@ public static class FileTypes public static readonly FileTypeFilter TEXT = FileTypeFilter.Leaf(TB("Text"), "txt", "md", "rtf"); public static readonly FileTypeFilter MS_WORD = FileTypeFilter.Leaf("Microsoft Word", "docx"); public static readonly FileTypeFilter WORD = FileTypeFilter.Composite("Word", ["odt"], MS_WORD); - public static readonly FileTypeFilter EXCEL = FileTypeFilter.Leaf("Excel", "xls", "xlsx"); + public static readonly FileTypeFilter EXCEL = FileTypeFilter.Leaf("Excel", "xls", "xlsx", "xlsm", "xlsb", "xla", "xlam"); + public static readonly FileTypeFilter OPEN_DOCUMENT_SPREADSHEET = FileTypeFilter.Leaf("OpenDocument Spreadsheet", "ods"); + public static readonly FileTypeFilter SPREADSHEET = FileTypeFilter.Parent(TB("Spreadsheet"), EXCEL, OPEN_DOCUMENT_SPREADSHEET); + public static readonly FileTypeFilter DELIMITED_TABLE = FileTypeFilter.Leaf(TB("Delimited table"), "csv", "tsv"); public static readonly FileTypeFilter POWER_POINT = FileTypeFilter.Leaf("PowerPoint", "ppt", "pptx", "odp"); public static readonly FileTypeFilter MAIL = FileTypeFilter.Leaf(TB("Mail"), "eml", "msg", "mbox"); public static readonly FileTypeFilter LATEX = FileTypeFilter.Leaf("LaTeX", "tex", "bib", "sty", "cls", "log"); public static readonly FileTypeFilter OFFICE_FILES = FileTypeFilter.Parent(TB("Office Files"), - WORD, EXCEL, POWER_POINT, PDF); + WORD, SPREADSHEET, POWER_POINT, PDF); public static readonly FileTypeFilter DOCUMENT = FileTypeFilter.Parent(TB("Document"), - TEXT, OFFICE_FILES, SOURCE_CODE, LATEX); + TEXT, OFFICE_FILES, SOURCE_CODE, LATEX, DELIMITED_TABLE); // Media hierarchy public static readonly FileTypeFilter IMAGE = FileTypeFilter.Leaf(TB("Image"), @@ -81,6 +86,7 @@ public static class FileTypes // Other standalone types public static readonly FileTypeFilter CERTIFICATE_BUNDLE = FileTypeFilter.Leaf(TB("Certificate bundle"), "pem", "crt", "cer"); public static readonly FileTypeFilter EXECUTABLES = FileTypeFilter.Leaf(TB("Executable"), "exe", "app", "bin", "appimage"); + public static readonly FileTypeFilter SHORTCUT = FileTypeFilter.Leaf(TB("Shortcut"), "lnk"); public static FileTypeFilter? AsOneFileType(params FileTypeFilter[]? types) { @@ -106,6 +112,14 @@ public static class FileTypes .ToArray(); } + public static bool IsAllowedExtension(string extension, params FileTypeFilter[]? types) + { + if (types == null || types.Length == 0 || string.IsNullOrWhiteSpace(extension)) + return false; + + return OnlyAllowTypes(types).Contains(extension.TrimStart('.'), StringComparer.OrdinalIgnoreCase); + } + /// /// Validates a file path against the provided filters. /// Supports extension-based matching and source-like file names (e.g. Dockerfile). @@ -116,11 +130,8 @@ public static class FileTypes return false; var extension = Path.GetExtension(filePath).TrimStart('.'); - if (!string.IsNullOrWhiteSpace(extension)) - { - if (OnlyAllowTypes(types).Contains(extension, StringComparer.OrdinalIgnoreCase)) - return true; - } + if (IsAllowedExtension(extension, types)) + return true; var fileName = Path.GetFileName(filePath); if (string.IsNullOrWhiteSpace(fileName)) diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs index ca3d4ff0..44366557 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs @@ -17,11 +17,6 @@ public sealed partial class DataSourceEmbeddingService private const int DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH = 300; private const bool IMAGE_EMBEDDING_ENABLED = false; - private static readonly string[] RAG_DELIMITED_TABLE_FILE_EXTENSIONS = ["csv", "tsv"]; - private static readonly string[] RAG_SPREADSHEET_FILE_EXTENSIONS = ["ods", "xlsm", "xlsb"]; - private static readonly string[] RAG_SPREADSHEET_ADD_IN_FILE_EXTENSIONS = ["xla", "xlam"]; - private static readonly string[] SKIPPED_RAG_FILE_EXTENSIONS = ["lnk"]; - private enum RagFileIndexingDecision { INDEXABLE, @@ -575,7 +570,7 @@ public sealed partial class DataSourceEmbeddingService ]); if (this.IsSourceCodeFilePath(filePath)) - return GetSourceCodeChunkingStrategy(filePath); + return GetSourceCodeChunkingStrategy(); return new("document", [ new("Page or extracted section", SplitBySourceSegments, true), @@ -587,24 +582,13 @@ public sealed partial class DataSourceEmbeddingService ]); } - private static ChunkingStrategy GetSourceCodeChunkingStrategy(string filePath) - { - var rules = new List - { + private static ChunkingStrategy GetSourceCodeChunkingStrategy() => + new("source-code", [ new("Extracted section", SplitBySourceSegments, true), - }; - rules.AddRange(GetSourceCodeDelimiterRules(filePath)); - rules.Add(new("Line break", SplitByLineBreaks)); - rules.Add(new("Whitespace", SplitByWhitespace)); - rules.Add(new("Hard cut", null)); - - return new("source-code", rules); - } - - private static IReadOnlyList GetSourceCodeDelimiterRules(string filePath) => Path.GetExtension(filePath).TrimStart('.') switch - { - _ => [], - }; + new("Line break", SplitByLineBreaks), + new("Whitespace", SplitByWhitespace), + new("Hard cut", null), + ]); private static List NormalizeSplitUnits(IReadOnlyList units, string fallbackText) { @@ -875,16 +859,12 @@ public sealed partial class DataSourceEmbeddingService private bool IsDelimitedTableFilePath(string filePath) { - var extension = Path.GetExtension(filePath).TrimStart('.'); - return RAG_DELIMITED_TABLE_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase); + return FileTypes.IsAllowedPath(filePath, FileTypes.DELIMITED_TABLE); } private bool IsSpreadsheetFilePath(string filePath) { - var extension = Path.GetExtension(filePath).TrimStart('.'); - return FileTypes.IsAllowedPath(filePath, FileTypes.EXCEL) - || RAG_SPREADSHEET_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase) - || RAG_SPREADSHEET_ADD_IN_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase); + return FileTypes.IsAllowedPath(filePath, FileTypes.SPREADSHEET); } private bool IsSourceCodeFilePath(string filePath) @@ -894,18 +874,12 @@ public sealed partial class DataSourceEmbeddingService private bool IsHtmlFilePath(string filePath) { - var extension = Path.GetExtension(filePath).TrimStart('.'); - return extension.Equals("html", StringComparison.OrdinalIgnoreCase) - || extension.Equals("htm", StringComparison.OrdinalIgnoreCase); + return FileTypes.IsAllowedPath(filePath, FileTypes.HTML); } private bool IsSupportedRagFilePath(string filePath) { - var extension = Path.GetExtension(filePath).TrimStart('.'); - return FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT) - || RAG_DELIMITED_TABLE_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase) - || RAG_SPREADSHEET_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase) - || RAG_SPREADSHEET_ADD_IN_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase); + return FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT); } private RagFileIndexingDecision GetRagFileIndexingDecision(FileInfo file) @@ -942,8 +916,7 @@ public sealed partial class DataSourceEmbeddingService private static bool IsSkippedRagFileName(string fileName) { - var extension = Path.GetExtension(fileName).TrimStart('.'); - return SKIPPED_RAG_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase) + return FileTypes.IsAllowedPath(fileName, FileTypes.SHORTCUT) || fileName.StartsWith(OFFICE_LOCK_FILE_PREFIX, StringComparison.Ordinal); } @@ -1126,9 +1099,6 @@ public sealed partial class DataSourceEmbeddingService : null; } - private string GetCollectionName(string dataSourceId) => - DataSourceEmbeddingNames.GetCollectionName(dataSourceId); - private string CreatePointId(string dataSourceId, string fingerprint, int chunkIndex) => CreateStableGuid($"{dataSourceId}:chunk:{fingerprint}:{chunkIndex}"); diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs index 65851b39..fc318f02 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs @@ -11,7 +11,7 @@ public sealed partial class DataSourceEmbeddingService EmbeddingStateClient? embeddingState, CancellationToken token) { - await this.DeleteCollectionAsync(this.GetCollectionName(dataSourceId), vectorStore, token); + await this.DeleteCollectionAsync(DataSourceEmbeddingNames.GetCollectionName(dataSourceId), vectorStore, token); embeddingState ??= await databaseClientProvider.GetEmbeddingStateAsync(token); if (!embeddingState.IsAvailable) diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index 244dcf9e..8bf87010 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -428,11 +428,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM return; } - var collectionName = this.GetCollectionName(dataSource.Id); + var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id); var persistedManifest = await embeddingState.GetManifestAsync(dataSource.Id, token); if (persistedManifest.VectorSize > 0) { - var ensureResult = await this.EnsureCollectionExistsAsync(vectorStore, collectionName, dataSource.Name, persistedManifest.VectorSize, token); + var ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, persistedManifest.VectorSize, token); if (ensureResult.Created) { logger.LogWarning( @@ -685,7 +685,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM VectorStoreOptimizationTracker optimizationTracker, CancellationToken token) { - var collectionName = this.GetCollectionName(dataSource.Id); + var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id); logger.LogDebug( "Resetting stored embeddings for file '{FilePath}' in collection '{CollectionName}' before re-indexing.", file.FullName, @@ -783,7 +783,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM if (manifest.VectorSize == 0) { token.ThrowIfCancellationRequested(); - var ensureResult = await this.EnsureCollectionExistsAsync(vectorStore, collectionName, dataSource.Name, vectorSize, token); + var ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, vectorSize, token); if (!ensureResult.Created) { logger.LogWarning( @@ -792,7 +792,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM dataSource.Name, dataSource.Id); await vectorStore.DeleteVectorStore(collectionName, token); - ensureResult = await this.EnsureCollectionExistsAsync(vectorStore, collectionName, dataSource.Name, vectorSize, token); + ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, vectorSize, token); if (!ensureResult.Created) throw new InvalidOperationException($"Vector store '{collectionName}' could not be recreated cleanly."); } @@ -845,11 +845,6 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM batch.Clear(); } - private async Task EnsureCollectionExistsAsync(VectorStoreClient vectorStore, string collectionName, string dataSourceName, int vectorSize, CancellationToken token) - { - return await vectorStore.EnsureVectorStoreExists(collectionName, dataSourceName, vectorSize, token); - } - private async Task UpsertPointsAsync( VectorStoreClient vectorStore, string collectionName, diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs index 580b4946..f81788a2 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs @@ -6,6 +6,7 @@ using AIStudio.Tools.Databases; using AIStudio.Tools.Databases.EmbeddingState; using AIStudio.Tools.Databases.VectorStore; using AIStudio.Tools.RAG; +using AIStudio.Tools.Rust; namespace AIStudio.Tools.Services; @@ -371,13 +372,18 @@ public sealed class DataSourceLocalRetrievalService( } } - private static RetrievalContentType GetRetrievalContentType(string fileType) => fileType.TrimStart('.').ToLowerInvariant() switch + private static RetrievalContentType GetRetrievalContentType(string fileType) { - "csv" or "tsv" or "ods" or "xls" or "xlsx" or "xlsm" or "xlsb" => RetrievalContentType.TEXT_SPREADSHEET, - "odp" or "ppt" or "pptx" => RetrievalContentType.TEXT_PRESENTATION, - "htm" or "html" => RetrievalContentType.TEXT_WEBSITE, - _ => RetrievalContentType.TEXT_DOCUMENT - }; + if (FileTypes.IsAllowedExtension(fileType, FileTypes.DELIMITED_TABLE, FileTypes.SPREADSHEET)) + return RetrievalContentType.TEXT_SPREADSHEET; + + if (FileTypes.IsAllowedExtension(fileType, FileTypes.POWER_POINT)) + return RetrievalContentType.TEXT_PRESENTATION; + + return FileTypes.IsAllowedExtension(fileType, FileTypes.HTML) + ? RetrievalContentType.TEXT_WEBSITE + : RetrievalContentType.TEXT_DOCUMENT; + } private static string GetQueryText(IContent lastUserPrompt) => lastUserPrompt switch { diff --git a/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs b/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs index 56b85230..ad52ede0 100644 --- a/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs +++ b/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs @@ -8,7 +8,11 @@ namespace AIStudio.Tools.Validation; public sealed class DataSourceValidation { + public const int MAX_NAME_LENGTH = 40; + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceValidation).Namespace, nameof(DataSourceValidation)); + + public static bool IsNameValid(string name) => !string.IsNullOrWhiteSpace(name) && name.Length <= MAX_NAME_LENGTH && !name.Any(char.IsControl); public Func GetSecretStorageIssue { get; init; } = () => string.Empty; @@ -116,10 +120,10 @@ public sealed class DataSourceValidation public string? ValidatingName(string dataSourceName) { - if(string.IsNullOrWhiteSpace(dataSourceName)) + if (string.IsNullOrWhiteSpace(dataSourceName)) return TB("The name must not be empty."); - - if (dataSourceName.Length > 40) + + if (dataSourceName.Length > MAX_NAME_LENGTH) return TB("The name must not exceed 40 characters."); if (dataSourceName.Any(char.IsControl)) diff --git a/runtime/src/qdrant_edge_database.rs b/runtime/src/qdrant_edge_database.rs index b92ae490..83e23f4a 100644 --- a/runtime/src/qdrant_edge_database.rs +++ b/runtime/src/qdrant_edge_database.rs @@ -127,16 +127,10 @@ pub struct DeleteQdrantEdgeStoreRequest { } #[derive(Serialize)] -pub struct QdrantEdgeOperationResponse { +pub struct QdrantEdgeResponse { pub success: bool, pub issue: String, -} - -#[derive(Serialize)] -pub struct QdrantEdgeEnsureStoreResponse { - pub success: bool, - pub issue: String, - pub data: Option, + pub data: Option, } #[derive(Serialize)] @@ -144,13 +138,6 @@ pub struct QdrantEdgeEnsureStoreResult { pub created: bool, } -#[derive(Serialize)] -pub struct QdrantEdgeSearchResponse { - pub success: bool, - pub issue: String, - pub data: Vec, -} - #[derive(Serialize)] pub struct QdrantEdgeSearchResult { pub point_id: String, @@ -203,23 +190,10 @@ impl QdrantEdgeDatabase { } // To ensure a shard exists and that you can insert a vector - fn get_or_create_store(&mut self, store_name: &str, vector_size: usize) -> QdrantEdgeResult<&EdgeShard> { - let path = self.store_path(store_name)?; - let is_initialized = store_is_initialized(&path, store_name)?; + fn get_or_create_store(&mut self, store_name: &str, vector_size: usize) -> QdrantEdgeResult<(&EdgeShard, bool)> { + let (path, is_initialized) = self.reconcile_store_state(store_name)?; if self.shards.contains_key(store_name) { - if is_initialized { - return Ok(self.shards.get(store_name).unwrap()); - } - - warn!(Source = "Qdrant Edge"; "Removing stale cached vector store '{}' because its initialized data directory no longer exists.", store_name); - self.shards.remove(store_name); - } - - if path.exists() && !is_initialized { - warn!(Source = "Qdrant Edge"; "Removing incompletely initialized vector store '{}' before recreating it.", store_name); - fs::remove_dir_all(&path).map_err(|error| { - format!("Failed to remove incomplete vector store '{store_name}' at '{}': {error}", path.display()) - })?; + return Ok((self.shards.get(store_name).unwrap(), false)); } let shard = if is_initialized { @@ -248,27 +222,14 @@ impl QdrantEdgeDatabase { }; self.shards.insert(store_name.to_string(), shard); - Ok(self.shards.get(store_name).unwrap()) + Ok((self.shards.get(store_name).unwrap(), !is_initialized)) } // To check whether a shard exists so you can delete a file from it fn get_existing_store(&mut self, store_name: &str) -> QdrantEdgeResult> { - let path = self.store_path(store_name)?; - let is_initialized = store_is_initialized(&path, store_name)?; + let (path, is_initialized) = self.reconcile_store_state(store_name)?; if self.shards.contains_key(store_name) { - if is_initialized { - return Ok(self.shards.get(store_name)); - } - - warn!(Source = "Qdrant Edge"; "Removing stale cached vector store '{}' because its initialized data directory no longer exists.", store_name); - self.shards.remove(store_name); - } - - if path.exists() && !is_initialized { - warn!(Source = "Qdrant Edge"; "Removing incompletely initialized vector store '{}' before continuing.", store_name); - fs::remove_dir_all(&path).map_err(|error| { - format!("Failed to remove incomplete vector store '{store_name}' at '{}': {error}", path.display()) - })?; + return Ok(self.shards.get(store_name)); } if !is_initialized { @@ -282,6 +243,25 @@ impl QdrantEdgeDatabase { Ok(self.shards.get(store_name)) } + fn reconcile_store_state(&mut self, store_name: &str) -> QdrantEdgeResult<(PathBuf, bool)> { + let path = self.store_path(store_name)?; + let is_initialized = store_is_initialized(&path, store_name)?; + + if self.shards.contains_key(store_name) && !is_initialized { + warn!(Source = "Qdrant Edge"; "Removing stale cached vector store '{}' because its initialized data directory no longer exists.", store_name); + self.shards.remove(store_name); + } + + if path.exists() && !is_initialized { + warn!(Source = "Qdrant Edge"; "Removing incompletely initialized vector store '{}' before continuing.", store_name); + fs::remove_dir_all(&path).map_err(|error| { + format!("Failed to remove incomplete vector store '{store_name}' at '{}': {error}", path.display()) + })?; + } + + Ok((path, is_initialized)) + } + fn info(&self) -> QdrantEdgeResult { let stores_path = self.base_path.join("stores"); let stores_count = if stores_path.exists() { @@ -305,11 +285,10 @@ impl QdrantEdgeDatabase { validate_vector_size(vector_size)?; validate_data_source_name(data_source_name)?; let store_path = self.store_path(store_name)?; - let store_existed = store_is_initialized(&store_path, store_name)?; - self.get_or_create_store(store_name, vector_size)?; + let (_, created) = self.get_or_create_store(store_name, vector_size)?; write_store_display_name(&store_path, data_source_name)?; Ok(QdrantEdgeEnsureStoreResult { - created: !store_existed, + created, }) } @@ -331,7 +310,7 @@ impl QdrantEdgeDatabase { } let store_path = self.store_path(store_name)?; - let shard = self.get_or_create_store(store_name, vector_size)?; + let (shard, _) = self.get_or_create_store(store_name, vector_size)?; write_store_display_name(&store_path, &data_source_name)?; let points = points .into_iter() @@ -454,60 +433,38 @@ pub async fn qdrant_edge_info(_token: APIToken) -> Json { }) } -pub async fn ensure_qdrant_edge_store(_token: APIToken, Json(request): Json) -> Json { - let mut database_guard = QDRANT_EDGE_DATABASE.lock().unwrap(); - let Some(database) = database_guard.as_mut() else { - return Json(QdrantEdgeEnsureStoreResponse { - success: false, - issue: "Qdrant Edge is not available.".to_string(), - data: None, - }); - }; - - match database.ensure_store_exists(&request.store_name, &request.data_source_name, request.vector_size) { - Ok(result) => Json(QdrantEdgeEnsureStoreResponse { - success: true, - issue: String::new(), - data: Some(result), - }), - Err(error) => { - let issue = error.to_string(); - error!(Source = "Qdrant Edge"; "Qdrant Edge operation failed: {issue}"); - Json(QdrantEdgeEnsureStoreResponse { - success: false, - issue, - data: None, - }) - }, - } +pub async fn ensure_qdrant_edge_store(_token: APIToken, Json(request): Json) -> Json> { + execute_qdrant_edge_request(|database| { + database.ensure_store_exists(&request.store_name, &request.data_source_name, request.vector_size) + }) } -pub async fn insert_qdrant_edge_embedding(_token: APIToken, Json(request): Json) -> Json { - execute_qdrant_edge_operation(|database| { +pub async fn insert_qdrant_edge_embedding(_token: APIToken, Json(request): Json) -> Json> { + execute_qdrant_edge_request(|database| { database.insert_embedding(&request.store_name, request.points) }) } -pub async fn search_qdrant_edge_embeddings(_token: APIToken, Json(request): Json) -> Json { - execute_qdrant_edge_query(|database| { +pub async fn search_qdrant_edge_embeddings(_token: APIToken, Json(request): Json) -> Json>> { + execute_qdrant_edge_request(|database| { database.search_embedding(&request.store_name, request.vector, request.max_matches) }) } -pub async fn delete_qdrant_edge_embedding_by_file(_token: APIToken, Json(request): Json) -> Json { - execute_qdrant_edge_operation(|database| { +pub async fn delete_qdrant_edge_embedding_by_file(_token: APIToken, Json(request): Json) -> Json> { + execute_qdrant_edge_request(|database| { database.delete_embedding_by_file(&request.store_name, &request.file_path) }) } -pub async fn optimize_qdrant_edge_store(_token: APIToken, Json(request): Json) -> Json { - execute_qdrant_edge_operation(|database| { +pub async fn optimize_qdrant_edge_store(_token: APIToken, Json(request): Json) -> Json> { + execute_qdrant_edge_request(|database| { database.optimize_store(&request.store_name) }) } -pub async fn delete_qdrant_edge_store(_token: APIToken, Json(request): Json) -> Json { - execute_qdrant_edge_operation(|database| { +pub async fn delete_qdrant_edge_store(_token: APIToken, Json(request): Json) -> Json> { + execute_qdrant_edge_request(|database| { database.delete_store(&request.store_name) }) } @@ -550,60 +507,33 @@ pub fn stop_qdrant_edge_database() { set_qdrant_edge_unavailable("Qdrant Edge was stopped.".to_string()); } -fn execute_qdrant_edge_operation(operation: F) -> Json +fn execute_qdrant_edge_request(operation: F) -> Json> where - F: FnOnce(&mut QdrantEdgeDatabase) -> QdrantEdgeResult<()>, + T: Serialize, + F: FnOnce(&mut QdrantEdgeDatabase) -> QdrantEdgeResult, { let mut database_guard = QDRANT_EDGE_DATABASE.lock().unwrap(); let Some(database) = database_guard.as_mut() else { - return Json(QdrantEdgeOperationResponse { + return Json(QdrantEdgeResponse { success: false, issue: "Qdrant Edge is not available.".to_string(), + data: None, }); }; match operation(database) { - Ok(_) => Json(QdrantEdgeOperationResponse { + Ok(data) => Json(QdrantEdgeResponse { success: true, issue: String::new(), + data: Some(data), }), Err(e) => { let issue = e.to_string(); - error!(Source = "Qdrant Edge"; "Qdrant Edge operation failed: {issue}"); - Json(QdrantEdgeOperationResponse { + error!(Source = "Qdrant Edge"; "Qdrant Edge request failed: {issue}"); + Json(QdrantEdgeResponse { success: false, issue, - }) - }, - } -} - -fn execute_qdrant_edge_query(operation: F) -> Json -where - F: FnOnce(&mut QdrantEdgeDatabase) -> QdrantEdgeResult>, -{ - let mut database_guard = QDRANT_EDGE_DATABASE.lock().unwrap(); - let Some(database) = database_guard.as_mut() else { - return Json(QdrantEdgeSearchResponse { - success: false, - issue: "Qdrant Edge is not available.".to_string(), - data: vec![], - }); - }; - - match operation(database) { - Ok(data) => Json(QdrantEdgeSearchResponse { - success: true, - issue: String::new(), - data, - }), - Err(e) => { - let issue = e.to_string(); - error!(Source = "Qdrant Edge"; "Qdrant Edge query failed: {issue}"); - Json(QdrantEdgeSearchResponse { - success: false, - issue, - data: vec![], + data: None, }) }, } @@ -741,20 +671,21 @@ fn store_is_initialized(path: &Path, store_name: &str) -> QdrantEdgeResult } fn write_store_initialization_marker(path: &Path, store_name: &str) -> std::io::Result<()> { - let marker_path = path.join(STORE_INITIALIZATION_MARKER); - let temporary_marker_path = path.join(STORE_INITIALIZATION_MARKER_TEMP); - fs::write(&temporary_marker_path, store_name)?; - fs::rename(temporary_marker_path, marker_path) + write_store_marker(path, STORE_INITIALIZATION_MARKER, STORE_INITIALIZATION_MARKER_TEMP, store_name) } fn write_store_display_name(path: &Path, data_source_name: &str) -> std::io::Result<()> { - let marker_path = path.join(STORE_DISPLAY_NAME_MARKER); - if fs::read_to_string(&marker_path).is_ok_and(|current_name| current_name == data_source_name) { + write_store_marker(path, STORE_DISPLAY_NAME_MARKER, STORE_DISPLAY_NAME_MARKER_TEMP, data_source_name) +} + +fn write_store_marker(path: &Path, marker_name: &str, temporary_marker_name: &str, value: &str) -> std::io::Result<()> { + let marker_path = path.join(marker_name); + if fs::read_to_string(&marker_path).is_ok_and(|current_value| current_value == value) { return Ok(()); } - let temporary_marker_path = path.join(STORE_DISPLAY_NAME_MARKER_TEMP); - fs::write(&temporary_marker_path, data_source_name)?; + let temporary_marker_path = path.join(temporary_marker_name); + fs::write(&temporary_marker_path, value)?; if marker_path.exists() { fs::remove_file(&marker_path)?; }