From fd2d99872a35ecc2c9e74da6f50d50b67b968a50 Mon Sep 17 00:00:00 2001 From: PaulKoudelka Date: Tue, 28 Jul 2026 20:22:37 +0200 Subject: [PATCH] added warning and improved error handling --- .../Assistants/I18N/allTexts.lua | 75 +++++++++++-------- .../Components/ChatComponent.razor.cs | 1 - .../DataSourceCloudEmbeddingWarning.razor | 9 +++ .../DataSourceCloudEmbeddingWarning.razor.cs | 52 +++++++++++++ .../DataSourceLocalDirectoryDialog.razor | 12 +-- .../DataSourceLocalDirectoryDialog.razor.cs | 3 +- .../Dialogs/DataSourceLocalFileDialog.razor | 12 +-- .../DataSourceLocalFileDialog.razor.cs | 3 +- .../SettingsDialogDataSources.razor.cs | 2 +- app/MindWork AI Studio/Pages/Embeddings.razor | 33 +++++++- .../Pages/Embeddings.razor.cs | 14 ++++ .../Services/DataSourceEmbeddingModels.cs | 16 +++- .../DataSourceEmbeddingService.Files.cs | 19 ++--- .../Services/DataSourceEmbeddingService.cs | 72 +++++++++++++++--- 14 files changed, 240 insertions(+), 83 deletions(-) create mode 100644 app/MindWork AI Studio/Components/DataSourceCloudEmbeddingWarning.razor create mode 100644 app/MindWork AI Studio/Components/DataSourceCloudEmbeddingWarning.razor.cs diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index 7ebcc47f..96f9748d 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -2206,6 +2206,36 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONFIGURATIONSHORTCUT::T4081853237"] = "C -- Configure Keyboard Shortcut UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONFIGURATIONSHORTCUT::T636303786"] = "Configure Keyboard Shortcut" +-- Yes, please send my data to the external embedding provider +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T1159107763"] = "Yes, please send my data to the external embedding provider" + +-- No, I will choose another embedding +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T1246976418"] = "No, I will choose another embedding" + +-- The data source '{0}' +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T2503488371"] = "The data source '{0}'" + +-- The file '{0}' +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T2794508936"] = "The file '{0}'" + +-- Warning: The selected embedding provider is not self-hosted. Creating embeddings can cost money and may need to run multiple times, for example after errors or file changes. {0} will be sent to an external third party. MindWork AI Studio has no control over what that third party does with the data after it is sent. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T3457494593"] = "Warning: The selected embedding provider is not self-hosted. Creating embeddings can cost money and may need to run multiple times, for example after errors or file changes. {0} will be sent to an external third party. MindWork AI Studio has no control over what that third party does with the data after it is sent." + +-- I confirm that I have read and understood the above +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T3683380716"] = "I confirm that I have read and understood the above" + +-- The selected data +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T3793916111"] = "The selected data" + +-- The selected file +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T3999057817"] = "The selected file" + +-- All files in the folder '{0}' and its subfolders +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T661754597"] = "All files in the folder '{0}' and its subfolders" + +-- All files in this folder and its subfolders +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T916879200"] = "All files in this folder and its subfolders" + -- Yes, let the AI decide which data sources are needed. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T1031370894"] = "Yes, let the AI decide which data sources are needed." @@ -3805,15 +3835,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1827669611" -- Update UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1847791252"] = "Update" --- Please note: the embedding you selected runs in the cloud. All your data will be sent to the cloud. Please confirm that you have read and understood this. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1922618794"] = "Please note: the embedding you selected runs in the cloud. All your data will be sent to the cloud. Please confirm that you have read and understood this." - -- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1948697886"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method." --- Please note: the embedding you selected runs in the cloud. All your data from the folder '{0}' and all its subdirectories will be sent to the cloud. Please confirm that you have read and understood this. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2403121734"] = "Please note: the embedding you selected runs in the cloud. All your data from the folder '{0}' and all its subdirectories will be sent to the cloud. Please confirm that you have read and understood this." - -- Add UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2646845972"] = "Add" @@ -3826,18 +3850,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2838542994" -- For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3359366900"] = "For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc." --- Yes, please send my data to the cloud -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3572613009"] = "Yes, please send my data to the cloud" - --- I confirm that I have read and understood the above -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3683380716"] = "I confirm that I have read and understood the above" - -- Your security policy UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T4081226330"] = "Your security policy" --- No, I will chose another embedding -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T4253147533"] = "No, I will chose another embedding" - -- Select the base directory UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T562479068"] = "Select the base directory" @@ -3931,15 +3946,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1827669611"] = " -- Update UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1847791252"] = "Update" --- Please note: the embedding you selected runs in the cloud. All your data will be sent to the cloud. Please confirm that you have read and understood this. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1922618794"] = "Please note: the embedding you selected runs in the cloud. All your data will be sent to the cloud. Please confirm that you have read and understood this." - -- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1948697886"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method." --- Please note: the embedding you selected runs in the cloud. All your data within the file '{0}' will be sent to the cloud. Please confirm that you have read and understood this. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2090178026"] = "Please note: the embedding you selected runs in the cloud. All your data within the file '{0}' will be sent to the cloud. Please confirm that you have read and understood this." - -- Add UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2646845972"] = "Add" @@ -3955,21 +3964,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2859265837"] = " -- For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3359366900"] = "For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc." --- Yes, please send my data to the cloud -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3572613009"] = "Yes, please send my data to the cloud" - --- I confirm that I have read and understood the above -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3683380716"] = "I confirm that I have read and understood the above" - -- Select the file UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3740148848"] = "Select the file" -- Your security policy UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T4081226330"] = "Your security policy" --- No, I will chose another embedding -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T4253147533"] = "No, I will chose another embedding" - -- Data Source Name UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T813773421"] = "Data Source Name" @@ -4168,6 +4168,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(O -- Invalid tokenizer: UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2448302543"] = "Invalid tokenizer:" +-- Maximum number of tokens sent to the embedding model per chunk. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2544519277"] = "Maximum number of tokens sent to the embedding model per chunk." + -- Add UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2646845972"] = "Add" @@ -4183,6 +4186,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2842060373"] = "In -- Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T290547799"] = "Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually." +-- Token limit +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2961294165"] = "Token limit" + +-- Please enter a token limit greater than 0. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3316544737"] = "Please enter a token limit greater than 0." + -- Choose a custom tokenizer here UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3787466119"] = "Choose a custom tokenizer here" @@ -6160,6 +6169,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2525374657"] = "{0} of {1} files -- Background embeddings UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2547971789"] = "Background embeddings" +-- Refresh this data source +UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2901874229"] = "Refresh this data source" + -- Failed files: {0} UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T309404893"] = "Failed files: {0}" @@ -6169,6 +6181,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T3473125711"] = "Indexed files: {0 -- No local data source has been queued for embedding yet. UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T3774205531"] = "No local data source has been queued for embedding yet." +-- Failure details ({0}) +UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T843144866"] = "Failure details ({0})" + -- Unlike services like ChatGPT, which impose limits after intensive use, MindWork AI Studio offers unlimited usage through the providers API. UI_TEXT_CONTENT["AISTUDIO::PAGES::HOME::T1009708591"] = "Unlike services like ChatGPT, which impose limits after intensive use, MindWork AI Studio offers unlimited usage through the providers API." diff --git a/app/MindWork AI Studio/Components/ChatComponent.razor.cs b/app/MindWork AI Studio/Components/ChatComponent.razor.cs index 7f4a3103..4294cfa7 100644 --- a/app/MindWork AI Studio/Components/ChatComponent.razor.cs +++ b/app/MindWork AI Studio/Components/ChatComponent.razor.cs @@ -81,7 +81,6 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable private Guid loadedParameterWorkspaceId = Guid.Empty; private Guid foregroundChatId = Guid.Empty; private int workspaceHeaderSyncVersion; - private CancellationTokenSource? cancellationTokenSource; private HashSet chatDocumentPaths = []; private string tokenCount = "0"; private string TokenCountMessage => $"{this.T("Estimated amount of tokens:")} {this.tokenCount}"; diff --git a/app/MindWork AI Studio/Components/DataSourceCloudEmbeddingWarning.razor b/app/MindWork AI Studio/Components/DataSourceCloudEmbeddingWarning.razor new file mode 100644 index 00000000..2ad8c465 --- /dev/null +++ b/app/MindWork AI Studio/Components/DataSourceCloudEmbeddingWarning.razor @@ -0,0 +1,9 @@ +@inherits MSGComponentBase + + + + @this.WarningText + + + + diff --git a/app/MindWork AI Studio/Components/DataSourceCloudEmbeddingWarning.razor.cs b/app/MindWork AI Studio/Components/DataSourceCloudEmbeddingWarning.razor.cs new file mode 100644 index 00000000..293685b7 --- /dev/null +++ b/app/MindWork AI Studio/Components/DataSourceCloudEmbeddingWarning.razor.cs @@ -0,0 +1,52 @@ +using AIStudio.Settings.DataModel; + +using Microsoft.AspNetCore.Components; + +namespace AIStudio.Components; + +public partial class DataSourceCloudEmbeddingWarning : MSGComponentBase +{ + [Parameter] + public DataSourceType DataSourceType { get; set; } + + [Parameter] + public string SourcePath { get; set; } = string.Empty; + + [Parameter] + public bool UserAcknowledged { get; set; } + + [Parameter] + public EventCallback UserAcknowledgedChanged { get; set; } + + [Parameter] + public Func Validation { get; set; } = _ => null; + + private string WarningText + { + get + { + var subject = this.GetSubjectText(); + return string.Format( + T("Warning: The selected embedding provider is not self-hosted. Creating embeddings can cost money and may need to run multiple times, for example after errors or file changes. {0} will be sent to an external third party. MindWork AI Studio has no control over what that third party does with the data after it is sent."), + subject); + } + } + + private string GetSubjectText() + { + if (string.IsNullOrWhiteSpace(this.SourcePath)) + return this.DataSourceType switch + { + DataSourceType.LOCAL_DIRECTORY => T("All files in this folder and its subfolders"), + DataSourceType.LOCAL_FILE => T("The selected file"), + _ => T("The selected data") + }; + + return this.DataSourceType switch + { + DataSourceType.LOCAL_DIRECTORY => string.Format(T("All files in the folder '{0}' and its subfolders"), this.SourcePath), + DataSourceType.LOCAL_FILE => string.Format(T("The file '{0}'"), this.SourcePath), + _ => string.Format(T("The data source '{0}'"), this.SourcePath) + }; + } +} diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor index 7cdae497..8154b4fa 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor @@ -59,17 +59,7 @@ { if (this.SelectedCloudEmbedding) { - - @if (string.IsNullOrWhiteSpace(this.dataPath)) - { - @T("Please note: the embedding you selected runs in the cloud. All your data will be sent to the cloud. Please confirm that you have read and understood this.") - } - else - { - @string.Format(T("Please note: the embedding you selected runs in the cloud. All your data from the folder '{0}' and all its subdirectories will be sent to the cloud. Please confirm that you have read and understood this."), this.dataPath) - } - - + } else { diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs index 42463e38..151350b9 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs @@ -96,7 +96,8 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase #endregion - private bool SelectedCloudEmbedding => !(this.SettingsManager.ConfigurationData.EmbeddingProviders.FirstOrDefault(x => x.Id == this.dataEmbeddingId)?.IsTrustedForDataSourceSecurityChecks(this.SettingsManager) ?? false); + private bool SelectedCloudEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders + .FirstOrDefault(x => x.Id == this.dataEmbeddingId) is { IsSelfHosted: false }; private DataSourceLocalDirectory CreateDataSource() => new() { diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor index d360b0de..8ae41651 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor @@ -59,17 +59,7 @@ { if (this.SelectedCloudEmbedding) { - - @if (string.IsNullOrWhiteSpace(this.dataFilePath)) - { - @T("Please note: the embedding you selected runs in the cloud. All your data will be sent to the cloud. Please confirm that you have read and understood this.") - } - else - { - @string.Format(T("Please note: the embedding you selected runs in the cloud. All your data within the file '{0}' will be sent to the cloud. Please confirm that you have read and understood this."), this.dataFilePath) - } - - + } else { diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs index 13b8df1e..b57ca0b8 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs @@ -96,7 +96,8 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase #endregion - private bool SelectedCloudEmbedding => !(this.SettingsManager.ConfigurationData.EmbeddingProviders.FirstOrDefault(x => x.Id == this.dataEmbeddingId)?.IsTrustedForDataSourceSecurityChecks(this.SettingsManager) ?? false); + private bool SelectedCloudEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders + .FirstOrDefault(x => x.Id == this.dataEmbeddingId) is { IsSelfHosted: false }; private DataSourceLocalFile CreateDataSource() => new() { diff --git a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogDataSources.razor.cs b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogDataSources.razor.cs index 0fb2a32a..7b5c7876 100644 --- a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogDataSources.razor.cs +++ b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogDataSources.razor.cs @@ -31,7 +31,7 @@ public partial class SettingsDialogDataSources : SettingsDialogBase private bool CanRefreshDataSource(IDataSource dataSource) { - return dataSource is DataSourceLocalDirectory or DataSourceLocalFile; + return this.DataSourceEmbeddingService.CanRefreshDataSource(dataSource); } private bool HasRefreshableDataSources() diff --git a/app/MindWork AI Studio/Pages/Embeddings.razor b/app/MindWork AI Studio/Pages/Embeddings.razor index 8d1fd01e..b0caa9fe 100644 --- a/app/MindWork AI Studio/Pages/Embeddings.razor +++ b/app/MindWork AI Studio/Pages/Embeddings.razor @@ -28,7 +28,15 @@ @status.DataSourceName - @status.StateLabel + + @status.StateLabel + @if (CanRefresh(status)) + { + + + + } + @@ -51,7 +59,28 @@ } - @if (!string.IsNullOrWhiteSpace(status.LastError)) + @if (status.Failures.Count > 0) + { + + + + @foreach (var failure in status.Failures) + { + + + @failure.FilePath + + + @failure.Reason + + + } + + + + } + + @if (!string.IsNullOrWhiteSpace(status.LastError) && status.Failures.Count == 0) { @status.LastError diff --git a/app/MindWork AI Studio/Pages/Embeddings.razor.cs b/app/MindWork AI Studio/Pages/Embeddings.razor.cs index 80e0bcbe..b9664d26 100644 --- a/app/MindWork AI Studio/Pages/Embeddings.razor.cs +++ b/app/MindWork AI Studio/Pages/Embeddings.razor.cs @@ -54,4 +54,18 @@ public partial class Embeddings : MSGComponentBase DataSourceEmbeddingState.COMPLETED => Color.Success, _ => Color.Default, }; + + private bool CanRefresh(DataSourceEmbeddingStatus status) + { + return this.DataSourceEmbeddingService.CanRefreshDataSource(status.DataSourceId) && + status.State is not DataSourceEmbeddingState.RUNNING and not DataSourceEmbeddingState.QUEUED && + (status.State is DataSourceEmbeddingState.FAILED || status.FailedFiles > 0); + } + + private async Task RefreshDataSource(DataSourceEmbeddingStatus status) + { + await this.DataSourceEmbeddingService.QueueDataSourceAsync(status.DataSourceId); + this.ReloadStatuses(); + await this.InvokeAsync(this.StateHasChanged); + } } diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingModels.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingModels.cs index 6275aef2..9546d547 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingModels.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingModels.cs @@ -28,7 +28,8 @@ public sealed record DataSourceEmbeddingStatus( int IndexedFiles, int FailedFiles, string CurrentFile, - string LastError) + string LastError, + IReadOnlyList Failures) { private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingService).Namespace, nameof(DataSourceEmbeddingService)); @@ -53,13 +54,26 @@ public sealed record DataSourceEmbeddingStatus( }; } +public sealed record DataSourceEmbeddingFailure( + string FilePath, + string Reason); + public sealed class FileEnumerationResult { public List Files { get; } = []; + public List Failures { get; } = []; + public int FailedFiles { get; set; } public string LastError { get; set; } = string.Empty; + + public void AddFailure(string filePath, string reason) + { + this.Failures.Add(new DataSourceEmbeddingFailure(filePath, reason)); + this.FailedFiles = this.Failures.Count; + this.LastError = reason; + } } public sealed class DataSourceEmbeddingManifest diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs index 29477bdb..e353951b 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs @@ -50,8 +50,8 @@ public sealed partial class DataSourceEmbeddingService var chunk = currentChunk.ToString().Trim(); if (!string.IsNullOrWhiteSpace(chunk)) { - await foreach (var finalChunk in this.SplitChunkByEmbeddingTokenLimitAsync(chunk, embeddingProvider, token)) - yield return finalChunk; + await foreach (var splitChunk in this.SplitChunkByEmbeddingTokenLimitAsync(chunk, embeddingProvider, token)) + yield return splitChunk; } var overlap = chunk.Length > CHUNK_OVERLAP_LENGTH @@ -239,8 +239,7 @@ public sealed partial class DataSourceEmbeddingService break; default: - result.FailedFiles = 1; - result.LastError = $"The selected file '{localFile.FilePath}' is not supported for background embeddings."; + result.AddFailure(localFile.FilePath, $"The selected file '{localFile.FilePath}' is not supported for background embeddings."); break; } @@ -254,13 +253,11 @@ public sealed partial class DataSourceEmbeddingService switch (dataSource) { case DataSourceLocalFile localFile: - result.FailedFiles = 1; - result.LastError = $"The selected file '{localFile.FilePath}' does not exist."; + result.AddFailure(localFile.FilePath, $"The selected file '{localFile.FilePath}' does not exist."); break; case DataSourceLocalDirectory localDirectory: - result.FailedFiles = 1; - result.LastError = $"The selected directory '{localDirectory.Path}' does not exist."; + result.AddFailure(localDirectory.Path, $"The selected directory '{localDirectory.Path}' does not exist."); break; } @@ -286,8 +283,7 @@ public sealed partial class DataSourceEmbeddingService catch (Exception exception) { logger.LogWarning(exception, "Cannot access directory '{DirectoryPath}' while indexing.", currentPath); - result.FailedFiles++; - result.LastError = $"The directory '{currentPath}' could not be accessed."; + result.AddFailure(currentPath, $"The directory '{currentPath}' could not be accessed."); continue; } @@ -303,8 +299,7 @@ public sealed partial class DataSourceEmbeddingService catch (Exception exception) { logger.LogWarning(exception, "Cannot inspect file '{FilePath}' while indexing.", filePath); - result.FailedFiles++; - result.LastError = $"The file '{filePath}' could not be inspected."; + result.AddFailure(filePath, $"The file '{filePath}' could not be inspected."); continue; } diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index b2dfa4fa..a7dd3404 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -103,11 +103,29 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM this.RefreshWatchers(); } + public bool CanRefreshDataSource(IDataSource dataSource) + { + return this.IsSupportedInternalDataSource(dataSource); + } + + public bool CanRefreshDataSource(string dataSourceId) + { + return this.TryGetConfiguredDataSource(dataSourceId, out var dataSource) && + this.CanRefreshDataSource(dataSource); + } + public Task QueueDataSourceAsync(IDataSource dataSource) { return this.QueueDataSourceAsync(dataSource, true); } + public Task QueueDataSourceAsync(string dataSourceId) + { + return this.TryGetConfiguredDataSource(dataSourceId, out var dataSource) + ? this.QueueDataSourceAsync(dataSource) + : Task.CompletedTask; + } + private async Task QueueDataSourceAsync(IDataSource dataSource, bool queueAfterCurrentRun) { if (!this.IsSupportedInternalDataSource(dataSource)) @@ -134,7 +152,15 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM logger.LogInformation("Queueing data source '{DataSourceName}' ({DataSourceId}) for background embeddings.", dataSource.Name, dataSource.Id); if (!this.statuses.TryGetValue(dataSource.Id, out var currentStatus) || currentStatus.State is not DataSourceEmbeddingState.RUNNING) - this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.QUEUED, currentStatus?.TotalFiles ?? 0, currentStatus?.IndexedFiles ?? 0, currentStatus?.FailedFiles ?? 0)); + { + this.UpsertStatus(this.CreateStatus( + dataSource, + DataSourceEmbeddingState.QUEUED, + currentStatus?.TotalFiles ?? 0, + currentStatus?.IndexedFiles ?? 0, + currentStatus?.FailedFiles ?? 0, + failures: currentStatus?.Failures ?? [])); + } logger.LogDebug("Upserting status for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id); await this.queue.Writer.WriteAsync(dataSource.Id); logger.LogDebug("Queued data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id); @@ -266,13 +292,15 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM totalFiles, 0, inputFiles.FailedFiles, - lastError: inputFiles.LastError)); + lastError: inputFiles.LastError, + failures: inputFiles.Failures)); var provider = embeddingProvider.CreateProvider(); var skippedFiles = 0; var completedFiles = 0; var failedFiles = inputFiles.FailedFiles; var lastError = inputFiles.LastError; + var failureDetails = inputFiles.Failures.ToList(); foreach (var file in indexedFiles) { @@ -288,11 +316,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM dataSource.Name, dataSource.Id); skippedFiles++; - this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, lastError: lastError)); + this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, lastError: lastError, failures: failureDetails)); continue; } - this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, lastError)); + this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, lastError, failureDetails)); try { @@ -338,16 +366,17 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM { failedFiles++; lastError = exception.Message; + failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, exception.Message)); manifest.Files.Remove(file.FullName); await this.DeleteFilePointsAsync(vectorStore, collectionName, file.FullName, token); await embeddingState.DeleteFileAsync(dataSource.Id, file.FullName, token); logger.LogWarning(exception, "Failed to embed file '{FilePath}' for data source '{DataSourceName}'.", file.FullName, dataSource.Name); - this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, exception.Message)); + this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, exception.Message, failureDetails)); } } - this.UpsertStatus(this.CreateCompletedStatus(dataSource, totalFiles, skippedFiles + completedFiles, failedFiles, lastError)); + this.UpsertStatus(this.CreateCompletedStatus(dataSource, totalFiles, skippedFiles + completedFiles, failedFiles, lastError, failureDetails)); logger.LogInformation( "Finished background embeddings for data source '{DataSourceName}' ({DataSourceId}). Indexed={IndexedFiles}, Failed={FailedFiles}, Total={TotalFiles}.", dataSource.Name, @@ -558,6 +587,14 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM return dataSource is DataSourceLocalDirectory or DataSourceLocalFile; } + private bool TryGetConfiguredDataSource(string dataSourceId, [NotNullWhen(true)] out IDataSource? dataSource) + { + dataSource = settingsManager.ConfigurationData.DataSources + .FirstOrDefault(source => source.Id.Equals(dataSourceId, StringComparison.OrdinalIgnoreCase)); + + return dataSource is not null; + } + private bool TryResolveEmbeddingProvider(IDataSource dataSource, [NotNullWhen(true)] out EmbeddingProvider? embeddingProvider) { embeddingProvider = settingsManager.ConfigurationData.EmbeddingProviders.FirstOrDefault(provider => @@ -641,7 +678,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM int indexedFiles, int failedFiles, string currentFile = "", - string lastError = "") + string lastError = "", + IReadOnlyList? failures = null) { return new DataSourceEmbeddingStatus( dataSource.Id, @@ -652,10 +690,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM indexedFiles, failedFiles, currentFile, - lastError); + lastError, + failures?.ToList() ?? []); } - private DataSourceEmbeddingStatus CreateCompletedStatus(IDataSource dataSource, int totalFiles, int indexedFiles, int failedFiles, string lastError) + private DataSourceEmbeddingStatus CreateCompletedStatus(IDataSource dataSource, int totalFiles, int indexedFiles, int failedFiles, string lastError, IReadOnlyList? failures = null) { return this.CreateStatus( dataSource, @@ -667,12 +706,20 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM ? string.IsNullOrWhiteSpace(lastError) ? "Some files could not be embedded. See the logs for details." : lastError - : string.Empty); + : string.Empty, + failures: failures); } private DataSourceEmbeddingStatus GetFallbackStatus(IDataSource dataSource, string errorMessage) { - return this.CreateStatus(dataSource, DataSourceEmbeddingState.FAILED, 0, 0, 1, lastError: errorMessage); + return this.CreateStatus( + dataSource, + DataSourceEmbeddingState.FAILED, + 0, + 0, + 1, + lastError: errorMessage, + failures: [new DataSourceEmbeddingFailure(dataSource.Name, errorMessage)]); } private DataSourceQueueRequestResult TryReserveDataSourceQueueSlot(string dataSourceId, bool queueAfterCurrentRun) @@ -749,7 +796,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM currentStatus?.TotalFiles ?? 0, currentStatus?.IndexedFiles ?? 0, currentStatus?.FailedFiles ?? 0, - lastError: currentStatus?.LastError ?? string.Empty)); + lastError: currentStatus?.LastError ?? string.Empty, + failures: currentStatus?.Failures ?? [])); try {