MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-08-12 15:32:11 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

added warning and improved error handling

Browse Source
This commit is contained in:
PaulKoudelka 2026-07-28 20:22:37 +02:00
parent 759c7f9d89
commit fd2d99872a
14 changed files with 240 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
app/MindWork AI Studio/Assistants/I18N/allTexts.lua
View File

@ -2206,6 +2206,36 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONFIGURATIONSHORTCUT::T4081853237"] = "C
-- Configure Keyboard Shortcut
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONFIGURATIONSHORTCUT::T636303786"] = "Configure Keyboard Shortcut"
-- Yes, please send my data to the external embedding provider
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T1159107763"] = "Yes, please send my data to the external embedding provider"
-- No, I will choose another embedding
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T1246976418"] = "No, I will choose another embedding"
-- The data source '{0}'
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T2503488371"] = "The data source '{0}'"
-- The file '{0}'
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T2794508936"] = "The file '{0}'"
-- Warning: The selected embedding provider is not self-hosted. Creating embeddings can cost money and may need to run multiple times, for example after errors or file changes. {0} will be sent to an external third party. MindWork AI Studio has no control over what that third party does with the data after it is sent.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T3457494593"] = "Warning: The selected embedding provider is not self-hosted. Creating embeddings can cost money and may need to run multiple times, for example after errors or file changes. {0} will be sent to an external third party. MindWork AI Studio has no control over what that third party does with the data after it is sent."
-- I confirm that I have read and understood the above
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T3683380716"] = "I confirm that I have read and understood the above"
-- The selected data
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T3793916111"] = "The selected data"
-- The selected file
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T3999057817"] = "The selected file"
-- All files in the folder '{0}' and its subfolders
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T661754597"] = "All files in the folder '{0}' and its subfolders"
-- All files in this folder and its subfolders
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T916879200"] = "All files in this folder and its subfolders"
-- Yes, let the AI decide which data sources are needed.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T1031370894"] = "Yes, let the AI decide which data sources are needed."
@ -3805,15 +3835,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1827669611"
-- Update
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1847791252"] = "Update"
-- Please note: the embedding you selected runs in the cloud. All your data will be sent to the cloud. Please confirm that you have read and understood this.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1922618794"] = "Please note: the embedding you selected runs in the cloud. All your data will be sent to the cloud. Please confirm that you have read and understood this."
-- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1948697886"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method."
-- Please note: the embedding you selected runs in the cloud. All your data from the folder '{0}' and all its subdirectories will be sent to the cloud. Please confirm that you have read and understood this.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2403121734"] = "Please note: the embedding you selected runs in the cloud. All your data from the folder '{0}' and all its subdirectories will be sent to the cloud. Please confirm that you have read and understood this."
-- Add
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2646845972"] = "Add"
@ -3826,18 +3850,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2838542994"
-- For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3359366900"] = "For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc."
-- Yes, please send my data to the cloud
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3572613009"] = "Yes, please send my data to the cloud"
-- I confirm that I have read and understood the above
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3683380716"] = "I confirm that I have read and understood the above"
-- Your security policy
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T4081226330"] = "Your security policy"
-- No, I will chose another embedding
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T4253147533"] = "No, I will chose another embedding"
-- Select the base directory
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T562479068"] = "Select the base directory"
@ -3931,15 +3946,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1827669611"] = "
-- Update
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1847791252"] = "Update"
-- Please note: the embedding you selected runs in the cloud. All your data will be sent to the cloud. Please confirm that you have read and understood this.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1922618794"] = "Please note: the embedding you selected runs in the cloud. All your data will be sent to the cloud. Please confirm that you have read and understood this."
-- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1948697886"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method."
-- Please note: the embedding you selected runs in the cloud. All your data within the file '{0}' will be sent to the cloud. Please confirm that you have read and understood this.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2090178026"] = "Please note: the embedding you selected runs in the cloud. All your data within the file '{0}' will be sent to the cloud. Please confirm that you have read and understood this."
-- Add
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2646845972"] = "Add"
@ -3955,21 +3964,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2859265837"] = "
-- For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3359366900"] = "For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc."
-- Yes, please send my data to the cloud
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3572613009"] = "Yes, please send my data to the cloud"
-- I confirm that I have read and understood the above
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3683380716"] = "I confirm that I have read and understood the above"
-- Select the file
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3740148848"] = "Select the file"
-- Your security policy
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T4081226330"] = "Your security policy"
-- No, I will chose another embedding
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T4253147533"] = "No, I will chose another embedding"
-- Data Source Name
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T813773421"] = "Data Source Name"
@ -4168,6 +4168,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(O
-- Invalid tokenizer:
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2448302543"] = "Invalid tokenizer:"
-- Maximum number of tokens sent to the embedding model per chunk.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2544519277"] = "Maximum number of tokens sent to the embedding model per chunk."
-- Add
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2646845972"] = "Add"
@ -4183,6 +4186,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2842060373"] = "In
-- Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T290547799"] = "Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually."
-- Token limit
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2961294165"] = "Token limit"
-- Please enter a token limit greater than 0.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3316544737"] = "Please enter a token limit greater than 0."
-- Choose a custom tokenizer here
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3787466119"] = "Choose a custom tokenizer here"
@ -6160,6 +6169,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2525374657"] = "{0} of {1} files
-- Background embeddings
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2547971789"] = "Background embeddings"
-- Refresh this data source
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2901874229"] = "Refresh this data source"
-- Failed files: {0}
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T309404893"] = "Failed files: {0}"
@ -6169,6 +6181,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T3473125711"] = "Indexed files: {0
-- No local data source has been queued for embedding yet.
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T3774205531"] = "No local data source has been queued for embedding yet."
-- Failure details ({0})
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T843144866"] = "Failure details ({0})"
-- Unlike services like ChatGPT, which impose limits after intensive use, MindWork AI Studio offers unlimited usage through the providers API.
UI_TEXT_CONTENT["AISTUDIO::PAGES::HOME::T1009708591"] = "Unlike services like ChatGPT, which impose limits after intensive use, MindWork AI Studio offers unlimited usage through the providers API."

1
app/MindWork AI Studio/Components/ChatComponent.razor.cs
View File

@ -81,7 +81,6 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
private Guid loadedParameterWorkspaceId = Guid.Empty;
private Guid foregroundChatId = Guid.Empty;
private int workspaceHeaderSyncVersion;
private CancellationTokenSource? cancellationTokenSource;
private HashSet<FileAttachment> chatDocumentPaths = [];
private string tokenCount = "0";
private string TokenCountMessage => $"{this.T("Estimated amount of tokens:")} {this.tokenCount}";

9
app/MindWork AI Studio/Components/DataSourceCloudEmbeddingWarning.razor Normal file
View File

@ -0,0 +1,9 @@
@inherits MSGComponentBase
<MudAlert Severity="Severity.Warning" Variant="Variant.Outlined" Dense="@true" Class="mb-3">
<MudText Typo="Typo.body2">
@this.WarningText
</MudText>
</MudAlert>
<MudTextSwitch Value="@this.UserAcknowledged" ValueChanged="@this.UserAcknowledgedChanged" Label="@T("I confirm that I have read and understood the above")" LabelOn="@T("Yes, please send my data to the external embedding provider")" LabelOff="@T("No, I will choose another embedding")" Validation="@this.Validation"/>

52
app/MindWork AI Studio/Components/DataSourceCloudEmbeddingWarning.razor.cs Normal file
View File

@ -0,0 +1,52 @@
using AIStudio.Settings.DataModel;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Components;
public partial class DataSourceCloudEmbeddingWarning : MSGComponentBase
{
[Parameter]
public DataSourceType DataSourceType { get; set; }
[Parameter]
public string SourcePath { get; set; } = string.Empty;
[Parameter]
public bool UserAcknowledged { get; set; }
[Parameter]
public EventCallback<bool> UserAcknowledgedChanged { get; set; }
[Parameter]
public Func<bool, string?> Validation { get; set; } = _ => null;
private string WarningText
{
get
{
var subject = this.GetSubjectText();
return string.Format(
T("Warning: The selected embedding provider is not self-hosted. Creating embeddings can cost money and may need to run multiple times, for example after errors or file changes. {0} will be sent to an external third party. MindWork AI Studio has no control over what that third party does with the data after it is sent."),
subject);
}
}
private string GetSubjectText()
{
if (string.IsNullOrWhiteSpace(this.SourcePath))
return this.DataSourceType switch
{
DataSourceType.LOCAL_DIRECTORY => T("All files in this folder and its subfolders"),
DataSourceType.LOCAL_FILE => T("The selected file"),
_ => T("The selected data")
};
return this.DataSourceType switch
{
DataSourceType.LOCAL_DIRECTORY => string.Format(T("All files in the folder '{0}' and its subfolders"), this.SourcePath),
DataSourceType.LOCAL_FILE => string.Format(T("The file '{0}'"), this.SourcePath),
_ => string.Format(T("The data source '{0}'"), this.SourcePath)
};
}
}

12
app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor
View File

@ -59,17 +59,7 @@
{
if (this.SelectedCloudEmbedding)
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Error" Class="mb-3">
@if (string.IsNullOrWhiteSpace(this.dataPath))
{
@T("Please note: the embedding you selected runs in the cloud. All your data will be sent to the cloud. Please confirm that you have read and understood this.")
}
else
{
@string.Format(T("Please note: the embedding you selected runs in the cloud. All your data from the folder '{0}' and all its subdirectories will be sent to the cloud. Please confirm that you have read and understood this."), this.dataPath)
}
</MudJustifiedText>
<MudTextSwitch @bind-Value="@this.dataUserAcknowledgedCloudEmbedding" Label="@T("I confirm that I have read and understood the above")" LabelOn="@T("Yes, please send my data to the cloud")" LabelOff="@T("No, I will chose another embedding")" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
<DataSourceCloudEmbeddingWarning DataSourceType="DataSourceType.LOCAL_DIRECTORY" SourcePath="@this.dataPath" @bind-UserAcknowledged="@this.dataUserAcknowledgedCloudEmbedding" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
}
else
{

3
app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs
View File

@ -96,7 +96,8 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
#endregion
private bool SelectedCloudEmbedding => !(this.SettingsManager.ConfigurationData.EmbeddingProviders.FirstOrDefault(x => x.Id == this.dataEmbeddingId)?.IsTrustedForDataSourceSecurityChecks(this.SettingsManager) ?? false);
private bool SelectedCloudEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders
.FirstOrDefault(x => x.Id == this.dataEmbeddingId) is { IsSelfHosted: false };
private DataSourceLocalDirectory CreateDataSource() => new()
{

12
app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor
View File

@ -59,17 +59,7 @@
{
if (this.SelectedCloudEmbedding)
{
<MudJustifiedText Typo="Typo.body1" Color="Color.Error" Class="mb-3">
@if (string.IsNullOrWhiteSpace(this.dataFilePath))
{
@T("Please note: the embedding you selected runs in the cloud. All your data will be sent to the cloud. Please confirm that you have read and understood this.")
}
else
{
@string.Format(T("Please note: the embedding you selected runs in the cloud. All your data within the file '{0}' will be sent to the cloud. Please confirm that you have read and understood this."), this.dataFilePath)
}
</MudJustifiedText>
<MudTextSwitch @bind-Value="@this.dataUserAcknowledgedCloudEmbedding" Label="@T("I confirm that I have read and understood the above")" LabelOn="@T("Yes, please send my data to the cloud")" LabelOff="@T("No, I will chose another embedding")" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
<DataSourceCloudEmbeddingWarning DataSourceType="DataSourceType.LOCAL_FILE" SourcePath="@this.dataFilePath" @bind-UserAcknowledged="@this.dataUserAcknowledgedCloudEmbedding" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
}
else
{

3
app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs
View File

@ -96,7 +96,8 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
#endregion
private bool SelectedCloudEmbedding => !(this.SettingsManager.ConfigurationData.EmbeddingProviders.FirstOrDefault(x => x.Id == this.dataEmbeddingId)?.IsTrustedForDataSourceSecurityChecks(this.SettingsManager) ?? false);
private bool SelectedCloudEmbedding => this.SettingsManager.ConfigurationData.EmbeddingProviders
.FirstOrDefault(x => x.Id == this.dataEmbeddingId) is { IsSelfHosted: false };
private DataSourceLocalFile CreateDataSource() => new()
{

2
app/MindWork AI Studio/Dialogs/Settings/SettingsDialogDataSources.razor.cs
View File

@ -31,7 +31,7 @@ public partial class SettingsDialogDataSources : SettingsDialogBase
private bool CanRefreshDataSource(IDataSource dataSource)
{
return dataSource is DataSourceLocalDirectory or DataSourceLocalFile;
return this.DataSourceEmbeddingService.CanRefreshDataSource(dataSource);
}
private bool HasRefreshableDataSources()

33
app/MindWork AI Studio/Pages/Embeddings.razor
View File

@ -28,7 +28,15 @@
<MudStack Spacing="2">
<MudStack Row="true" Justify="Justify.SpaceBetween" AlignItems="AlignItems.Center">
<MudText Typo="Typo.h6">@status.DataSourceName</MudText>
<MudChip T="string" Color="@GetStatusColor(status)" Variant="Variant.Filled">@status.StateLabel</MudChip>
<MudStack Row="true" Spacing="1" AlignItems="AlignItems.Center">
<MudChip T="string" Color="@GetStatusColor(status)" Variant="Variant.Filled">@status.StateLabel</MudChip>
@if (CanRefresh(status))
{
<MudTooltip Text="@T("Refresh this data source")">
<MudIconButton Icon="@Icons.Material.Filled.Sync" Color="Color.Info" Size="Size.Small" OnClick="@(async () => await this.RefreshDataSource(status))" />
</MudTooltip>
}
</MudStack>
</MudStack>
<MudProgressLinear Value="@status.ProgressPercent" Rounded="@true" Color="@GetStatusColor(status)" />
@ -51,7 +59,28 @@
</MudText>
}
@if (!string.IsNullOrWhiteSpace(status.LastError))
@if (status.Failures.Count > 0)
{
<MudExpansionPanels MultiExpansion="@false" Class="mt-2">
<ExpansionPanel HeaderIcon="@Icons.Material.Filled.ReportProblem" HeaderText="@string.Format(T("Failure details ({0})"), status.Failures.Count)" MaxHeight="360">
<MudStack Spacing="2">
@foreach (var failure in status.Failures)
{
<MudStack Spacing="1" Class="py-1">
<MudText Typo="Typo.body2" Style="word-break: break-word;">
@failure.FilePath
</MudText>
<MudText Typo="Typo.caption" Color="Color.Warning" Style="word-break: break-word;">
@failure.Reason
</MudText>
</MudStack>
}
</MudStack>
</ExpansionPanel>
</MudExpansionPanels>
}
@if (!string.IsNullOrWhiteSpace(status.LastError) && status.Failures.Count == 0)
{
<MudAlert Severity="Severity.Warning" Variant="Variant.Text">
@status.LastError

14
app/MindWork AI Studio/Pages/Embeddings.razor.cs
View File

@ -54,4 +54,18 @@ public partial class Embeddings : MSGComponentBase
DataSourceEmbeddingState.COMPLETED => Color.Success,
_ => Color.Default,
};
private bool CanRefresh(DataSourceEmbeddingStatus status)
{
return this.DataSourceEmbeddingService.CanRefreshDataSource(status.DataSourceId) &&
status.State is not DataSourceEmbeddingState.RUNNING and not DataSourceEmbeddingState.QUEUED &&
(status.State is DataSourceEmbeddingState.FAILED || status.FailedFiles > 0);
}
private async Task RefreshDataSource(DataSourceEmbeddingStatus status)
{
await this.DataSourceEmbeddingService.QueueDataSourceAsync(status.DataSourceId);
this.ReloadStatuses();
await this.InvokeAsync(this.StateHasChanged);
}
}

16
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingModels.cs
View File

@ -28,7 +28,8 @@ public sealed record DataSourceEmbeddingStatus(
int IndexedFiles,
int FailedFiles,
string CurrentFile,
string LastError)
string LastError,
IReadOnlyList<DataSourceEmbeddingFailure> Failures)
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingService).Namespace, nameof(DataSourceEmbeddingService));
@ -53,13 +54,26 @@ public sealed record DataSourceEmbeddingStatus(
};
}
public sealed record DataSourceEmbeddingFailure(
string FilePath,
string Reason);
public sealed class FileEnumerationResult
{
public List<FileInfo> Files { get; } = [];
public List<DataSourceEmbeddingFailure> Failures { get; } = [];
public int FailedFiles { get; set; }
public string LastError { get; set; } = string.Empty;
public void AddFailure(string filePath, string reason)
{
this.Failures.Add(new DataSourceEmbeddingFailure(filePath, reason));
this.FailedFiles = this.Failures.Count;
this.LastError = reason;
}
}
public sealed class DataSourceEmbeddingManifest

19
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs
View File

@ -50,8 +50,8 @@ public sealed partial class DataSourceEmbeddingService
var chunk = currentChunk.ToString().Trim();
if (!string.IsNullOrWhiteSpace(chunk))
{
await foreach (var finalChunk in this.SplitChunkByEmbeddingTokenLimitAsync(chunk, embeddingProvider, token))
yield return finalChunk;
await foreach (var splitChunk in this.SplitChunkByEmbeddingTokenLimitAsync(chunk, embeddingProvider, token))
yield return splitChunk;
}
var overlap = chunk.Length > CHUNK_OVERLAP_LENGTH
@ -239,8 +239,7 @@ public sealed partial class DataSourceEmbeddingService
break;
default:
result.FailedFiles = 1;
result.LastError = $"The selected file '{localFile.FilePath}' is not supported for background embeddings.";
result.AddFailure(localFile.FilePath, $"The selected file '{localFile.FilePath}' is not supported for background embeddings.");
break;
}
@ -254,13 +253,11 @@ public sealed partial class DataSourceEmbeddingService
switch (dataSource)
{
case DataSourceLocalFile localFile:
result.FailedFiles = 1;
result.LastError = $"The selected file '{localFile.FilePath}' does not exist.";
result.AddFailure(localFile.FilePath, $"The selected file '{localFile.FilePath}' does not exist.");
break;
case DataSourceLocalDirectory localDirectory:
result.FailedFiles = 1;
result.LastError = $"The selected directory '{localDirectory.Path}' does not exist.";
result.AddFailure(localDirectory.Path, $"The selected directory '{localDirectory.Path}' does not exist.");
break;
}
@ -286,8 +283,7 @@ public sealed partial class DataSourceEmbeddingService
catch (Exception exception)
{
logger.LogWarning(exception, "Cannot access directory '{DirectoryPath}' while indexing.", currentPath);
result.FailedFiles++;
result.LastError = $"The directory '{currentPath}' could not be accessed.";
result.AddFailure(currentPath, $"The directory '{currentPath}' could not be accessed.");
continue;
}
@ -303,8 +299,7 @@ public sealed partial class DataSourceEmbeddingService
catch (Exception exception)
{
logger.LogWarning(exception, "Cannot inspect file '{FilePath}' while indexing.", filePath);
result.FailedFiles++;
result.LastError = $"The file '{filePath}' could not be inspected.";
result.AddFailure(filePath, $"The file '{filePath}' could not be inspected.");
continue;
}

72
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs
View File

@ -103,11 +103,29 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
this.RefreshWatchers();
}
public bool CanRefreshDataSource(IDataSource dataSource)
{
return this.IsSupportedInternalDataSource(dataSource);
}
public bool CanRefreshDataSource(string dataSourceId)
{
return this.TryGetConfiguredDataSource(dataSourceId, out var dataSource) &&
this.CanRefreshDataSource(dataSource);
}
public Task QueueDataSourceAsync(IDataSource dataSource)
{
return this.QueueDataSourceAsync(dataSource, true);
}
public Task QueueDataSourceAsync(string dataSourceId)
{
return this.TryGetConfiguredDataSource(dataSourceId, out var dataSource)
? this.QueueDataSourceAsync(dataSource)
: Task.CompletedTask;
}
private async Task QueueDataSourceAsync(IDataSource dataSource, bool queueAfterCurrentRun)
{
if (!this.IsSupportedInternalDataSource(dataSource))
@ -134,7 +152,15 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
logger.LogInformation("Queueing data source '{DataSourceName}' ({DataSourceId}) for background embeddings.", dataSource.Name, dataSource.Id);
if (!this.statuses.TryGetValue(dataSource.Id, out var currentStatus) || currentStatus.State is not DataSourceEmbeddingState.RUNNING)
this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.QUEUED, currentStatus?.TotalFiles ?? 0, currentStatus?.IndexedFiles ?? 0, currentStatus?.FailedFiles ?? 0));
{
this.UpsertStatus(this.CreateStatus(
dataSource,
DataSourceEmbeddingState.QUEUED,
currentStatus?.TotalFiles ?? 0,
currentStatus?.IndexedFiles ?? 0,
currentStatus?.FailedFiles ?? 0,
failures: currentStatus?.Failures ?? []));
}
logger.LogDebug("Upserting status for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
await this.queue.Writer.WriteAsync(dataSource.Id);
logger.LogDebug("Queued data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
@ -266,13 +292,15 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
totalFiles,
0,
inputFiles.FailedFiles,
lastError: inputFiles.LastError));
lastError: inputFiles.LastError,
failures: inputFiles.Failures));
var provider = embeddingProvider.CreateProvider();
var skippedFiles = 0;
var completedFiles = 0;
var failedFiles = inputFiles.FailedFiles;
var lastError = inputFiles.LastError;
var failureDetails = inputFiles.Failures.ToList();
foreach (var file in indexedFiles)
{
@ -288,11 +316,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
dataSource.Name,
dataSource.Id);
skippedFiles++;
this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, lastError: lastError));
this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, lastError: lastError, failures: failureDetails));
continue;
}
this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, lastError));
this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, lastError, failureDetails));
try
{
@ -338,16 +366,17 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
{
failedFiles++;
lastError = exception.Message;
failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, exception.Message));
manifest.Files.Remove(file.FullName);
await this.DeleteFilePointsAsync(vectorStore, collectionName, file.FullName, token);
await embeddingState.DeleteFileAsync(dataSource.Id, file.FullName, token);
logger.LogWarning(exception, "Failed to embed file '{FilePath}' for data source '{DataSourceName}'.", file.FullName, dataSource.Name);
this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, exception.Message));
this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, exception.Message, failureDetails));
}
}
this.UpsertStatus(this.CreateCompletedStatus(dataSource, totalFiles, skippedFiles + completedFiles, failedFiles, lastError));
this.UpsertStatus(this.CreateCompletedStatus(dataSource, totalFiles, skippedFiles + completedFiles, failedFiles, lastError, failureDetails));
logger.LogInformation(
"Finished background embeddings for data source '{DataSourceName}' ({DataSourceId}). Indexed={IndexedFiles}, Failed={FailedFiles}, Total={TotalFiles}.",
dataSource.Name,
@ -558,6 +587,14 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
return dataSource is DataSourceLocalDirectory or DataSourceLocalFile;
}
private bool TryGetConfiguredDataSource(string dataSourceId, [NotNullWhen(true)] out IDataSource? dataSource)
{
dataSource = settingsManager.ConfigurationData.DataSources
.FirstOrDefault(source => source.Id.Equals(dataSourceId, StringComparison.OrdinalIgnoreCase));
return dataSource is not null;
}
private bool TryResolveEmbeddingProvider(IDataSource dataSource, [NotNullWhen(true)] out EmbeddingProvider? embeddingProvider)
{
embeddingProvider = settingsManager.ConfigurationData.EmbeddingProviders.FirstOrDefault(provider =>
@ -641,7 +678,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
int indexedFiles,
int failedFiles,
string currentFile = "",
string lastError = "")
string lastError = "",
IReadOnlyList<DataSourceEmbeddingFailure>? failures = null)
{
return new DataSourceEmbeddingStatus(
dataSource.Id,
@ -652,10 +690,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
indexedFiles,
failedFiles,
currentFile,
lastError);
lastError,
failures?.ToList() ?? []);
}
private DataSourceEmbeddingStatus CreateCompletedStatus(IDataSource dataSource, int totalFiles, int indexedFiles, int failedFiles, string lastError)
private DataSourceEmbeddingStatus CreateCompletedStatus(IDataSource dataSource, int totalFiles, int indexedFiles, int failedFiles, string lastError, IReadOnlyList<DataSourceEmbeddingFailure>? failures = null)
{
return this.CreateStatus(
dataSource,
@ -667,12 +706,20 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
? string.IsNullOrWhiteSpace(lastError)
? "Some files could not be embedded. See the logs for details."
: lastError
: string.Empty);
: string.Empty,
failures: failures);
}
private DataSourceEmbeddingStatus GetFallbackStatus(IDataSource dataSource, string errorMessage)
{
return this.CreateStatus(dataSource, DataSourceEmbeddingState.FAILED, 0, 0, 1, lastError: errorMessage);
return this.CreateStatus(
dataSource,
DataSourceEmbeddingState.FAILED,
0,
0,
1,
lastError: errorMessage,
failures: [new DataSourceEmbeddingFailure(dataSource.Name, errorMessage)]);
}
private DataSourceQueueRequestResult TryReserveDataSourceQueueSlot(string dataSourceId, bool queueAfterCurrentRun)
@ -749,7 +796,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
currentStatus?.TotalFiles ?? 0,
currentStatus?.IndexedFiles ?? 0,
currentStatus?.FailedFiles ?? 0,
lastError: currentStatus?.LastError ?? string.Empty));
lastError: currentStatus?.LastError ?? string.Empty,
failures: currentStatus?.Failures ?? []));
try
{