MindWorkAI/AI-Studio
1
0 Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-10-18 02:57:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added mailboxes as local data sources (#1022)
Build and Release / Determine run mode (push) Waiting to run
Build and Release / Read metadata (push) Blocked by required conditions
Build and Release / Sync Flatpak repo (push) Blocked by required conditions
Build and Release / Collect Flatpak artifacts (push) Blocked by required conditions
Build and Release / Verify (push) Waiting to run
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions

Browse Source
This commit is contained in:
Thorsten Sommer authored and GitHub committed 2026-10-04 12:26:26 +02:00
1 parent 93be539e50
commit c4400c0ff6
343 files changed
+26051 -3122
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/MindWork AI Studio/Pages/Embeddings.razor
+65 -16
View File
@@ -1,4 +1,5 @@
@attribute [Route(Routes.EMBEDDINGS)]
@using AIStudio.Tools.Services
@inherits MSGComponentBase
<MudStack Spacing="3" Class="pr-2 pb-4" Style="height: 100%; min-height: 0; overflow-y: auto;">
@@ -13,11 +14,17 @@
<MudText Typo="Typo.body1" Class="mt-2">
@T("AI Studio indexes local RAG data sources in the background. Finished files stay recorded so unchanged files can be skipped after a restart, while added or deleted files are detected during the next run. The same applies to documents without readable text, such as scanned pages: AI Studio remembers them and reads them again only once they change.")
</MudText>
@if (this.AreMailboxesEnabled)
{
<MudText Typo="Typo.body1" Class="mt-2">
@T("Mailboxes are synced every 16 minutes while the automatic refresh is on. Each mail is embedded only once: when it moves to another folder, AI Studio merely notes where it lies now.")
</MudText>
}
<MudStack Row="true" Class="mt-3" Wrap="Wrap.Wrap" Spacing="2">
<MudChip T="string" Color="Color.Success" Variant="Variant.Filled">@string.Format(T("Indexed files: {0}"), this.TotalIndexedFiles)</MudChip>
<MudChip T="string" Color="Color.Info" Variant="Variant.Filled">@string.Format(T("Pending files: {0}"), this.TotalPendingFiles)</MudChip>
<MudChip T="string" Color="Color.Default" Variant="Variant.Filled">@string.Format(T("Skipped files: {0}"), this.TotalPermanentlySkippedFiles)</MudChip>
<MudChip T="string" Color="Color.Error" Variant="Variant.Filled">@string.Format(T("Failed files: {0}"), this.TotalFailedFiles)</MudChip>
<MudChip T="string" Color="Color.Success" Variant="Variant.Filled">@string.Format(T("Indexed: {0}"), this.FormatNumber(this.TotalIndexedDocuments))</MudChip>
<MudChip T="string" Color="Color.Info" Variant="Variant.Filled">@string.Format(T("Pending: {0}"), this.FormatNumber(this.TotalPendingDocuments))</MudChip>
<MudChip T="string" Color="Color.Default" Variant="Variant.Filled">@string.Format(T("Skipped: {0}"), this.FormatNumber(this.TotalPermanentlySkippedDocuments))</MudChip>
<MudChip T="string" Color="Color.Error" Variant="Variant.Filled">@string.Format(T("Failed: {0}"), this.FormatNumber(this.TotalFailedDocuments))</MudChip>
</MudStack>
@if (this.IsWorkingThroughDataSources)
{
@@ -74,27 +81,34 @@
<MudProgressLinear Value="@status.ProgressPercent" Rounded="@true" Color="@GetStatusColor(status)" />
<MudText Typo="Typo.body2">
@this.GetFileProgressText(status)
@this.GetProgressText(status)
</MudText>
@if (status.PermanentlySkippedFiles > 0)
@if (IsMailbox(status))
{
<MudText Typo="Typo.body2">
@string.Format(T("Skipped files: {0}. AI Studio reads them again once they change."), status.PermanentlySkippedFiles)
@this.GetLastSyncText(status)
</MudText>
}
@if (status.FailedFiles > 0)
@if (status.PermanentlySkippedDocuments > 0)
{
<MudText Typo="Typo.body2">
@string.Format(T("Failed files: {0}"), status.FailedFiles)
@this.GetSkippedText(status)
</MudText>
}
@if (!string.IsNullOrWhiteSpace(status.CurrentFile))
@if (status.FailedDocuments > 0)
{
<MudText Typo="Typo.body2">
@string.Format(T("Current file: {0}"), status.CurrentFile)
@this.GetFailedText(status)
</MudText>
}
@if (!string.IsNullOrWhiteSpace(status.CurrentDocument))
{
<MudText Typo="Typo.body2">
@this.GetCurrentDocumentText(status)
</MudText>
}
@@ -112,7 +126,7 @@
<MudStack Spacing="3">
<MudStack Row="@true" Wrap="Wrap.Wrap" Spacing="2" AlignItems="AlignItems.Center">
<MudChip T="string" Size="Size.Small" Color="@group.Cause.Color" Variant="Variant.Outlined">
@(group.Cause.IsPermanent ? T("Skipped until the file changes") : T("Tried again during the next run"))
@(group.Cause.IsPermanent ? (IsMailbox(status) ? T("Skipped") : T("Skipped until the file changes")) : T("Tried again during the next run"))
</MudChip>
@if (!string.IsNullOrWhiteSpace(group.EmbeddingProviderName))
{
@@ -132,15 +146,22 @@
<col style="width: 7em;"/>
</ColGroup>
<HeaderContent>
<MudTh>@T("File")</MudTh>
<MudTh>@(IsMailbox(status) ? T("Mail") : T("File"))</MudTh>
<MudTh>@T("Noticed")</MudTh>
<MudTh>@T("Actions")</MudTh>
</HeaderContent>
<RowTemplate>
<MudTd Style="white-space: normal; overflow-wrap: anywhere;">
@* Captions render as spans, so each of them needs to be told to take its own line. *@
<MudText Typo="Typo.body2">@GetFileName(context.FilePath)</MudText>
<MudText Typo="Typo.caption" Class="mud-text-secondary d-block">@context.FilePath</MudText>
@if (IsMailbox(status))
{
<MudText Typo="Typo.body2">@GetMailName(context)</MudText>
}
else
{
<MudText Typo="Typo.body2">@GetFileName(context.DocumentKey)</MudText>
<MudText Typo="Typo.caption" Class="mud-text-secondary d-block">@context.DocumentKey</MudText>
}
@if (group.Cause.ShowsMessagePerFile)
{
<MudText Typo="Typo.caption" Color="Color.Warning" Class="d-block">@context.Reason</MudText>
@@ -150,7 +171,7 @@
<MudText Typo="Typo.caption">@GetOccurrenceText(context)</MudText>
</MudTd>
<MudTd>
@if (CanShowInFileManager(context))
@if (CanShowInFileManager(status, context))
{
<MudTooltip Text="@T("Show this file in the file browser of your system")">
<MudIconButton Icon="@Icons.Material.Filled.FolderOpen" Color="Color.Info" Size="Size.Small" OnClick="@(async () => await this.ShowInFileManager(context))"/>
@@ -179,6 +200,34 @@
@status.LastError
</MudAlert>
}
@*
What the user has to decide comes with the buttons to decide it. The
sentence above says what is at stake; reading it and then having to find
the settings of the mailbox is where people give up.
*@
@if (status.Attention is DataSourceAttention.AUTH_FAILED)
{
<MudStack Row="@true" Wrap="Wrap.Wrap" Spacing="2">
<MudButton Variant="Variant.Filled" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Key" OnClick="@(async () => await this.ChangeMailboxSettings(status))">
@T("Change the password")
</MudButton>
<MudButton Variant="Variant.Outlined" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Sync" OnClick="@(async () => await this.RefreshDataSource(status))">
@T("Try to sign in again")
</MudButton>
</MudStack>
}
else if (status is { Attention: DataSourceAttention.MASS_REMOVAL_PENDING, PendingRemovalCount: not null })
{
<MudStack Row="@true" Wrap="Wrap.Wrap" Spacing="2">
<MudButton Variant="Variant.Filled" Color="Color.Warning" StartIcon="@Icons.Material.Filled.PlaylistRemove" OnClick="@(async () => await this.ApproveMailRemoval(status))">
@T("Remove them from the index")
</MudButton>
<MudButton Variant="Variant.Outlined" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Settings" OnClick="@(async () => await this.ChangeMailboxSettings(status))">
@T("Change the settings of the mailbox")
</MudButton>
</MudStack>
}
</MudStack>
</ChildContent>
</MudExpansionPanel>
app/MindWork AI Studio/Pages/Embeddings.razor.cs
+103 -22
View File
@@ -44,18 +44,24 @@ public partial class Embeddings : MSGComponentBase
/// </remarks>
private CultureInfo currentCulture = CultureInfo.InvariantCulture;
private int TotalIndexedFiles => this.Statuses.Sum(status => status.IndexedFiles);
/// <remarks>
/// The sums above every data source count files and mails alike, which is why their chips name
/// neither.
/// </remarks>
private int TotalIndexedDocuments => this.Statuses.Sum(status => status.IndexedDocuments);
private int TotalPendingFiles => this.Statuses.Sum(status => Math.Max(0, status.TotalFiles - status.IndexedFiles - status.FailedFiles - status.PermanentlySkippedFiles));
private int TotalPendingDocuments => this.Statuses.Sum(status => Math.Max(0, status.TotalDocuments - status.IndexedDocuments - status.FailedDocuments - status.PermanentlySkippedDocuments));
private int TotalFailedFiles => this.Statuses.Sum(status => status.FailedFiles);
private int TotalFailedDocuments => this.Statuses.Sum(status => status.FailedDocuments);
private int TotalPermanentlySkippedFiles => this.Statuses.Sum(status => status.PermanentlySkippedFiles);
private int TotalPermanentlySkippedDocuments => this.Statuses.Sum(status => status.PermanentlySkippedDocuments);
private bool AreMailboxesEnabled => PreviewFeatures.PRE_MAILBOXES_2026.IsEnabled(this.SettingsManager);
/// <remarks>
/// The chips above count files, which says nothing about how far the list of data sources itself
/// has come. While several of them wait their turn, this is the one line saying so. With a single
/// data source there is nothing to say: its own row already tells the whole story.
/// The chips above count documents, which says nothing about how far the list of data sources
/// itself has come. While several of them wait their turn, this is the one line saying so. With a
/// single data source there is nothing to say: its own row already tells the whole story.
/// </remarks>
private bool IsWorkingThroughDataSources => this.Statuses.Count > 1 && this.Statuses.Any(status => status.State is DataSourceEmbeddingState.RUNNING or DataSourceEmbeddingState.QUEUED);
@@ -141,7 +147,7 @@ public partial class Embeddings : MSGComponentBase
/// data source at once.
/// </remarks>
private static bool IsWorthOpening(DataSourceEmbeddingStatus status) =>
status.State is DataSourceEmbeddingState.RUNNING or DataSourceEmbeddingState.QUEUED or DataSourceEmbeddingState.FAILED || status.FailedFiles > 0;
status.State is DataSourceEmbeddingState.RUNNING or DataSourceEmbeddingState.QUEUED or DataSourceEmbeddingState.FAILED || status.FailedDocuments > 0;
/// <remarks>
/// MudBlazor keeps track of which panel is open on its own, so this only records the decision.
@@ -173,7 +179,7 @@ public partial class Embeddings : MSGComponentBase
}
/// <summary>
/// What the panel of a data source says about its progress through the files.
/// What the panel of a data source says about its progress through its files or mails.
/// </summary>
/// <remarks>
/// While a file is being worked on, the sentence names that file and how far into it we are.
@@ -186,26 +192,63 @@ public partial class Embeddings : MSGComponentBase
/// while it is being read, and hanging the choice on the block number let the line jump back
/// and forth between two entirely different sentences at every file. Now the beginning of the
/// sentence stays put and the blocks are appended to it as soon as the first one arrives.
///
/// A mail has no pages: those of its attachments would not say which attachment they are in.
/// </remarks>
private string GetFileProgressText(DataSourceEmbeddingStatus status)
private string GetProgressText(DataSourceEmbeddingStatus status)
{
if (status.State is not DataSourceEmbeddingState.RUNNING || string.IsNullOrWhiteSpace(status.CurrentFile))
return string.Format(T("{0} of {1} files are indexed."), this.FormatNumber(status.IndexedFiles), this.FormatNumber(status.TotalFiles));
var isMailbox = IsMailbox(status);
if (status.State is not DataSourceEmbeddingState.RUNNING || string.IsNullOrWhiteSpace(status.CurrentDocument))
{
return isMailbox
? string.Format(T("{0} of {1} mails are indexed."), this.FormatNumber(status.IndexedDocuments), this.FormatNumber(status.TotalDocuments))
: string.Format(T("{0} of {1} files are indexed."), this.FormatNumber(status.IndexedDocuments), this.FormatNumber(status.TotalDocuments));
}
//
// Everything already dealt with, plus the one in hand. Skipped and failed files are part of
// that: they are behind us in the folder, and leaving them out would let the number fall
// behind the file whose name is shown right next to it.
//
var currentFileNumber = Math.Min(status.TotalFiles, status.IndexedFiles + status.PermanentlySkippedFiles + status.FailedFiles + 1);
var currentNumber = this.FormatNumber(Math.Min(status.TotalDocuments, status.IndexedDocuments + status.PermanentlySkippedDocuments + status.FailedDocuments + 1));
var total = this.FormatNumber(status.TotalDocuments);
if (isMailbox)
{
return status.CurrentDocumentBlock is { } mailBlock
? string.Format(T("Mail {0} of {1} is being indexed: block {2}."), currentNumber, total, this.FormatNumber(mailBlock))
: string.Format(T("Mail {0} of {1} is being indexed."), currentNumber, total);
}
return status switch
{
{ CurrentFileBlock: { } block, CurrentFilePage: { } page } => string.Format(T("File {0} of {1} is being indexed: block {2}, page {3}."), this.FormatNumber(currentFileNumber), this.FormatNumber(status.TotalFiles), this.FormatNumber(block), this.FormatNumber(page)),
{ CurrentFileBlock: { } block } => string.Format(T("File {0} of {1} is being indexed: block {2}."), this.FormatNumber(currentFileNumber), this.FormatNumber(status.TotalFiles), this.FormatNumber(block)),
_ => string.Format(T("File {0} of {1} is being indexed."), this.FormatNumber(currentFileNumber), this.FormatNumber(status.TotalFiles)),
{ CurrentDocumentBlock: { } block, CurrentDocumentPage: { } page } => string.Format(T("File {0} of {1} is being indexed: block {2}, page {3}."), currentNumber, total, this.FormatNumber(block), this.FormatNumber(page)),
{ CurrentDocumentBlock: { } block } => string.Format(T("File {0} of {1} is being indexed: block {2}."), currentNumber, total, this.FormatNumber(block)),
_ => string.Format(T("File {0} of {1} is being indexed."), currentNumber, total),
};
}
private string GetSkippedText(DataSourceEmbeddingStatus status) => IsMailbox(status)
? string.Format(T("Skipped mails: {0}."), this.FormatNumber(status.PermanentlySkippedDocuments))
: string.Format(T("Skipped files: {0}. AI Studio reads them again once they change."), this.FormatNumber(status.PermanentlySkippedDocuments));
private string GetFailedText(DataSourceEmbeddingStatus status) => IsMailbox(status)
? string.Format(T("Failed mails: {0}"), this.FormatNumber(status.FailedDocuments))
: string.Format(T("Failed files: {0}"), this.FormatNumber(status.FailedDocuments));
private string GetCurrentDocumentText(DataSourceEmbeddingStatus status) => IsMailbox(status)
? string.Format(T("Current mail: {0}"), status.CurrentDocument)
: string.Format(T("Current file: {0}"), status.CurrentDocument);
private static bool IsMailbox(DataSourceEmbeddingStatus status) => status.DataSourceType is DataSourceType.MAILBOX;
/// <remarks>
/// A mailbox can be searched long before it was worked through once: its first sync takes
/// hours with a large one, newest mails first. What the index lacks until then are older mails.
/// </remarks>
private string GetLastSyncText(DataSourceEmbeddingStatus status) => status.LastSyncUtc is { } lastSyncUtc
? string.Format(T("Last complete sync: {0}"), lastSyncUtc.ToLocalTime().ToString("g", this.currentCulture))
: T("Not synced completely yet. AI Studio works through the mailbox piece by piece, the newest mails first, and older mails are still missing.");
private string FormatNumber(int value) => value.ToString("N0", this.currentCulture);
private static Color GetStatusColor(DataSourceEmbeddingStatus status) => status.State switch
@@ -213,7 +256,7 @@ public partial class Embeddings : MSGComponentBase
DataSourceEmbeddingState.RUNNING => Color.Warning,
DataSourceEmbeddingState.QUEUED => Color.Info,
DataSourceEmbeddingState.FAILED => Color.Error,
DataSourceEmbeddingState.COMPLETED when status.FailedFiles > 0 => Color.Warning,
DataSourceEmbeddingState.COMPLETED when status.FailedDocuments > 0 => Color.Warning,
DataSourceEmbeddingState.COMPLETED => Color.Success,
_ => Color.Default,
};
@@ -238,7 +281,7 @@ public partial class Embeddings : MSGComponentBase
/// </remarks>
private IReadOnlyList<FailureGroup> GetFailureGroups(DataSourceEmbeddingStatus status) => status.Failures
.GroupBy(this.GetFailureCause)
.Select(group => new FailureGroup(group.Key, GetEmbeddingProviderName(group), group.OrderBy(failure => failure.FilePath, StringComparer.OrdinalIgnoreCase).ToList()))
.Select(group => new FailureGroup(group.Key, GetEmbeddingProviderName(group), group.OrderBy(failure => IsMailbox(status) ? GetMailName(failure) : failure.DocumentKey, StringComparer.OrdinalIgnoreCase).ToList()))
.OrderBy(group => group.Cause.Priority)
.ThenByDescending(group => group.Failures.Count)
.ThenBy(group => group.Cause.Title, StringComparer.OrdinalIgnoreCase)
@@ -285,9 +328,19 @@ public partial class Embeddings : MSGComponentBase
/// <remarks>
/// A failure which was not about one file, such as a folder which is gone, carries the name of
/// the data source instead of a path. There is nothing to show for those.
/// the data source instead of a path. There is nothing to show for those, nor for a mail, which
/// lies on a server.
/// </remarks>
private static bool CanShowInFileManager(DataSourceEmbeddingFailure failure) => !string.IsNullOrWhiteSpace(failure.FilePath) && Path.IsPathRooted(failure.FilePath);
private static bool CanShowInFileManager(DataSourceEmbeddingStatus status, DataSourceEmbeddingFailure failure) => !IsMailbox(status) && !string.IsNullOrWhiteSpace(failure.DocumentKey) && Path.IsPathRooted(failure.DocumentKey);
/// <summary>
/// How a failed mail is called in the list: by its subject, which is all the user knows it by.
/// </summary>
/// <remarks>
/// The key of a mail is a hash, which means nothing to anybody. A failure which was not about
/// one mail carries the name of the mailbox instead.
/// </remarks>
private static string GetMailName(DataSourceEmbeddingFailure failure) => string.IsNullOrWhiteSpace(failure.DisplayName) ? failure.DocumentKey : failure.DisplayName;
/// <summary>
/// Opens the file browser of the system and selects the file in it.
@@ -302,7 +355,7 @@ public partial class Embeddings : MSGComponentBase
OpenPathResponse response;
try
{
response = await this.RustService.TryOpenPathInRuntimeFileManager(failure.FilePath);
response = await this.RustService.TryOpenPathInRuntimeFileManager(failure.DocumentKey);
}
catch (Exception e)
{
@@ -326,7 +379,7 @@ public partial class Embeddings : MSGComponentBase
{
return this.DataSourceEmbeddingService.CanRefreshDataSource(status.DataSourceId) &&
status is { VectorStoreUnreadable: false, State: not DataSourceEmbeddingState.RUNNING and not DataSourceEmbeddingState.QUEUED } &&
(status.State is DataSourceEmbeddingState.FAILED || status.FailedFiles > 0);
(status.State is DataSourceEmbeddingState.FAILED || status.FailedDocuments > 0);
}
/// <remarks>
@@ -356,6 +409,34 @@ public partial class Embeddings : MSGComponentBase
await this.InvokeAsync(this.StateHasChanged);
}
/// <summary>
/// Opens the settings of a mailbox, e.g. to enter the current password after a refused sign-in.
/// </summary>
/// <remarks>
/// Saving them syncs the mailbox right away. With a new password, that sync is allowed to sign
/// in. The page reloads through CONFIGURATION_CHANGED, which it receives like everybody else.
/// </remarks>
private async Task ChangeMailboxSettings(DataSourceEmbeddingStatus status)
{
if (await MailboxEditing.EditAsync(this.DialogService, this.SettingsManager, this.DataSourceEmbeddingService, status.DataSourceId))
await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
}
/// <summary>
/// Agrees to the removal a mailbox held back, for the very number the user read.
/// </summary>
private async Task ApproveMailRemoval(DataSourceEmbeddingStatus status)
{
if (status.PendingRemovalCount is not { } removalCount)
return;
if (!await this.DataSourceEmbeddingService.ApprovePendingMailRemovalAsync(status.DataSourceId, removalCount))
await this.MessageBus.SendWarning(new(Icons.Material.Filled.PlaylistRemove, T("Your agreement could not be recorded: either the number of mails to remove changed in the meantime, or the index cannot be reached. AI Studio asks you again after the next sync.")));
this.ReloadStatuses();
await this.InvokeAsync(this.StateHasChanged);
}
private async Task RepairDataSource(DataSourceEmbeddingStatus status)
{
if (!await DataSourceRepair.ConfirmAndRepairAsync(this.DialogService, this.DataSourceEmbeddingService, status.DataSourceId, status.DataSourceName))