MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-09-27 00:53:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Show block and page progress while indexing a file

Browse Source
This commit is contained in:
Thorsten Sommer 2026-09-16 19:48:26 +02:00
parent 5f438cbfbe
commit a8fded4df5
Signed by untrusted user who does not match committer: tsommer
GPG Key ID: 371BBA77A02C0108
4 changed files with 97 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/MindWork AI Studio/Pages/Embeddings.razor
View File

@ -68,7 +68,7 @@
<MudProgressLinear Value="@status.ProgressPercent" Rounded="@true" Color="@GetStatusColor(status)" />
<MudText Typo="Typo.body2">
@string.Format(T("{0} of {1} files are indexed."), status.IndexedFiles, status.TotalFiles)
@this.GetFileProgressText(status)
</MudText>
@if (status.PermanentlySkippedFiles > 0)

53
app/MindWork AI Studio/Pages/Embeddings.razor.cs
View File

@ -1,3 +1,5 @@
using System.Globalization;
using AIStudio.Components;
using AIStudio.Dialogs.Settings;
using AIStudio.Provider;
@ -35,6 +37,13 @@ public partial class Embeddings : MSGComponentBase
private string? expandedDataSourceId;
private bool userChoseExpansion;
/// <remarks>
/// The language of AI Studio is chosen in its settings and does not move the thread's culture
/// along with it. Without this, a German reading a German page would find a file count written
/// with English separators.
/// </remarks>
private CultureInfo currentCulture = CultureInfo.InvariantCulture;
private int TotalIndexedFiles => this.Statuses.Sum(status => status.IndexedFiles);
private int TotalPendingFiles => this.Statuses.Sum(status => Math.Max(0, status.TotalFiles - status.IndexedFiles - status.FailedFiles - status.PermanentlySkippedFiles));
@ -69,20 +78,28 @@ public partial class Embeddings : MSGComponentBase
return;
}
this.ApplyFilters([], [ Event.RAG_EMBEDDING_STATUS_CHANGED, Event.CONFIGURATION_CHANGED ]);
this.ApplyFilters([], [ Event.RAG_EMBEDDING_STATUS_CHANGED, Event.CONFIGURATION_CHANGED, Event.PLUGINS_RELOADED ]);
await this.RefreshCulture();
await base.OnInitializedAsync();
this.ReloadStatuses();
}
protected override Task ProcessIncomingMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default
protected override async Task ProcessIncomingMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default
{
if (triggeredEvent is Event.RAG_EMBEDDING_STATUS_CHANGED or Event.CONFIGURATION_CHANGED)
if (triggeredEvent is Event.CONFIGURATION_CHANGED or Event.PLUGINS_RELOADED)
await this.RefreshCulture();
if (triggeredEvent is Event.RAG_EMBEDDING_STATUS_CHANGED or Event.CONFIGURATION_CHANGED or Event.PLUGINS_RELOADED)
{
this.ReloadStatuses();
this.StateHasChanged();
}
}
return Task.CompletedTask;
private async Task RefreshCulture()
{
var activeLanguagePlugin = await this.SettingsManager.GetActiveLanguagePlugin();
this.currentCulture = CommonTools.DeriveActiveCultureOrInvariant(activeLanguagePlugin.IETFTag);
}
private void ReloadStatuses()
@ -155,6 +172,34 @@ public partial class Embeddings : MSGComponentBase
await dialogReference.Result;
}
/// <summary>
/// What the panel of a data source says about its progress through the files.
/// </summary>
/// <remarks>
/// While a file is being worked on, the sentence names that file and how far into it we are.
/// Counting finished files alone leaves the same sentence standing for hours on a document of
/// several thousand pages, and a progress which never moves cannot be told apart from one which
/// is stuck. The total number of blocks is not part of it: the blocks are produced while the
/// file is read, so nobody knows how many there will be until the file is done.
/// </remarks>
private string GetFileProgressText(DataSourceEmbeddingStatus status)
{
if (status.State is not DataSourceEmbeddingState.RUNNING || status.CurrentFileBlock is not { } block)
return string.Format(T("{0} of {1} files are indexed."), this.FormatNumber(status.IndexedFiles), this.FormatNumber(status.TotalFiles));
//
// Everything already dealt with, plus the one in hand. Skipped and failed files are part of
// that: they are behind us in the folder, and leaving them out would let the number fall
// behind the file whose name is shown right next to it.
//
var currentFileNumber = Math.Min(status.TotalFiles, status.IndexedFiles + status.PermanentlySkippedFiles + status.FailedFiles + 1);
return status.CurrentFilePage is { } page
? string.Format(T("File {0} of {1} is being indexed: block {2}, page {3}."), this.FormatNumber(currentFileNumber), this.FormatNumber(status.TotalFiles), this.FormatNumber(block), this.FormatNumber(page))
: string.Format(T("File {0} of {1} is being indexed: block {2}."), this.FormatNumber(currentFileNumber), this.FormatNumber(status.TotalFiles), this.FormatNumber(block));
}
private string FormatNumber(int value) => value.ToString("N0", this.currentCulture);
private static Color GetStatusColor(DataSourceEmbeddingStatus status) => status.State switch
{
DataSourceEmbeddingState.RUNNING => Color.Warning,

42
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs
View File

@ -18,6 +18,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
{
private const int VECTOR_STORE_OPTIMIZATION_CHUNK_THRESHOLD = 100_000;
/// <summary>
/// How often the block progress within one file is reported to the user interface at most.
/// </summary>
private static readonly TimeSpan BLOCK_PROGRESS_INTERVAL = TimeSpan.FromSeconds(3);
private readonly Channel<DataSourceEmbeddingQueueItem> queue = Channel.CreateUnbounded<DataSourceEmbeddingQueueItem>();
private readonly ConcurrentDictionary<string, byte> queuedIds = new(StringComparer.OrdinalIgnoreCase);
private readonly ConcurrentDictionary<string, byte> runningIds = new(StringComparer.OrdinalIgnoreCase);
@ -646,6 +651,13 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, lastError, failureDetails, permanentlySkippedFiles));
//
// What the page says while one file is being worked on. Without it, a document of
// several thousand pages leaves the same sentence standing for hours, and a progress
// which never moves cannot be told apart from one which is stuck.
//
var lastBlockReportUtc = DateTimeOffset.MinValue;
try
{
logger.LogInformation(
@ -658,7 +670,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
skippedFiles + completedFiles + 1,
totalFiles);
var startedAtUtc = DateTimeOffset.UtcNow;
var chunkCount = await this.IndexOneFileAsync(indexStore, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, optimizationTracker, token);
var chunkCount = await this.IndexOneFileAsync(indexStore, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, optimizationTracker, ReportBlockProgress, token);
token.ThrowIfCancellationRequested();
var fingerprintAfterEmbedding = BuildFileMetadataHash(file);
if (!string.Equals(fingerprint, fingerprintAfterEmbedding, StringComparison.Ordinal))
@ -780,6 +792,24 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
logger.LogWarning(exception, "Failed to embed file '{FilePath}' for data source '{DataSourceName}'.", file.FullName, dataSource.Name);
this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, failureMessage, failureDetails, permanentlySkippedFiles));
}
continue;
void ReportBlockProgress(int blockNumber, int? pageNumber)
{
//
// The first block goes out at once, so the line is there instead of blank. After
// that, at most one message every BLOCK_PROGRESS_INTERVAL: each one re-renders the
// embedding page, the navigation bar and the table in the settings, and the blocks
// of a large file arrive far faster than anybody can read them.
//
var nowUtc = DateTimeOffset.UtcNow;
if (blockNumber > 1 && nowUtc - lastBlockReportUtc < BLOCK_PROGRESS_INTERVAL)
return;
lastBlockReportUtc = nowUtc;
this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, lastError, failureDetails, permanentlySkippedFiles, blockNumber, pageNumber));
}
}
manifest.SourceHash = metadataSnapshot.SourceHash;
@ -823,6 +853,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
IProvider provider,
DataSourceEmbeddingManifest manifest,
VectorStoreOptimizationTracker optimizationTracker,
Action<int, int?> reportBlockProgress,
CancellationToken token)
{
var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id);
@ -845,6 +876,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
{
batch.Add(new(this.CreatePointId(dataSource.Id, fingerprint, totalChunkCount), chunk.Text, totalChunkCount, chunk.PageNumber));
totalChunkCount++;
reportBlockProgress(totalChunkCount, chunk.PageNumber);
if (batch.Count >= embeddingBatchSize)
await this.FlushBatchAsync(indexStore, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, optimizationTracker, collectionName, batch, token);
@ -1443,7 +1475,9 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
string currentFile = "",
string lastError = "",
IReadOnlyList<DataSourceEmbeddingFailure>? failures = null,
int permanentlySkippedFiles = 0)
int permanentlySkippedFiles = 0,
int? currentFileBlock = null,
int? currentFilePage = null)
{
return new DataSourceEmbeddingStatus(
dataSource.Id,
@ -1456,7 +1490,9 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
currentFile,
lastError,
failures?.ToList() ?? [],
permanentlySkippedFiles);
permanentlySkippedFiles,
currentFileBlock,
currentFilePage);
}
/// <remarks>

9
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs
View File

@ -3,6 +3,11 @@ using AIStudio.Tools.PluginSystem;
namespace AIStudio.Tools.Services;
/// <remarks>
/// CurrentFileBlock and CurrentFilePage are null rather than zero while nothing is known about
/// them: a file which is only about to start has no first block, and not every kind of document
/// has pages to count. Block numbers start at one, the way the page states them.
/// </remarks>
public sealed record DataSourceEmbeddingStatus(
string DataSourceId,
string DataSourceName,
@ -14,7 +19,9 @@ public sealed record DataSourceEmbeddingStatus(
string CurrentFile,
string LastError,
IReadOnlyList<DataSourceEmbeddingFailure> Failures,
int PermanentlySkippedFiles = 0)
int PermanentlySkippedFiles = 0,
int? CurrentFileBlock = null,
int? CurrentFilePage = null)
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingStatus).Namespace, nameof(DataSourceEmbeddingStatus));