diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingFailure.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingFailure.cs
index e73bb5a4..8d77fad7 100644
--- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingFailure.cs
+++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingFailure.cs
@@ -1,3 +1,22 @@
+using System.Net;
+
+using AIStudio.Provider;
+
namespace AIStudio.Tools.Services;
-public sealed record DataSourceEmbeddingFailure(string FilePath, string Reason);
+///
+/// One input which could not be embedded, together with everything known about why.
+///
+///
+/// The reason alone used to be all we kept, which made every failure look alike in the UI: a
+/// rejected API key, an unreachable provider, and a file nobody may read were one and the same
+/// list entry. The surrounding fields are what lets the UI offer the matching way out.
+///
+/// The file that failed or the name of the data source when the failure was not about one file.
+/// What to tell the user about it, ready to show.
+/// When it happened, so the list still makes sense when the user looks at it later.
+/// What kind of failure it was. Everything that did not come from a provider stays at NONE.
+/// What the provider answered, where it answered at all.
+/// The embedding provider that was asked.
+public sealed record DataSourceEmbeddingFailure(string FilePath, string Reason, DateTimeOffset OccurredAtUtc, ProviderRequestFailureReason FailureReason = ProviderRequestFailureReason.NONE,
+ HttpStatusCode? StatusCode = null, string EmbeddingProviderName = "");
\ No newline at end of file
diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs
index f92bd76f..15f107e5 100644
--- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs
+++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs
@@ -450,7 +450,7 @@ public sealed partial class DataSourceEmbeddingService
var smallestCandidate = AddOverlapPrefix(text[startIndex..Math.Min(startIndex + 1, text.Length)].Trim(), overlapPrefix);
var smallestCandidateTokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, smallestCandidate, token);
- throw new InvalidOperationException($"The max chunk length for embedding provider '{embeddingProvider.Name}' is too low. The smallest possible split still has {smallestCandidateTokenCount} tokens, but the configured limit is {options.MaxChunkTokenLength}.");
+ throw new InvalidOperationException(string.Format(TB("The chunk size configured for the embedding provider '{0}' is too small: the smallest piece the text can be cut into still has {1} tokens, while the limit is {2}."), embeddingProvider.Name, smallestCandidateTokenCount, options.MaxChunkTokenLength));
}
var chunk = AddOverlapPrefix(text[startIndex..bestEndIndex].Trim(), overlapPrefix);
@@ -525,7 +525,7 @@ public sealed partial class DataSourceEmbeddingService
return response.Value.TokenCount;
var message = response?.Message ?? "No response was returned by the tokenizer service.";
- throw new InvalidOperationException($"Could not count tokens for embedding provider '{embeddingProvider.Name}'. {message}");
+ throw new InvalidOperationException(string.Format(TB("The tokens of the text could not be counted for the embedding provider '{0}'. {1}"), embeddingProvider.Name, message));
}
private ChunkingOptions GetChunkingOptions(IDataSource dataSource, EmbeddingProvider embeddingProvider)
@@ -742,7 +742,7 @@ public sealed partial class DataSourceEmbeddingService
break;
default:
- result.AddFailure(localFile.FilePath, $"The selected file '{localFile.FilePath}' is not supported for background embeddings.");
+ result.AddFailure(localFile.FilePath, string.Format(TB("The file '{0}' has a type AI Studio cannot index."), localFile.FilePath));
break;
}
@@ -756,11 +756,11 @@ public sealed partial class DataSourceEmbeddingService
switch (dataSource)
{
case DataSourceLocalFile localFile:
- result.AddFailure(localFile.FilePath, $"The selected file '{localFile.FilePath}' does not exist.");
+ result.AddFailure(localFile.FilePath, string.Format(TB("The file '{0}' does not exist."), localFile.FilePath));
break;
case DataSourceLocalDirectory localDirectory:
- result.AddFailure(localDirectory.Path, $"The selected directory '{localDirectory.Path}' does not exist.");
+ result.AddFailure(localDirectory.Path, string.Format(TB("The folder '{0}' does not exist."), localDirectory.Path));
break;
}
@@ -786,7 +786,7 @@ public sealed partial class DataSourceEmbeddingService
catch (Exception exception)
{
logger.LogWarning(exception, "Cannot access directory '{DirectoryPath}' while indexing.", currentPath);
- result.AddFailure(currentPath, $"The directory '{currentPath}' could not be accessed.");
+ result.AddFailure(currentPath, string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it."), currentPath));
continue;
}
@@ -802,7 +802,7 @@ public sealed partial class DataSourceEmbeddingService
catch (Exception exception)
{
logger.LogWarning(exception, "Cannot inspect file '{FilePath}' while indexing.", filePath);
- result.AddFailure(filePath, $"The file '{filePath}' could not be inspected.");
+ result.AddFailure(filePath, string.Format(TB("The file '{0}' could not be read. Please check whether you are allowed to read it."), filePath));
continue;
}
diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs
index 1250313a..29f80338 100644
--- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs
+++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs
@@ -456,13 +456,13 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
if (!this.TryResolveEmbeddingProvider(dataSource, out var embeddingProvider))
{
token.ThrowIfCancellationRequested();
- this.UpsertStatus(this.GetFallbackStatus(dataSource, "The selected embedding provider is not available."));
+ this.UpsertStatus(this.GetFallbackStatus(dataSource, TB("The selected embedding provider is not available. Please check it in the settings.")));
return;
}
if (!embeddingProvider.GetConfidenceLevel(settingsManager).AllowsDataSourceConfidenceLevel(internalDataSource.ConfidenceLevel))
{
- var errorMessage = $"The selected embedding provider is not allowed to embed this data source. The data source requires provider confidence '{internalDataSource.ConfidenceLevel.GetName()}'. The embedding provider has confidence '{embeddingProvider.GetConfidenceLevel(settingsManager).GetName()}'.";
+ var errorMessage = string.Format(TB("The selected embedding provider is not allowed to index this data source. The data source asks for the confidence level '{0}', while the embedding provider has '{1}'."), internalDataSource.ConfidenceLevel.GetName(), embeddingProvider.GetConfidenceLevel(settingsManager).GetName());
logger.LogWarning(
"Skipping background embeddings for data source '{DataSourceName}' ({DataSourceId}) because embedding provider '{EmbeddingProviderName}' ({EmbeddingProviderId}) does not meet the required confidence. RequiredConfidence={RequiredConfidence}, EmbeddingProviderConfidence={EmbeddingProviderConfidence}.",
dataSource.Name,
@@ -567,6 +567,13 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
var lastError = inputFiles.LastError;
var failureDetails = inputFiles.Failures.ToList();
+ //
+ // Which kinds of provider failure the user was already told about in this run. A rejected
+ // API key is the same problem for every one of a few thousand documents, and one message
+ // is what it takes to send the user to the settings.
+ //
+ var reportedFailureReasons = new HashSet();
+
//
// Everything the runtime filters out of these files is reported once for the whole data
// source. A run over a few thousand documents which removes something in forty of them
@@ -614,7 +621,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
token.ThrowIfCancellationRequested();
var fingerprintAfterEmbedding = BuildFileMetadataHash(file);
if (!string.Equals(fingerprint, fingerprintAfterEmbedding, StringComparison.Ordinal))
- throw new IOException($"The file '{file.FullName}' changed while it was being embedded. Its partial embeddings will be discarded and the file will be retried on the next refresh.");
+ throw new IOException(string.Format(TB("The file '{0}' changed while it was being indexed. What was indexed of it is discarded, and the file is tried again during the next run."), file.FullName));
var embeddedAtUtc = DateTimeOffset.UtcNow;
var record = new EmbeddedFileRecord(
@@ -646,11 +653,44 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
{
throw;
}
+ catch (ProviderRequestException exception)
+ {
+ //
+ // The provider said what went wrong and what the user can do about it. That
+ // sentence is what goes into the status, together with the classification the UI
+ // needs to offer the matching way out.
+ //
+ failedFiles++;
+ lastError = exception.UserMessage;
+ failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, exception.UserMessage, DateTimeOffset.UtcNow, exception.FailureReason, exception.StatusCode, embeddingProvider.Name));
+ manifest.Files.Remove(file.FullName);
+ await this.CleanupFailedFileAsync(indexStore, vectorStore, dataSource, collectionName, file.FullName, optimizationTracker, token);
+
+ logger.LogWarning(
+ exception,
+ "Failed to embed file '{FilePath}' for data source '{DataSourceName}' because the embedding provider '{EmbeddingProviderName}' failed. FailureReason={FailureReason}, StatusCode={StatusCode}.",
+ file.FullName,
+ dataSource.Name,
+ embeddingProvider.Name,
+ exception.FailureReason,
+ exception.StatusCode);
+ this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, exception.UserMessage, failureDetails));
+
+ // Once per kind of failure, not once per file:
+ if (reportedFailureReasons.Add(exception.FailureReason))
+ await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.CloudOff, exception.UserMessage));
+ }
catch (Exception exception)
{
+ //
+ // Everything which is not the provider's doing: a file which changed while it was
+ // read, one which yielded no text, a vector store which refused to store. These
+ // are about this one file, so they go into the list and not into a message which
+ // would interrupt whatever the user is doing right now.
+ //
failedFiles++;
lastError = exception.Message;
- failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, exception.Message));
+ failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, exception.Message, DateTimeOffset.UtcNow, EmbeddingProviderName: embeddingProvider.Name));
manifest.Files.Remove(file.FullName);
await this.CleanupFailedFileAsync(indexStore, vectorStore, dataSource, collectionName, file.FullName, optimizationTracker, token);
@@ -730,7 +770,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
await this.FlushBatchAsync(indexStore, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, optimizationTracker, collectionName, batch, token);
if (totalChunkCount == 0)
- throw new InvalidOperationException($"The file '{file.Name}' did not yield any text chunks.");
+ throw new InvalidOperationException(string.Format(TB("No text could be read from the file '{0}'."), file.Name));
logger.LogDebug(
"Generated {ChunkCount} chunks for file '{FilePath}' in data source '{DataSourceName}' ({DataSourceId}).",
@@ -775,26 +815,35 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
{
throw;
}
+ catch (ProviderRequestException)
+ {
+ //
+ // The provider already named the cause and what to do about it. Wrapping that in a
+ // sentence about a batch of chunks would replace the one thing the user can act on
+ // with the fact that something failed:
+ //
+ throw;
+ }
catch (Exception exception)
{
- throw new InvalidOperationException($"The embedding provider failed to embed {batch.Count} chunk(s) for file '{file.Name}'. Provider message: {exception.Message}", exception);
+ throw new InvalidOperationException(string.Format(TB("The embedding provider was not able to embed {0} part(s) of the file '{1}'. The provider reported: {2}"), batch.Count, file.Name, exception.Message), exception);
}
if (vectors.Count != batch.Count)
- throw new InvalidOperationException($"The embedding provider returned {vectors.Count} vectors for {batch.Count} text chunks.");
+ throw new InvalidOperationException(string.Format(TB("The embedding provider answered with {0} vectors for {1} parts of the file '{2}'. Please select another embedding model or provider."), vectors.Count, batch.Count, file.Name));
var vectorSize = vectors.FirstOrDefault()?.Count ?? 0;
if (vectorSize <= 0)
- throw new InvalidOperationException("The embedding provider returned an empty vector.");
+ throw new InvalidOperationException(TB("The embedding provider answered with an empty vector. Please select another embedding model or provider."));
if (vectors.Any(vector => vector.Count != vectorSize))
- throw new InvalidOperationException("The embedding provider returned vectors with inconsistent dimensions.");
+ throw new InvalidOperationException(TB("The embedding provider answered with vectors of different sizes. Please select another embedding model or provider."));
if (vectors.Any(vector => vector.Any(value => !float.IsFinite(value))))
- throw new InvalidOperationException("The embedding provider returned a vector containing a non-finite value.");
+ throw new InvalidOperationException(TB("The embedding provider answered with a vector containing an invalid number. Please select another embedding model or provider."));
if (manifest.VectorSize > 0 && manifest.VectorSize != vectorSize)
- throw new InvalidOperationException($"The embedding vector size changed from {manifest.VectorSize} to {vectorSize}. Please re-save the data source to trigger a clean re-index.");
+ throw new InvalidOperationException(string.Format(TB("The size of the embedding vectors changed from {0} to {1}. Please save the data source again to index it from scratch."), manifest.VectorSize, vectorSize));
if (manifest.VectorSize == 0)
{
@@ -810,7 +859,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
await vectorStore.DeleteVectorStore(collectionName, token);
ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, vectorSize, token);
if (!ensureResult.Created)
- throw new InvalidOperationException($"Vector store '{collectionName}' could not be recreated cleanly.");
+ throw new InvalidOperationException(string.Format(TB("The local index '{0}' could not be created again. Please restart AI Studio and try once more."), collectionName));
}
await indexStore.UpdateVectorSizeAsync(dataSource.Id, vectorSize, token);
@@ -1262,7 +1311,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
failedFiles,
lastError: failedFiles > 0
? string.IsNullOrWhiteSpace(lastError)
- ? "Some files could not be embedded. See the logs for details."
+ ? TB("Some files could not be indexed. The list below says which ones and why.")
: lastError
: string.Empty,
failures: failures);
@@ -1277,7 +1326,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
0,
1,
lastError: errorMessage,
- failures: [new DataSourceEmbeddingFailure(dataSource.Name, errorMessage)]);
+ failures: [new DataSourceEmbeddingFailure(dataSource.Name, errorMessage, DateTimeOffset.UtcNow)]);
}
private DataSourceQueueRequestResult TryReserveDataSourceQueueSlot(string dataSourceId, bool queueAfterCurrentRun)
diff --git a/app/MindWork AI Studio/Tools/Services/FileEnumerationResult.cs b/app/MindWork AI Studio/Tools/Services/FileEnumerationResult.cs
index e0837179..bce3fc84 100644
--- a/app/MindWork AI Studio/Tools/Services/FileEnumerationResult.cs
+++ b/app/MindWork AI Studio/Tools/Services/FileEnumerationResult.cs
@@ -12,8 +12,8 @@ public sealed class FileEnumerationResult
public void AddFailure(string filePath, string reason)
{
- this.Failures.Add(new DataSourceEmbeddingFailure(filePath, reason));
+ this.Failures.Add(new DataSourceEmbeddingFailure(filePath, reason, DateTimeOffset.UtcNow));
this.FailedFiles = this.Failures.Count;
this.LastError = reason;
}
-}
+}
\ No newline at end of file