From f094cb6ac52cd7537134872ad7563c674872fcde Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Wed, 9 Sep 2026 13:23:12 +0200 Subject: [PATCH] Localize the messages of failed indexing attempts --- .../Assistants/I18N/allTexts.lua | 12 +++++++++++ app/MindWork AI Studio/Pages/Embeddings.razor | 5 +++-- .../plugin.lua | 12 +++++++++++ .../plugin.lua | 12 +++++++++++ .../FileExtractionErrorCodeExtensions.cs | 8 +++++++ .../Services/DataSourceEmbeddingService.cs | 21 ++++++++++++------- .../Tools/Services/RustService.Retrieval.cs | 6 +++++- 7 files changed, 66 insertions(+), 10 deletions(-) diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index fc698b1a..b691858b 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -10732,6 +10732,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONERRORCODEEXTENSIONS::T173921008" -- The file '{0}' could not be read and was not indexed. When the file is stored on a network drive, the drive might be unavailable, or another program might be blocking the file. AI Studio tries again during the next run. UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONERRORCODEEXTENSIONS::T1888709599"] = "The file '{0}' could not be read and was not indexed. When the file is stored on a network drive, the drive might be unavailable, or another program might be blocking the file. AI Studio tries again during the next run." +-- Internal error +UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONERRORCODEEXTENSIONS::T1891925702"] = "Internal error" + -- File could not be read UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONERRORCODEEXTENSIONS::T1931822272"] = "File could not be read" @@ -11677,6 +11680,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T1542963 -- The embedding provider answered with a vector containing an invalid number. Please select another embedding model or provider. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T1663635773"] = "The embedding provider answered with a vector containing an invalid number. Please select another embedding model or provider." +-- The local RAG index database is not available. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T1738200026"] = "The local RAG index database is not available." + -- The file '{0}' changed while it was being indexed. What was indexed of it is discarded, and the file is tried again during the next run. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T1935191670"] = "The file '{0}' changed while it was being indexed. What was indexed of it is discarded, and the file is tried again during the next run." @@ -11695,9 +11701,15 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T2424608 -- The embedding provider was not able to embed {0} part(s) of the file '{1}'. The provider reported: {2} UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T2456390987"] = "The embedding provider was not able to embed {0} part(s) of the file '{1}'. The provider reported: {2}" +-- The vector database is not available. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T2489270584"] = "The vector database is not available." + -- The selected embedding provider is not available. Please check it in the settings. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T2494993815"] = "The selected embedding provider is not available. Please check it in the settings." +-- The data source '{0}' could not be processed. The log file holds the details. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T268763982"] = "The data source '{0}' could not be processed. The log file holds the details." + -- The folder '{0}' could not be opened. Please check whether you are allowed to read it. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T3230000698"] = "The folder '{0}' could not be opened. Please check whether you are allowed to read it." diff --git a/app/MindWork AI Studio/Pages/Embeddings.razor b/app/MindWork AI Studio/Pages/Embeddings.razor index 6a2806ef..6022b4aa 100644 --- a/app/MindWork AI Studio/Pages/Embeddings.razor +++ b/app/MindWork AI Studio/Pages/Embeddings.razor @@ -107,11 +107,12 @@ + @* Captions render as spans, so each of them needs to be told to take its own line. *@ @GetFileName(context.FilePath) - @context.FilePath + @context.FilePath @if (group.Cause.ShowsMessagePerFile) { - @context.Reason + @context.Reason } diff --git a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua index 63c9c6ae..515797b3 100644 --- a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua @@ -10734,6 +10734,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONERRORCODEEXTENSIONS::T173921008" -- The file '{0}' could not be read and was not indexed. When the file is stored on a network drive, the drive might be unavailable, or another program might be blocking the file. AI Studio tries again during the next run. UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONERRORCODEEXTENSIONS::T1888709599"] = "Die Datei „{0}“ konnte nicht gelesen und daher nicht indexiert werden. Wenn die Datei auf einem Netzlaufwerk gespeichert ist, ist das Laufwerk möglicherweise nicht verfügbar oder ein anderes Programm blockiert die Datei. AI Studio versucht es beim nächsten Durchlauf erneut." +-- Internal error +UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONERRORCODEEXTENSIONS::T1891925702"] = "Interner Fehler" + -- File could not be read UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONERRORCODEEXTENSIONS::T1931822272"] = "Datei konnte nicht gelesen werden" @@ -11679,6 +11682,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T1542963 -- The embedding provider answered with a vector containing an invalid number. Please select another embedding model or provider. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T1663635773"] = "Der Einbettungsanbieter hat einen Vektor mit einer ungültigen Zahl zurückgegeben. Bitte wählen Sie ein anderes Einbettungsmodell oder einen anderen Anbieter aus." +-- The local RAG index database is not available. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T1738200026"] = "Die lokale RAG-Indexdatenbank ist nicht verfügbar." + -- The file '{0}' changed while it was being indexed. What was indexed of it is discarded, and the file is tried again during the next run. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T1935191670"] = "Die Datei „{0}“ wurde während der Indexierung geändert. Die bereits indexierten Inhalte werden verworfen, und die Datei wird beim nächsten Durchlauf erneut verarbeitet." @@ -11697,9 +11703,15 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T2424608 -- The embedding provider was not able to embed {0} part(s) of the file '{1}'. The provider reported: {2} UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T2456390987"] = "Der Einbettungsanbieter konnte {0} Teil(e) der Datei „{1}“ nicht einbetten. Der Anbieter meldete: {2}" +-- The vector database is not available. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T2489270584"] = "Die Vektordatenbank ist nicht verfügbar." + -- The selected embedding provider is not available. Please check it in the settings. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T2494993815"] = "Der ausgewählte Einbettungsanbieter ist nicht verfügbar. Bitte überprüfen Sie ihn in den Einstellungen." +-- The data source '{0}' could not be processed. The log file holds the details. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T268763982"] = "Die Datenquelle „{0}“ konnte nicht verarbeitet werden. Details finden Sie in der Protokolldatei." + -- The folder '{0}' could not be opened. Please check whether you are allowed to read it. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T3230000698"] = "Der Ordner „{0}“ konnte nicht geöffnet werden. Bitte prüfen Sie, ob Sie ihn lesen dürfen." diff --git a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua index 6a2510f4..455699e6 100644 --- a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua @@ -10734,6 +10734,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONERRORCODEEXTENSIONS::T173921008" -- The file '{0}' could not be read and was not indexed. When the file is stored on a network drive, the drive might be unavailable, or another program might be blocking the file. AI Studio tries again during the next run. UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONERRORCODEEXTENSIONS::T1888709599"] = "The file '{0}' could not be read and was not indexed. When the file is stored on a network drive, the drive might be unavailable, or another program might be blocking the file. AI Studio tries again during the next run." +-- Internal error +UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONERRORCODEEXTENSIONS::T1891925702"] = "Internal error" + -- File could not be read UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONERRORCODEEXTENSIONS::T1931822272"] = "File could not be read" @@ -11679,6 +11682,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T1542963 -- The embedding provider answered with a vector containing an invalid number. Please select another embedding model or provider. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T1663635773"] = "The embedding provider answered with a vector containing an invalid number. Please select another embedding model or provider." +-- The local RAG index database is not available. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T1738200026"] = "The local RAG index database is not available." + -- The file '{0}' changed while it was being indexed. What was indexed of it is discarded, and the file is tried again during the next run. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T1935191670"] = "The file '{0}' changed while it was being indexed. What was indexed of it is discarded, and the file is tried again during the next run." @@ -11697,9 +11703,15 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T2424608 -- The embedding provider was not able to embed {0} part(s) of the file '{1}'. The provider reported: {2} UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T2456390987"] = "The embedding provider was not able to embed {0} part(s) of the file '{1}'. The provider reported: {2}" +-- The vector database is not available. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T2489270584"] = "The vector database is not available." + -- The selected embedding provider is not available. Please check it in the settings. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T2494993815"] = "The selected embedding provider is not available. Please check it in the settings." +-- The data source '{0}' could not be processed. The log file holds the details. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T268763982"] = "The data source '{0}' could not be processed. The log file holds the details." + -- The folder '{0}' could not be opened. Please check whether you are allowed to read it. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T3230000698"] = "The folder '{0}' could not be opened. Please check whether you are allowed to read it." diff --git a/app/MindWork AI Studio/Tools/FileExtractionErrorCodeExtensions.cs b/app/MindWork AI Studio/Tools/FileExtractionErrorCodeExtensions.cs index bf5a0a0f..a5366827 100644 --- a/app/MindWork AI Studio/Tools/FileExtractionErrorCodeExtensions.cs +++ b/app/MindWork AI Studio/Tools/FileExtractionErrorCodeExtensions.cs @@ -78,6 +78,14 @@ internal static class FileExtractionErrorCodeExtensions FileExtractionErrorCode.PDFIUM_UNAVAILABLE => TB("PDF system unavailable"), FileExtractionErrorCode.PANDOC_UNAVAILABLE => TB("Pandoc unavailable"), + // Nothing about these lies in the file: AI Studio asked its runtime for the content and + // got back something it cannot work with. One name for all of them, because that is the + // one thing the user can tell from them: + FileExtractionErrorCode.INVALID_RESPONSE => TB("Internal error"), + FileExtractionErrorCode.INVALID_REQUEST => TB("Internal error"), + FileExtractionErrorCode.REQUEST_FAILED => TB("Internal error"), + FileExtractionErrorCode.INTERNAL => TB("Internal error"), + // Codes which say nothing beyond the message of the single file. The caller names those // files itself and shows their messages instead: _ => string.Empty, diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index 6cac7238..1cc8ac61 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -329,7 +329,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM else { logger.LogError(exception, "Background embedding failed for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id); - this.UpsertStatus(this.GetFallbackStatus(dataSource, exception.Message)); + this.UpsertStatus(this.GetFallbackStatus(dataSource, string.Format(TB("The data source '{0}' could not be processed. The log file holds the details."), dataSource.Name))); } } finally @@ -425,7 +425,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM dataSource.Id, vectorStore.Name); token.ThrowIfCancellationRequested(); - this.UpsertStatus(this.GetFallbackStatus(dataSource, "The vector database is not available.")); + this.UpsertStatus(this.GetFallbackStatus(dataSource, TB("The vector database is not available."))); return; } @@ -437,7 +437,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM dataSource.Id, indexStore.Name); token.ThrowIfCancellationRequested(); - this.UpsertStatus(this.GetFallbackStatus(dataSource, "The local RAG index database is not available.")); + this.UpsertStatus(this.GetFallbackStatus(dataSource, TB("The local RAG index database is not available."))); return; } @@ -765,14 +765,21 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM // would interrupt whatever the user is doing right now. // failedFiles++; - lastError = exception.Message; - failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, exception.Message, DateTimeOffset.UtcNow, EmbeddingProviderName: embeddingProvider.Name, ExtractionCode: exception is FileExtractionException extractionFailure ? extractionFailure.Code : FileExtractionErrorCode.NONE)); + var extractionCode = exception is FileExtractionException extractionFailure ? extractionFailure.Code : FileExtractionErrorCode.NONE; + + // + // Deliberately not the message of the exception: that one is written for the log + // file, in English, and repeats the path which the list shows anyway. + // + var failureMessage = extractionCode.ToIndexingUserMessage(file.Name); + lastError = failureMessage; + failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, failureMessage, DateTimeOffset.UtcNow, EmbeddingProviderName: embeddingProvider.Name, ExtractionCode: extractionCode)); manifest.Files.Remove(file.FullName); await this.ForgetPermanentFailureAsync(indexStore, dataSource, manifest, file.FullName, token); await this.CleanupFailedFileAsync(indexStore, vectorStore, dataSource, collectionName, file.FullName, optimizationTracker, token); logger.LogWarning(exception, "Failed to embed file '{FilePath}' for data source '{DataSourceName}'.", file.FullName, dataSource.Name); - this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, exception.Message, failureDetails, permanentlySkippedFiles)); + this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, failureMessage, failureDetails, permanentlySkippedFiles)); } } @@ -1175,7 +1182,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM catch (Exception exception) { logger.LogError(exception, "Initial embedding hash check failed for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id); - this.UpsertStatus(this.GetFallbackStatus(dataSource, exception.Message)); + this.UpsertStatus(this.GetFallbackStatus(dataSource, string.Format(TB("The data source '{0}' could not be processed. The log file holds the details."), dataSource.Name))); } } diff --git a/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs b/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs index b518ab09..89cda150 100644 --- a/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs +++ b/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs @@ -267,8 +267,12 @@ public sealed partial class RustService { await foreach (var segment in this.StreamArbitraryFileDataCore(path, false, true, embeddingProvider.TokenizerPath, token)) { + // + // Carries a code so callers can classify it: the file itself is fine, the answer of + // the runtime was not, which makes this worth another attempt. + // if (segment.TokenCount is null) - throw new InvalidOperationException($"Rust did not return a token count for an extracted segment from '{path}' using provider '{embeddingProvider.Name}'."); + throw new FileExtractionException(FileExtractionErrorCode.INVALID_RESPONSE, $"Rust did not return a token count for an extracted segment from '{path}' using provider '{embeddingProvider.Name}'."); yield return new(segment.Content, segment.TokenCount.Value); }