From 6dd2389a994b928b8dd95fbc2ed363fd34c632c1 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Wed, 9 Sep 2026 10:42:02 +0200 Subject: [PATCH] Carry the extraction failure code into the indexer --- .../FileExtractionErrorCodeExtensions.cs | 95 +++++++++++++++++++ .../Tools/FileExtractionException.cs | 27 ++++++ .../Services/DataSourceEmbeddingService.cs | 7 +- .../Tools/Services/RustService.Retrieval.cs | 2 +- 4 files changed, 129 insertions(+), 2 deletions(-) create mode 100644 app/MindWork AI Studio/Tools/FileExtractionErrorCodeExtensions.cs create mode 100644 app/MindWork AI Studio/Tools/FileExtractionException.cs diff --git a/app/MindWork AI Studio/Tools/FileExtractionErrorCodeExtensions.cs b/app/MindWork AI Studio/Tools/FileExtractionErrorCodeExtensions.cs new file mode 100644 index 00000000..9901b15d --- /dev/null +++ b/app/MindWork AI Studio/Tools/FileExtractionErrorCodeExtensions.cs @@ -0,0 +1,95 @@ +using AIStudio.Tools.PluginSystem; + +namespace AIStudio.Tools; + +/// +/// Tells failures which lie in the file apart from failures which lie in its surroundings, and +/// puts both into words for the indexing user interface. +/// +/// +/// A file without readable text fails the same way on every run, so the indexer remembers it and +/// waits for the file to change. An offline network drive or an overloaded provider says nothing +/// about the file itself, which is why those keep being retried. +/// +internal static class FileExtractionErrorCodeExtensions +{ + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(FileExtractionErrorCodeExtensions).Namespace, nameof(FileExtractionErrorCodeExtensions)); + + /// + /// Gets a value indicating whether reading the file again will fail again, as long as the file + /// itself does not change. + /// + /// The stable failure code. + /// True, when the reason lies in the file itself. + internal static bool IsPermanentIndexingFailure(this FileExtractionErrorCode code) => code switch + { + // + // The reason lies in the file. Reading it again without changing it produces the same + // outcome, so the indexer waits for a new fingerprint: + // + FileExtractionErrorCode.NO_TEXT_EXTRACTED => true, + FileExtractionErrorCode.NO_CONTENT => true, + FileExtractionErrorCode.NOT_TEXT_CONTENT => true, + FileExtractionErrorCode.NOT_A_VALID_PDF => true, + FileExtractionErrorCode.NOT_A_VALID_SPREADSHEET => true, + FileExtractionErrorCode.PDF_ENCRYPTED => true, + FileExtractionErrorCode.FORMAT_DETECTION_FAILED => true, + FileExtractionErrorCode.EXECUTABLE_REJECTED => true, + FileExtractionErrorCode.UNSUPPORTED => true, + + // Pages holding nothing but images are one of the recurring cases here, and that is a + // property of the document, not of the environment: + FileExtractionErrorCode.PAGE_EXTRACTION_FAILED => true, + + // + // Everything else depends on the surroundings: an unavailable drive, a file someone else + // has open, a missing engine, or a runtime which did not answer in time. All of them are + // worth another attempt during the next run: + // + _ => false, + }; + + /// + /// Gets the localized message which explains why a file was not indexed. + /// + /// + /// These texts are the counterpart of the ones used for chat attachments: there, a file which + /// cannot be read is simply not sent, while here it stays out of the index and the user needs + /// to know whether AI Studio will come back to it on its own. + /// + /// The stable failure code. + /// The name of the file, as shown to the user. + /// The localized message. + internal static string ToIndexingUserMessage(this FileExtractionErrorCode code, string fileName) => string.Format(ToIndexingMessageFormat(code), fileName); + + private static string ToIndexingMessageFormat(FileExtractionErrorCode code) => code switch + { + // + // Permanent failures. Each of them names what is wrong with the file and says that AI + // Studio comes back to it once the file changes: + // + FileExtractionErrorCode.NO_TEXT_EXTRACTED => TB("No text could be read from the file '{0}', so it was not indexed. It might contain images only, such as a scanned PDF without a text layer. AI Studio reads it again as soon as the file changes."), + FileExtractionErrorCode.NO_CONTENT => TB("The file '{0}' did not provide any content, so it was not indexed. AI Studio reads it again as soon as the file changes."), + FileExtractionErrorCode.NOT_TEXT_CONTENT => TB("The file '{0}' is not a text file, so it was not indexed. Its content could not be read as text, which means it might have a wrong file extension. AI Studio reads it again as soon as the file changes."), + FileExtractionErrorCode.NOT_A_VALID_PDF => TB("The file '{0}' is not a readable PDF, so it was not indexed. It might be damaged or transferred incompletely. AI Studio reads it again as soon as the file changes."), + FileExtractionErrorCode.NOT_A_VALID_SPREADSHEET => TB("The file '{0}' is not a readable spreadsheet, so it was not indexed. It might be damaged or transferred incompletely. AI Studio reads it again as soon as the file changes."), + FileExtractionErrorCode.PDF_ENCRYPTED => TB("The file '{0}' is protected and could not be opened, so it was not indexed. AI Studio reads it again as soon as the file changes."), + FileExtractionErrorCode.FORMAT_DETECTION_FAILED => TB("The file type of '{0}' could not be determined, so the file was not indexed. AI Studio reads it again as soon as the file changes."), + FileExtractionErrorCode.EXECUTABLE_REJECTED => TB("The file '{0}' is an executable program and was not indexed, regardless of its file extension."), + FileExtractionErrorCode.UNSUPPORTED => TB("The file type of '{0}' is not supported, so the file was not indexed. AI Studio reads it again as soon as the file changes."), + FileExtractionErrorCode.PAGE_EXTRACTION_FAILED => TB("Pages of the file '{0}' could not be read, so it was not indexed. They might contain images only. AI Studio reads it again as soon as the file changes."), + + // + // Temporary failures. They name what the user can act on, and every one of them is tried + // again during the next run: + // + FileExtractionErrorCode.FILE_NOT_FOUND => TB("The file '{0}' does not exist anymore and was not indexed."), + FileExtractionErrorCode.FILE_NOT_READABLE => TB("The file '{0}' could not be read and was not indexed. When the file is stored on a network drive, the drive might be unavailable, or another program might be blocking the file. AI Studio tries again during the next run."), + FileExtractionErrorCode.FILE_LOCKED => TB("The file '{0}' is currently open in another program, which is why it was not indexed. When the file is stored on a shared network drive, a colleague might have it open. AI Studio tries again during the next run."), + FileExtractionErrorCode.TIMEOUT => TB("Reading the file '{0}' took too long and was stopped, so the file was not indexed. When the file is stored on a network drive, the connection might be slow or interrupted. AI Studio tries again during the next run."), + FileExtractionErrorCode.PDFIUM_UNAVAILABLE => TB("AI Studio was not able to start its PDF engine, so the file '{0}' was not indexed. AI Studio tries again during the next run."), + FileExtractionErrorCode.PANDOC_UNAVAILABLE => TB("Reading the file '{0}' needs Pandoc, which is not available, so the file was not indexed. AI Studio tries again during the next run."), + + _ => TB("The file '{0}' could not be read and was not indexed. AI Studio tries again during the next run."), + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/FileExtractionException.cs b/app/MindWork AI Studio/Tools/FileExtractionException.cs new file mode 100644 index 00000000..32f221c7 --- /dev/null +++ b/app/MindWork AI Studio/Tools/FileExtractionException.cs @@ -0,0 +1,27 @@ +namespace AIStudio.Tools; + +/// +/// Thrown when a file could not be read, carrying the stable failure code along with the message. +/// +/// +/// The plain message alone does not say whether another attempt is worth anything. The code does, +/// which is what the indexer needs to tell a file without readable text apart from a network drive +/// which happens to be offline. +/// +public sealed class FileExtractionException(FileExtractionErrorCode code, string message, int? pageNumber = null, string? detectedFormat = null) : Exception(message) +{ + /// + /// Gets the stable failure code. + /// + public FileExtractionErrorCode Code { get; } = code; + + /// + /// Gets the page the failure belongs to, when the failure affects a single page only. + /// + public int? PageNumber { get; } = pageNumber; + + /// + /// Gets the format the runtime identified by looking at the content. + /// + public string? DetectedFormat { get; } = detectedFormat; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index 29f80338..3a5df36b 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -769,8 +769,13 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM if (batch.Count > 0) await this.FlushBatchAsync(indexStore, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, optimizationTracker, collectionName, batch, token); + // + // The extraction itself did not report a failure, but nothing usable came out of it. For + // the index this is the same case as a scanned page without a text layer, which is why it + // carries a code of its own instead of an unclassified exception: + // if (totalChunkCount == 0) - throw new InvalidOperationException(string.Format(TB("No text could be read from the file '{0}'."), file.Name)); + throw new FileExtractionException(FileExtractionErrorCode.NO_CONTENT, string.Format(TB("No text could be read from the file '{0}'."), file.Name)); logger.LogDebug( "Generated {ChunkCount} chunks for file '{FilePath}' in data source '{DataSourceName}' ({DataSourceId}).", diff --git a/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs b/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs index 2cdc4b9b..b518ab09 100644 --- a/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs +++ b/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs @@ -372,7 +372,7 @@ public sealed partial class RustService error.DetectedFormat, error.Message); - throw new InvalidOperationException($"Rust could not extract '{path}': {error.Message}"); + throw new FileExtractionException(error.ParsedCode, $"Rust could not extract '{path}': {error.Message}", error.PageNumber, error.DetectedFormat); } if (processedEvent.PromptInjection is { } promptInjection)