mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-17 16:58:00 +00:00
Show permanently skipped files as their own category
This commit is contained in:
6 files changed
+64
-10
No files matched your search
@@ -18,5 +18,7 @@ namespace AIStudio.Tools.Services;
|
||||
/// <param name="FailureReason">What kind of failure it was. Everything that did not come from a provider stays at NONE.</param>
|
||||
/// <param name="StatusCode">What the provider answered, where it answered at all.</param>
|
||||
/// <param name="EmbeddingProviderName">The embedding provider that was asked.</param>
|
||||
/// <param name="ExtractionCode">Why reading the file failed, where the failure was about reading it at all.</param>
|
||||
/// <param name="IsPermanent">Whether the file stays out of the index until it changes.</param>
|
||||
public sealed record DataSourceEmbeddingFailure(string FilePath, string Reason, DateTimeOffset OccurredAtUtc, ProviderRequestFailureReason FailureReason = ProviderRequestFailureReason.NONE,
|
||||
HttpStatusCode? StatusCode = null, string EmbeddingProviderName = "");
|
||||
HttpStatusCode? StatusCode = null, string EmbeddingProviderName = "", FileExtractionErrorCode ExtractionCode = FileExtractionErrorCode.NONE, bool IsPermanent = false);
|
||||
@@ -640,7 +640,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
permanentlySkippedFiles++;
|
||||
|
||||
// The stored reason keeps its place in the list, so the user still sees why:
|
||||
failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, permanentFailure.Message, permanentFailure.OccurredAtUtc));
|
||||
failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, permanentFailure.Message, permanentFailure.OccurredAtUtc, ExtractionCode: permanentFailure.Code, IsPermanent: true));
|
||||
this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, lastError: lastError, failures: failureDetails, permanentlySkippedFiles: permanentlySkippedFiles));
|
||||
continue;
|
||||
}
|
||||
@@ -735,7 +735,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
permanentlySkippedFiles++;
|
||||
var occurredAtUtc = DateTimeOffset.UtcNow;
|
||||
var indexingMessage = exception.Code.ToIndexingUserMessage(file.Name);
|
||||
failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, indexingMessage, occurredAtUtc));
|
||||
failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, indexingMessage, occurredAtUtc, ExtractionCode: exception.Code, IsPermanent: true));
|
||||
manifest.Files.Remove(file.FullName);
|
||||
await this.CleanupFailedFileAsync(indexStore, vectorStore, dataSource, collectionName, file.FullName, optimizationTracker, token);
|
||||
|
||||
@@ -766,7 +766,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
//
|
||||
failedFiles++;
|
||||
lastError = exception.Message;
|
||||
failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, exception.Message, DateTimeOffset.UtcNow, EmbeddingProviderName: embeddingProvider.Name));
|
||||
failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, exception.Message, DateTimeOffset.UtcNow, EmbeddingProviderName: embeddingProvider.Name, ExtractionCode: exception is FileExtractionException extractionFailure ? extractionFailure.Code : FileExtractionErrorCode.NONE));
|
||||
manifest.Files.Remove(file.FullName);
|
||||
await this.ForgetPermanentFailureAsync(indexStore, dataSource, manifest, file.FullName, token);
|
||||
await this.CleanupFailedFileAsync(indexStore, vectorStore, dataSource, collectionName, file.FullName, optimizationTracker, token);
|
||||
@@ -1376,7 +1376,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
}
|
||||
|
||||
private static List<DataSourceEmbeddingFailure> CreatePermanentFailureDetails(DataSourceEmbeddingManifest manifest) => manifest.PermanentFailures
|
||||
.Select(failure => new DataSourceEmbeddingFailure(failure.Key, failure.Value.Message, failure.Value.OccurredAtUtc))
|
||||
.Select(failure => new DataSourceEmbeddingFailure(failure.Key, failure.Value.Message, failure.Value.OccurredAtUtc, ExtractionCode: failure.Value.Code, IsPermanent: true))
|
||||
.ToList();
|
||||
|
||||
private static string GetFileEmbeddingReason(FileInfo file, string currentHash, EmbeddedFileRecord? existingRecord)
|
||||
|
||||
Reference in new issue
Block a user