diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateDataSourceEntity.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateDataSourceEntity.cs index 660a897f..4b04487b 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateDataSourceEntity.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateDataSourceEntity.cs @@ -19,4 +19,6 @@ internal sealed class EmbeddingStateDataSourceEntity public DateTimeOffset UpdatedAtUtc { get; set; } public List Files { get; set; } = []; + + public List PermanentIndexingFailures { get; set; } = []; } diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs index 6bef7bf9..9bfea694 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs @@ -24,6 +24,10 @@ public abstract class IndexStoreClient(string name, string path) : DatabaseClien public abstract Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token); + public abstract Task UpsertPermanentFailureAsync(string dataSourceId, PermanentIndexingFailure failure, CancellationToken token); + + public abstract Task DeletePermanentFailureAsync(string dataSourceId, string filePath, CancellationToken token); + public abstract Task UpsertChunksAsync(string dataSourceId, IReadOnlyList chunks, CancellationToken token); public abstract Task> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs index 7d0d3ab5..d07b977b 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs @@ -15,6 +15,8 @@ internal sealed class IndexStoreDbContext(DbContextOptions public DbSet EmbeddingChunks => this.Set(); + public DbSet PermanentIndexingFailures => this.Set(); + public DbSet SearchResults => this.Set(); protected override void OnModelCreating(ModelBuilder modelBuilder) @@ -40,6 +42,12 @@ internal sealed class IndexStoreDbContext(DbContextOptions .WithOne(file => file.DataSource) .HasForeignKey(file => file.DataSourceId) .OnDelete(DeleteBehavior.Cascade); + + entity + .HasMany(dataSource => dataSource.PermanentIndexingFailures) + .WithOne(failure => failure.DataSource) + .HasForeignKey(failure => failure.DataSourceId) + .OnDelete(DeleteBehavior.Cascade); }); modelBuilder.Entity(entity => @@ -94,6 +102,23 @@ internal sealed class IndexStoreDbContext(DbContextOptions entity.HasIndex(chunk => new { chunk.ParentFileId, chunk.ChunkIndex }).HasDatabaseName("idx_embedding_chunks_parent_file_chunk_index").IsUnique(); }); + modelBuilder.Entity(entity => + { + entity.ToTable("permanent_indexing_failures"); + entity.HasKey(failure => failure.ParentFileId); + + entity.Property(failure => failure.ParentFileId).HasColumnName("parent_file_id"); + entity.Property(failure => failure.DataSourceId).HasColumnName("data_source_id").IsRequired(); + entity.Property(failure => failure.AbsolutePath).HasColumnName("absolute_path").UseCollation("NOCASE").IsRequired(); + entity.Property(failure => failure.Fingerprint).HasColumnName("fingerprint").IsRequired(); + entity.Property(failure => failure.FailureCode).HasColumnName("failure_code").IsRequired(); + entity.Property(failure => failure.FailureMessage).HasColumnName("failure_message").IsRequired(); + entity.Property(failure => failure.OccurredAtUtc).HasColumnName("occurred_at_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired(); + + entity.HasIndex(failure => failure.DataSourceId).HasDatabaseName("idx_permanent_indexing_failures_data_source"); + entity.HasIndex(failure => new { failure.DataSourceId, failure.AbsolutePath }).HasDatabaseName("idx_permanent_indexing_failures_data_source_absolute_path").IsUnique(); + }); + modelBuilder.Entity(entity => { entity.HasNoKey(); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs index bf44f43b..f684e6db 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs @@ -7,9 +7,10 @@ namespace AIStudio.Tools.Databases.IndexStore; internal static class IndexStoreSchemaMigrator { [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))] + [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.PermanentIndexingFailures))] public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token) { await context.Database.MigrateAsync(token); await context.Database.ExecuteSqlRawAsync("PRAGMA journal_mode=WAL;", token); } -} +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexingFailureEntity.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexingFailureEntity.cs new file mode 100644 index 00000000..83a4eba3 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexingFailureEntity.cs @@ -0,0 +1,27 @@ +namespace AIStudio.Tools.Databases.IndexStore; + +internal sealed class IndexingFailureEntity +{ + public string ParentFileId { get; set; } = string.Empty; + + public string DataSourceId { get; set; } = string.Empty; + + public string AbsolutePath { get; set; } = string.Empty; + + public string Fingerprint { get; set; } = string.Empty; + + /// + /// The failure code, stored by name. + /// + /// + /// The enum has no explicit numbers, so storing the name keeps the rows readable across + /// versions which add or reorder codes. + /// + public string FailureCode { get; set; } = string.Empty; + + public string FailureMessage { get; set; } = string.Empty; + + public DateTimeOffset OccurredAtUtc { get; set; } + + public EmbeddingStateDataSourceEntity? DataSource { get; set; } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260909000000_PermanentIndexingFailures.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260909000000_PermanentIndexingFailures.cs new file mode 100644 index 00000000..40929dde --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260909000000_PermanentIndexingFailures.cs @@ -0,0 +1,53 @@ +#nullable disable + +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace AIStudio.Tools.Databases.IndexStore.Migrations; + +[DbContext(typeof(IndexStoreDbContext))] +[Migration("20260909000000_PermanentIndexingFailures")] +public partial class PermanentIndexingFailures : Migration +{ + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "permanent_indexing_failures", + columns: table => new + { + parent_file_id = table.Column(type: "TEXT", nullable: false), + data_source_id = table.Column(type: "TEXT", nullable: false), + absolute_path = table.Column(type: "TEXT", nullable: false, collation: "NOCASE"), + fingerprint = table.Column(type: "TEXT", nullable: false), + failure_code = table.Column(type: "TEXT", nullable: false), + failure_message = table.Column(type: "TEXT", nullable: false), + occurred_at_utc = table.Column(type: "TEXT", nullable: false), + }, + constraints: table => + { + table.PrimaryKey("PK_permanent_indexing_failures", failure => failure.parent_file_id); + table.ForeignKey( + name: "FK_permanent_indexing_failures_data_sources_data_source_id", + column: failure => failure.data_source_id, + principalTable: "data_sources", + principalColumn: "data_source_id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "idx_permanent_indexing_failures_data_source", + table: "permanent_indexing_failures", + column: "data_source_id"); + + migrationBuilder.CreateIndex( + name: "idx_permanent_indexing_failures_data_source_absolute_path", + table: "permanent_indexing_failures", + columns: ["data_source_id", "absolute_path"], + unique: true); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable(name: "permanent_indexing_failures"); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs index c02521e7..cf9015d6 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs @@ -209,6 +209,55 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot entity.ToTable("embedding_chunks"); }); + modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.IndexingFailureEntity", entity => + { + entity.Property("ParentFileId") + .HasColumnType("TEXT") + .HasColumnName("parent_file_id"); + + entity.Property("AbsolutePath") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("absolute_path") + .UseCollation("NOCASE"); + + entity.Property("DataSourceId") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("data_source_id"); + + entity.Property("FailureCode") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("failure_code"); + + entity.Property("FailureMessage") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("failure_message"); + + entity.Property("Fingerprint") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("fingerprint"); + + entity.Property("OccurredAtUtc") + .HasConversion(utcDateTimeOffsetConverter) + .HasColumnType("TEXT") + .HasColumnName("occurred_at_utc"); + + entity.HasKey("ParentFileId"); + + entity.HasIndex("DataSourceId") + .HasDatabaseName("idx_permanent_indexing_failures_data_source"); + + entity.HasIndex("DataSourceId", "AbsolutePath") + .IsUnique() + .HasDatabaseName("idx_permanent_indexing_failures_data_source_absolute_path"); + + entity.ToTable("permanent_indexing_failures"); + }); + modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.IndexStoreSearchResultEntity", entity => { entity.Property("AbsolutePath") @@ -316,9 +365,22 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot entity.Navigation("File"); }); + modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.IndexingFailureEntity", entity => + { + entity.HasOne("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", "DataSource") + .WithMany("PermanentIndexingFailures") + .HasForeignKey("DataSourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + entity.Navigation("DataSource"); + }); + modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", entity => { entity.Navigation("Files"); + + entity.Navigation("PermanentIndexingFailures"); }); modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", entity => diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs index 76f93f25..cac66a85 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs @@ -44,6 +44,10 @@ public sealed class NoIndexStoreClient(string name, string? unavailableReason, D public override Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token) => Task.CompletedTask; + public override Task UpsertPermanentFailureAsync(string dataSourceId, PermanentIndexingFailure failure, CancellationToken token) => Task.CompletedTask; + + public override Task DeletePermanentFailureAsync(string dataSourceId, string filePath, CancellationToken token) => Task.CompletedTask; + public override Task UpsertChunksAsync(string dataSourceId, IReadOnlyList chunks, CancellationToken token) => Task.CompletedTask; public override Task> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token) => diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/PermanentIndexingFailure.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/PermanentIndexingFailure.cs new file mode 100644 index 00000000..5f0316f0 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/PermanentIndexingFailure.cs @@ -0,0 +1,3 @@ +namespace AIStudio.Tools.Databases.IndexStore; + +public sealed record PermanentIndexingFailure(string ParentFileId, string AbsolutePath, string Fingerprint, FileExtractionErrorCode Code, string Message, DateTimeOffset OccurredAtUtc); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs index b0669e33..1e8d44cc 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs @@ -66,6 +66,7 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat yield return (TB("Indexed data sources"), (await context.DataSources.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture)); yield return (TB("Indexed files"), (await context.EmbeddedFiles.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture)); yield return (TB("Search chunks"), (await context.EmbeddingChunks.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture)); + yield return (TB("Permanently skipped files"), (await context.PermanentIndexingFailures.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture)); } public override async Task GetManifestAsync(string dataSourceId, CancellationToken token) @@ -99,6 +100,19 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat file.ChunkCount); } + var permanentFailures = await context.PermanentIndexingFailures + .AsNoTracking() + .Where(failure => failure.DataSourceId == dataSourceId) + .ToListAsync(token); + foreach (var failure in permanentFailures) + { + manifest.PermanentFailures[failure.AbsolutePath] = new PermanentIndexingFailureRecord( + failure.Fingerprint, + ParseFailureCode(failure.FailureCode), + failure.FailureMessage, + failure.OccurredAtUtc); + } + return manifest; } @@ -190,6 +204,31 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat await transaction.CommitAsync(token); } + public override async Task UpsertPermanentFailureAsync(string dataSourceId, PermanentIndexingFailure failure, CancellationToken token) + { + await using var context = this.CreateContext(); + var failureEntity = await context.PermanentIndexingFailures.FirstOrDefaultAsync(entity => entity.ParentFileId == failure.ParentFileId, token); + if (failureEntity is null) + { + failureEntity = new IndexingFailureEntity + { + ParentFileId = failure.ParentFileId, + }; + context.PermanentIndexingFailures.Add(failureEntity); + } + + ApplyPermanentFailure(failureEntity, dataSourceId, failure); + await context.SaveChangesAsync(token); + } + + public override async Task DeletePermanentFailureAsync(string dataSourceId, string filePath, CancellationToken token) + { + await using var context = this.CreateContext(); + await context.PermanentIndexingFailures + .Where(failure => failure.DataSourceId == dataSourceId && failure.AbsolutePath == filePath) + .ExecuteDeleteAsync(token); + } + public override async Task UpsertChunksAsync(string dataSourceId, IReadOnlyList chunks, CancellationToken token) { if (chunks.Count == 0) @@ -359,6 +398,24 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat fileEntity.ConfidenceLevelRank = file.ConfidenceLevelRank; } + private static void ApplyPermanentFailure(IndexingFailureEntity failureEntity, string dataSourceId, PermanentIndexingFailure failure) + { + failureEntity.DataSourceId = dataSourceId; + failureEntity.AbsolutePath = failure.AbsolutePath; + failureEntity.Fingerprint = failure.Fingerprint; + failureEntity.FailureCode = failure.Code.ToString(); + failureEntity.FailureMessage = failure.Message; + failureEntity.OccurredAtUtc = failure.OccurredAtUtc; + } + + /// + /// A row written by a newer version may name a code this one does not know. Such a row still + /// says that the file failed permanently, so it keeps its place in the manifest and only loses + /// the reason it names. + /// + private static FileExtractionErrorCode ParseFailureCode(string failureCode) => + Enum.TryParse(failureCode, ignoreCase: true, out var parsedCode) ? parsedCode : FileExtractionErrorCode.UNKNOWN; + private static void ApplyChunk(EmbeddingStateChunkEntity chunkEntity, EmbeddingStateChunk chunk) { chunkEntity.ChunkId = chunk.ChunkId; diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingManifest.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingManifest.cs index 44b0d526..7b40a47e 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingManifest.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingManifest.cs @@ -11,4 +11,15 @@ public sealed class DataSourceEmbeddingManifest public int VectorSize { get; set; } public Dictionary Files { get; init; } = new(StringComparer.OrdinalIgnoreCase); + + /// + /// The files whose indexing failed for a reason which lies in the file itself, keyed by their + /// absolute path. + /// + /// + /// These files are not read again as long as their fingerprint stays the same. Without this, + /// a folder holding hundreds of scanned documents without a text layer would be read again on + /// every single run, with the outcome known in advance. + /// + public Dictionary PermanentFailures { get; init; } = new(StringComparer.OrdinalIgnoreCase); } diff --git a/app/MindWork AI Studio/Tools/Services/PermanentIndexingFailureRecord.cs b/app/MindWork AI Studio/Tools/Services/PermanentIndexingFailureRecord.cs new file mode 100644 index 00000000..b913cf19 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/PermanentIndexingFailureRecord.cs @@ -0,0 +1,3 @@ +namespace AIStudio.Tools.Services; + +public sealed record PermanentIndexingFailureRecord(string Fingerprint, FileExtractionErrorCode Code, string Message, DateTimeOffset OccurredAtUtc); \ No newline at end of file