MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-09-27 19:13:37 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Persist permanent indexing failures in the RAG index

Browse Source
This commit is contained in:
Thorsten Sommer 2026-09-09 11:12:30 +02:00
parent 6dd2389a99
commit 0e22d58796
Signed by untrusted user who does not match committer: tsommer
GPG Key ID: 371BBA77A02C0108
12 changed files with 253 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateDataSourceEntity.cs
View File

@ -19,4 +19,6 @@ internal sealed class EmbeddingStateDataSourceEntity
public DateTimeOffset UpdatedAtUtc { get; set; }
public List<EmbeddingStateFileEntity> Files { get; set; } = [];
public List<IndexingFailureEntity> PermanentIndexingFailures { get; set; } = [];
}

4
app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs
View File

@ -24,6 +24,10 @@ public abstract class IndexStoreClient(string name, string path) : DatabaseClien
public abstract Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token);
public abstract Task UpsertPermanentFailureAsync(string dataSourceId, PermanentIndexingFailure failure, CancellationToken token);
public abstract Task DeletePermanentFailureAsync(string dataSourceId, string filePath, CancellationToken token);
public abstract Task UpsertChunksAsync(string dataSourceId, IReadOnlyList<EmbeddingStateChunk> chunks, CancellationToken token);
public abstract Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token);

25
app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs
View File

@ -15,6 +15,8 @@ internal sealed class IndexStoreDbContext(DbContextOptions<IndexStoreDbContext>
public DbSet<EmbeddingStateChunkEntity> EmbeddingChunks => this.Set<EmbeddingStateChunkEntity>();
public DbSet<IndexingFailureEntity> PermanentIndexingFailures => this.Set<IndexingFailureEntity>();
public DbSet<IndexStoreSearchResultEntity> SearchResults => this.Set<IndexStoreSearchResultEntity>();
protected override void OnModelCreating(ModelBuilder modelBuilder)
@ -40,6 +42,12 @@ internal sealed class IndexStoreDbContext(DbContextOptions<IndexStoreDbContext>
.WithOne(file => file.DataSource)
.HasForeignKey(file => file.DataSourceId)
.OnDelete(DeleteBehavior.Cascade);
entity
.HasMany(dataSource => dataSource.PermanentIndexingFailures)
.WithOne(failure => failure.DataSource)
.HasForeignKey(failure => failure.DataSourceId)
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity<EmbeddingStateFileEntity>(entity =>
@ -94,6 +102,23 @@ internal sealed class IndexStoreDbContext(DbContextOptions<IndexStoreDbContext>
entity.HasIndex(chunk => new { chunk.ParentFileId, chunk.ChunkIndex }).HasDatabaseName("idx_embedding_chunks_parent_file_chunk_index").IsUnique();
});
modelBuilder.Entity<IndexingFailureEntity>(entity =>
{
entity.ToTable("permanent_indexing_failures");
entity.HasKey(failure => failure.ParentFileId);
entity.Property(failure => failure.ParentFileId).HasColumnName("parent_file_id");
entity.Property(failure => failure.DataSourceId).HasColumnName("data_source_id").IsRequired();
entity.Property(failure => failure.AbsolutePath).HasColumnName("absolute_path").UseCollation("NOCASE").IsRequired();
entity.Property(failure => failure.Fingerprint).HasColumnName("fingerprint").IsRequired();
entity.Property(failure => failure.FailureCode).HasColumnName("failure_code").IsRequired();
entity.Property(failure => failure.FailureMessage).HasColumnName("failure_message").IsRequired();
entity.Property(failure => failure.OccurredAtUtc).HasColumnName("occurred_at_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
entity.HasIndex(failure => failure.DataSourceId).HasDatabaseName("idx_permanent_indexing_failures_data_source");
entity.HasIndex(failure => new { failure.DataSourceId, failure.AbsolutePath }).HasDatabaseName("idx_permanent_indexing_failures_data_source_absolute_path").IsUnique();
});
modelBuilder.Entity<IndexStoreSearchResultEntity>(entity =>
{
entity.HasNoKey();

3
app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs
View File

@ -7,9 +7,10 @@ namespace AIStudio.Tools.Databases.IndexStore;
internal static class IndexStoreSchemaMigrator
{
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))]
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.PermanentIndexingFailures))]
public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token)
{
await context.Database.MigrateAsync(token);
await context.Database.ExecuteSqlRawAsync("PRAGMA journal_mode=WAL;", token);
}
}
}

27
app/MindWork AI Studio/Tools/Databases/IndexStore/IndexingFailureEntity.cs Normal file
View File

@ -0,0 +1,27 @@
namespace AIStudio.Tools.Databases.IndexStore;
internal sealed class IndexingFailureEntity
{
public string ParentFileId { get; set; } = string.Empty;
public string DataSourceId { get; set; } = string.Empty;
public string AbsolutePath { get; set; } = string.Empty;
public string Fingerprint { get; set; } = string.Empty;
/// <summary>
/// The failure code, stored by name.
/// </summary>
/// <remarks>
/// The enum has no explicit numbers, so storing the name keeps the rows readable across
/// versions which add or reorder codes.
/// </remarks>
public string FailureCode { get; set; } = string.Empty;
public string FailureMessage { get; set; } = string.Empty;
public DateTimeOffset OccurredAtUtc { get; set; }
public EmbeddingStateDataSourceEntity? DataSource { get; set; }
}

53
app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260909000000_PermanentIndexingFailures.cs Normal file
View File

@ -0,0 +1,53 @@
#nullable disable
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
namespace AIStudio.Tools.Databases.IndexStore.Migrations;
[DbContext(typeof(IndexStoreDbContext))]
[Migration("20260909000000_PermanentIndexingFailures")]
public partial class PermanentIndexingFailures : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "permanent_indexing_failures",
columns: table => new
{
parent_file_id = table.Column<string>(type: "TEXT", nullable: false),
data_source_id = table.Column<string>(type: "TEXT", nullable: false),
absolute_path = table.Column<string>(type: "TEXT", nullable: false, collation: "NOCASE"),
fingerprint = table.Column<string>(type: "TEXT", nullable: false),
failure_code = table.Column<string>(type: "TEXT", nullable: false),
failure_message = table.Column<string>(type: "TEXT", nullable: false),
occurred_at_utc = table.Column<string>(type: "TEXT", nullable: false),
},
constraints: table =>
{
table.PrimaryKey("PK_permanent_indexing_failures", failure => failure.parent_file_id);
table.ForeignKey(
name: "FK_permanent_indexing_failures_data_sources_data_source_id",
column: failure => failure.data_source_id,
principalTable: "data_sources",
principalColumn: "data_source_id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateIndex(
name: "idx_permanent_indexing_failures_data_source",
table: "permanent_indexing_failures",
column: "data_source_id");
migrationBuilder.CreateIndex(
name: "idx_permanent_indexing_failures_data_source_absolute_path",
table: "permanent_indexing_failures",
columns: ["data_source_id", "absolute_path"],
unique: true);
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(name: "permanent_indexing_failures");
}
}

62
app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs
View File

@ -209,6 +209,55 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
entity.ToTable("embedding_chunks");
});
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.IndexingFailureEntity", entity =>
{
entity.Property<string>("ParentFileId")
.HasColumnType("TEXT")
.HasColumnName("parent_file_id");
entity.Property<string>("AbsolutePath")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("absolute_path")
.UseCollation("NOCASE");
entity.Property<string>("DataSourceId")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("data_source_id");
entity.Property<string>("FailureCode")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("failure_code");
entity.Property<string>("FailureMessage")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("failure_message");
entity.Property<string>("Fingerprint")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("fingerprint");
entity.Property<DateTimeOffset>("OccurredAtUtc")
.HasConversion(utcDateTimeOffsetConverter)
.HasColumnType("TEXT")
.HasColumnName("occurred_at_utc");
entity.HasKey("ParentFileId");
entity.HasIndex("DataSourceId")
.HasDatabaseName("idx_permanent_indexing_failures_data_source");
entity.HasIndex("DataSourceId", "AbsolutePath")
.IsUnique()
.HasDatabaseName("idx_permanent_indexing_failures_data_source_absolute_path");
entity.ToTable("permanent_indexing_failures");
});
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.IndexStoreSearchResultEntity", entity =>
{
entity.Property<string>("AbsolutePath")
@ -316,9 +365,22 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
entity.Navigation("File");
});
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.IndexingFailureEntity", entity =>
{
entity.HasOne("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", "DataSource")
.WithMany("PermanentIndexingFailures")
.HasForeignKey("DataSourceId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
entity.Navigation("DataSource");
});
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", entity =>
{
entity.Navigation("Files");
entity.Navigation("PermanentIndexingFailures");
});
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", entity =>

4
app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs
View File

@ -44,6 +44,10 @@ public sealed class NoIndexStoreClient(string name, string? unavailableReason, D
public override Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token) => Task.CompletedTask;
public override Task UpsertPermanentFailureAsync(string dataSourceId, PermanentIndexingFailure failure, CancellationToken token) => Task.CompletedTask;
public override Task DeletePermanentFailureAsync(string dataSourceId, string filePath, CancellationToken token) => Task.CompletedTask;
public override Task UpsertChunksAsync(string dataSourceId, IReadOnlyList<EmbeddingStateChunk> chunks, CancellationToken token) => Task.CompletedTask;
public override Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token) =>

3
app/MindWork AI Studio/Tools/Databases/IndexStore/PermanentIndexingFailure.cs Normal file
View File

@ -0,0 +1,3 @@
namespace AIStudio.Tools.Databases.IndexStore;
public sealed record PermanentIndexingFailure(string ParentFileId, string AbsolutePath, string Fingerprint, FileExtractionErrorCode Code, string Message, DateTimeOffset OccurredAtUtc);

57
app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs
View File

@ -66,6 +66,7 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
yield return (TB("Indexed data sources"), (await context.DataSources.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
yield return (TB("Indexed files"), (await context.EmbeddedFiles.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
yield return (TB("Search chunks"), (await context.EmbeddingChunks.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
yield return (TB("Permanently skipped files"), (await context.PermanentIndexingFailures.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
}
public override async Task<DataSourceEmbeddingManifest> GetManifestAsync(string dataSourceId, CancellationToken token)
@ -99,6 +100,19 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
file.ChunkCount);
}
var permanentFailures = await context.PermanentIndexingFailures
.AsNoTracking()
.Where(failure => failure.DataSourceId == dataSourceId)
.ToListAsync(token);
foreach (var failure in permanentFailures)
{
manifest.PermanentFailures[failure.AbsolutePath] = new PermanentIndexingFailureRecord(
failure.Fingerprint,
ParseFailureCode(failure.FailureCode),
failure.FailureMessage,
failure.OccurredAtUtc);
}
return manifest;
}
@ -190,6 +204,31 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
await transaction.CommitAsync(token);
}
public override async Task UpsertPermanentFailureAsync(string dataSourceId, PermanentIndexingFailure failure, CancellationToken token)
{
await using var context = this.CreateContext();
var failureEntity = await context.PermanentIndexingFailures.FirstOrDefaultAsync(entity => entity.ParentFileId == failure.ParentFileId, token);
if (failureEntity is null)
{
failureEntity = new IndexingFailureEntity
{
ParentFileId = failure.ParentFileId,
};
context.PermanentIndexingFailures.Add(failureEntity);
}
ApplyPermanentFailure(failureEntity, dataSourceId, failure);
await context.SaveChangesAsync(token);
}
public override async Task DeletePermanentFailureAsync(string dataSourceId, string filePath, CancellationToken token)
{
await using var context = this.CreateContext();
await context.PermanentIndexingFailures
.Where(failure => failure.DataSourceId == dataSourceId && failure.AbsolutePath == filePath)
.ExecuteDeleteAsync(token);
}
public override async Task UpsertChunksAsync(string dataSourceId, IReadOnlyList<EmbeddingStateChunk> chunks, CancellationToken token)
{
if (chunks.Count == 0)
@ -359,6 +398,24 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
fileEntity.ConfidenceLevelRank = file.ConfidenceLevelRank;
}
private static void ApplyPermanentFailure(IndexingFailureEntity failureEntity, string dataSourceId, PermanentIndexingFailure failure)
{
failureEntity.DataSourceId = dataSourceId;
failureEntity.AbsolutePath = failure.AbsolutePath;
failureEntity.Fingerprint = failure.Fingerprint;
failureEntity.FailureCode = failure.Code.ToString();
failureEntity.FailureMessage = failure.Message;
failureEntity.OccurredAtUtc = failure.OccurredAtUtc;
}
/// <remarks>
/// A row written by a newer version may name a code this one does not know. Such a row still
/// says that the file failed permanently, so it keeps its place in the manifest and only loses
/// the reason it names.
/// </remarks>
private static FileExtractionErrorCode ParseFailureCode(string failureCode) =>
Enum.TryParse<FileExtractionErrorCode>(failureCode, ignoreCase: true, out var parsedCode) ? parsedCode : FileExtractionErrorCode.UNKNOWN;
private static void ApplyChunk(EmbeddingStateChunkEntity chunkEntity, EmbeddingStateChunk chunk)
{
chunkEntity.ChunkId = chunk.ChunkId;

11
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingManifest.cs
View File

@ -11,4 +11,15 @@ public sealed class DataSourceEmbeddingManifest
public int VectorSize { get; set; }
public Dictionary<string, EmbeddedFileRecord> Files { get; init; } = new(StringComparer.OrdinalIgnoreCase);
/// <summary>
/// The files whose indexing failed for a reason which lies in the file itself, keyed by their
/// absolute path.
/// </summary>
/// <remarks>
/// These files are not read again as long as their fingerprint stays the same. Without this,
/// a folder holding hundreds of scanned documents without a text layer would be read again on
/// every single run, with the outcome known in advance.
/// </remarks>
public Dictionary<string, PermanentIndexingFailureRecord> PermanentFailures { get; init; } = new(StringComparer.OrdinalIgnoreCase);
}

3
app/MindWork AI Studio/Tools/Services/PermanentIndexingFailureRecord.cs Normal file
View File

@ -0,0 +1,3 @@
namespace AIStudio.Tools.Services;
public sealed record PermanentIndexingFailureRecord(string Fingerprint, FileExtractionErrorCode Code, string Message, DateTimeOffset OccurredAtUtc);