From ff3927dcde8a8acd0808900e734b8d042487a1e5 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Tue, 15 Sep 2026 22:28:08 +0200 Subject: [PATCH] Removed the confidence level from the RAG index database --- .../IndexStore/EmbeddingStateFile.cs | 4 +- .../IndexStore/EmbeddingStateFileEntity.cs | 4 -- .../IndexStore/IndexStoreDbContext.cs | 3 -- .../IndexStore/IndexStoreSchemaMigrator.cs | 1 + .../IndexStore/IndexStoreSearchResult.cs | 4 +- .../IndexStoreSearchResultEntity.cs | 4 -- .../20260915000000_DropFileConfidenceLevel.cs | 51 +++++++++++++++++++ .../IndexStoreDbContextModelSnapshot.cs | 19 ------- .../SqliteIndexStoreClientImplementation.cs | 10 +--- .../DataSourceEmbeddingService.Files.cs | 10 +--- 10 files changed, 57 insertions(+), 53 deletions(-) create mode 100644 app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260915000000_DropFileConfidenceLevel.cs diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs index cbf268b4..8089ea18 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs @@ -11,6 +11,4 @@ public sealed record EmbeddingStateFile( DateTimeOffset CreationUtc, DateTimeOffset LastWriteUtc, DateTimeOffset EmbeddedAtUtc, - int ChunkCount, - string ConfidenceLevel, - int ConfidenceLevelRank); + int ChunkCount); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs index bd116ba2..1f33c2e8 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs @@ -26,10 +26,6 @@ internal sealed class EmbeddingStateFileEntity public int ChunkCount { get; set; } - public string ConfidenceLevel { get; set; } = string.Empty; - - public int ConfidenceLevelRank { get; set; } - public EmbeddingStateDataSourceEntity? DataSource { get; set; } public List Chunks { get; set; } = []; diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs index d07b977b..870837d5 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs @@ -67,13 +67,10 @@ internal sealed class IndexStoreDbContext(DbContextOptions entity.Property(file => file.LastWriteUtc).HasColumnName("last_write_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired(); entity.Property(file => file.EmbeddedAtUtc).HasColumnName("embedded_at_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired(); entity.Property(file => file.ChunkCount).HasColumnName("chunk_count"); - entity.Property(file => file.ConfidenceLevel).HasColumnName("confidence_level").IsRequired(); - entity.Property(file => file.ConfidenceLevelRank).HasColumnName("confidence_level_rank"); entity.HasIndex(file => file.DataSourceId).HasDatabaseName("idx_embedded_files_data_source"); entity.HasIndex(file => file.AbsolutePath).HasDatabaseName("idx_embedded_files_absolute_path"); entity.HasIndex(file => file.FileType).HasDatabaseName("idx_embedded_files_file_type"); - entity.HasIndex(file => file.ConfidenceLevelRank).HasDatabaseName("idx_embedded_files_confidence"); entity.HasIndex(file => new { file.DataSourceId, file.AbsolutePath }).HasDatabaseName("idx_embedded_files_data_source_absolute_path").IsUnique(); entity diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs index f684e6db..7ae6e4f2 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs @@ -8,6 +8,7 @@ internal static class IndexStoreSchemaMigrator { [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))] [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.PermanentIndexingFailures))] + [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropFileConfidenceLevel))] public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token) { await context.Database.MigrateAsync(token); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs index 0ae8d884..58d1d767 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs @@ -19,6 +19,4 @@ public sealed record IndexStoreSearchResult( DateTimeOffset CreationUtc, DateTimeOffset LastWriteUtc, DateTimeOffset EmbeddedAtUtc, - int ChunkCount, - string ConfidenceLevel, - int ConfidenceLevelRank); + int ChunkCount); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs index 1385c9f5..51ea7796 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs @@ -39,8 +39,4 @@ internal sealed class IndexStoreSearchResultEntity public DateTimeOffset EmbeddedAtUtc { get; set; } public int ChunkCount { get; set; } - - public string ConfidenceLevel { get; set; } = string.Empty; - - public int ConfidenceLevelRank { get; set; } } diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260915000000_DropFileConfidenceLevel.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260915000000_DropFileConfidenceLevel.cs new file mode 100644 index 00000000..84fa066d --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260915000000_DropFileConfidenceLevel.cs @@ -0,0 +1,51 @@ +#nullable disable + +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace AIStudio.Tools.Databases.IndexStore.Migrations; + +/// +/// Drops the copy of the data source confidence level which every indexed file carried. +/// +/// +/// The confidence level is what a data source asks of a provider. It is a property of the data +/// source, it is enforced live before anything is indexed or answered, and it changes no vector. +/// Keeping a copy per file only meant the index had to be thrown away whenever the setting changed. +/// +[DbContext(typeof(IndexStoreDbContext))] +[Migration("20260915000000_DropFileConfidenceLevel")] +public partial class DropFileConfidenceLevel : Migration +{ + /// + /// The columns go through raw SQL instead of DropColumn on purpose. The SQLite provider answers + /// DropColumn by rebuilding the table, and a rebuild drops the table the trigger + /// embedded_files_file_name_au hangs on, which would silently stop the full-text index from + /// following a renamed file. A native ALTER TABLE ... DROP COLUMN leaves the table itself alone. + /// It does refuse a column an index names, so the index has to go first. + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "idx_embedded_files_confidence", + table: "embedded_files"); + + migrationBuilder.Sql(""" + ALTER TABLE embedded_files DROP COLUMN confidence_level; + ALTER TABLE embedded_files DROP COLUMN confidence_level_rank; + """); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql(""" + ALTER TABLE embedded_files ADD COLUMN confidence_level TEXT NOT NULL DEFAULT ''; + ALTER TABLE embedded_files ADD COLUMN confidence_level_rank INTEGER NOT NULL DEFAULT 0; + """); + + migrationBuilder.CreateIndex( + name: "idx_embedded_files_confidence", + table: "embedded_files", + column: "confidence_level_rank"); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs index cf9015d6..3d8336c1 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs @@ -80,15 +80,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot .HasColumnType("INTEGER") .HasColumnName("chunk_count"); - entity.Property("ConfidenceLevel") - .IsRequired() - .HasColumnType("TEXT") - .HasColumnName("confidence_level"); - - entity.Property("ConfidenceLevelRank") - .HasColumnType("INTEGER") - .HasColumnName("confidence_level_rank"); - entity.Property("CreationUtc") .HasConversion(utcDateTimeOffsetConverter) .HasColumnType("TEXT") @@ -138,9 +129,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot entity.HasIndex("AbsolutePath") .HasDatabaseName("idx_embedded_files_absolute_path"); - entity.HasIndex("ConfidenceLevelRank") - .HasDatabaseName("idx_embedded_files_confidence"); - entity.HasIndex("DataSourceId") .HasDatabaseName("idx_embedded_files_data_source"); @@ -278,13 +266,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot .IsRequired() .HasColumnType("TEXT"); - entity.Property("ConfidenceLevel") - .IsRequired() - .HasColumnType("TEXT"); - - entity.Property("ConfidenceLevelRank") - .HasColumnType("INTEGER"); - entity.Property("CreationUtc") .HasConversion(utcDateTimeOffsetConverter) .HasColumnType("TEXT"); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs index 1e8d44cc..513bc969 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs @@ -302,9 +302,7 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat f.creation_utc AS CreationUtc, f.last_write_utc AS LastWriteUtc, c.embedded_at_utc AS EmbeddedAtUtc, - f.chunk_count AS ChunkCount, - f.confidence_level AS ConfidenceLevel, - f.confidence_level_rank AS ConfidenceLevelRank + f.chunk_count AS ChunkCount FROM embedding_chunks_fts JOIN embedding_chunks c ON c.id = embedding_chunks_fts.rowid JOIN embedded_files f ON f.parent_file_id = c.parent_file_id @@ -394,8 +392,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat fileEntity.LastWriteUtc = file.LastWriteUtc; fileEntity.EmbeddedAtUtc = file.EmbeddedAtUtc; fileEntity.ChunkCount = file.ChunkCount; - fileEntity.ConfidenceLevel = file.ConfidenceLevel; - fileEntity.ConfidenceLevelRank = file.ConfidenceLevelRank; } private static void ApplyPermanentFailure(IndexingFailureEntity failureEntity, string dataSourceId, PermanentIndexingFailure failure) @@ -445,9 +441,7 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat result.CreationUtc, result.LastWriteUtc, result.EmbeddedAtUtc, - result.ChunkCount, - result.ConfidenceLevel, - result.ConfidenceLevelRank); + result.ChunkCount); private static string BuildFtsQuery(string query) { diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs index 6033a283..6b735487 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs @@ -1116,7 +1116,6 @@ public sealed partial class DataSourceEmbeddingService { file.Refresh(); var absolutePath = Path.GetFullPath(file.FullName); - var confidenceLevel = GetDataSourceConfidenceLevel(dataSource); return new( this.CreateParentFileId(dataSource.Id, absolutePath), absolutePath, @@ -1128,9 +1127,7 @@ public sealed partial class DataSourceEmbeddingService file.Exists ? new DateTimeOffset(file.CreationTimeUtc) : DateTimeOffset.UnixEpoch, file.Exists ? new DateTimeOffset(file.LastWriteTimeUtc) : DateTimeOffset.UnixEpoch, embeddedAtUtc, - chunkCount, - confidenceLevel.ToString(), - (int)confidenceLevel); + chunkCount); } private IReadOnlyList CreateEmbeddingStateChunks(EmbeddingStateFile parentFile, IReadOnlyList batch, DateTimeOffset embeddedAtUtc) @@ -1146,11 +1143,6 @@ public sealed partial class DataSourceEmbeddingService .ToList(); } - private static ConfidenceLevel GetDataSourceConfidenceLevel(IDataSource dataSource) => - dataSource is not IInternalDataSource internalDataSource || internalDataSource.ConfidenceLevel is ConfidenceLevel.NONE - ? ConfidenceLevel.UNKNOWN - : internalDataSource.ConfidenceLevel; - private static string GetFileType(FileInfo file) { var extension = file.Extension.TrimStart('.').ToLowerInvariant();