Fixed your documents being indexed again after a confidence level change (#976)

2026-10-17 03:17:59 +00:00 · 2026-09-16 10:20:28 +02:00
1 parent 1ebe8eb2a4
commit 186f10cee2
@@ -11,6 +11,4 @@ public sealed record EmbeddingStateFile(
    DateTimeOffset CreationUtc,
    DateTimeOffset LastWriteUtc,
    DateTimeOffset EmbeddedAtUtc,
-    int ChunkCount,
-    string ConfidenceLevel,
-    int ConfidenceLevelRank);
+    int ChunkCount);
@@ -26,10 +26,6 @@ internal sealed class EmbeddingStateFileEntity

    public int ChunkCount { get; set; }

-    public string ConfidenceLevel { get; set; } = string.Empty;
-
-    public int ConfidenceLevelRank { get; set; }
-
    public EmbeddingStateDataSourceEntity? DataSource { get; set; }

    public List<EmbeddingStateChunkEntity> Chunks { get; set; } = [];
@@ -67,13 +67,10 @@ internal sealed class IndexStoreDbContext(DbContextOptions<IndexStoreDbContext>
            entity.Property(file => file.LastWriteUtc).HasColumnName("last_write_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
            entity.Property(file => file.EmbeddedAtUtc).HasColumnName("embedded_at_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
            entity.Property(file => file.ChunkCount).HasColumnName("chunk_count");
-            entity.Property(file => file.ConfidenceLevel).HasColumnName("confidence_level").IsRequired();
-            entity.Property(file => file.ConfidenceLevelRank).HasColumnName("confidence_level_rank");

            entity.HasIndex(file => file.DataSourceId).HasDatabaseName("idx_embedded_files_data_source");
            entity.HasIndex(file => file.AbsolutePath).HasDatabaseName("idx_embedded_files_absolute_path");
            entity.HasIndex(file => file.FileType).HasDatabaseName("idx_embedded_files_file_type");
-            entity.HasIndex(file => file.ConfidenceLevelRank).HasDatabaseName("idx_embedded_files_confidence");
            entity.HasIndex(file => new { file.DataSourceId, file.AbsolutePath }).HasDatabaseName("idx_embedded_files_data_source_absolute_path").IsUnique();

            entity
@@ -8,6 +8,7 @@ internal static class IndexStoreSchemaMigrator
 {
    [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))]
    [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.PermanentIndexingFailures))]
+    [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropFileConfidenceLevel))]
    public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token)
    {
        await context.Database.MigrateAsync(token);
@@ -19,6 +19,4 @@ public sealed record IndexStoreSearchResult(
    DateTimeOffset CreationUtc,
    DateTimeOffset LastWriteUtc,
    DateTimeOffset EmbeddedAtUtc,
-    int ChunkCount,
-    string ConfidenceLevel,
-    int ConfidenceLevelRank);
+    int ChunkCount);
@@ -39,8 +39,4 @@ internal sealed class IndexStoreSearchResultEntity
    public DateTimeOffset EmbeddedAtUtc { get; set; }

    public int ChunkCount { get; set; }
-
-    public string ConfidenceLevel { get; set; } = string.Empty;
-
-    public int ConfidenceLevelRank { get; set; }
 }
@@ -0,0 +1,51 @@
+#nullable disable
+
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace AIStudio.Tools.Databases.IndexStore.Migrations;
+
+/// <summary>
+/// Drops the copy of the data source confidence level which every indexed file carried.
+/// </summary>
+/// <remarks>
+/// The confidence level is what a data source asks of a provider. It is a property of the data
+/// source, it is enforced live before anything is indexed or answered, and it changes no vector.
+/// Keeping a copy per file only meant the index had to be thrown away whenever the setting changed.
+/// </remarks>
+[DbContext(typeof(IndexStoreDbContext))]
+[Migration("20260915000000_DropFileConfidenceLevel")]
+public partial class DropFileConfidenceLevel : Migration
+{
+    /// <remarks>
+    /// The columns go through raw SQL instead of DropColumn on purpose. The SQLite provider answers
+    /// DropColumn by rebuilding the table, and a rebuild drops the table the trigger
+    /// embedded_files_file_name_au hangs on, which would silently stop the full-text index from
+    /// following a renamed file. A native ALTER TABLE ... DROP COLUMN leaves the table itself alone.
+    /// It does refuse a column an index names, so the index has to go first.
+    /// </remarks>
+    protected override void Up(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.DropIndex(
+            name: "idx_embedded_files_confidence",
+            table: "embedded_files");
+
+        migrationBuilder.Sql("""
+                             ALTER TABLE embedded_files DROP COLUMN confidence_level;
+                             ALTER TABLE embedded_files DROP COLUMN confidence_level_rank;
+                             """);
+    }
+
+    protected override void Down(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.Sql("""
+                             ALTER TABLE embedded_files ADD COLUMN confidence_level TEXT NOT NULL DEFAULT '';
+                             ALTER TABLE embedded_files ADD COLUMN confidence_level_rank INTEGER NOT NULL DEFAULT 0;
+                             """);
+
+        migrationBuilder.CreateIndex(
+            name: "idx_embedded_files_confidence",
+            table: "embedded_files",
+            column: "confidence_level_rank");
+    }
+}
@@ -80,15 +80,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
                .HasColumnType("INTEGER")
                .HasColumnName("chunk_count");

-            entity.Property<string>("ConfidenceLevel")
-                .IsRequired()
-                .HasColumnType("TEXT")
-                .HasColumnName("confidence_level");
-
-            entity.Property<int>("ConfidenceLevelRank")
-                .HasColumnType("INTEGER")
-                .HasColumnName("confidence_level_rank");
-
            entity.Property<DateTimeOffset>("CreationUtc")
                .HasConversion(utcDateTimeOffsetConverter)
                .HasColumnType("TEXT")
@@ -138,9 +129,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
            entity.HasIndex("AbsolutePath")
                .HasDatabaseName("idx_embedded_files_absolute_path");

-            entity.HasIndex("ConfidenceLevelRank")
-                .HasDatabaseName("idx_embedded_files_confidence");
-
            entity.HasIndex("DataSourceId")
                .HasDatabaseName("idx_embedded_files_data_source");

@@ -278,13 +266,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
                .IsRequired()
                .HasColumnType("TEXT");

-            entity.Property<string>("ConfidenceLevel")
-                .IsRequired()
-                .HasColumnType("TEXT");
-
-            entity.Property<int>("ConfidenceLevelRank")
-                .HasColumnType("INTEGER");
-
            entity.Property<DateTimeOffset>("CreationUtc")
                .HasConversion(utcDateTimeOffsetConverter)
                .HasColumnType("TEXT");
@@ -302,9 +302,7 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
                                      f.creation_utc AS CreationUtc,
                                      f.last_write_utc AS LastWriteUtc,
                                      c.embedded_at_utc AS EmbeddedAtUtc,
-                                      f.chunk_count AS ChunkCount,
-                                      f.confidence_level AS ConfidenceLevel,
-                                      f.confidence_level_rank AS ConfidenceLevelRank
+                                      f.chunk_count AS ChunkCount
                                  FROM embedding_chunks_fts
                                  JOIN embedding_chunks c ON c.id = embedding_chunks_fts.rowid
                                  JOIN embedded_files f ON f.parent_file_id = c.parent_file_id
@@ -394,8 +392,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
        fileEntity.LastWriteUtc = file.LastWriteUtc;
        fileEntity.EmbeddedAtUtc = file.EmbeddedAtUtc;
        fileEntity.ChunkCount = file.ChunkCount;
-        fileEntity.ConfidenceLevel = file.ConfidenceLevel;
-        fileEntity.ConfidenceLevelRank = file.ConfidenceLevelRank;
    }

    private static void ApplyPermanentFailure(IndexingFailureEntity failureEntity, string dataSourceId, PermanentIndexingFailure failure)
@@ -445,9 +441,7 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
        result.CreationUtc,
        result.LastWriteUtc,
        result.EmbeddedAtUtc,
-        result.ChunkCount,
-        result.ConfidenceLevel,
-        result.ConfidenceLevelRank);
+        result.ChunkCount);

    private static string BuildFtsQuery(string query)
    {
@@ -19,6 +19,4 @@ public sealed record VectorSearchResult(
    string Fingerprint,
    string CreationUtc,
    string LastWriteUtc,
-    string EmbeddedAtUtc,
-    string ConfidenceLevel,
-    int ConfidenceLevelRank);
+    string EmbeddedAtUtc);
@@ -19,6 +19,4 @@ public sealed record VectorStoragePoint(
    string Fingerprint,
    DateTimeOffset CreationUtc,
    DateTimeOffset LastWriteUtc,
-    DateTimeOffset EmbeddedAtUtc,
-    string ConfidenceLevel,
-    int ConfidenceLevelRank);
+    DateTimeOffset EmbeddedAtUtc);