Removed the confidence level from the RAG index database

app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs

@@ -11,6 +11,4 @@ public sealed record EmbeddingStateFile(
     DateTimeOffset CreationUtc,
     DateTimeOffset LastWriteUtc,
     DateTimeOffset EmbeddedAtUtc,
-    int ChunkCount,
+    int ChunkCount);
-    string ConfidenceLevel,
-    int ConfidenceLevelRank);

app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs

@@ -26,10 +26,6 @@ internal sealed class EmbeddingStateFileEntity
 
     public int ChunkCount { get; set; }
 
-    public string ConfidenceLevel { get; set; } = string.Empty;
-
-    public int ConfidenceLevelRank { get; set; }
-
     public EmbeddingStateDataSourceEntity? DataSource { get; set; }
 
     public List<EmbeddingStateChunkEntity> Chunks { get; set; } = [];

app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs

@@ -67,13 +67,10 @@ internal sealed class IndexStoreDbContext(DbContextOptions<IndexStoreDbContext>
             entity.Property(file => file.LastWriteUtc).HasColumnName("last_write_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
             entity.Property(file => file.EmbeddedAtUtc).HasColumnName("embedded_at_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
             entity.Property(file => file.ChunkCount).HasColumnName("chunk_count");
-            entity.Property(file => file.ConfidenceLevel).HasColumnName("confidence_level").IsRequired();
-            entity.Property(file => file.ConfidenceLevelRank).HasColumnName("confidence_level_rank");
 
             entity.HasIndex(file => file.DataSourceId).HasDatabaseName("idx_embedded_files_data_source");
             entity.HasIndex(file => file.AbsolutePath).HasDatabaseName("idx_embedded_files_absolute_path");
             entity.HasIndex(file => file.FileType).HasDatabaseName("idx_embedded_files_file_type");
-            entity.HasIndex(file => file.ConfidenceLevelRank).HasDatabaseName("idx_embedded_files_confidence");
             entity.HasIndex(file => new { file.DataSourceId, file.AbsolutePath }).HasDatabaseName("idx_embedded_files_data_source_absolute_path").IsUnique();
 
             entity

app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs

@@ -8,6 +8,7 @@ internal static class IndexStoreSchemaMigrator
 {
     [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))]
     [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.PermanentIndexingFailures))]
+    [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropFileConfidenceLevel))]
     public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token)
     {
         await context.Database.MigrateAsync(token);

app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs

@@ -19,6 +19,4 @@ public sealed record IndexStoreSearchResult(
     DateTimeOffset CreationUtc,
     DateTimeOffset LastWriteUtc,
     DateTimeOffset EmbeddedAtUtc,
-    int ChunkCount,
+    int ChunkCount);
-    string ConfidenceLevel,
-    int ConfidenceLevelRank);

app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs

@@ -39,8 +39,4 @@ internal sealed class IndexStoreSearchResultEntity
     public DateTimeOffset EmbeddedAtUtc { get; set; }
 
     public int ChunkCount { get; set; }
-
-    public string ConfidenceLevel { get; set; } = string.Empty;
-
-    public int ConfidenceLevelRank { get; set; }
 }

app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260915000000_DropFileConfidenceLevel.cs

@@ -0,0 +1,51 @@
+#nullable disable
+
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace AIStudio.Tools.Databases.IndexStore.Migrations;
+
+/// <summary>
+/// Drops the copy of the data source confidence level which every indexed file carried.
+/// </summary>
+/// <remarks>
+/// The confidence level is what a data source asks of a provider. It is a property of the data
+/// source, it is enforced live before anything is indexed or answered, and it changes no vector.
+/// Keeping a copy per file only meant the index had to be thrown away whenever the setting changed.
+/// </remarks>
+[DbContext(typeof(IndexStoreDbContext))]
+[Migration("20260915000000_DropFileConfidenceLevel")]
+public partial class DropFileConfidenceLevel : Migration
+{
+    /// <remarks>
+    /// The columns go through raw SQL instead of DropColumn on purpose. The SQLite provider answers
+    /// DropColumn by rebuilding the table, and a rebuild drops the table the trigger
+    /// embedded_files_file_name_au hangs on, which would silently stop the full-text index from
+    /// following a renamed file. A native ALTER TABLE ... DROP COLUMN leaves the table itself alone.
+    /// It does refuse a column an index names, so the index has to go first.
+    /// </remarks>
+    protected override void Up(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.DropIndex(
+            name: "idx_embedded_files_confidence",
+            table: "embedded_files");
+
+        migrationBuilder.Sql("""
+                             ALTER TABLE embedded_files DROP COLUMN confidence_level;
+                             ALTER TABLE embedded_files DROP COLUMN confidence_level_rank;
+                             """);
+    }
+
+    protected override void Down(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.Sql("""
+                             ALTER TABLE embedded_files ADD COLUMN confidence_level TEXT NOT NULL DEFAULT '';
+                             ALTER TABLE embedded_files ADD COLUMN confidence_level_rank INTEGER NOT NULL DEFAULT 0;
+                             """);
+
+        migrationBuilder.CreateIndex(
+            name: "idx_embedded_files_confidence",
+            table: "embedded_files",
+            column: "confidence_level_rank");
+    }
+}

app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs

@@ -80,15 +80,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
                 .HasColumnType("INTEGER")
                 .HasColumnName("chunk_count");
 
-            entity.Property<string>("ConfidenceLevel")
-                .IsRequired()
-                .HasColumnType("TEXT")
-                .HasColumnName("confidence_level");
-
-            entity.Property<int>("ConfidenceLevelRank")
-                .HasColumnType("INTEGER")
-                .HasColumnName("confidence_level_rank");
-
             entity.Property<DateTimeOffset>("CreationUtc")
                 .HasConversion(utcDateTimeOffsetConverter)
                 .HasColumnType("TEXT")
@@ -138,9 +129,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
             entity.HasIndex("AbsolutePath")
                 .HasDatabaseName("idx_embedded_files_absolute_path");
 
-            entity.HasIndex("ConfidenceLevelRank")
-                .HasDatabaseName("idx_embedded_files_confidence");
-
             entity.HasIndex("DataSourceId")
                 .HasDatabaseName("idx_embedded_files_data_source");
 
@@ -278,13 +266,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
                 .IsRequired()
                 .HasColumnType("TEXT");
 
-            entity.Property<string>("ConfidenceLevel")
-                .IsRequired()
-                .HasColumnType("TEXT");
-
-            entity.Property<int>("ConfidenceLevelRank")
-                .HasColumnType("INTEGER");
-
             entity.Property<DateTimeOffset>("CreationUtc")
                 .HasConversion(utcDateTimeOffsetConverter)
                 .HasColumnType("TEXT");

app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs

@@ -302,9 +302,7 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
                                       f.creation_utc AS CreationUtc,
                                       f.last_write_utc AS LastWriteUtc,
                                       c.embedded_at_utc AS EmbeddedAtUtc,
-                                      f.chunk_count AS ChunkCount,
+                                      f.chunk_count AS ChunkCount
-                                      f.confidence_level AS ConfidenceLevel,
-                                      f.confidence_level_rank AS ConfidenceLevelRank
                                   FROM embedding_chunks_fts
                                   JOIN embedding_chunks c ON c.id = embedding_chunks_fts.rowid
                                   JOIN embedded_files f ON f.parent_file_id = c.parent_file_id
@@ -394,8 +392,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
         fileEntity.LastWriteUtc = file.LastWriteUtc;
         fileEntity.EmbeddedAtUtc = file.EmbeddedAtUtc;
         fileEntity.ChunkCount = file.ChunkCount;
-        fileEntity.ConfidenceLevel = file.ConfidenceLevel;
-        fileEntity.ConfidenceLevelRank = file.ConfidenceLevelRank;
     }
 
     private static void ApplyPermanentFailure(IndexingFailureEntity failureEntity, string dataSourceId, PermanentIndexingFailure failure)
@@ -445,9 +441,7 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
         result.CreationUtc,
         result.LastWriteUtc,
         result.EmbeddedAtUtc,
-        result.ChunkCount,
+        result.ChunkCount);
-        result.ConfidenceLevel,
-        result.ConfidenceLevelRank);
 
     private static string BuildFtsQuery(string query)
     {

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs

@@ -1116,7 +1116,6 @@ public sealed partial class DataSourceEmbeddingService
     {
         file.Refresh();
         var absolutePath = Path.GetFullPath(file.FullName);
-        var confidenceLevel = GetDataSourceConfidenceLevel(dataSource);
         return new(
             this.CreateParentFileId(dataSource.Id, absolutePath),
             absolutePath,
@@ -1128,9 +1127,7 @@ public sealed partial class DataSourceEmbeddingService
             file.Exists ? new DateTimeOffset(file.CreationTimeUtc) : DateTimeOffset.UnixEpoch,
             file.Exists ? new DateTimeOffset(file.LastWriteTimeUtc) : DateTimeOffset.UnixEpoch,
             embeddedAtUtc,
-            chunkCount,
+            chunkCount);
-            confidenceLevel.ToString(),
-            (int)confidenceLevel);
     }
 
     private IReadOnlyList<EmbeddingStateChunk> CreateEmbeddingStateChunks(EmbeddingStateFile parentFile, IReadOnlyList<EmbeddingChunkDraft> batch, DateTimeOffset embeddedAtUtc)
@@ -1146,11 +1143,6 @@ public sealed partial class DataSourceEmbeddingService
             .ToList();
     }
 
-    private static ConfidenceLevel GetDataSourceConfidenceLevel(IDataSource dataSource) =>
-        dataSource is not IInternalDataSource internalDataSource || internalDataSource.ConfidenceLevel is ConfidenceLevel.NONE
-            ? ConfidenceLevel.UNKNOWN
-            : internalDataSource.ConfidenceLevel;
-
     private static string GetFileType(FileInfo file)
     {
         var extension = file.Extension.TrimStart('.').ToLowerInvariant();