Removed the denormalized data source name from the RAG storage (#977)

2026-10-05 05:49:40 +00:00 · 2026-09-16 11:15:32 +02:00
1 parent 186f10cee2
commit b8d971c586
@@ -4,8 +4,6 @@ internal sealed class EmbeddingStateDataSourceEntity
 {
    public string DataSourceId { get; set; } = string.Empty;

-    public string DataSourceName { get; set; } = string.Empty;
-
    public string DataSourceType { get; set; } = string.Empty;

    public string EmbeddingProviderId { get; set; } = string.Empty;
@@ -8,7 +8,6 @@ public abstract class IndexStoreClient(string name, string path) : DatabaseClien

    public abstract Task UpsertDataSourceAsync(
        string dataSourceId,
-        string dataSourceName,
        string dataSourceType,
        string embeddingProviderId,
        string embeddingSignature,
@@ -29,7 +29,6 @@ internal sealed class IndexStoreDbContext(DbContextOptions<IndexStoreDbContext>
            entity.HasKey(dataSource => dataSource.DataSourceId);

            entity.Property(dataSource => dataSource.DataSourceId).HasColumnName("data_source_id");
-            entity.Property(dataSource => dataSource.DataSourceName).HasColumnName("data_source_name").IsRequired();
            entity.Property(dataSource => dataSource.DataSourceType).HasColumnName("data_source_type").IsRequired();
            entity.Property(dataSource => dataSource.EmbeddingProviderId).HasColumnName("embedding_provider_id").IsRequired();
            entity.Property(dataSource => dataSource.EmbeddingSignature).HasColumnName("embedding_signature").IsRequired();
@@ -9,6 +9,7 @@ internal static class IndexStoreSchemaMigrator
    [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))]
    [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.PermanentIndexingFailures))]
    [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropFileConfidenceLevel))]
+    [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropDataSourceName))]
    public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token)
    {
        await context.Database.MigrateAsync(token);
@@ -4,7 +4,6 @@ public sealed record IndexStoreSearchResult(
    string ChunkId,
    string ParentFileId,
    string DataSourceId,
-    string DataSourceName,
    string DataSourceType,
    string AbsolutePath,
    string FileName,
@@ -8,8 +8,6 @@ internal sealed class IndexStoreSearchResultEntity

    public string DataSourceId { get; set; } = string.Empty;

-    public string DataSourceName { get; set; } = string.Empty;
-
    public string DataSourceType { get; set; } = string.Empty;

    public string AbsolutePath { get; set; } = string.Empty;
@@ -0,0 +1,36 @@
+#nullable disable
+
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace AIStudio.Tools.Databases.IndexStore.Migrations;
+
+/// <summary>
+/// Drops the copy of the data source name which the index kept next to each indexed data source.
+/// </summary>
+/// <remarks>
+/// The name a user gives a data source lives in the configuration and is read from there whenever
+/// it is needed. The copy here was only ever written, never read, and a copy of a name people are
+/// free to change can do nothing but go stale.
+/// </remarks>
+[DbContext(typeof(IndexStoreDbContext))]
+[Migration("20260916000000_DropDataSourceName")]
+public partial class DropDataSourceName : Migration
+{
+    /// <remarks>
+    /// The column goes through raw SQL instead of DropColumn on purpose. The SQLite provider answers
+    /// DropColumn by rebuilding the table, and dropping the old data_sources table would let the
+    /// cascade of the foreign key in embedded_files take every indexed file and chunk with it. A
+    /// native ALTER TABLE ... DROP COLUMN leaves the table itself alone. No index names this column,
+    /// so nothing has to be dropped first.
+    /// </remarks>
+    protected override void Up(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.Sql("ALTER TABLE data_sources DROP COLUMN data_source_name;");
+    }
+
+    protected override void Down(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.Sql("ALTER TABLE data_sources ADD COLUMN data_source_name TEXT NOT NULL DEFAULT '';");
+    }
+}
@@ -23,11 +23,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
                .HasColumnType("TEXT")
                .HasColumnName("data_source_id");

-            entity.Property<string>("DataSourceName")
-                .IsRequired()
-                .HasColumnType("TEXT")
-                .HasColumnName("data_source_name");
-
            entity.Property<string>("DataSourceType")
                .IsRequired()
                .HasColumnType("TEXT")
@@ -274,10 +269,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
                .IsRequired()
                .HasColumnType("TEXT");

-            entity.Property<string>("DataSourceName")
-                .IsRequired()
-                .HasColumnType("TEXT");
-
            entity.Property<string>("DataSourceType")
                .IsRequired()
                .HasColumnType("TEXT");
@@ -28,7 +28,6 @@ public sealed class NoIndexStoreClient(string name, string? unavailableReason, D

    public override Task UpsertDataSourceAsync(
        string dataSourceId,
-        string dataSourceName,
        string dataSourceType,
        string embeddingProviderId,
        string embeddingSignature,
@@ -118,7 +118,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat

    public override async Task UpsertDataSourceAsync(
        string dataSourceId,
-        string dataSourceName,
        string dataSourceType,
        string embeddingProviderId,
        string embeddingSignature,
@@ -137,7 +136,7 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
            context.DataSources.Add(dataSource);
        }

-        ApplyDataSource(dataSource, dataSourceName, dataSourceType, embeddingProviderId, embeddingSignature, sourceHash, vectorSize);
+        ApplyDataSource(dataSource, dataSourceType, embeddingProviderId, embeddingSignature, sourceHash, vectorSize);
        await context.SaveChangesAsync(token);
    }

@@ -287,7 +286,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
                                      c.chunk_id AS ChunkId,
                                      c.parent_file_id AS ParentFileId,
                                      ds.data_source_id AS DataSourceId,
-                                      ds.data_source_name AS DataSourceName,
                                      ds.data_source_type AS DataSourceType,
                                      f.absolute_path AS AbsolutePath,
                                      f.file_name AS FileName,
@@ -363,14 +361,12 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat

    private static void ApplyDataSource(
        EmbeddingStateDataSourceEntity dataSource,
-        string dataSourceName,
        string dataSourceType,
        string embeddingProviderId,
        string embeddingSignature,
        string sourceHash,
        int vectorSize)
    {
-        dataSource.DataSourceName = dataSourceName;
        dataSource.DataSourceType = dataSourceType;
        dataSource.EmbeddingProviderId = embeddingProviderId;
        dataSource.EmbeddingSignature = embeddingSignature;
@@ -426,7 +422,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
        result.ChunkId,
        result.ParentFileId,
        result.DataSourceId,
-        result.DataSourceName,
        result.DataSourceType,
        result.AbsolutePath,
        result.FileName,
@@ -4,7 +4,6 @@ public sealed record VectorSearchResult(
    string PointId,
    double Score,
    string DataSourceId,
-    string DataSourceName,
    string DataSourceType,
    string ChunkId,
    string ParentFileId,
@@ -4,7 +4,6 @@ public sealed record VectorStoragePoint(
    string PointId,
    IReadOnlyList<float> Vector,
    string DataSourceId,
-    string DataSourceName,
    string DataSourceType,
    string ChunkId,
    string ParentFileId,
@@ -2,7 +2,6 @@ using System.Security.Cryptography;
 using System.Text;
 using System.Text.RegularExpressions;

-using AIStudio.Provider;
 using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
 using AIStudio.Tools.Databases.IndexStore;
@@ -1015,7 +1015,6 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
            item.ChunkId,
            vectors[index],
            dataSource.Id,
-            dataSource.Name,
            dataSource.Type.ToString(),
            item.ChunkId,
            parentFile.ParentFileId,
@@ -1274,7 +1273,6 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM

        await indexStore.UpsertDataSourceAsync(
            dataSource.Id,
-            dataSource.Name,
            dataSource.Type.ToString(),
            manifest.EmbeddingProviderId,
            manifest.EmbeddingSignature,
@@ -42,7 +42,6 @@ public sealed class DataSourceLocalRetrievalService(
        string ChunkId,
        string ParentFileId,
        string DataSourceId,
-        string DataSourceName,
        string DataSourceType,
        string AbsolutePath,
        string FileName,
@@ -93,7 +92,7 @@ public sealed class DataSourceLocalRetrievalService(

        return hits
            .Where(hit => !string.IsNullOrWhiteSpace(hit.Text))
-            .Select(ToRetrievalContext)
+            .Select(hit => ToRetrievalContext(hit, dataSource))
            .ToList();
    }

@@ -342,7 +341,6 @@ public sealed class DataSourceLocalRetrievalService(
            result.ChunkId,
            result.ParentFileId,
            result.DataSourceId,
-            result.DataSourceName,
            result.DataSourceType,
            FirstNonEmpty(result.AbsolutePath, result.FilePath),
            result.FileName,
@@ -360,7 +358,6 @@ public sealed class DataSourceLocalRetrievalService(
            result.ChunkId,
            result.ParentFileId,
            result.DataSourceId,
-            result.DataSourceName,
            result.DataSourceType,
            result.AbsolutePath,
            result.FileName,
@@ -372,9 +369,9 @@ public sealed class DataSourceLocalRetrievalService(
            result.Score,
            rank);

-    private static RetrievalTextContext ToRetrievalContext(LocalRetrievalHit hit)
+    private static RetrievalTextContext ToRetrievalContext(LocalRetrievalHit hit, IInternalDataSource dataSource)
    {
-        var sourceName = FirstNonEmpty(hit.FileName, hit.DataSourceName);
+        var sourceName = FirstNonEmpty(hit.FileName, dataSource.Name);
        var path = FirstNonEmpty(hit.AbsolutePath, hit.RelativePath);
        var referenceLink = string.IsNullOrWhiteSpace(path) ? string.Empty : BuildReferenceLink(path, hit);

@@ -387,15 +384,15 @@ public sealed class DataSourceLocalRetrievalService(
            Links = [],
            MatchedText = hit.Text,
            SurroundingContent = [],
-            ReferenceTitle = BuildReferenceTitle(hit),
+            ReferenceTitle = BuildReferenceTitle(hit, dataSource),
            ReferenceLink = referenceLink,
            PageNumber = hit.PageNumber is > 0 ? hit.PageNumber : null,
        };
    }

-    private static string BuildReferenceTitle(LocalRetrievalHit hit)
+    private static string BuildReferenceTitle(LocalRetrievalHit hit, IInternalDataSource dataSource)
    {
-        var sourceName = FirstNonEmpty(hit.FileName, hit.DataSourceName);
+        var sourceName = FirstNonEmpty(hit.FileName, dataSource.Name);
        return BuildLocatedReferenceTitle(sourceName, hit.ChunkIndex, hit.PageNumber);
    }