From d7bdd5292174e9186b6907faf81194db0c37b0a5 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Wed, 16 Sep 2026 10:58:57 +0200 Subject: [PATCH] Removed the data source name from the RAG index database --- .../EmbeddingStateDataSourceEntity.cs | 2 -- .../Databases/IndexStore/IndexStoreClient.cs | 1 - .../IndexStore/IndexStoreDbContext.cs | 1 - .../IndexStore/IndexStoreSchemaMigrator.cs | 1 + .../IndexStore/IndexStoreSearchResult.cs | 1 - .../IndexStoreSearchResultEntity.cs | 2 -- .../20260916000000_DropDataSourceName.cs | 36 +++++++++++++++++++ .../IndexStoreDbContextModelSnapshot.cs | 9 ----- .../IndexStore/NoIndexStoreClient.cs | 1 - .../SqliteIndexStoreClientImplementation.cs | 7 +--- .../Services/DataSourceEmbeddingService.cs | 1 - 11 files changed, 38 insertions(+), 24 deletions(-) create mode 100644 app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260916000000_DropDataSourceName.cs diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateDataSourceEntity.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateDataSourceEntity.cs index 4b04487b..29728294 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateDataSourceEntity.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateDataSourceEntity.cs @@ -4,8 +4,6 @@ internal sealed class EmbeddingStateDataSourceEntity { public string DataSourceId { get; set; } = string.Empty; - public string DataSourceName { get; set; } = string.Empty; - public string DataSourceType { get; set; } = string.Empty; public string EmbeddingProviderId { get; set; } = string.Empty; diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs index 9bfea694..1cf0ae49 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs @@ -8,7 +8,6 @@ public abstract class IndexStoreClient(string name, string path) : DatabaseClien public abstract Task UpsertDataSourceAsync( string dataSourceId, - string dataSourceName, string dataSourceType, string embeddingProviderId, string embeddingSignature, diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs index 870837d5..642d43bd 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs @@ -29,7 +29,6 @@ internal sealed class IndexStoreDbContext(DbContextOptions entity.HasKey(dataSource => dataSource.DataSourceId); entity.Property(dataSource => dataSource.DataSourceId).HasColumnName("data_source_id"); - entity.Property(dataSource => dataSource.DataSourceName).HasColumnName("data_source_name").IsRequired(); entity.Property(dataSource => dataSource.DataSourceType).HasColumnName("data_source_type").IsRequired(); entity.Property(dataSource => dataSource.EmbeddingProviderId).HasColumnName("embedding_provider_id").IsRequired(); entity.Property(dataSource => dataSource.EmbeddingSignature).HasColumnName("embedding_signature").IsRequired(); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs index 7ae6e4f2..8d148e3e 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs @@ -9,6 +9,7 @@ internal static class IndexStoreSchemaMigrator [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))] [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.PermanentIndexingFailures))] [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropFileConfidenceLevel))] + [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropDataSourceName))] public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token) { await context.Database.MigrateAsync(token); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs index 58d1d767..6a4304fe 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs @@ -4,7 +4,6 @@ public sealed record IndexStoreSearchResult( string ChunkId, string ParentFileId, string DataSourceId, - string DataSourceName, string DataSourceType, string AbsolutePath, string FileName, diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs index 51ea7796..12307280 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs @@ -8,8 +8,6 @@ internal sealed class IndexStoreSearchResultEntity public string DataSourceId { get; set; } = string.Empty; - public string DataSourceName { get; set; } = string.Empty; - public string DataSourceType { get; set; } = string.Empty; public string AbsolutePath { get; set; } = string.Empty; diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260916000000_DropDataSourceName.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260916000000_DropDataSourceName.cs new file mode 100644 index 00000000..9a603836 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260916000000_DropDataSourceName.cs @@ -0,0 +1,36 @@ +#nullable disable + +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace AIStudio.Tools.Databases.IndexStore.Migrations; + +/// +/// Drops the copy of the data source name which the index kept next to each indexed data source. +/// +/// +/// The name a user gives a data source lives in the configuration and is read from there whenever +/// it is needed. The copy here was only ever written, never read, and a copy of a name people are +/// free to change can do nothing but go stale. +/// +[DbContext(typeof(IndexStoreDbContext))] +[Migration("20260916000000_DropDataSourceName")] +public partial class DropDataSourceName : Migration +{ + /// + /// The column goes through raw SQL instead of DropColumn on purpose. The SQLite provider answers + /// DropColumn by rebuilding the table, and dropping the old data_sources table would let the + /// cascade of the foreign key in embedded_files take every indexed file and chunk with it. A + /// native ALTER TABLE ... DROP COLUMN leaves the table itself alone. No index names this column, + /// so nothing has to be dropped first. + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql("ALTER TABLE data_sources DROP COLUMN data_source_name;"); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql("ALTER TABLE data_sources ADD COLUMN data_source_name TEXT NOT NULL DEFAULT '';"); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs index 3d8336c1..11d9a6e0 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs @@ -23,11 +23,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot .HasColumnType("TEXT") .HasColumnName("data_source_id"); - entity.Property("DataSourceName") - .IsRequired() - .HasColumnType("TEXT") - .HasColumnName("data_source_name"); - entity.Property("DataSourceType") .IsRequired() .HasColumnType("TEXT") @@ -274,10 +269,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot .IsRequired() .HasColumnType("TEXT"); - entity.Property("DataSourceName") - .IsRequired() - .HasColumnType("TEXT"); - entity.Property("DataSourceType") .IsRequired() .HasColumnType("TEXT"); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs index cac66a85..4ab41065 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs @@ -28,7 +28,6 @@ public sealed class NoIndexStoreClient(string name, string? unavailableReason, D public override Task UpsertDataSourceAsync( string dataSourceId, - string dataSourceName, string dataSourceType, string embeddingProviderId, string embeddingSignature, diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs index 513bc969..1593d4a1 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs @@ -118,7 +118,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat public override async Task UpsertDataSourceAsync( string dataSourceId, - string dataSourceName, string dataSourceType, string embeddingProviderId, string embeddingSignature, @@ -137,7 +136,7 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat context.DataSources.Add(dataSource); } - ApplyDataSource(dataSource, dataSourceName, dataSourceType, embeddingProviderId, embeddingSignature, sourceHash, vectorSize); + ApplyDataSource(dataSource, dataSourceType, embeddingProviderId, embeddingSignature, sourceHash, vectorSize); await context.SaveChangesAsync(token); } @@ -287,7 +286,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat c.chunk_id AS ChunkId, c.parent_file_id AS ParentFileId, ds.data_source_id AS DataSourceId, - ds.data_source_name AS DataSourceName, ds.data_source_type AS DataSourceType, f.absolute_path AS AbsolutePath, f.file_name AS FileName, @@ -363,14 +361,12 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat private static void ApplyDataSource( EmbeddingStateDataSourceEntity dataSource, - string dataSourceName, string dataSourceType, string embeddingProviderId, string embeddingSignature, string sourceHash, int vectorSize) { - dataSource.DataSourceName = dataSourceName; dataSource.DataSourceType = dataSourceType; dataSource.EmbeddingProviderId = embeddingProviderId; dataSource.EmbeddingSignature = embeddingSignature; @@ -426,7 +422,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat result.ChunkId, result.ParentFileId, result.DataSourceId, - result.DataSourceName, result.DataSourceType, result.AbsolutePath, result.FileName, diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index c777de22..0ca86c9e 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -1273,7 +1273,6 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM await indexStore.UpsertDataSourceAsync( dataSource.Id, - dataSource.Name, dataSource.Type.ToString(), manifest.EmbeddingProviderId, manifest.EmbeddingSignature,