Removed the data source name from the RAG index database

This commit is contained in:
Thorsten Sommer committed 2026-09-16 10:58:57 +02:00
1 parent 7e99dd66dc
commit d7bdd52921
11 files changed
+38 -24

No files matched your search

@@ -4,8 +4,6 @@ internal sealed class EmbeddingStateDataSourceEntity
{ {
public string DataSourceId { get; set; } = string.Empty; public string DataSourceId { get; set; } = string.Empty;
public string DataSourceName { get; set; } = string.Empty;
public string DataSourceType { get; set; } = string.Empty; public string DataSourceType { get; set; } = string.Empty;
public string EmbeddingProviderId { get; set; } = string.Empty; public string EmbeddingProviderId { get; set; } = string.Empty;
@@ -8,7 +8,6 @@ public abstract class IndexStoreClient(string name, string path) : DatabaseClien
public abstract Task UpsertDataSourceAsync( public abstract Task UpsertDataSourceAsync(
string dataSourceId, string dataSourceId,
string dataSourceName,
string dataSourceType, string dataSourceType,
string embeddingProviderId, string embeddingProviderId,
string embeddingSignature, string embeddingSignature,
@@ -29,7 +29,6 @@ internal sealed class IndexStoreDbContext(DbContextOptions<IndexStoreDbContext>
entity.HasKey(dataSource => dataSource.DataSourceId); entity.HasKey(dataSource => dataSource.DataSourceId);
entity.Property(dataSource => dataSource.DataSourceId).HasColumnName("data_source_id"); entity.Property(dataSource => dataSource.DataSourceId).HasColumnName("data_source_id");
entity.Property(dataSource => dataSource.DataSourceName).HasColumnName("data_source_name").IsRequired();
entity.Property(dataSource => dataSource.DataSourceType).HasColumnName("data_source_type").IsRequired(); entity.Property(dataSource => dataSource.DataSourceType).HasColumnName("data_source_type").IsRequired();
entity.Property(dataSource => dataSource.EmbeddingProviderId).HasColumnName("embedding_provider_id").IsRequired(); entity.Property(dataSource => dataSource.EmbeddingProviderId).HasColumnName("embedding_provider_id").IsRequired();
entity.Property(dataSource => dataSource.EmbeddingSignature).HasColumnName("embedding_signature").IsRequired(); entity.Property(dataSource => dataSource.EmbeddingSignature).HasColumnName("embedding_signature").IsRequired();
@@ -9,6 +9,7 @@ internal static class IndexStoreSchemaMigrator
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))] [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))]
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.PermanentIndexingFailures))] [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.PermanentIndexingFailures))]
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropFileConfidenceLevel))] [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropFileConfidenceLevel))]
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropDataSourceName))]
public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token) public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token)
{ {
await context.Database.MigrateAsync(token); await context.Database.MigrateAsync(token);
@@ -4,7 +4,6 @@ public sealed record IndexStoreSearchResult(
string ChunkId, string ChunkId,
string ParentFileId, string ParentFileId,
string DataSourceId, string DataSourceId,
string DataSourceName,
string DataSourceType, string DataSourceType,
string AbsolutePath, string AbsolutePath,
string FileName, string FileName,
@@ -8,8 +8,6 @@ internal sealed class IndexStoreSearchResultEntity
public string DataSourceId { get; set; } = string.Empty; public string DataSourceId { get; set; } = string.Empty;
public string DataSourceName { get; set; } = string.Empty;
public string DataSourceType { get; set; } = string.Empty; public string DataSourceType { get; set; } = string.Empty;
public string AbsolutePath { get; set; } = string.Empty; public string AbsolutePath { get; set; } = string.Empty;
@@ -0,0 +1,36 @@
#nullable disable
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
namespace AIStudio.Tools.Databases.IndexStore.Migrations;
/// <summary>
/// Drops the copy of the data source name which the index kept next to each indexed data source.
/// </summary>
/// <remarks>
/// The name a user gives a data source lives in the configuration and is read from there whenever
/// it is needed. The copy here was only ever written, never read, and a copy of a name people are
/// free to change can do nothing but go stale.
/// </remarks>
[DbContext(typeof(IndexStoreDbContext))]
[Migration("20260916000000_DropDataSourceName")]
public partial class DropDataSourceName : Migration
{
/// <remarks>
/// The column goes through raw SQL instead of DropColumn on purpose. The SQLite provider answers
/// DropColumn by rebuilding the table, and dropping the old data_sources table would let the
/// cascade of the foreign key in embedded_files take every indexed file and chunk with it. A
/// native ALTER TABLE ... DROP COLUMN leaves the table itself alone. No index names this column,
/// so nothing has to be dropped first.
/// </remarks>
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.Sql("ALTER TABLE data_sources DROP COLUMN data_source_name;");
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.Sql("ALTER TABLE data_sources ADD COLUMN data_source_name TEXT NOT NULL DEFAULT '';");
}
}
@@ -23,11 +23,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
.HasColumnType("TEXT") .HasColumnType("TEXT")
.HasColumnName("data_source_id"); .HasColumnName("data_source_id");
entity.Property<string>("DataSourceName")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("data_source_name");
entity.Property<string>("DataSourceType") entity.Property<string>("DataSourceType")
.IsRequired() .IsRequired()
.HasColumnType("TEXT") .HasColumnType("TEXT")
@@ -274,10 +269,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
.IsRequired() .IsRequired()
.HasColumnType("TEXT"); .HasColumnType("TEXT");
entity.Property<string>("DataSourceName")
.IsRequired()
.HasColumnType("TEXT");
entity.Property<string>("DataSourceType") entity.Property<string>("DataSourceType")
.IsRequired() .IsRequired()
.HasColumnType("TEXT"); .HasColumnType("TEXT");
@@ -28,7 +28,6 @@ public sealed class NoIndexStoreClient(string name, string? unavailableReason, D
public override Task UpsertDataSourceAsync( public override Task UpsertDataSourceAsync(
string dataSourceId, string dataSourceId,
string dataSourceName,
string dataSourceType, string dataSourceType,
string embeddingProviderId, string embeddingProviderId,
string embeddingSignature, string embeddingSignature,
@@ -118,7 +118,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
public override async Task UpsertDataSourceAsync( public override async Task UpsertDataSourceAsync(
string dataSourceId, string dataSourceId,
string dataSourceName,
string dataSourceType, string dataSourceType,
string embeddingProviderId, string embeddingProviderId,
string embeddingSignature, string embeddingSignature,
@@ -137,7 +136,7 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
context.DataSources.Add(dataSource); context.DataSources.Add(dataSource);
} }
ApplyDataSource(dataSource, dataSourceName, dataSourceType, embeddingProviderId, embeddingSignature, sourceHash, vectorSize); ApplyDataSource(dataSource, dataSourceType, embeddingProviderId, embeddingSignature, sourceHash, vectorSize);
await context.SaveChangesAsync(token); await context.SaveChangesAsync(token);
} }
@@ -287,7 +286,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
c.chunk_id AS ChunkId, c.chunk_id AS ChunkId,
c.parent_file_id AS ParentFileId, c.parent_file_id AS ParentFileId,
ds.data_source_id AS DataSourceId, ds.data_source_id AS DataSourceId,
ds.data_source_name AS DataSourceName,
ds.data_source_type AS DataSourceType, ds.data_source_type AS DataSourceType,
f.absolute_path AS AbsolutePath, f.absolute_path AS AbsolutePath,
f.file_name AS FileName, f.file_name AS FileName,
@@ -363,14 +361,12 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
private static void ApplyDataSource( private static void ApplyDataSource(
EmbeddingStateDataSourceEntity dataSource, EmbeddingStateDataSourceEntity dataSource,
string dataSourceName,
string dataSourceType, string dataSourceType,
string embeddingProviderId, string embeddingProviderId,
string embeddingSignature, string embeddingSignature,
string sourceHash, string sourceHash,
int vectorSize) int vectorSize)
{ {
dataSource.DataSourceName = dataSourceName;
dataSource.DataSourceType = dataSourceType; dataSource.DataSourceType = dataSourceType;
dataSource.EmbeddingProviderId = embeddingProviderId; dataSource.EmbeddingProviderId = embeddingProviderId;
dataSource.EmbeddingSignature = embeddingSignature; dataSource.EmbeddingSignature = embeddingSignature;
@@ -426,7 +422,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
result.ChunkId, result.ChunkId,
result.ParentFileId, result.ParentFileId,
result.DataSourceId, result.DataSourceId,
result.DataSourceName,
result.DataSourceType, result.DataSourceType,
result.AbsolutePath, result.AbsolutePath,
result.FileName, result.FileName,
@@ -1273,7 +1273,6 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
await indexStore.UpsertDataSourceAsync( await indexStore.UpsertDataSourceAsync(
dataSource.Id, dataSource.Id,
dataSource.Name,
dataSource.Type.ToString(), dataSource.Type.ToString(),
manifest.EmbeddingProviderId, manifest.EmbeddingProviderId,
manifest.EmbeddingSignature, manifest.EmbeddingSignature,