diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateDataSourceEntity.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateDataSourceEntity.cs index 4b04487b..29728294 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateDataSourceEntity.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateDataSourceEntity.cs @@ -4,8 +4,6 @@ internal sealed class EmbeddingStateDataSourceEntity { public string DataSourceId { get; set; } = string.Empty; - public string DataSourceName { get; set; } = string.Empty; - public string DataSourceType { get; set; } = string.Empty; public string EmbeddingProviderId { get; set; } = string.Empty; diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs index 9bfea694..1cf0ae49 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs @@ -8,7 +8,6 @@ public abstract class IndexStoreClient(string name, string path) : DatabaseClien public abstract Task UpsertDataSourceAsync( string dataSourceId, - string dataSourceName, string dataSourceType, string embeddingProviderId, string embeddingSignature, diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs index 870837d5..642d43bd 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs @@ -29,7 +29,6 @@ internal sealed class IndexStoreDbContext(DbContextOptions entity.HasKey(dataSource => dataSource.DataSourceId); entity.Property(dataSource => dataSource.DataSourceId).HasColumnName("data_source_id"); - entity.Property(dataSource => dataSource.DataSourceName).HasColumnName("data_source_name").IsRequired(); entity.Property(dataSource => dataSource.DataSourceType).HasColumnName("data_source_type").IsRequired(); entity.Property(dataSource => dataSource.EmbeddingProviderId).HasColumnName("embedding_provider_id").IsRequired(); entity.Property(dataSource => dataSource.EmbeddingSignature).HasColumnName("embedding_signature").IsRequired(); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs index 7ae6e4f2..8d148e3e 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs @@ -9,6 +9,7 @@ internal static class IndexStoreSchemaMigrator [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))] [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.PermanentIndexingFailures))] [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropFileConfidenceLevel))] + [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropDataSourceName))] public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token) { await context.Database.MigrateAsync(token); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs index 58d1d767..6a4304fe 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs @@ -4,7 +4,6 @@ public sealed record IndexStoreSearchResult( string ChunkId, string ParentFileId, string DataSourceId, - string DataSourceName, string DataSourceType, string AbsolutePath, string FileName, diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs index 51ea7796..12307280 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs @@ -8,8 +8,6 @@ internal sealed class IndexStoreSearchResultEntity public string DataSourceId { get; set; } = string.Empty; - public string DataSourceName { get; set; } = string.Empty; - public string DataSourceType { get; set; } = string.Empty; public string AbsolutePath { get; set; } = string.Empty; diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260916000000_DropDataSourceName.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260916000000_DropDataSourceName.cs new file mode 100644 index 00000000..9a603836 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260916000000_DropDataSourceName.cs @@ -0,0 +1,36 @@ +#nullable disable + +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace AIStudio.Tools.Databases.IndexStore.Migrations; + +/// +/// Drops the copy of the data source name which the index kept next to each indexed data source. +/// +/// +/// The name a user gives a data source lives in the configuration and is read from there whenever +/// it is needed. The copy here was only ever written, never read, and a copy of a name people are +/// free to change can do nothing but go stale. +/// +[DbContext(typeof(IndexStoreDbContext))] +[Migration("20260916000000_DropDataSourceName")] +public partial class DropDataSourceName : Migration +{ + /// + /// The column goes through raw SQL instead of DropColumn on purpose. The SQLite provider answers + /// DropColumn by rebuilding the table, and dropping the old data_sources table would let the + /// cascade of the foreign key in embedded_files take every indexed file and chunk with it. A + /// native ALTER TABLE ... DROP COLUMN leaves the table itself alone. No index names this column, + /// so nothing has to be dropped first. + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql("ALTER TABLE data_sources DROP COLUMN data_source_name;"); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql("ALTER TABLE data_sources ADD COLUMN data_source_name TEXT NOT NULL DEFAULT '';"); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs index 3d8336c1..11d9a6e0 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs @@ -23,11 +23,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot .HasColumnType("TEXT") .HasColumnName("data_source_id"); - entity.Property("DataSourceName") - .IsRequired() - .HasColumnType("TEXT") - .HasColumnName("data_source_name"); - entity.Property("DataSourceType") .IsRequired() .HasColumnType("TEXT") @@ -274,10 +269,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot .IsRequired() .HasColumnType("TEXT"); - entity.Property("DataSourceName") - .IsRequired() - .HasColumnType("TEXT"); - entity.Property("DataSourceType") .IsRequired() .HasColumnType("TEXT"); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs index cac66a85..4ab41065 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs @@ -28,7 +28,6 @@ public sealed class NoIndexStoreClient(string name, string? unavailableReason, D public override Task UpsertDataSourceAsync( string dataSourceId, - string dataSourceName, string dataSourceType, string embeddingProviderId, string embeddingSignature, diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs index 513bc969..1593d4a1 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs @@ -118,7 +118,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat public override async Task UpsertDataSourceAsync( string dataSourceId, - string dataSourceName, string dataSourceType, string embeddingProviderId, string embeddingSignature, @@ -137,7 +136,7 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat context.DataSources.Add(dataSource); } - ApplyDataSource(dataSource, dataSourceName, dataSourceType, embeddingProviderId, embeddingSignature, sourceHash, vectorSize); + ApplyDataSource(dataSource, dataSourceType, embeddingProviderId, embeddingSignature, sourceHash, vectorSize); await context.SaveChangesAsync(token); } @@ -287,7 +286,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat c.chunk_id AS ChunkId, c.parent_file_id AS ParentFileId, ds.data_source_id AS DataSourceId, - ds.data_source_name AS DataSourceName, ds.data_source_type AS DataSourceType, f.absolute_path AS AbsolutePath, f.file_name AS FileName, @@ -363,14 +361,12 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat private static void ApplyDataSource( EmbeddingStateDataSourceEntity dataSource, - string dataSourceName, string dataSourceType, string embeddingProviderId, string embeddingSignature, string sourceHash, int vectorSize) { - dataSource.DataSourceName = dataSourceName; dataSource.DataSourceType = dataSourceType; dataSource.EmbeddingProviderId = embeddingProviderId; dataSource.EmbeddingSignature = embeddingSignature; @@ -426,7 +422,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat result.ChunkId, result.ParentFileId, result.DataSourceId, - result.DataSourceName, result.DataSourceType, result.AbsolutePath, result.FileName, diff --git a/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorSearchResult.cs b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorSearchResult.cs index c0b127de..1158ea34 100644 --- a/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorSearchResult.cs +++ b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorSearchResult.cs @@ -4,7 +4,6 @@ public sealed record VectorSearchResult( string PointId, double Score, string DataSourceId, - string DataSourceName, string DataSourceType, string ChunkId, string ParentFileId, diff --git a/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoragePoint.cs b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoragePoint.cs index 87a6f334..c559a74c 100644 --- a/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoragePoint.cs +++ b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoragePoint.cs @@ -4,7 +4,6 @@ public sealed record VectorStoragePoint( string PointId, IReadOnlyList Vector, string DataSourceId, - string DataSourceName, string DataSourceType, string ChunkId, string ParentFileId, diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs index 6b735487..0348c558 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs @@ -2,7 +2,6 @@ using System.Security.Cryptography; using System.Text; using System.Text.RegularExpressions; -using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools.Databases.IndexStore; diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index db4ca76f..0ca86c9e 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -1015,7 +1015,6 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM item.ChunkId, vectors[index], dataSource.Id, - dataSource.Name, dataSource.Type.ToString(), item.ChunkId, parentFile.ParentFileId, @@ -1274,7 +1273,6 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM await indexStore.UpsertDataSourceAsync( dataSource.Id, - dataSource.Name, dataSource.Type.ToString(), manifest.EmbeddingProviderId, manifest.EmbeddingSignature, diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs index 7e5f2645..448b6e70 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs @@ -42,7 +42,6 @@ public sealed class DataSourceLocalRetrievalService( string ChunkId, string ParentFileId, string DataSourceId, - string DataSourceName, string DataSourceType, string AbsolutePath, string FileName, @@ -93,7 +92,7 @@ public sealed class DataSourceLocalRetrievalService( return hits .Where(hit => !string.IsNullOrWhiteSpace(hit.Text)) - .Select(ToRetrievalContext) + .Select(hit => ToRetrievalContext(hit, dataSource)) .ToList(); } @@ -342,7 +341,6 @@ public sealed class DataSourceLocalRetrievalService( result.ChunkId, result.ParentFileId, result.DataSourceId, - result.DataSourceName, result.DataSourceType, FirstNonEmpty(result.AbsolutePath, result.FilePath), result.FileName, @@ -360,7 +358,6 @@ public sealed class DataSourceLocalRetrievalService( result.ChunkId, result.ParentFileId, result.DataSourceId, - result.DataSourceName, result.DataSourceType, result.AbsolutePath, result.FileName, @@ -372,9 +369,9 @@ public sealed class DataSourceLocalRetrievalService( result.Score, rank); - private static RetrievalTextContext ToRetrievalContext(LocalRetrievalHit hit) + private static RetrievalTextContext ToRetrievalContext(LocalRetrievalHit hit, IInternalDataSource dataSource) { - var sourceName = FirstNonEmpty(hit.FileName, hit.DataSourceName); + var sourceName = FirstNonEmpty(hit.FileName, dataSource.Name); var path = FirstNonEmpty(hit.AbsolutePath, hit.RelativePath); var referenceLink = string.IsNullOrWhiteSpace(path) ? string.Empty : BuildReferenceLink(path, hit); @@ -387,15 +384,15 @@ public sealed class DataSourceLocalRetrievalService( Links = [], MatchedText = hit.Text, SurroundingContent = [], - ReferenceTitle = BuildReferenceTitle(hit), + ReferenceTitle = BuildReferenceTitle(hit, dataSource), ReferenceLink = referenceLink, PageNumber = hit.PageNumber is > 0 ? hit.PageNumber : null, }; } - private static string BuildReferenceTitle(LocalRetrievalHit hit) + private static string BuildReferenceTitle(LocalRetrievalHit hit, IInternalDataSource dataSource) { - var sourceName = FirstNonEmpty(hit.FileName, hit.DataSourceName); + var sourceName = FirstNonEmpty(hit.FileName, dataSource.Name); return BuildLocatedReferenceTitle(sourceName, hit.ChunkIndex, hit.PageNumber); } diff --git a/runtime/src/qdrant_edge_database.rs b/runtime/src/qdrant_edge_database.rs index 8b54c9c2..c4543d63 100644 --- a/runtime/src/qdrant_edge_database.rs +++ b/runtime/src/qdrant_edge_database.rs @@ -70,7 +70,6 @@ pub struct QdrantEdgeStoragePoint { pub point_id: String, pub vector: Vec, pub data_source_id: String, - pub data_source_name: String, pub data_source_type: String, pub chunk_id: String, pub parent_file_id: String, @@ -141,7 +140,6 @@ pub struct QdrantEdgeSearchResult { pub point_id: String, pub score: f32, pub data_source_id: String, - pub data_source_name: String, pub data_source_type: String, pub chunk_id: String, pub parent_file_id: String, @@ -299,15 +297,7 @@ impl QdrantEdgeDatabase { return Err("All vectors in one insert request must have the same size.".into()); } - let data_source_name = first_point.data_source_name.clone(); - validate_data_source_name(&data_source_name)?; - if points.iter().any(|point| point.data_source_name != data_source_name) { - return Err("All points in one insert request must belong to the same data source name.".into()); - } - - let store_path = self.store_path(store_name)?; let (shard, _) = self.get_or_create_store(store_name, vector_size)?; - write_store_display_name(&store_path, &data_source_name)?; let points = points .into_iter() .map(to_qdrant_edge_point) @@ -738,7 +728,6 @@ fn to_qdrant_edge_point(point: QdrantEdgeStoragePoint) -> QdrantEdgeResult QdrantEdgeSearchResult { point_id: point_id_to_string(point.id), score: point.score, data_source_id: payload_string(&payload, "data_source_id"), - data_source_name: payload_string(&payload, "data_source_name"), data_source_type: payload_string(&payload, "data_source_type"), chunk_id: payload_string(&payload, "chunk_id"), parent_file_id: payload_string(&payload, "parent_file_id"),