Merge branch 'main' into improve_document_analysis

app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateDataSourceEntity.cs

@@ -4,8 +4,6 @@ internal sealed class EmbeddingStateDataSourceEntity
 {
     public string DataSourceId { get; set; } = string.Empty;
 
-    public string DataSourceName { get; set; } = string.Empty;
-
     public string DataSourceType { get; set; } = string.Empty;
 
     public string EmbeddingProviderId { get; set; } = string.Empty;

app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs

@@ -8,7 +8,6 @@ public abstract class IndexStoreClient(string name, string path) : DatabaseClien
 
     public abstract Task UpsertDataSourceAsync(
         string dataSourceId,
-        string dataSourceName,
         string dataSourceType,
         string embeddingProviderId,
         string embeddingSignature,

app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs

@@ -29,7 +29,6 @@ internal sealed class IndexStoreDbContext(DbContextOptions<IndexStoreDbContext>
             entity.HasKey(dataSource => dataSource.DataSourceId);
 
             entity.Property(dataSource => dataSource.DataSourceId).HasColumnName("data_source_id");
-            entity.Property(dataSource => dataSource.DataSourceName).HasColumnName("data_source_name").IsRequired();
             entity.Property(dataSource => dataSource.DataSourceType).HasColumnName("data_source_type").IsRequired();
             entity.Property(dataSource => dataSource.EmbeddingProviderId).HasColumnName("embedding_provider_id").IsRequired();
             entity.Property(dataSource => dataSource.EmbeddingSignature).HasColumnName("embedding_signature").IsRequired();

app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs

@@ -9,6 +9,7 @@ internal static class IndexStoreSchemaMigrator
     [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))]
     [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.PermanentIndexingFailures))]
     [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropFileConfidenceLevel))]
+    [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropDataSourceName))]
     public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token)
     {
         await context.Database.MigrateAsync(token);

app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs

@@ -4,7 +4,6 @@ public sealed record IndexStoreSearchResult(
     string ChunkId,
     string ParentFileId,
     string DataSourceId,
-    string DataSourceName,
     string DataSourceType,
     string AbsolutePath,
     string FileName,

app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs

@@ -8,8 +8,6 @@ internal sealed class IndexStoreSearchResultEntity
 
     public string DataSourceId { get; set; } = string.Empty;
 
-    public string DataSourceName { get; set; } = string.Empty;
-
     public string DataSourceType { get; set; } = string.Empty;
 
     public string AbsolutePath { get; set; } = string.Empty;

app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260916000000_DropDataSourceName.cs

@@ -0,0 +1,36 @@
+#nullable disable
+
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace AIStudio.Tools.Databases.IndexStore.Migrations;
+
+/// <summary>
+/// Drops the copy of the data source name which the index kept next to each indexed data source.
+/// </summary>
+/// <remarks>
+/// The name a user gives a data source lives in the configuration and is read from there whenever
+/// it is needed. The copy here was only ever written, never read, and a copy of a name people are
+/// free to change can do nothing but go stale.
+/// </remarks>
+[DbContext(typeof(IndexStoreDbContext))]
+[Migration("20260916000000_DropDataSourceName")]
+public partial class DropDataSourceName : Migration
+{
+    /// <remarks>
+    /// The column goes through raw SQL instead of DropColumn on purpose. The SQLite provider answers
+    /// DropColumn by rebuilding the table, and dropping the old data_sources table would let the
+    /// cascade of the foreign key in embedded_files take every indexed file and chunk with it. A
+    /// native ALTER TABLE ... DROP COLUMN leaves the table itself alone. No index names this column,
+    /// so nothing has to be dropped first.
+    /// </remarks>
+    protected override void Up(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.Sql("ALTER TABLE data_sources DROP COLUMN data_source_name;");
+    }
+
+    protected override void Down(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.Sql("ALTER TABLE data_sources ADD COLUMN data_source_name TEXT NOT NULL DEFAULT '';");
+    }
+}

app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs

@@ -23,11 +23,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
                 .HasColumnType("TEXT")
                 .HasColumnName("data_source_id");
 
-            entity.Property<string>("DataSourceName")
-                .IsRequired()
-                .HasColumnType("TEXT")
-                .HasColumnName("data_source_name");
-
             entity.Property<string>("DataSourceType")
                 .IsRequired()
                 .HasColumnType("TEXT")
@@ -274,10 +269,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
                 .IsRequired()
                 .HasColumnType("TEXT");
 
-            entity.Property<string>("DataSourceName")
-                .IsRequired()
-                .HasColumnType("TEXT");
-
             entity.Property<string>("DataSourceType")
                 .IsRequired()
                 .HasColumnType("TEXT");

app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs

@@ -28,7 +28,6 @@ public sealed class NoIndexStoreClient(string name, string? unavailableReason, D
 
     public override Task UpsertDataSourceAsync(
         string dataSourceId,
-        string dataSourceName,
         string dataSourceType,
         string embeddingProviderId,
         string embeddingSignature,

app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs

@@ -118,7 +118,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
 
     public override async Task UpsertDataSourceAsync(
         string dataSourceId,
-        string dataSourceName,
         string dataSourceType,
         string embeddingProviderId,
         string embeddingSignature,
@@ -137,7 +136,7 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
             context.DataSources.Add(dataSource);
         }
 
-        ApplyDataSource(dataSource, dataSourceName, dataSourceType, embeddingProviderId, embeddingSignature, sourceHash, vectorSize);
+        ApplyDataSource(dataSource, dataSourceType, embeddingProviderId, embeddingSignature, sourceHash, vectorSize);
         await context.SaveChangesAsync(token);
     }
 
@@ -287,7 +286,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
                                       c.chunk_id AS ChunkId,
                                       c.parent_file_id AS ParentFileId,
                                       ds.data_source_id AS DataSourceId,
-                                      ds.data_source_name AS DataSourceName,
                                       ds.data_source_type AS DataSourceType,
                                       f.absolute_path AS AbsolutePath,
                                       f.file_name AS FileName,
@@ -363,14 +361,12 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
 
     private static void ApplyDataSource(
         EmbeddingStateDataSourceEntity dataSource,
-        string dataSourceName,
         string dataSourceType,
         string embeddingProviderId,
         string embeddingSignature,
         string sourceHash,
         int vectorSize)
     {
-        dataSource.DataSourceName = dataSourceName;
         dataSource.DataSourceType = dataSourceType;
         dataSource.EmbeddingProviderId = embeddingProviderId;
         dataSource.EmbeddingSignature = embeddingSignature;
@@ -426,7 +422,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
         result.ChunkId,
         result.ParentFileId,
         result.DataSourceId,
-        result.DataSourceName,
         result.DataSourceType,
         result.AbsolutePath,
         result.FileName,

app/MindWork AI Studio/Tools/Databases/VectorStore/VectorSearchResult.cs

@@ -4,7 +4,6 @@ public sealed record VectorSearchResult(
     string PointId,
     double Score,
     string DataSourceId,
-    string DataSourceName,
     string DataSourceType,
     string ChunkId,
     string ParentFileId,

app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoragePoint.cs

@@ -4,7 +4,6 @@ public sealed record VectorStoragePoint(
     string PointId,
     IReadOnlyList<float> Vector,
     string DataSourceId,
-    string DataSourceName,
     string DataSourceType,
     string ChunkId,
     string ParentFileId,

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs

@@ -2,7 +2,6 @@ using System.Security.Cryptography;
 using System.Text;
 using System.Text.RegularExpressions;
 
-using AIStudio.Provider;
 using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
 using AIStudio.Tools.Databases.IndexStore;

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs

@@ -1015,7 +1015,6 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             item.ChunkId,
             vectors[index],
             dataSource.Id,
-            dataSource.Name,
             dataSource.Type.ToString(),
             item.ChunkId,
             parentFile.ParentFileId,
@@ -1274,7 +1273,6 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
 
         await indexStore.UpsertDataSourceAsync(
             dataSource.Id,
-            dataSource.Name,
             dataSource.Type.ToString(),
             manifest.EmbeddingProviderId,
             manifest.EmbeddingSignature,

app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs

@@ -42,7 +42,6 @@ public sealed class DataSourceLocalRetrievalService(
         string ChunkId,
         string ParentFileId,
         string DataSourceId,
-        string DataSourceName,
         string DataSourceType,
         string AbsolutePath,
         string FileName,
@@ -93,7 +92,7 @@ public sealed class DataSourceLocalRetrievalService(
 
         return hits
             .Where(hit => !string.IsNullOrWhiteSpace(hit.Text))
-            .Select(ToRetrievalContext)
+            .Select(hit => ToRetrievalContext(hit, dataSource))
             .ToList();
     }
 
@@ -342,7 +341,6 @@ public sealed class DataSourceLocalRetrievalService(
             result.ChunkId,
             result.ParentFileId,
             result.DataSourceId,
-            result.DataSourceName,
             result.DataSourceType,
             FirstNonEmpty(result.AbsolutePath, result.FilePath),
             result.FileName,
@@ -360,7 +358,6 @@ public sealed class DataSourceLocalRetrievalService(
             result.ChunkId,
             result.ParentFileId,
             result.DataSourceId,
-            result.DataSourceName,
             result.DataSourceType,
             result.AbsolutePath,
             result.FileName,
@@ -372,9 +369,9 @@ public sealed class DataSourceLocalRetrievalService(
             result.Score,
             rank);
 
-    private static RetrievalTextContext ToRetrievalContext(LocalRetrievalHit hit)
+    private static RetrievalTextContext ToRetrievalContext(LocalRetrievalHit hit, IInternalDataSource dataSource)
     {
-        var sourceName = FirstNonEmpty(hit.FileName, hit.DataSourceName);
+        var sourceName = FirstNonEmpty(hit.FileName, dataSource.Name);
         var path = FirstNonEmpty(hit.AbsolutePath, hit.RelativePath);
         var referenceLink = string.IsNullOrWhiteSpace(path) ? string.Empty : BuildReferenceLink(path, hit);
 
@@ -387,15 +384,15 @@ public sealed class DataSourceLocalRetrievalService(
             Links = [],
             MatchedText = hit.Text,
             SurroundingContent = [],
-            ReferenceTitle = BuildReferenceTitle(hit),
+            ReferenceTitle = BuildReferenceTitle(hit, dataSource),
             ReferenceLink = referenceLink,
             PageNumber = hit.PageNumber is > 0 ? hit.PageNumber : null,
         };
     }
 
-    private static string BuildReferenceTitle(LocalRetrievalHit hit)
+    private static string BuildReferenceTitle(LocalRetrievalHit hit, IInternalDataSource dataSource)
     {
-        var sourceName = FirstNonEmpty(hit.FileName, hit.DataSourceName);
+        var sourceName = FirstNonEmpty(hit.FileName, dataSource.Name);
         return BuildLocatedReferenceTitle(sourceName, hit.ChunkIndex, hit.PageNumber);
     }
 

runtime/src/qdrant_edge_database.rs

@@ -70,7 +70,6 @@ pub struct QdrantEdgeStoragePoint {
     pub point_id: String,
     pub vector: Vec<f32>,
     pub data_source_id: String,
-    pub data_source_name: String,
     pub data_source_type: String,
     pub chunk_id: String,
     pub parent_file_id: String,
@@ -141,7 +140,6 @@ pub struct QdrantEdgeSearchResult {
     pub point_id: String,
     pub score: f32,
     pub data_source_id: String,
-    pub data_source_name: String,
     pub data_source_type: String,
     pub chunk_id: String,
     pub parent_file_id: String,
@@ -299,15 +297,7 @@ impl QdrantEdgeDatabase {
             return Err("All vectors in one insert request must have the same size.".into());
         }
 
-        let data_source_name = first_point.data_source_name.clone();
-        validate_data_source_name(&data_source_name)?;
-        if points.iter().any(|point| point.data_source_name != data_source_name) {
-            return Err("All points in one insert request must belong to the same data source name.".into());
-        }
-
-        let store_path = self.store_path(store_name)?;
         let (shard, _) = self.get_or_create_store(store_name, vector_size)?;
-        write_store_display_name(&store_path, &data_source_name)?;
         let points = points
             .into_iter()
             .map(to_qdrant_edge_point)
@@ -738,7 +728,6 @@ fn to_qdrant_edge_point(point: QdrantEdgeStoragePoint) -> QdrantEdgeResult<qdran
         Vectors::new_named([(VECTOR_NAME, point.vector)]),
         json!({
             "data_source_id": point.data_source_id,
-            "data_source_name": point.data_source_name,
             "data_source_type": point.data_source_type,
             "chunk_id": point.chunk_id,
             "parent_file_id": point.parent_file_id,
@@ -765,7 +754,6 @@ fn to_qdrant_edge_search_result(point: ScoredPoint) -> QdrantEdgeSearchResult {
         point_id: point_id_to_string(point.id),
         score: point.score,
         data_source_id: payload_string(&payload, "data_source_id"),
-        data_source_name: payload_string(&payload, "data_source_name"),
         data_source_type: payload_string(&payload, "data_source_type"),
         chunk_id: payload_string(&payload, "chunk_id"),
         parent_file_id: payload_string(&payload, "parent_file_id"),