mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-18 10:39:07 +00:00
Removed the data source name from the RAG index database
This commit is contained in:
11 files changed
+38
-24
No files matched your search
@@ -4,8 +4,6 @@ internal sealed class EmbeddingStateDataSourceEntity
|
|||||||
{
|
{
|
||||||
public string DataSourceId { get; set; } = string.Empty;
|
public string DataSourceId { get; set; } = string.Empty;
|
||||||
|
|
||||||
public string DataSourceName { get; set; } = string.Empty;
|
|
||||||
|
|
||||||
public string DataSourceType { get; set; } = string.Empty;
|
public string DataSourceType { get; set; } = string.Empty;
|
||||||
|
|
||||||
public string EmbeddingProviderId { get; set; } = string.Empty;
|
public string EmbeddingProviderId { get; set; } = string.Empty;
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ public abstract class IndexStoreClient(string name, string path) : DatabaseClien
|
|||||||
|
|
||||||
public abstract Task UpsertDataSourceAsync(
|
public abstract Task UpsertDataSourceAsync(
|
||||||
string dataSourceId,
|
string dataSourceId,
|
||||||
string dataSourceName,
|
|
||||||
string dataSourceType,
|
string dataSourceType,
|
||||||
string embeddingProviderId,
|
string embeddingProviderId,
|
||||||
string embeddingSignature,
|
string embeddingSignature,
|
||||||
|
|||||||
@@ -29,7 +29,6 @@ internal sealed class IndexStoreDbContext(DbContextOptions<IndexStoreDbContext>
|
|||||||
entity.HasKey(dataSource => dataSource.DataSourceId);
|
entity.HasKey(dataSource => dataSource.DataSourceId);
|
||||||
|
|
||||||
entity.Property(dataSource => dataSource.DataSourceId).HasColumnName("data_source_id");
|
entity.Property(dataSource => dataSource.DataSourceId).HasColumnName("data_source_id");
|
||||||
entity.Property(dataSource => dataSource.DataSourceName).HasColumnName("data_source_name").IsRequired();
|
|
||||||
entity.Property(dataSource => dataSource.DataSourceType).HasColumnName("data_source_type").IsRequired();
|
entity.Property(dataSource => dataSource.DataSourceType).HasColumnName("data_source_type").IsRequired();
|
||||||
entity.Property(dataSource => dataSource.EmbeddingProviderId).HasColumnName("embedding_provider_id").IsRequired();
|
entity.Property(dataSource => dataSource.EmbeddingProviderId).HasColumnName("embedding_provider_id").IsRequired();
|
||||||
entity.Property(dataSource => dataSource.EmbeddingSignature).HasColumnName("embedding_signature").IsRequired();
|
entity.Property(dataSource => dataSource.EmbeddingSignature).HasColumnName("embedding_signature").IsRequired();
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ internal static class IndexStoreSchemaMigrator
|
|||||||
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))]
|
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))]
|
||||||
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.PermanentIndexingFailures))]
|
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.PermanentIndexingFailures))]
|
||||||
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropFileConfidenceLevel))]
|
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropFileConfidenceLevel))]
|
||||||
|
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropDataSourceName))]
|
||||||
public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token)
|
public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token)
|
||||||
{
|
{
|
||||||
await context.Database.MigrateAsync(token);
|
await context.Database.MigrateAsync(token);
|
||||||
|
|||||||
@@ -4,7 +4,6 @@ public sealed record IndexStoreSearchResult(
|
|||||||
string ChunkId,
|
string ChunkId,
|
||||||
string ParentFileId,
|
string ParentFileId,
|
||||||
string DataSourceId,
|
string DataSourceId,
|
||||||
string DataSourceName,
|
|
||||||
string DataSourceType,
|
string DataSourceType,
|
||||||
string AbsolutePath,
|
string AbsolutePath,
|
||||||
string FileName,
|
string FileName,
|
||||||
|
|||||||
@@ -8,8 +8,6 @@ internal sealed class IndexStoreSearchResultEntity
|
|||||||
|
|
||||||
public string DataSourceId { get; set; } = string.Empty;
|
public string DataSourceId { get; set; } = string.Empty;
|
||||||
|
|
||||||
public string DataSourceName { get; set; } = string.Empty;
|
|
||||||
|
|
||||||
public string DataSourceType { get; set; } = string.Empty;
|
public string DataSourceType { get; set; } = string.Empty;
|
||||||
|
|
||||||
public string AbsolutePath { get; set; } = string.Empty;
|
public string AbsolutePath { get; set; } = string.Empty;
|
||||||
|
|||||||
+36
@@ -0,0 +1,36 @@
|
|||||||
|
#nullable disable
|
||||||
|
|
||||||
|
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||||
|
using Microsoft.EntityFrameworkCore.Migrations;
|
||||||
|
|
||||||
|
namespace AIStudio.Tools.Databases.IndexStore.Migrations;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Drops the copy of the data source name which the index kept next to each indexed data source.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// The name a user gives a data source lives in the configuration and is read from there whenever
|
||||||
|
/// it is needed. The copy here was only ever written, never read, and a copy of a name people are
|
||||||
|
/// free to change can do nothing but go stale.
|
||||||
|
/// </remarks>
|
||||||
|
[DbContext(typeof(IndexStoreDbContext))]
|
||||||
|
[Migration("20260916000000_DropDataSourceName")]
|
||||||
|
public partial class DropDataSourceName : Migration
|
||||||
|
{
|
||||||
|
/// <remarks>
|
||||||
|
/// The column goes through raw SQL instead of DropColumn on purpose. The SQLite provider answers
|
||||||
|
/// DropColumn by rebuilding the table, and dropping the old data_sources table would let the
|
||||||
|
/// cascade of the foreign key in embedded_files take every indexed file and chunk with it. A
|
||||||
|
/// native ALTER TABLE ... DROP COLUMN leaves the table itself alone. No index names this column,
|
||||||
|
/// so nothing has to be dropped first.
|
||||||
|
/// </remarks>
|
||||||
|
protected override void Up(MigrationBuilder migrationBuilder)
|
||||||
|
{
|
||||||
|
migrationBuilder.Sql("ALTER TABLE data_sources DROP COLUMN data_source_name;");
|
||||||
|
}
|
||||||
|
|
||||||
|
protected override void Down(MigrationBuilder migrationBuilder)
|
||||||
|
{
|
||||||
|
migrationBuilder.Sql("ALTER TABLE data_sources ADD COLUMN data_source_name TEXT NOT NULL DEFAULT '';");
|
||||||
|
}
|
||||||
|
}
|
||||||
-9
@@ -23,11 +23,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
|
|||||||
.HasColumnType("TEXT")
|
.HasColumnType("TEXT")
|
||||||
.HasColumnName("data_source_id");
|
.HasColumnName("data_source_id");
|
||||||
|
|
||||||
entity.Property<string>("DataSourceName")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT")
|
|
||||||
.HasColumnName("data_source_name");
|
|
||||||
|
|
||||||
entity.Property<string>("DataSourceType")
|
entity.Property<string>("DataSourceType")
|
||||||
.IsRequired()
|
.IsRequired()
|
||||||
.HasColumnType("TEXT")
|
.HasColumnType("TEXT")
|
||||||
@@ -274,10 +269,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
|
|||||||
.IsRequired()
|
.IsRequired()
|
||||||
.HasColumnType("TEXT");
|
.HasColumnType("TEXT");
|
||||||
|
|
||||||
entity.Property<string>("DataSourceName")
|
|
||||||
.IsRequired()
|
|
||||||
.HasColumnType("TEXT");
|
|
||||||
|
|
||||||
entity.Property<string>("DataSourceType")
|
entity.Property<string>("DataSourceType")
|
||||||
.IsRequired()
|
.IsRequired()
|
||||||
.HasColumnType("TEXT");
|
.HasColumnType("TEXT");
|
||||||
|
|||||||
@@ -28,7 +28,6 @@ public sealed class NoIndexStoreClient(string name, string? unavailableReason, D
|
|||||||
|
|
||||||
public override Task UpsertDataSourceAsync(
|
public override Task UpsertDataSourceAsync(
|
||||||
string dataSourceId,
|
string dataSourceId,
|
||||||
string dataSourceName,
|
|
||||||
string dataSourceType,
|
string dataSourceType,
|
||||||
string embeddingProviderId,
|
string embeddingProviderId,
|
||||||
string embeddingSignature,
|
string embeddingSignature,
|
||||||
|
|||||||
+1
-6
@@ -118,7 +118,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
|||||||
|
|
||||||
public override async Task UpsertDataSourceAsync(
|
public override async Task UpsertDataSourceAsync(
|
||||||
string dataSourceId,
|
string dataSourceId,
|
||||||
string dataSourceName,
|
|
||||||
string dataSourceType,
|
string dataSourceType,
|
||||||
string embeddingProviderId,
|
string embeddingProviderId,
|
||||||
string embeddingSignature,
|
string embeddingSignature,
|
||||||
@@ -137,7 +136,7 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
|||||||
context.DataSources.Add(dataSource);
|
context.DataSources.Add(dataSource);
|
||||||
}
|
}
|
||||||
|
|
||||||
ApplyDataSource(dataSource, dataSourceName, dataSourceType, embeddingProviderId, embeddingSignature, sourceHash, vectorSize);
|
ApplyDataSource(dataSource, dataSourceType, embeddingProviderId, embeddingSignature, sourceHash, vectorSize);
|
||||||
await context.SaveChangesAsync(token);
|
await context.SaveChangesAsync(token);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -287,7 +286,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
|||||||
c.chunk_id AS ChunkId,
|
c.chunk_id AS ChunkId,
|
||||||
c.parent_file_id AS ParentFileId,
|
c.parent_file_id AS ParentFileId,
|
||||||
ds.data_source_id AS DataSourceId,
|
ds.data_source_id AS DataSourceId,
|
||||||
ds.data_source_name AS DataSourceName,
|
|
||||||
ds.data_source_type AS DataSourceType,
|
ds.data_source_type AS DataSourceType,
|
||||||
f.absolute_path AS AbsolutePath,
|
f.absolute_path AS AbsolutePath,
|
||||||
f.file_name AS FileName,
|
f.file_name AS FileName,
|
||||||
@@ -363,14 +361,12 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
|||||||
|
|
||||||
private static void ApplyDataSource(
|
private static void ApplyDataSource(
|
||||||
EmbeddingStateDataSourceEntity dataSource,
|
EmbeddingStateDataSourceEntity dataSource,
|
||||||
string dataSourceName,
|
|
||||||
string dataSourceType,
|
string dataSourceType,
|
||||||
string embeddingProviderId,
|
string embeddingProviderId,
|
||||||
string embeddingSignature,
|
string embeddingSignature,
|
||||||
string sourceHash,
|
string sourceHash,
|
||||||
int vectorSize)
|
int vectorSize)
|
||||||
{
|
{
|
||||||
dataSource.DataSourceName = dataSourceName;
|
|
||||||
dataSource.DataSourceType = dataSourceType;
|
dataSource.DataSourceType = dataSourceType;
|
||||||
dataSource.EmbeddingProviderId = embeddingProviderId;
|
dataSource.EmbeddingProviderId = embeddingProviderId;
|
||||||
dataSource.EmbeddingSignature = embeddingSignature;
|
dataSource.EmbeddingSignature = embeddingSignature;
|
||||||
@@ -426,7 +422,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
|||||||
result.ChunkId,
|
result.ChunkId,
|
||||||
result.ParentFileId,
|
result.ParentFileId,
|
||||||
result.DataSourceId,
|
result.DataSourceId,
|
||||||
result.DataSourceName,
|
|
||||||
result.DataSourceType,
|
result.DataSourceType,
|
||||||
result.AbsolutePath,
|
result.AbsolutePath,
|
||||||
result.FileName,
|
result.FileName,
|
||||||
|
|||||||
@@ -1273,7 +1273,6 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
|
|
||||||
await indexStore.UpsertDataSourceAsync(
|
await indexStore.UpsertDataSourceAsync(
|
||||||
dataSource.Id,
|
dataSource.Id,
|
||||||
dataSource.Name,
|
|
||||||
dataSource.Type.ToString(),
|
dataSource.Type.ToString(),
|
||||||
manifest.EmbeddingProviderId,
|
manifest.EmbeddingProviderId,
|
||||||
manifest.EmbeddingSignature,
|
manifest.EmbeddingSignature,
|
||||||
|
|||||||
Reference in new issue
Block a user