mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-13 09:33:48 +00:00
Removed the denormalized data source name from the RAG storage (#977)
Build and Release / Determine run mode (push) Waiting to run
Build and Release / Read metadata (push) Blocked by required conditions
Build and Release / Sync Flatpak repo (push) Blocked by required conditions
Build and Release / Collect Flatpak artifacts (push) Blocked by required conditions
Build and Release / Verify (push) Waiting to run
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions
Build and Release / Determine run mode (push) Waiting to run
Build and Release / Read metadata (push) Blocked by required conditions
Build and Release / Sync Flatpak repo (push) Blocked by required conditions
Build and Release / Collect Flatpak artifacts (push) Blocked by required conditions
Build and Release / Verify (push) Waiting to run
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions
This commit is contained in:
1 parent
186f10cee2
commit
b8d971c586
16 files changed
+44
-49
No files matched your search
@@ -4,8 +4,6 @@ internal sealed class EmbeddingStateDataSourceEntity
|
||||
{
|
||||
public string DataSourceId { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceName { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceType { get; set; } = string.Empty;
|
||||
|
||||
public string EmbeddingProviderId { get; set; } = string.Empty;
|
||||
|
||||
@@ -8,7 +8,6 @@ public abstract class IndexStoreClient(string name, string path) : DatabaseClien
|
||||
|
||||
public abstract Task UpsertDataSourceAsync(
|
||||
string dataSourceId,
|
||||
string dataSourceName,
|
||||
string dataSourceType,
|
||||
string embeddingProviderId,
|
||||
string embeddingSignature,
|
||||
|
||||
@@ -29,7 +29,6 @@ internal sealed class IndexStoreDbContext(DbContextOptions<IndexStoreDbContext>
|
||||
entity.HasKey(dataSource => dataSource.DataSourceId);
|
||||
|
||||
entity.Property(dataSource => dataSource.DataSourceId).HasColumnName("data_source_id");
|
||||
entity.Property(dataSource => dataSource.DataSourceName).HasColumnName("data_source_name").IsRequired();
|
||||
entity.Property(dataSource => dataSource.DataSourceType).HasColumnName("data_source_type").IsRequired();
|
||||
entity.Property(dataSource => dataSource.EmbeddingProviderId).HasColumnName("embedding_provider_id").IsRequired();
|
||||
entity.Property(dataSource => dataSource.EmbeddingSignature).HasColumnName("embedding_signature").IsRequired();
|
||||
|
||||
@@ -9,6 +9,7 @@ internal static class IndexStoreSchemaMigrator
|
||||
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))]
|
||||
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.PermanentIndexingFailures))]
|
||||
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropFileConfidenceLevel))]
|
||||
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropDataSourceName))]
|
||||
public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token)
|
||||
{
|
||||
await context.Database.MigrateAsync(token);
|
||||
|
||||
@@ -4,7 +4,6 @@ public sealed record IndexStoreSearchResult(
|
||||
string ChunkId,
|
||||
string ParentFileId,
|
||||
string DataSourceId,
|
||||
string DataSourceName,
|
||||
string DataSourceType,
|
||||
string AbsolutePath,
|
||||
string FileName,
|
||||
|
||||
@@ -8,8 +8,6 @@ internal sealed class IndexStoreSearchResultEntity
|
||||
|
||||
public string DataSourceId { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceName { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceType { get; set; } = string.Empty;
|
||||
|
||||
public string AbsolutePath { get; set; } = string.Empty;
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
#nullable disable
|
||||
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
namespace AIStudio.Tools.Databases.IndexStore.Migrations;
|
||||
|
||||
/// <summary>
|
||||
/// Drops the copy of the data source name which the index kept next to each indexed data source.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The name a user gives a data source lives in the configuration and is read from there whenever
|
||||
/// it is needed. The copy here was only ever written, never read, and a copy of a name people are
|
||||
/// free to change can do nothing but go stale.
|
||||
/// </remarks>
|
||||
[DbContext(typeof(IndexStoreDbContext))]
|
||||
[Migration("20260916000000_DropDataSourceName")]
|
||||
public partial class DropDataSourceName : Migration
|
||||
{
|
||||
/// <remarks>
|
||||
/// The column goes through raw SQL instead of DropColumn on purpose. The SQLite provider answers
|
||||
/// DropColumn by rebuilding the table, and dropping the old data_sources table would let the
|
||||
/// cascade of the foreign key in embedded_files take every indexed file and chunk with it. A
|
||||
/// native ALTER TABLE ... DROP COLUMN leaves the table itself alone. No index names this column,
|
||||
/// so nothing has to be dropped first.
|
||||
/// </remarks>
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.Sql("ALTER TABLE data_sources DROP COLUMN data_source_name;");
|
||||
}
|
||||
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.Sql("ALTER TABLE data_sources ADD COLUMN data_source_name TEXT NOT NULL DEFAULT '';");
|
||||
}
|
||||
}
|
||||
-9
@@ -23,11 +23,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("data_source_id");
|
||||
|
||||
entity.Property<string>("DataSourceName")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("data_source_name");
|
||||
|
||||
entity.Property<string>("DataSourceType")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
@@ -274,10 +269,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<string>("DataSourceName")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<string>("DataSourceType")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
@@ -28,7 +28,6 @@ public sealed class NoIndexStoreClient(string name, string? unavailableReason, D
|
||||
|
||||
public override Task UpsertDataSourceAsync(
|
||||
string dataSourceId,
|
||||
string dataSourceName,
|
||||
string dataSourceType,
|
||||
string embeddingProviderId,
|
||||
string embeddingSignature,
|
||||
|
||||
+1
-6
@@ -118,7 +118,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
||||
|
||||
public override async Task UpsertDataSourceAsync(
|
||||
string dataSourceId,
|
||||
string dataSourceName,
|
||||
string dataSourceType,
|
||||
string embeddingProviderId,
|
||||
string embeddingSignature,
|
||||
@@ -137,7 +136,7 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
||||
context.DataSources.Add(dataSource);
|
||||
}
|
||||
|
||||
ApplyDataSource(dataSource, dataSourceName, dataSourceType, embeddingProviderId, embeddingSignature, sourceHash, vectorSize);
|
||||
ApplyDataSource(dataSource, dataSourceType, embeddingProviderId, embeddingSignature, sourceHash, vectorSize);
|
||||
await context.SaveChangesAsync(token);
|
||||
}
|
||||
|
||||
@@ -287,7 +286,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
||||
c.chunk_id AS ChunkId,
|
||||
c.parent_file_id AS ParentFileId,
|
||||
ds.data_source_id AS DataSourceId,
|
||||
ds.data_source_name AS DataSourceName,
|
||||
ds.data_source_type AS DataSourceType,
|
||||
f.absolute_path AS AbsolutePath,
|
||||
f.file_name AS FileName,
|
||||
@@ -363,14 +361,12 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
||||
|
||||
private static void ApplyDataSource(
|
||||
EmbeddingStateDataSourceEntity dataSource,
|
||||
string dataSourceName,
|
||||
string dataSourceType,
|
||||
string embeddingProviderId,
|
||||
string embeddingSignature,
|
||||
string sourceHash,
|
||||
int vectorSize)
|
||||
{
|
||||
dataSource.DataSourceName = dataSourceName;
|
||||
dataSource.DataSourceType = dataSourceType;
|
||||
dataSource.EmbeddingProviderId = embeddingProviderId;
|
||||
dataSource.EmbeddingSignature = embeddingSignature;
|
||||
@@ -426,7 +422,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
||||
result.ChunkId,
|
||||
result.ParentFileId,
|
||||
result.DataSourceId,
|
||||
result.DataSourceName,
|
||||
result.DataSourceType,
|
||||
result.AbsolutePath,
|
||||
result.FileName,
|
||||
|
||||
@@ -4,7 +4,6 @@ public sealed record VectorSearchResult(
|
||||
string PointId,
|
||||
double Score,
|
||||
string DataSourceId,
|
||||
string DataSourceName,
|
||||
string DataSourceType,
|
||||
string ChunkId,
|
||||
string ParentFileId,
|
||||
|
||||
@@ -4,7 +4,6 @@ public sealed record VectorStoragePoint(
|
||||
string PointId,
|
||||
IReadOnlyList<float> Vector,
|
||||
string DataSourceId,
|
||||
string DataSourceName,
|
||||
string DataSourceType,
|
||||
string ChunkId,
|
||||
string ParentFileId,
|
||||
|
||||
Reference in new issue
Block a user